From 194267eb498bf4afbcc3f2307308f5b2c8a37965 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 00:31:16 +0530 Subject: [PATCH 001/147] feat: checkpoint phase 1 foundation hardening --- CHANGELOG.md | 39 +++ cmd/stacyvm-agent/main.go | 2 + cmd/stacyvm-agent/main_unsupported.go | 9 + cmd/stacyvm/cmd_serve.go | 3 + docs/provider-contract.md | 80 ++++++ internal/api/routes/environments.go | 54 +--- internal/api/routes/environments_test.go | 34 +++ internal/api/routes/errors.go | 30 +++ internal/api/routes/providers.go | 4 +- internal/api/routes/sandboxes.go | 81 +----- internal/api/routes/sandboxes_test.go | 17 +- internal/api/routes/templates.go | 35 +-- internal/api/routes/templates_test.go | 75 ++++++ internal/httputil/response.go | 14 +- internal/orchestrator/errors.go | 12 + internal/orchestrator/manager.go | 226 ++++++++++++++++- internal/orchestrator/manager_test.go | 129 ++++++++++ internal/providers/custom.go | 51 +++- internal/providers/custom_conformance_test.go | 240 ++++++++++++++++++ internal/providers/docker.go | 132 +++++++++- internal/providers/docker_test.go | 51 +++- internal/providers/errors.go | 42 +++ internal/providers/firecracker.go | 34 ++- internal/providers/firecracker_test.go | 54 ++++ internal/providers/mock.go | 11 +- internal/providers/mock_test.go | 7 + internal/providers/proot.go | 20 +- internal/providers/proot_test.go | 15 ++ internal/providers/provider.go | 20 ++ .../providers/provider_conformance_test.go | 157 ++++++++++++ internal/providers/registry.go | 2 +- internal/store/errors.go | 33 +++ internal/store/sqlite.go | 37 ++- internal/store/sqlite_test.go | 17 ++ 34 files changed, 1557 insertions(+), 210 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 cmd/stacyvm-agent/main_unsupported.go create mode 100644 docs/provider-contract.md create mode 100644 internal/api/routes/errors.go create mode 100644 internal/api/routes/templates_test.go create mode 100644 internal/orchestrator/errors.go create mode 100644 internal/providers/custom_conformance_test.go create mode 100644 internal/providers/errors.go create mode 100644 internal/providers/provider_conformance_test.go create mode 100644 internal/store/errors.go diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b0eaaf8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,39 @@ +# Changelog + +## Phase 1 Foundation Hardening - 2026-05-08 + +This checkpoint closes the Phase 1 reliability and production-readiness foundation. + +### Added + +- Provider contract documentation in `docs/provider-contract.md`. +- Typed provider errors for sandbox lifecycle, provider availability, exec timeout, and resource-limit failures. +- Typed store errors for not-found and conflict cases. +- Shared API route error mapping with explicit `404`, `408`, `429`, and `503` responses. +- Provider conformance harness covering lifecycle, exec, streaming exec, and file operations. +- Mock, Docker, Custom, PRoot, and Firecracker conformance coverage, with PRoot and Firecracker gated on platform dependencies. +- Startup reconciliation that refreshes persisted sandbox state from provider runtime state. +- Docker runtime inventory and adoption for StacyVM containers missing from SQLite after process restart. +- Streaming exec timeout handling that emits an explicit stderr timeout chunk. +- Non-Linux `stacyvm-agent` stub so repository builds work on macOS while the real agent remains Linux-only. + +### Changed + +- Sandbox, template, environment, and provider routes now use typed errors instead of string matching. +- Docker sandboxes now include richer `stacyvm.*` labels for reconciliation and metadata recovery. +- Docker missing-container paths now map to `ErrSandboxNotFound`. +- Manager `Exec` and `ExecStream` now consistently honor caller-supplied timeouts. +- Provider comments now point implementers to the documented contract and conformance tests. + +### Verified + +- `make test` +- `make build` +- `cd web && npm run build` +- Docker provider conformance and runtime inventory tests with Docker daemon access + +### Platform Notes + +- Firecracker conformance is available on Linux hosts with `/dev/kvm`, Firecracker, kernel, rootfs, and agent paths configured. +- PRoot conformance is available when `proot` and a usable rootfs are installed. +- Local sandboxed test runs still need permission to bind `httptest` sockets for the full integration suite. diff --git a/cmd/stacyvm-agent/main.go b/cmd/stacyvm-agent/main.go index 1a105f7..0d4cbdf 100644 --- a/cmd/stacyvm-agent/main.go +++ b/cmd/stacyvm-agent/main.go @@ -1,3 +1,5 @@ +//go:build linux + // stacyvm-agent runs inside a Firecracker VM and serves exec/file requests // over vsock. Build with: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/stacyvm-agent ./cmd/stacyvm-agent package main diff --git a/cmd/stacyvm-agent/main_unsupported.go b/cmd/stacyvm-agent/main_unsupported.go new file mode 100644 index 0000000..bd90e76 --- /dev/null +++ b/cmd/stacyvm-agent/main_unsupported.go @@ -0,0 +1,9 @@ +//go:build !linux + +package main + +import "fmt" + +func main() { + fmt.Println("stacyvm-agent only runs on Linux") +} diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index 52c0763..f409b3c 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -171,6 +171,9 @@ func runServe() error { Pool: cfg.Pool, PreviewDomain: cfg.Server.PreviewDomain, }) + if err := mgr.Reconcile(context.Background()); err != nil { + return err + } mgr.Start() mgr.InitVMPool() defer mgr.Stop() diff --git a/docs/provider-contract.md b/docs/provider-contract.md new file mode 100644 index 0000000..5c43696 --- /dev/null +++ b/docs/provider-contract.md @@ -0,0 +1,80 @@ +# StacyVM Provider Contract + +Providers are the runtime boundary for StacyVM. The orchestrator, API, SDKs, and +dashboard must be able to treat every provider the same way, whether the runtime +is Docker, Firecracker, PRoot, E2B, a custom HTTP service, or a test double. + +The Go source of truth is `internal/providers/provider.go`. This document +spells out the behavioral contract expected by the shared provider conformance +tests. + +## Lifecycle + +- `Name` returns a stable unique identifier used in config, API responses, and + persisted sandbox records. +- `Spawn` creates a running sandbox and returns a non-empty sandbox ID. +- `Status` returns `running` for an active sandbox. +- `Destroy` tears down runtime resources. Providers may return `ErrSandboxNotFound` + if the runtime object is already gone. +- After destroy, exec and file operations must fail with either + `ErrSandboxDestroyed` or `ErrSandboxNotFound`. +- Provider implementations should be best-effort idempotent around external + cleanup. The orchestrator may call destroy during TTL reaping, explicit API + requests, or recovery flows. + +## Exec + +- `Exec` runs the requested command and returns stdout, stderr, and exit code. +- A nonzero command exit is not a provider error. It must return an `ExecResult` + with the nonzero exit code. +- Provider errors are reserved for runtime failures: missing sandbox, provider + unavailable, transport failure, timeout, or invalid provider state. +- Context cancellation and deadlines should be honored. Deadline expiration + should map to `ErrExecTimeout` where the provider can detect it. +- `ExecStream` emits stdout/stderr chunks and closes its channel when the command + finishes or the stream fails. + +## Files + +- File paths are interpreted inside the sandbox filesystem. +- Providers must support write, read, list, delete, move, chmod, stat, and glob. +- `WriteFile` should create missing parent directories when the runtime can do so. +- File reads return an `io.ReadCloser`; callers own closing it. +- Missing sandbox errors should use `ErrSandboxNotFound` or `ErrSandboxDestroyed`. +- Missing file behavior can remain provider-specific unless an API route maps it + into a user-facing error. + +## Health + +- `Healthy` should be fast and side-effect free. +- It should return false when the runtime dependency is unreachable, for example + Docker daemon unavailable, PRoot binary missing, Firecracker binary missing, or + a custom HTTP backend failing its health endpoint. + +## Typed Errors + +Providers should use these sentinel errors from `internal/providers/errors.go`: + +- `ErrSandboxNotFound` +- `ErrSandboxDestroyed` +- `ErrProviderNotFound` +- `ErrProviderUnavailable` +- `ErrExecTimeout` +- `ErrResourceLimit` + +Wrapping is encouraged with `fmt.Errorf("context: %w", err)` so callers can use +`errors.Is`. + +## Conformance Tests + +Shared conformance tests live in +`internal/providers/provider_conformance_test.go`. + +Current coverage: + +- Mock provider +- Docker provider, when Docker is available +- Custom provider through an in-process fake HTTP backend + +Future providers should be wired into the same harness whenever their runtime +dependencies are available. diff --git a/internal/api/routes/environments.go b/internal/api/routes/environments.go index 5386f0e..dea9a85 100644 --- a/internal/api/routes/environments.go +++ b/internal/api/routes/environments.go @@ -181,11 +181,7 @@ func (e *EnvironmentRoutes) CreateSpec(w http.ResponseWriter, r *http.Request) { UpdatedAt: now, } if err := e.store.CreateEnvironmentSpec(r.Context(), rec); err != nil { - if strings.Contains(err.Error(), "UNIQUE constraint") { - httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, "spec name already exists for this owner") - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -216,11 +212,7 @@ func (e *EnvironmentRoutes) GetSpec(w http.ResponseWriter, r *http.Request) { specID := chi.URLParam(r, "specID") rec, err := e.store.GetEnvironmentSpec(r.Context(), specID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, toSpecResponse(rec)) @@ -230,11 +222,7 @@ func (e *EnvironmentRoutes) Suggestions(w http.ResponseWriter, r *http.Request) specID := chi.URLParam(r, "specID") rec, err := e.store.GetEnvironmentSpec(r.Context(), specID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -265,11 +253,7 @@ func (e *EnvironmentRoutes) StartBuild(w http.ResponseWriter, r *http.Request) { spec, err := e.store.GetEnvironmentSpec(r.Context(), req.SpecID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -327,11 +311,7 @@ func (e *EnvironmentRoutes) GetBuild(w http.ResponseWriter, r *http.Request) { buildID := chi.URLParam(r, "buildID") resp, err := e.getBuildResponse(r, buildID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, resp) @@ -394,11 +374,7 @@ func (e *EnvironmentRoutes) CancelBuild(w http.ResponseWriter, r *http.Request) buildID := chi.URLParam(r, "buildID") build, err := e.store.GetEnvironmentBuild(r.Context(), buildID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -432,11 +408,7 @@ func (e *EnvironmentRoutes) SpawnConfig(w http.ResponseWriter, r *http.Request) buildID := chi.URLParam(r, "buildID") build, err := e.store.GetEnvironmentBuild(r.Context(), buildID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -507,11 +479,7 @@ func (e *EnvironmentRoutes) SaveRegistryConnection(w http.ResponseWriter, r *htt UpdatedAt: now, } if err := e.store.SaveRegistryConnection(r.Context(), rec); err != nil { - if strings.Contains(err.Error(), "UNIQUE constraint") { - httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, "registry connection already exists") - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, toRegistryConnectionResponse(rec)) @@ -538,11 +506,7 @@ func (e *EnvironmentRoutes) ListRegistryConnections(w http.ResponseWriter, r *ht func (e *EnvironmentRoutes) DeleteRegistryConnection(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "connectionID") if err := e.store.DeleteRegistryConnection(r.Context(), id); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) diff --git a/internal/api/routes/environments_test.go b/internal/api/routes/environments_test.go index 1f4e5e5..cac5555 100644 --- a/internal/api/routes/environments_test.go +++ b/internal/api/routes/environments_test.go @@ -109,6 +109,40 @@ func TestEnvironmentFlow_CreateBuildSpawnConfig(t *testing.T) { } } +func TestEnvironmentCreateSpecDuplicateReturnsConflict(t *testing.T) { + r := setupEnvTestRouter(t) + + specReq := map[string]any{ + "owner_id": "user-dupe", + "name": "same-name", + "base_image": "python:3.12-slim", + } + body, _ := json.Marshal(specReq) + for i := 0; i < 2; i++ { + req := httptest.NewRequest("POST", "/api/v1/environments/specs", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if i == 0 && w.Code != http.StatusCreated { + t.Fatalf("first create: expected 201, got %d: %s", w.Code, w.Body.String()) + } + if i == 1 && w.Code != http.StatusConflict { + t.Fatalf("second create: expected 409, got %d: %s", w.Code, w.Body.String()) + } + } +} + +func TestEnvironmentGetSpecMissingReturnsNotFound(t *testing.T) { + r := setupEnvTestRouter(t) + + req := httptest.NewRequest("GET", "/api/v1/environments/specs/spec-nope", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusNotFound { + t.Fatalf("expected 404, got %d: %s", w.Code, w.Body.String()) + } +} + func TestEnvironmentBuildCancel(t *testing.T) { r := setupEnvTestRouter(t) diff --git a/internal/api/routes/errors.go b/internal/api/routes/errors.go new file mode 100644 index 0000000..4992017 --- /dev/null +++ b/internal/api/routes/errors.go @@ -0,0 +1,30 @@ +package routes + +import ( + "errors" + "net/http" + + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/store" +) + +func writeRouteError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, orchestrator.ErrSandboxNotFound), + errors.Is(err, orchestrator.ErrSandboxDestroyed), + errors.Is(err, orchestrator.ErrProviderNotFound), + errors.Is(err, store.ErrNotFound): + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) + case errors.Is(err, store.ErrConflict): + httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, err.Error()) + case errors.Is(err, orchestrator.ErrExecTimeout): + httputil.WriteError(w, http.StatusRequestTimeout, httputil.CodeTimeout, err.Error()) + case errors.Is(err, orchestrator.ErrResourceLimit): + httputil.WriteError(w, http.StatusTooManyRequests, httputil.CodeResourceLimit, err.Error()) + case errors.Is(err, orchestrator.ErrProviderUnavailable): + httputil.WriteError(w, http.StatusServiceUnavailable, httputil.CodeUnavailable, err.Error()) + default: + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + } +} diff --git a/internal/api/routes/providers.go b/internal/api/routes/providers.go index 1752085..35b72e0 100644 --- a/internal/api/routes/providers.go +++ b/internal/api/routes/providers.go @@ -4,9 +4,9 @@ import ( "context" "net/http" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/providers" + "github.com/go-chi/chi/v5" ) type sandboxCounter interface { @@ -117,7 +117,7 @@ func (p *ProviderRoutes) Detail(w http.ResponseWriter, r *http.Request) { prov, err := p.registry.Get(name) if err != nil { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "provider not found") + writeRouteError(w, err) return } diff --git a/internal/api/routes/sandboxes.go b/internal/api/routes/sandboxes.go index ea0ca8f..382a150 100644 --- a/internal/api/routes/sandboxes.go +++ b/internal/api/routes/sandboxes.go @@ -4,12 +4,11 @@ import ( "encoding/json" "net/http" "strconv" - "strings" "time" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/go-chi/chi/v5" "nhooyr.io/websocket" "nhooyr.io/websocket/wsjson" ) @@ -118,11 +117,7 @@ func (s *SandboxRoutes) Get(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") sb, err := s.manager.Get(r.Context(), id) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, sb) @@ -143,11 +138,7 @@ func (s *SandboxRoutes) Get(w http.ResponseWriter, r *http.Request) { func (s *SandboxRoutes) Destroy(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") if err := s.manager.Destroy(r.Context(), id); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "destroyed"}) @@ -196,11 +187,7 @@ func (s *SandboxRoutes) Extend(w http.ResponseWriter, r *http.Request) { sb, err := s.manager.ExtendTTL(r.Context(), id, duration) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -236,11 +223,7 @@ func (s *SandboxRoutes) Exec(w http.ResponseWriter, r *http.Request) { result, err := s.manager.Exec(r.Context(), id, req) if err != nil { - if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "destroyed") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, result) @@ -292,11 +275,7 @@ func (s *SandboxRoutes) WriteFile(w http.ResponseWriter, r *http.Request) { } if err := s.manager.WriteFile(r.Context(), id, req); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "written"}) @@ -326,11 +305,7 @@ func (s *SandboxRoutes) ReadFile(w http.ResponseWriter, r *http.Request) { data, err := s.manager.ReadFile(r.Context(), id, path) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -361,11 +336,7 @@ func (s *SandboxRoutes) ListFiles(w http.ResponseWriter, r *http.Request) { files, err := s.manager.ListFiles(r.Context(), id, path) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, files) @@ -386,11 +357,7 @@ func (s *SandboxRoutes) DeleteFile(w http.ResponseWriter, r *http.Request) { Path: path, Recursive: recursive, }); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) @@ -406,11 +373,7 @@ func (s *SandboxRoutes) MoveFile(w http.ResponseWriter, r *http.Request) { } if err := s.manager.MoveFile(r.Context(), id, req); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "moved"}) @@ -426,11 +389,7 @@ func (s *SandboxRoutes) ChmodFile(w http.ResponseWriter, r *http.Request) { } if err := s.manager.ChmodFile(r.Context(), id, req); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "chmod applied"}) @@ -447,11 +406,7 @@ func (s *SandboxRoutes) StatFile(w http.ResponseWriter, r *http.Request) { fi, err := s.manager.StatFile(r.Context(), id, path) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, fi) @@ -468,11 +423,7 @@ func (s *SandboxRoutes) GlobFiles(w http.ResponseWriter, r *http.Request) { matches, err := s.manager.GlobFiles(r.Context(), id, pattern) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, matches) @@ -534,11 +485,7 @@ func (s *SandboxRoutes) ConsoleLog(w http.ResponseWriter, r *http.Request) { log, err := s.manager.ConsoleLog(r.Context(), id, lines) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, log) diff --git a/internal/api/routes/sandboxes_test.go b/internal/api/routes/sandboxes_test.go index b319158..d72e547 100644 --- a/internal/api/routes/sandboxes_test.go +++ b/internal/api/routes/sandboxes_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/orchestrator" "github.com/StacyOs/stacyvm/internal/providers" "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" "github.com/rs/zerolog" ) @@ -128,6 +128,21 @@ func TestExecInSandbox(t *testing.T) { } } +func TestExecInSandbox_Timeout(t *testing.T) { + r, _ := setupTestRouter(t) + + sbID := createTestSandbox(t, r) + execBody := `{"command":"sleep 1","timeout":"1ms"}` + req := httptest.NewRequest("POST", "/api/v1/sandboxes/"+sbID+"/exec", bytes.NewBufferString(execBody)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusRequestTimeout { + t.Fatalf("expected 408, got %d: %s", w.Code, w.Body.String()) + } +} + func TestDestroyAndGet404(t *testing.T) { r, _ := setupTestRouter(t) diff --git a/internal/api/routes/templates.go b/internal/api/routes/templates.go index 6f23e84..2d48e70 100644 --- a/internal/api/routes/templates.go +++ b/internal/api/routes/templates.go @@ -3,11 +3,10 @@ package routes import ( "encoding/json" "net/http" - "strings" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/go-chi/chi/v5" ) type TemplateRoutes struct { @@ -53,11 +52,7 @@ func (t *TemplateRoutes) Create(w http.ResponseWriter, r *http.Request) { return } if err := t.registry.Create(r.Context(), &tmpl); err != nil { - if strings.Contains(err.Error(), "UNIQUE constraint") { - httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, "template already exists") - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, tmpl) @@ -101,11 +96,7 @@ func (t *TemplateRoutes) Get(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") tmpl, err := t.registry.Get(r.Context(), name) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, tmpl) @@ -135,11 +126,7 @@ func (t *TemplateRoutes) Update(w http.ResponseWriter, r *http.Request) { } tmpl.Name = name if err := t.registry.Update(r.Context(), &tmpl); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, tmpl) @@ -160,11 +147,7 @@ func (t *TemplateRoutes) Update(w http.ResponseWriter, r *http.Request) { func (t *TemplateRoutes) Delete(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") if err := t.registry.Delete(r.Context(), name); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) @@ -188,11 +171,7 @@ func (t *TemplateRoutes) Spawn(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") tmpl, err := t.registry.Get(r.Context(), name) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -213,7 +192,7 @@ func (t *TemplateRoutes) Spawn(w http.ResponseWriter, r *http.Request) { sb, err := t.manager.Spawn(r.Context(), req) if err != nil { - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, sb) diff --git a/internal/api/routes/templates_test.go b/internal/api/routes/templates_test.go new file mode 100644 index 0000000..b457b33 --- /dev/null +++ b/internal/api/routes/templates_test.go @@ -0,0 +1,75 @@ +package routes + +import ( + "bytes" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog" +) + +func setupTemplateTestRouter(t *testing.T) chi.Router { + t.Helper() + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := providers.NewMockProvider() + reg.Register(mock) + if err := reg.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + events := orchestrator.NewEventBus() + mgr := orchestrator.NewManager(reg, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + mgr.Start() + t.Cleanup(func() { mgr.Stop() }) + + r := chi.NewRouter() + r.Mount("/api/v1/templates", NewTemplateRoutes(orchestrator.NewTemplateRegistry(st), mgr).Routes()) + return r +} + +func TestTemplateDuplicateReturnsConflict(t *testing.T) { + r := setupTemplateTestRouter(t) + body := `{"name":"node","image":"node:20","ttl_seconds":300}` + + for i := 0; i < 2; i++ { + req := httptest.NewRequest("POST", "/api/v1/templates", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if i == 0 && w.Code != http.StatusCreated { + t.Fatalf("first create: expected 201, got %d: %s", w.Code, w.Body.String()) + } + if i == 1 && w.Code != http.StatusConflict { + t.Fatalf("second create: expected 409, got %d: %s", w.Code, w.Body.String()) + } + } +} + +func TestTemplateMissingReturnsNotFound(t *testing.T) { + r := setupTemplateTestRouter(t) + + req := httptest.NewRequest("GET", "/api/v1/templates/missing-template", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusNotFound { + t.Fatalf("expected 404, got %d: %s", w.Code, w.Body.String()) + } +} diff --git a/internal/httputil/response.go b/internal/httputil/response.go index b78691d..2c9cc74 100644 --- a/internal/httputil/response.go +++ b/internal/httputil/response.go @@ -8,12 +8,14 @@ import ( type ErrorCode string const ( - CodeNotFound ErrorCode = "NOT_FOUND" - CodeBadRequest ErrorCode = "BAD_REQUEST" - CodeInternal ErrorCode = "INTERNAL_ERROR" - CodeUnauth ErrorCode = "UNAUTHORIZED" - CodeConflict ErrorCode = "CONFLICT" - CodeUnavailable ErrorCode = "UNAVAILABLE" + CodeNotFound ErrorCode = "NOT_FOUND" + CodeBadRequest ErrorCode = "BAD_REQUEST" + CodeInternal ErrorCode = "INTERNAL_ERROR" + CodeUnauth ErrorCode = "UNAUTHORIZED" + CodeConflict ErrorCode = "CONFLICT" + CodeUnavailable ErrorCode = "UNAVAILABLE" + CodeTimeout ErrorCode = "TIMEOUT" + CodeResourceLimit ErrorCode = "RESOURCE_LIMIT" ) type APIError struct { diff --git a/internal/orchestrator/errors.go b/internal/orchestrator/errors.go new file mode 100644 index 0000000..1c5c73b --- /dev/null +++ b/internal/orchestrator/errors.go @@ -0,0 +1,12 @@ +package orchestrator + +import "github.com/StacyOs/stacyvm/internal/providers" + +var ( + ErrSandboxNotFound = providers.ErrSandboxNotFound + ErrSandboxDestroyed = providers.ErrSandboxDestroyed + ErrProviderNotFound = providers.ErrProviderNotFound + ErrProviderUnavailable = providers.ErrProviderUnavailable + ErrExecTimeout = providers.ErrExecTimeout + ErrResourceLimit = providers.ErrResourceLimit +) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 171fb4a..c18b407 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/json" + "errors" "fmt" "io" "path/filepath" @@ -118,6 +119,156 @@ func (m *Manager) pruneExpired() { } } +// Reconcile refreshes persisted sandbox state against provider runtime state. +// It is intended for startup recovery after the server process restarts. +func (m *Manager) Reconcile(ctx context.Context) error { + records, err := m.store.ListSandboxes(ctx) + if err != nil { + return fmt.Errorf("listing sandboxes for reconciliation: %w", err) + } + + known := make(map[string]struct{}, len(records)) + for _, rec := range records { + known[rec.ID] = struct{}{} + if SandboxState(rec.State) == StateDestroyed { + continue + } + + prov, err := m.registry.Get(rec.Provider) + if err != nil { + m.logger.Warn().Err(err).Str("sandbox", rec.ID).Str("provider", rec.Provider).Msg("reconcile: provider unavailable") + if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateError)); updateErr != nil { + return fmt.Errorf("marking sandbox %s error: %w", rec.ID, updateErr) + } + continue + } + + status, err := prov.Status(ctx, rec.ID) + if err != nil { + if errors.Is(err, providers.ErrSandboxNotFound) || errors.Is(err, providers.ErrSandboxDestroyed) { + if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateDestroyed)); updateErr != nil { + return fmt.Errorf("marking stale sandbox %s destroyed: %w", rec.ID, updateErr) + } + m.mu.Lock() + delete(m.sandboxes, rec.ID) + m.mu.Unlock() + m.logger.Info().Str("sandbox", rec.ID).Msg("reconcile: stale sandbox marked destroyed") + continue + } + m.logger.Warn().Err(err).Str("sandbox", rec.ID).Msg("reconcile: provider status failed") + if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateError)); updateErr != nil { + return fmt.Errorf("marking sandbox %s error: %w", rec.ID, updateErr) + } + continue + } + + state := SandboxState(status.State) + if state == "" { + state = StateRunning + } + if state == StateDestroyed { + if err := m.store.UpdateSandboxState(ctx, rec.ID, string(StateDestroyed)); err != nil { + return fmt.Errorf("marking sandbox %s destroyed: %w", rec.ID, err) + } + continue + } + if state != SandboxState(rec.State) { + if err := m.store.UpdateSandboxState(ctx, rec.ID, string(state)); err != nil { + return fmt.Errorf("updating reconciled sandbox %s state: %w", rec.ID, err) + } + } + + sb := recordToSandbox(rec) + sb.State = state + sb.PreviewDomain = m.previewDomain + m.mu.Lock() + m.sandboxes[rec.ID] = sb + m.mu.Unlock() + } + + if err := m.reconcileProviderRuntimes(ctx, known); err != nil { + return err + } + + return nil +} + +func (m *Manager) reconcileProviderRuntimes(ctx context.Context, known map[string]struct{}) error { + for _, name := range m.registry.List() { + prov, err := m.registry.Get(name) + if err != nil { + continue + } + lister, ok := prov.(providers.RuntimeSandboxLister) + if !ok { + continue + } + + runtimes, err := lister.ListRuntimeSandboxes(ctx) + if err != nil { + m.logger.Warn().Err(err).Str("provider", name).Msg("reconcile: runtime inventory failed") + continue + } + for _, runtime := range runtimes { + if _, ok := known[runtime.ID]; ok { + continue + } + if runtime.State == "" { + runtime.State = string(StateRunning) + } + if SandboxState(runtime.State) == StateDestroyed { + continue + } + if runtime.Provider == "" { + runtime.Provider = prov.Name() + } + if runtime.Image == "" { + runtime.Image = m.defaultImage + } + if runtime.CreatedAt.IsZero() { + runtime.CreatedAt = time.Now().UTC() + } + expiresAt := runtime.CreatedAt.Add(m.defaultTTL) + if expiresAt.Before(time.Now()) { + expiresAt = time.Now().Add(m.defaultTTL) + } + + metaJSON, _ := json.Marshal(runtime.Metadata) + rec := &store.SandboxRecord{ + ID: runtime.ID, + State: runtime.State, + Provider: runtime.Provider, + Image: runtime.Image, + MemoryMB: m.defaultMemory, + VCPUs: m.defaultVCPUs, + Metadata: string(metaJSON), + CreatedAt: runtime.CreatedAt, + ExpiresAt: expiresAt, + UpdatedAt: time.Now().UTC(), + } + if err := m.store.CreateSandbox(ctx, rec); err != nil { + if errors.Is(err, store.ErrConflict) { + continue + } + return fmt.Errorf("adopting runtime sandbox %s: %w", runtime.ID, err) + } + + sb := recordToSandbox(rec) + sb.PreviewDomain = m.previewDomain + m.mu.Lock() + m.sandboxes[runtime.ID] = sb + m.mu.Unlock() + known[runtime.ID] = struct{}{} + + m.logger.Info(). + Str("sandbox", runtime.ID). + Str("provider", runtime.Provider). + Msg("reconcile: adopted provider runtime") + } + } + return nil +} + func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) { providerName := req.Provider prov, err := m.registry.Get(providerName) @@ -227,6 +378,9 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req // Acquire a VM slot (may spawn a new VM if needed). vmID, err := m.vmPoolMgr.Acquire(ctx, sandboxID) if err != nil { + if errors.Is(err, ErrVMPoolFull) { + return nil, providers.ResourceLimitError(err.Error()) + } return nil, fmt.Errorf("pool acquire: %w", err) } @@ -296,6 +450,17 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( return nil, err } + execCtx := ctx + var cancel context.CancelFunc + if req.Timeout != "" { + timeout, err := time.ParseDuration(req.Timeout) + if err != nil { + return nil, fmt.Errorf("parsing exec timeout: %w", err) + } + execCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + m.events.Publish(Event{ Type: EventExecStarted, SandboxID: sandboxID, @@ -308,13 +473,16 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( } start := time.Now() - result, err := prov.Exec(ctx, m.resolveVMID(sb), providers.ExecOptions{ + result, err := prov.Exec(execCtx, m.resolveVMID(sb), providers.ExecOptions{ Command: req.Command, Args: req.Args, Env: req.Env, WorkDir: workDir, }) if err != nil { + if execCtx.Err() == context.DeadlineExceeded { + return nil, providers.ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec: %w", err) } duration := time.Since(start) @@ -351,17 +519,60 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ return nil, err } + execCtx := ctx + var cancel context.CancelFunc + if req.Timeout != "" { + timeout, err := time.ParseDuration(req.Timeout) + if err != nil { + return nil, fmt.Errorf("parsing exec timeout: %w", err) + } + execCtx, cancel = context.WithTimeout(ctx, timeout) + } + workDir := req.WorkDir if workDir == "" && sb.VMID != "" { workDir = "/workspace/" + sandboxID } - return prov.ExecStream(ctx, m.resolveVMID(sb), providers.ExecOptions{ + ch, err := prov.ExecStream(execCtx, m.resolveVMID(sb), providers.ExecOptions{ Command: req.Command, Args: req.Args, Env: req.Env, WorkDir: workDir, }) + if err != nil { + if cancel != nil { + cancel() + } + if execCtx.Err() == context.DeadlineExceeded { + return nil, providers.ExecTimeoutError(sandboxID) + } + return nil, err + } + if cancel == nil { + return ch, nil + } + + out := make(chan providers.StreamChunk, 64) + go func() { + defer close(out) + defer cancel() + timedOut := false + for chunk := range ch { + select { + case out <- chunk: + case <-execCtx.Done(): + timedOut = true + } + } + if execCtx.Err() == context.DeadlineExceeded || timedOut { + select { + case out <- providers.StreamChunk{Stream: "stderr", Data: providers.ExecTimeoutError(sandboxID).Error()}: + case <-ctx.Done(): + } + } + }() + return out, nil } func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWriteRequest) error { @@ -626,11 +837,11 @@ func (m *Manager) Get(ctx context.Context, id string) (*Sandbox, error) { // Fall back to store rec, err := m.store.GetSandbox(ctx, id) if err != nil { - return nil, err + return nil, providers.SandboxNotFoundError(id) } sb = recordToSandbox(rec) if sb.State == StateDestroyed { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, providers.SandboxDestroyedError(id) } return sb, nil } @@ -793,13 +1004,13 @@ func (m *Manager) getSandboxAndProvider(id string) (*Sandbox, providers.Provider if !ok { rec, err := m.store.GetSandbox(context.Background(), id) if err != nil { - return nil, nil, fmt.Errorf("sandbox %q not found", id) + return nil, nil, providers.SandboxNotFoundError(id) } sb = recordToSandbox(rec) } if sb.State == StateDestroyed { - return nil, nil, fmt.Errorf("sandbox %q is destroyed", id) + return nil, nil, providers.SandboxDestroyedError(id) } prov, err := m.registry.Get(sb.Provider) @@ -818,7 +1029,7 @@ func (m *Manager) getProvider(id string) (providers.Provider, error) { if !ok { rec, err := m.store.GetSandbox(context.Background(), id) if err != nil { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, providers.SandboxNotFoundError(id) } sb = recordToSandbox(rec) } @@ -843,4 +1054,3 @@ func recordToSandbox(r *store.SandboxRecord) *Sandbox { Metadata: metadata, } } - diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 8578776..0a33951 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -2,7 +2,9 @@ package orchestrator import ( "context" + "errors" "path/filepath" + "strings" "testing" "time" @@ -76,6 +78,93 @@ func TestManager_List(t *testing.T) { } } +func TestManager_ReconcileMarksMissingRuntimeDestroyed(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + now := time.Now().UTC() + + if err := m.store.CreateSandbox(ctx, &store.SandboxRecord{ + ID: "sb-missing-runtime", + State: string(StateRunning), + Provider: "mock", + Image: "alpine:latest", + MemoryMB: 512, + VCPUs: 1, + Metadata: "{}", + CreatedAt: now, + ExpiresAt: now.Add(time.Hour), + UpdatedAt: now, + }); err != nil { + t.Fatalf("create stale sandbox record: %v", err) + } + + if err := m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + + rec, err := m.store.GetSandbox(ctx, "sb-missing-runtime") + if err != nil { + t.Fatalf("get reconciled sandbox: %v", err) + } + if rec.State != string(StateDestroyed) { + t.Fatalf("expected destroyed after reconcile, got %s", rec.State) + } +} + +type runtimeListerProvider struct { + providers.Provider + runtimes []providers.RuntimeSandbox +} + +func (p *runtimeListerProvider) ListRuntimeSandboxes(ctx context.Context) ([]providers.RuntimeSandbox, error) { + return p.runtimes, nil +} + +func TestManager_ReconcileAdoptsProviderRuntime(t *testing.T) { + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := &runtimeListerProvider{ + Provider: providers.NewMockProvider(), + runtimes: []providers.RuntimeSandbox{{ + ID: "sb-adopted-runtime", + State: string(StateRunning), + Provider: "mock", + Image: "alpine:latest", + CreatedAt: time.Now().UTC(), + Metadata: map[string]string{"source": "runtime"}, + }}, + } + reg.Register(mock) + reg.SetDefault("mock") + + m := NewManager(reg, st, NewEventBus(), zerolog.Nop(), ManagerConfig{ + DefaultTTL: time.Hour, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + + if err := m.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + rec, err := st.GetSandbox(context.Background(), "sb-adopted-runtime") + if err != nil { + t.Fatalf("get adopted sandbox: %v", err) + } + if rec.State != string(StateRunning) { + t.Fatalf("expected running adopted state, got %s", rec.State) + } + if rec.Provider != "mock" { + t.Fatalf("expected mock provider, got %s", rec.Provider) + } +} + func TestManager_Exec(t *testing.T) { m := setupManager(t) ctx := context.Background() @@ -94,6 +183,46 @@ func TestManager_Exec(t *testing.T) { } } +func TestManager_ExecTimeout(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + + _, err := m.Exec(ctx, sb.ID, ExecRequest{ + Command: "sleep 1", + Timeout: "1ms", + }) + if !errors.Is(err, ErrExecTimeout) { + t.Fatalf("expected ErrExecTimeout, got %v", err) + } +} + +func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + + ch, err := m.ExecStream(ctx, sb.ID, ExecRequest{ + Command: "sleep 1", + Timeout: "1ms", + }) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + + var sawTimeout bool + for chunk := range ch { + if chunk.Stream == "stderr" && strings.Contains(chunk.Data, ErrExecTimeout.Error()) { + sawTimeout = true + } + } + if !sawTimeout { + t.Fatal("expected timeout error chunk") + } +} + func TestManager_WriteAndReadFile(t *testing.T) { m := setupManager(t) ctx := context.Background() diff --git a/internal/providers/custom.go b/internal/providers/custom.go index 1b451a7..487c01a 100644 --- a/internal/providers/custom.go +++ b/internal/providers/custom.go @@ -63,6 +63,23 @@ func NewCustomProvider(cfg CustomProviderConfig) *CustomProvider { func (p *CustomProvider) Name() string { return p.name } +func customHTTPError(operation string, code int, data []byte, sandboxID string) error { + switch code { + case http.StatusNotFound: + return SandboxNotFoundError(sandboxID) + case http.StatusGone: + return SandboxDestroyedError(sandboxID) + case http.StatusRequestTimeout: + return ExecTimeoutError(sandboxID) + case http.StatusTooManyRequests: + return ResourceLimitError(operation) + case http.StatusServiceUnavailable: + return ProviderUnavailableError("custom", fmt.Errorf("%s", string(data))) + default: + return fmt.Errorf("%s failed (HTTP %d): %s", operation, code, string(data)) + } +} + // doRequest is a shared helper that builds, signs, and executes an HTTP // request against the remote custom endpoint. func (p *CustomProvider) doRequest(ctx context.Context, method, path string, body interface{}) ([]byte, int, error) { @@ -143,7 +160,7 @@ func (p *CustomProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, return "", fmt.Errorf("custom spawn: %w", err) } if code >= 400 { - return "", fmt.Errorf("custom spawn failed (HTTP %d): %s", code, string(data)) + return "", customHTTPError("custom spawn", code, data, "") } var result struct { @@ -181,7 +198,7 @@ func (p *CustomProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp return nil, fmt.Errorf("custom exec: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom exec failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom exec", code, data, sandboxID) } var result struct { @@ -225,7 +242,7 @@ func (p *CustomProvider) ExecStream(ctx context.Context, sandboxID string, opts if resp.StatusCode >= 400 { data, _ := io.ReadAll(resp.Body) resp.Body.Close() - return nil, fmt.Errorf("custom exec stream failed (HTTP %d): %s", resp.StatusCode, string(data)) + return nil, customHTTPError("custom exec stream", resp.StatusCode, data, sandboxID) } ch := make(chan StreamChunk, 64) @@ -244,6 +261,12 @@ func (p *CustomProvider) ExecStream(ctx context.Context, sandboxID string, opts ch <- chunk } } + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } }() return ch, nil @@ -271,7 +294,7 @@ func (p *CustomProvider) WriteFile(ctx context.Context, sandboxID string, path s return fmt.Errorf("custom write: %w", err) } if code >= 400 { - return fmt.Errorf("custom write failed (HTTP %d): %s", code, string(respData)) + return customHTTPError("custom write", code, respData, sandboxID) } return nil } @@ -288,7 +311,7 @@ func (p *CustomProvider) ReadFile(ctx context.Context, sandboxID string, path st return nil, fmt.Errorf("custom read: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom read failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom read", code, data, sandboxID) } return io.NopCloser(bytes.NewReader(data)), nil } @@ -305,7 +328,7 @@ func (p *CustomProvider) ListFiles(ctx context.Context, sandboxID string, path s return nil, fmt.Errorf("custom list: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom list failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom list", code, data, sandboxID) } var files []FileInfo @@ -326,7 +349,7 @@ func (p *CustomProvider) DeleteFile(ctx context.Context, sandboxID string, path return fmt.Errorf("custom delete: %w", err) } if code >= 400 { - return fmt.Errorf("custom delete failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom delete", code, data, sandboxID) } return nil } @@ -342,7 +365,7 @@ func (p *CustomProvider) MoveFile(ctx context.Context, sandboxID string, oldPath return fmt.Errorf("custom move: %w", err) } if code >= 400 { - return fmt.Errorf("custom move failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom move", code, data, sandboxID) } return nil } @@ -358,7 +381,7 @@ func (p *CustomProvider) ChmodFile(ctx context.Context, sandboxID string, path s return fmt.Errorf("custom chmod: %w", err) } if code >= 400 { - return fmt.Errorf("custom chmod failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom chmod", code, data, sandboxID) } return nil } @@ -373,7 +396,7 @@ func (p *CustomProvider) StatFile(ctx context.Context, sandboxID string, path st return nil, fmt.Errorf("custom stat: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom stat failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom stat", code, data, sandboxID) } var fi FileInfo @@ -393,7 +416,7 @@ func (p *CustomProvider) GlobFiles(ctx context.Context, sandboxID string, patter return nil, fmt.Errorf("custom glob: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom glob failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom glob", code, data, sandboxID) } var matches []string @@ -411,10 +434,10 @@ func (p *CustomProvider) Status(ctx context.Context, sandboxID string) (*Sandbox return nil, fmt.Errorf("custom status: %w", err) } if code == 404 { - return &SandboxStatus{ID: sandboxID, State: "destroyed"}, nil + return nil, SandboxNotFoundError(sandboxID) } if code >= 400 { - return nil, fmt.Errorf("custom status failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom status", code, data, sandboxID) } var result struct { @@ -445,7 +468,7 @@ func (p *CustomProvider) Destroy(ctx context.Context, sandboxID string) error { return fmt.Errorf("custom destroy: %w", err) } if code >= 400 && code != 404 { - return fmt.Errorf("custom destroy failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom destroy", code, data, sandboxID) } return nil } diff --git a/internal/providers/custom_conformance_test.go b/internal/providers/custom_conformance_test.go new file mode 100644 index 0000000..79413c4 --- /dev/null +++ b/internal/providers/custom_conformance_test.go @@ -0,0 +1,240 @@ +package providers + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "path" + "strconv" + "strings" + "testing" + "time" +) + +func TestCustomProvider_Conformance(t *testing.T) { + server := newFakeCustomProviderServer(t) + defer server.Close() + + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + return NewCustomProvider(CustomProviderConfig{ + BaseURL: server.URL, + Timeout: 5 * time.Second, + }) + }) +} + +type fakeCustomBackend struct { + mock *MockProvider +} + +func newFakeCustomProviderServer(t *testing.T) *httptest.Server { + t.Helper() + backend := &fakeCustomBackend{mock: NewMockProvider()} + mux := http.NewServeMux() + mux.HandleFunc("/health", backend.health) + mux.HandleFunc("/spawn", backend.spawn) + mux.HandleFunc("/exec", backend.exec) + mux.HandleFunc("/files", backend.files) + mux.HandleFunc("/files/list", backend.listFiles) + mux.HandleFunc("/files/move", backend.moveFile) + mux.HandleFunc("/files/chmod", backend.chmodFile) + mux.HandleFunc("/files/stat", backend.statFile) + mux.HandleFunc("/files/glob", backend.globFiles) + mux.HandleFunc("/status/", backend.status) + mux.HandleFunc("/sandboxes/", backend.destroy) + return httptest.NewServer(mux) +} + +func (b *fakeCustomBackend) health(w http.ResponseWriter, r *http.Request) { + writeFakeJSON(w, http.StatusOK, map[string]bool{"ok": true}) +} + +func (b *fakeCustomBackend) spawn(w http.ResponseWriter, r *http.Request) { + id, err := b.mock.Spawn(r.Context(), SpawnOptions{}) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, map[string]string{"id": id}) +} + +func (b *fakeCustomBackend) exec(w http.ResponseWriter, r *http.Request) { + var req struct { + SandboxID string `json:"sandbox_id"` + Command string `json:"command"` + Stream bool `json:"stream"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if req.Stream { + ch, err := b.mock.ExecStream(r.Context(), req.SandboxID, ExecOptions{Command: req.Command}) + if err != nil { + writeFakeError(w, err) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + enc := json.NewEncoder(w) + for chunk := range ch { + _ = enc.Encode(chunk) + } + return + } + result, err := b.mock.Exec(r.Context(), req.SandboxID, ExecOptions{Command: req.Command}) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, map[string]any{ + "exit_code": result.ExitCode, + "stdout": result.Stdout, + "stderr": result.Stderr, + }) +} + +func (b *fakeCustomBackend) files(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + rc, err := b.mock.ReadFile(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("path")) + if err != nil { + writeFakeError(w, err) + return + } + defer rc.Close() + _, _ = io.Copy(w, rc) + case http.MethodPost: + var req struct { + SandboxID string `json:"sandbox_id"` + Path string `json:"path"` + Content string `json:"content"` + Mode string `json:"mode"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.WriteFile(r.Context(), req.SandboxID, req.Path, strings.NewReader(req.Content), req.Mode); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) + case http.MethodDelete: + var req struct { + SandboxID string `json:"sandbox_id"` + Path string `json:"path"` + Recursive bool `json:"recursive"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.DeleteFile(r.Context(), req.SandboxID, req.Path, req.Recursive); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func (b *fakeCustomBackend) listFiles(w http.ResponseWriter, r *http.Request) { + files, err := b.mock.ListFiles(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("path")) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, files) +} + +func (b *fakeCustomBackend) moveFile(w http.ResponseWriter, r *http.Request) { + var req struct { + SandboxID string `json:"sandbox_id"` + OldPath string `json:"old_path"` + NewPath string `json:"new_path"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.MoveFile(r.Context(), req.SandboxID, req.OldPath, req.NewPath); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (b *fakeCustomBackend) chmodFile(w http.ResponseWriter, r *http.Request) { + var req struct { + SandboxID string `json:"sandbox_id"` + Path string `json:"path"` + Mode string `json:"mode"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.ChmodFile(r.Context(), req.SandboxID, req.Path, req.Mode); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (b *fakeCustomBackend) statFile(w http.ResponseWriter, r *http.Request) { + fi, err := b.mock.StatFile(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("path")) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, fi) +} + +func (b *fakeCustomBackend) globFiles(w http.ResponseWriter, r *http.Request) { + matches, err := b.mock.GlobFiles(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("pattern")) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, matches) +} + +func (b *fakeCustomBackend) status(w http.ResponseWriter, r *http.Request) { + id := path.Base(r.URL.Path) + status, err := b.mock.Status(r.Context(), id) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, status) +} + +func (b *fakeCustomBackend) destroy(w http.ResponseWriter, r *http.Request) { + id := path.Base(r.URL.Path) + if err := b.mock.Destroy(r.Context(), id); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func writeFakeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeFakeError(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + if errors.Is(err, ErrSandboxNotFound) { + status = http.StatusNotFound + } + if errors.Is(err, ErrSandboxDestroyed) { + status = http.StatusGone + } + http.Error(w, strconv.Quote(err.Error()), status) +} diff --git a/internal/providers/docker.go b/internal/providers/docker.go index 7d50942..abbc755 100644 --- a/internal/providers/docker.go +++ b/internal/providers/docker.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "crypto/rand" + "encoding/json" "fmt" "io" "path" @@ -15,6 +16,7 @@ import ( "time" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" dockerimage "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" @@ -150,7 +152,15 @@ func (d *DockerProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, sandboxID := fmt.Sprintf("sb-%x", b) labels := map[string]string{ - "stacyvm": "true", + "stacyvm": "true", + "stacyvm.provider": "docker", + "stacyvm.sandbox": sandboxID, + "stacyvm.image": image, + } + if len(opts.Metadata) > 0 { + if data, err := json.Marshal(opts.Metadata); err == nil { + labels["stacyvm.metadata"] = string(data) + } } if d.config.PreviewDomain != "" { @@ -241,22 +251,34 @@ func (d *DockerProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp execID, err := d.cli.ContainerExecCreate(ctx, sandboxID, execCfg) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec create: %w", err) } resp, err := d.cli.ContainerExecAttach(ctx, execID.ID, container.ExecStartOptions{}) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec attach: %w", err) } defer resp.Close() var stdout, stderr bytes.Buffer if _, err := stdcopy.StdCopy(&stdout, &stderr, resp.Reader); err != nil && err != io.EOF { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } d.logger.Debug().Err(err).Msg("stdcopy exec error") } inspect, err := d.cli.ContainerExecInspect(ctx, execID.ID) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec inspect: %w", err) } @@ -309,6 +331,12 @@ func (d *DockerProvider) ExecStream(ctx context.Context, sandboxID string, opts if _, err := stdcopy.StdCopy(stdoutW, stderrW, resp.Reader); err != nil && err != io.EOF { d.logger.Debug().Err(err).Msg("stdcopy stream error") } + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } }() return ch, nil @@ -529,6 +557,9 @@ func (d *DockerProvider) GlobFiles(ctx context.Context, sandboxID string, patter func (d *DockerProvider) Status(ctx context.Context, sandboxID string) (*SandboxStatus, error) { info, err := d.cli.ContainerInspect(ctx, sandboxID) if err != nil { + if client.IsErrNotFound(err) { + return nil, SandboxNotFoundError(sandboxID) + } return nil, fmt.Errorf("inspect container %q: %w", sandboxID, err) } @@ -560,6 +591,9 @@ func (d *DockerProvider) Destroy(ctx context.Context, sandboxID string) error { d.logger.Debug().Err(err).Msg("container stop (continuing with remove)") } if err := d.cli.ContainerRemove(ctx, sandboxID, container.RemoveOptions{Force: true}); err != nil { + if client.IsErrNotFound(err) { + return SandboxNotFoundError(sandboxID) + } return fmt.Errorf("removing container %q: %w", sandboxID, err) } @@ -605,20 +639,111 @@ func (d *DockerProvider) ConsoleLog(ctx context.Context, sandboxID string, lines return result, nil } +func (d *DockerProvider) ListRuntimeSandboxes(ctx context.Context) ([]RuntimeSandbox, error) { + args := filters.NewArgs(filters.Arg("label", "stacyvm=true")) + containers, err := d.cli.ContainerList(ctx, container.ListOptions{All: true, Filters: args}) + if err != nil { + return nil, fmt.Errorf("list stacyvm containers: %w", err) + } + + out := make([]RuntimeSandbox, 0, len(containers)) + for _, c := range containers { + id := c.Labels["stacyvm.sandbox"] + if id == "" && len(c.Names) > 0 { + id = strings.TrimPrefix(c.Names[0], "/") + } + if id == "" { + id = c.ID + } + + image := c.Labels["stacyvm.image"] + if image == "" { + image = c.Image + } + + metadata := map[string]string{} + if raw := c.Labels["stacyvm.metadata"]; raw != "" { + _ = json.Unmarshal([]byte(raw), &metadata) + } + + out = append(out, RuntimeSandbox{ + ID: id, + State: dockerContainerState(c.State), + Provider: d.Name(), + Image: image, + CreatedAt: time.Unix(c.Created, 0).UTC(), + Metadata: metadata, + }) + d.rememberSandbox(id, image, dockerContainerState(c.State)) + } + return out, nil +} + // --------------------------------------------------------------------------- // Private helpers // --------------------------------------------------------------------------- func (d *DockerProvider) getSandbox(id string) (*dockerSandbox, error) { d.mu.RLock() - defer d.mu.RUnlock() sb, ok := d.sandboxes[id] + d.mu.RUnlock() if !ok { - return nil, fmt.Errorf("sandbox %q not found (may have been destroyed)", id) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + info, err := d.cli.ContainerInspect(ctx, id) + if err != nil { + if client.IsErrNotFound(err) { + return nil, SandboxNotFoundError(id) + } + return nil, fmt.Errorf("inspect container %q: %w", id, err) + } + if info.Config == nil || info.Config.Labels["stacyvm"] != "true" { + return nil, SandboxNotFoundError(id) + } + image := info.Config.Labels["stacyvm.image"] + if image == "" { + image = info.Config.Image + } + state := "unknown" + if info.State != nil { + state = dockerContainerState(info.State.Status) + } + return d.rememberSandbox(id, image, state), nil } return sb, nil } +func (d *DockerProvider) rememberSandbox(id, image, state string) *dockerSandbox { + if state == "" { + state = "running" + } + d.mu.Lock() + defer d.mu.Unlock() + sb := &dockerSandbox{id: id, image: image, state: state} + d.sandboxes[id] = sb + return sb +} + +func dockerContainerState(state string) string { + switch state { + case "running": + return "running" + case "paused": + return "paused" + case "restarting": + return "restarting" + case "created": + return "creating" + case "exited", "dead", "removing": + return "stopped" + default: + if state == "" { + return "unknown" + } + return state + } +} + // containerExec runs a one-shot command inside the container and returns the result. func (d *DockerProvider) containerExec(ctx context.Context, containerID string, cmd string) (*ExecResult, error) { execCfg := container.ExecOptions{ @@ -801,4 +926,3 @@ func parseStatOutput(output string) []FileInfo { } return files } - diff --git a/internal/providers/docker_test.go b/internal/providers/docker_test.go index 55852b5..8de860a 100644 --- a/internal/providers/docker_test.go +++ b/internal/providers/docker_test.go @@ -2,6 +2,7 @@ package providers import ( "context" + "errors" "io" "os" "os/exec" @@ -181,6 +182,44 @@ func TestDockerIntegration_Healthy(t *testing.T) { } } +func TestDockerIntegration_Conformance(t *testing.T) { + skipIfNoDocker(t) + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + p, err := newTestDockerProvider(t) + if err != nil { + t.Fatalf("provider: %v", err) + } + return p + }) +} + +func TestDockerIntegration_ListRuntimeSandboxes(t *testing.T) { + skipIfNoDocker(t) + p, err := newTestDockerProvider(t) + if err != nil { + t.Fatalf("provider: %v", err) + } + id := spawnTestSandbox(t, p) + + runtimes, err := p.ListRuntimeSandboxes(context.Background()) + if err != nil { + t.Fatalf("list runtime sandboxes: %v", err) + } + for _, runtime := range runtimes { + if runtime.ID == id { + if runtime.Provider != "docker" { + t.Fatalf("provider = %q, want docker", runtime.Provider) + } + if runtime.Image == "" { + t.Fatal("runtime image is empty") + } + return + } + } + t.Fatalf("spawned sandbox %s not found in runtime inventory", id) +} + func TestDockerIntegration_SpawnAndDestroy(t *testing.T) { skipIfNoDocker(t) p, err := newTestDockerProvider(t) @@ -536,6 +575,9 @@ func TestDockerIntegration_StatusNotFound(t *testing.T) { if err == nil { t.Error("expected error for nonexistent container") } + if !errors.Is(err, ErrSandboxNotFound) { + t.Fatalf("expected ErrSandboxNotFound, got %v", err) + } } func TestDockerIntegration_DestroyNotFound(t *testing.T) { @@ -549,6 +591,9 @@ func TestDockerIntegration_DestroyNotFound(t *testing.T) { if err == nil { t.Error("expected error for nonexistent container") } + if !errors.Is(err, ErrSandboxNotFound) { + t.Fatalf("expected ErrSandboxNotFound, got %v", err) + } } func TestDockerIntegration_PoolWorkspaceIsolation(t *testing.T) { @@ -563,8 +608,8 @@ func TestDockerIntegration_PoolWorkspaceIsolation(t *testing.T) { userA, userB := "sb-aaaa0001", "sb-bbbb0002" // Create workspace dirs - p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userA}) //nolint:errcheck - p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userB}) //nolint:errcheck + p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userA}) //nolint:errcheck + p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userB}) //nolint:errcheck // User A writes secret p.Exec(ctx, vmID, ExecOptions{Command: "echo TOP_SECRET > /workspace/" + userA + "/secret.txt"}) //nolint:errcheck @@ -593,7 +638,7 @@ func TestDockerIntegration_ConcurrentUsers(t *testing.T) { users := []string{"sb-u001", "sb-u002", "sb-u003"} for _, u := range users { - p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + u}) //nolint:errcheck + p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + u}) //nolint:errcheck p.Exec(ctx, vmID, ExecOptions{Command: "echo " + u + " > /workspace/" + u + "/id.txt"}) //nolint:errcheck } diff --git a/internal/providers/errors.go b/internal/providers/errors.go new file mode 100644 index 0000000..dc8fc9c --- /dev/null +++ b/internal/providers/errors.go @@ -0,0 +1,42 @@ +package providers + +import ( + "errors" + "fmt" +) + +var ( + ErrSandboxNotFound = errors.New("sandbox not found") + ErrSandboxDestroyed = errors.New("sandbox destroyed") + ErrProviderNotFound = errors.New("provider not found") + ErrProviderUnavailable = errors.New("provider unavailable") + ErrExecTimeout = errors.New("exec timeout") + ErrResourceLimit = errors.New("resource limit exceeded") +) + +func SandboxNotFoundError(id string) error { + return fmt.Errorf("%w: %s", ErrSandboxNotFound, id) +} + +func SandboxDestroyedError(id string) error { + return fmt.Errorf("%w: %s", ErrSandboxDestroyed, id) +} + +func ProviderNotFoundError(name string) error { + return fmt.Errorf("%w: %s", ErrProviderNotFound, name) +} + +func ProviderUnavailableError(name string, err error) error { + if err == nil { + return fmt.Errorf("%w: %s", ErrProviderUnavailable, name) + } + return fmt.Errorf("%w: %s: %v", ErrProviderUnavailable, name, err) +} + +func ExecTimeoutError(sandboxID string) error { + return fmt.Errorf("%w: %s", ErrExecTimeout, sandboxID) +} + +func ResourceLimitError(resource string) error { + return fmt.Errorf("%w: %s", ErrResourceLimit, resource) +} diff --git a/internal/providers/firecracker.go b/internal/providers/firecracker.go index f10fcb3..cd091f4 100644 --- a/internal/providers/firecracker.go +++ b/internal/providers/firecracker.go @@ -37,7 +37,7 @@ type FirecrackerProvider struct { // snapshotInfo holds paths to a base snapshot's files. type snapshotInfo struct { - dir string // snapshot directory + dir string // snapshot directory vmstatePath string // CPU/device state memoryPath string // full RAM snapshot rootfsPath string // clean baseline rootfs @@ -489,7 +489,7 @@ func (p *FirecrackerProvider) Spawn(ctx context.Context, opts SpawnOptions) (str // Machine config. if err := api.put(ctx, "/machine-config", map[string]any{ - "vcpu_count": vcpus, + "vcpu_count": vcpus, "mem_size_mib": memMB, }); err != nil { cmd.Process.Kill() @@ -595,7 +595,7 @@ func (p *FirecrackerProvider) getVM(sandboxID string) (*vmInstance, error) { defer p.mu.RUnlock() vm, ok := p.vms[sandboxID] if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return vm, nil } @@ -665,13 +665,33 @@ func (p *FirecrackerProvider) ExecStream(ctx context.Context, sandboxID string, go func() { defer vm.connMu.Unlock() defer close(ch) + defer vm.conn.SetReadDeadline(time.Time{}) //nolint:errcheck for { + if deadline, ok := ctx.Deadline(); ok { + _ = vm.conn.SetReadDeadline(deadline) + } sresp, err := agentproto.ReadStreamResponse(vm.conn) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } return } if sresp.Data != "" { - ch <- StreamChunk{Stream: sresp.Stream, Data: sresp.Data} + select { + case ch <- StreamChunk{Stream: sresp.Stream, Data: sresp.Data}: + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } + return + } } if sresp.Done { return @@ -928,7 +948,7 @@ func (p *FirecrackerProvider) Destroy(ctx context.Context, sandboxID string) err vm, ok := p.vms[sandboxID] if !ok { p.mu.Unlock() - return fmt.Errorf("sandbox %q not found", sandboxID) + return SandboxNotFoundError(sandboxID) } delete(p.vms, sandboxID) p.mu.Unlock() @@ -962,7 +982,7 @@ func (p *FirecrackerProvider) ConsoleLog(ctx context.Context, sandboxID string, vm, ok := p.vms[sandboxID] p.mu.RUnlock() if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return vm.consoleBuf.Lines(lines), nil } @@ -1067,7 +1087,7 @@ var syscall0 = os.Signal(signalZero(0)) type signalZero int -func (signalZero) Signal() {} +func (signalZero) Signal() {} func (signalZero) String() string { return "signal 0" } func generateRequestID() string { diff --git a/internal/providers/firecracker_test.go b/internal/providers/firecracker_test.go index 5fecdac..232e3c9 100644 --- a/internal/providers/firecracker_test.go +++ b/internal/providers/firecracker_test.go @@ -1,6 +1,9 @@ package providers import ( + "os" + "os/exec" + "runtime" "strings" "testing" @@ -109,3 +112,54 @@ func TestStatusNotFound(t *testing.T) { t.Error("Status should return error for nonexistent sandbox") } } + +func TestFirecrackerProvider_Integration_Conformance(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("firecracker conformance requires Linux") + } + if _, err := os.Stat("/dev/kvm"); err != nil { + t.Skip("firecracker conformance requires /dev/kvm") + } + + firecrackerPath := os.Getenv("STACYVM_FIRECRACKER_PATH") + if firecrackerPath == "" { + var err error + firecrackerPath, err = exec.LookPath("firecracker") + if err != nil { + t.Skip("firecracker binary not found; set STACYVM_FIRECRACKER_PATH") + } + } + kernelPath := os.Getenv("STACYVM_KERNEL_PATH") + if kernelPath == "" { + t.Skip("set STACYVM_KERNEL_PATH to run firecracker conformance") + } + rootfsPath := os.Getenv("STACYVM_ROOTFS_PATH") + if rootfsPath == "" { + t.Skip("set STACYVM_ROOTFS_PATH to run firecracker conformance") + } + agentPath := os.Getenv("STACYVM_AGENT_PATH") + if agentPath == "" { + agentPath = "./bin/stacyvm-agent" + } + for name, path := range map[string]string{ + "kernel": kernelPath, + "rootfs": rootfsPath, + "agent": agentPath, + } { + if _, err := os.Stat(path); err != nil { + t.Skipf("%s path %q is unavailable: %v", name, path, err) + } + } + + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + return NewFirecrackerProvider(FirecrackerProviderConfig{ + FirecrackerPath: firecrackerPath, + KernelPath: kernelPath, + DefaultRootfs: rootfsPath, + AgentPath: agentPath, + DataDir: t.TempDir(), + DefaultMemoryMB: 256, + }, zerolog.Nop()) + }) +} diff --git a/internal/providers/mock.go b/internal/providers/mock.go index 341b4a2..fde4133 100644 --- a/internal/providers/mock.go +++ b/internal/providers/mock.go @@ -73,10 +73,10 @@ func (m *MockProvider) getSandbox(id string) (*mockSandbox, error) { defer m.mu.RUnlock() sb, ok := m.sandboxes[id] if !ok { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, SandboxNotFoundError(id) } if sb.state == "destroyed" { - return nil, fmt.Errorf("sandbox %q is destroyed", id) + return nil, SandboxDestroyedError(id) } return sb, nil } @@ -111,6 +111,9 @@ func (m *MockProvider) Exec(ctx context.Context, sandboxID string, opts ExecOpti err = cmd.Run() exitCode := 0 if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } if exitErr, ok := err.(*exec.ExitError); ok { exitCode = exitErr.ExitCode() } else { @@ -354,7 +357,7 @@ func (m *MockProvider) Status(ctx context.Context, sandboxID string) (*SandboxSt defer m.mu.RUnlock() sb, ok := m.sandboxes[sandboxID] if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return &SandboxStatus{ ID: sb.id, @@ -367,7 +370,7 @@ func (m *MockProvider) Destroy(ctx context.Context, sandboxID string) error { defer m.mu.Unlock() sb, ok := m.sandboxes[sandboxID] if !ok { - return fmt.Errorf("sandbox %q not found", sandboxID) + return SandboxNotFoundError(sandboxID) } sb.state = "destroyed" return os.RemoveAll(sb.root) diff --git a/internal/providers/mock_test.go b/internal/providers/mock_test.go index 4421b20..ff3748e 100644 --- a/internal/providers/mock_test.go +++ b/internal/providers/mock_test.go @@ -22,6 +22,13 @@ func TestMockProvider_Healthy(t *testing.T) { } } +func TestMockProvider_Conformance(t *testing.T) { + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + return NewMockProvider() + }) +} + func TestMockProvider_Spawn(t *testing.T) { p := NewMockProvider() ctx := context.Background() diff --git a/internal/providers/proot.go b/internal/providers/proot.go index 3879b44..08854cd 100644 --- a/internal/providers/proot.go +++ b/internal/providers/proot.go @@ -110,7 +110,7 @@ func (p *PRootProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, e } } if activeCount >= p.config.MaxSandboxes { - return "", fmt.Errorf("max sandboxes reached (%d)", p.config.MaxSandboxes) + return "", ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", p.config.MaxSandboxes)) } id := generatePRootSandboxID() @@ -139,10 +139,10 @@ func (p *PRootProvider) getSandbox(id string) (*prootSandbox, error) { defer p.mu.RUnlock() sb, ok := p.sandboxes[id] if !ok { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, SandboxNotFoundError(id) } if sb.state == "destroyed" { - return nil, fmt.Errorf("sandbox %q is destroyed", id) + return nil, SandboxDestroyedError(id) } return sb, nil } @@ -223,6 +223,9 @@ func (p *PRootProvider) Exec(ctx context.Context, sandboxID string, opts ExecOpt err = cmd.Run() exitCode := 0 if err != nil { + if execCtx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } if exitErr, ok := err.(*exec.ExitError); ok { exitCode = exitErr.ExitCode() } else { @@ -295,7 +298,12 @@ func (p *PRootProvider) ExecStream(ctx context.Context, sandboxID string, opts E go readStream("stderr", stderrPipe) wg.Wait() - cmd.Wait() + if err := cmd.Wait(); err != nil && execCtx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + case <-ctx.Done(): + } + } }() return ch, nil @@ -499,7 +507,7 @@ func (p *PRootProvider) Status(ctx context.Context, sandboxID string) (*SandboxS defer p.mu.RUnlock() sb, ok := p.sandboxes[sandboxID] if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return &SandboxStatus{ ID: sb.id, @@ -512,7 +520,7 @@ func (p *PRootProvider) Destroy(ctx context.Context, sandboxID string) error { sb, ok := p.sandboxes[sandboxID] if !ok { p.mu.Unlock() - return fmt.Errorf("sandbox %q not found", sandboxID) + return SandboxNotFoundError(sandboxID) } sb.state = "destroyed" delete(p.sandboxes, sandboxID) diff --git a/internal/providers/proot_test.go b/internal/providers/proot_test.go index 8ef590d..737a3b1 100644 --- a/internal/providers/proot_test.go +++ b/internal/providers/proot_test.go @@ -538,6 +538,21 @@ func TestPRootProvider_Integration_Exec(t *testing.T) { } } +func TestPRootProvider_Integration_Conformance(t *testing.T) { + prootPath := skipIfNoPRoot(t) + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + tmpDir := t.TempDir() + return NewPRootProvider(PRootProviderConfig{ + RootfsPath: "/", + PRootBinary: prootPath, + WorkspaceBase: filepath.Join(tmpDir, "workspaces"), + DefaultTimeout: 30 * time.Second, + MaxSandboxes: 5, + }, testPRootLogger()) + }) +} + func TestPRootProvider_Integration_ExecStream(t *testing.T) { prootPath := skipIfNoPRoot(t) tmpDir := t.TempDir() diff --git a/internal/providers/provider.go b/internal/providers/provider.go index dd6aef6..e2bad09 100644 --- a/internal/providers/provider.go +++ b/internal/providers/provider.go @@ -45,7 +45,21 @@ type SandboxStatus struct { State string } +// RuntimeSandbox describes a provider-owned runtime discovered outside the +// orchestrator's in-memory state, usually during startup reconciliation. +type RuntimeSandbox struct { + ID string + State string + Provider string + Image string + CreatedAt time.Time + Metadata map[string]string +} + // Provider defines the interface for sandbox execution backends. +// +// Implementations must satisfy the behavior documented in +// docs/provider-contract.md and exercised by provider_conformance_test.go. type Provider interface { // Name returns the unique provider identifier. Name() string @@ -108,3 +122,9 @@ type SnapshotSummary struct { type SnapshotLister interface { ListSnapshots() []SnapshotSummary } + +// RuntimeSandboxLister is implemented by providers that can enumerate +// already-running runtimes for startup reconciliation. +type RuntimeSandboxLister interface { + ListRuntimeSandboxes(ctx context.Context) ([]RuntimeSandbox, error) +} diff --git a/internal/providers/provider_conformance_test.go b/internal/providers/provider_conformance_test.go new file mode 100644 index 0000000..7b0f6c4 --- /dev/null +++ b/internal/providers/provider_conformance_test.go @@ -0,0 +1,157 @@ +package providers + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" +) + +type providerFactory func(t *testing.T) Provider + +func runProviderConformance(t *testing.T, factory providerFactory) { + t.Helper() + + t.Run("spawn status and destroy", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + + id, err := p.Spawn(ctx, SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if id == "" { + t.Fatal("spawn returned an empty sandbox ID") + } + + status, err := p.Status(ctx, id) + if err != nil { + t.Fatalf("status: %v", err) + } + if status.ID != id { + t.Fatalf("status ID = %q, want %q", status.ID, id) + } + if status.State != "running" { + t.Fatalf("status state = %q, want running", status.State) + } + + if err := p.Destroy(ctx, id); err != nil { + t.Fatalf("destroy: %v", err) + } + + _, err = p.Exec(ctx, id, ExecOptions{Command: "echo after destroy"}) + if !errors.Is(err, ErrSandboxDestroyed) && !errors.Is(err, ErrSandboxNotFound) { + t.Fatalf("exec after destroy error = %v, want sandbox lifecycle error", err) + } + }) + + t.Run("exec captures success and nonzero exit", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + id := spawnConformanceSandbox(t, p) + t.Cleanup(func() { _ = p.Destroy(context.Background(), id) }) + + result, err := p.Exec(ctx, id, ExecOptions{Command: "echo conformance-ok"}) + if err != nil { + t.Fatalf("exec success: %v", err) + } + if result.ExitCode != 0 { + t.Fatalf("exit code = %d, want 0", result.ExitCode) + } + if !strings.Contains(result.Stdout, "conformance-ok") { + t.Fatalf("stdout = %q, want conformance-ok", result.Stdout) + } + + result, err = p.Exec(ctx, id, ExecOptions{Command: "exit 7"}) + if err != nil { + t.Fatalf("exec nonzero: %v", err) + } + if result.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", result.ExitCode) + } + }) + + t.Run("exec stream emits output", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + id := spawnConformanceSandbox(t, p) + t.Cleanup(func() { _ = p.Destroy(context.Background(), id) }) + + ch, err := p.ExecStream(ctx, id, ExecOptions{Command: "printf stream-ok"}) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + var out strings.Builder + for chunk := range ch { + out.WriteString(chunk.Data) + } + if !strings.Contains(out.String(), "stream-ok") { + t.Fatalf("stream output = %q, want stream-ok", out.String()) + } + }) + + t.Run("file operations round trip", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + id := spawnConformanceSandbox(t, p) + t.Cleanup(func() { _ = p.Destroy(context.Background(), id) }) + + const originalPath = "/workspace/contract.txt" + const movedPath = "/workspace/contract-moved.txt" + const content = "provider contract file" + + if err := p.WriteFile(ctx, id, originalPath, bytes.NewReader([]byte(content)), "0644"); err != nil { + t.Fatalf("write file: %v", err) + } + + rc, err := p.ReadFile(ctx, id, originalPath) + if err != nil { + t.Fatalf("read file: %v", err) + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatalf("read content: %v", err) + } + if string(data) != content { + t.Fatalf("content = %q, want %q", string(data), content) + } + + if _, err := p.StatFile(ctx, id, originalPath); err != nil { + t.Fatalf("stat file: %v", err) + } + + matches, err := p.GlobFiles(ctx, id, "/workspace/contract*.txt") + if err != nil { + t.Fatalf("glob files: %v", err) + } + if len(matches) == 0 { + t.Fatal("glob files returned no matches") + } + + if err := p.MoveFile(ctx, id, originalPath, movedPath); err != nil { + t.Fatalf("move file: %v", err) + } + if err := p.ChmodFile(ctx, id, movedPath, "0755"); err != nil { + t.Fatalf("chmod file: %v", err) + } + if err := p.DeleteFile(ctx, id, movedPath, false); err != nil { + t.Fatalf("delete file: %v", err) + } + }) +} + +func spawnConformanceSandbox(t *testing.T, p Provider) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + id, err := p.Spawn(ctx, SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + return id +} diff --git a/internal/providers/registry.go b/internal/providers/registry.go index 5ab7261..02071a3 100644 --- a/internal/providers/registry.go +++ b/internal/providers/registry.go @@ -50,7 +50,7 @@ func (r *Registry) Get(name string) (Provider, error) { p, ok := r.providers[name] if !ok { - return nil, fmt.Errorf("provider %q not found", name) + return nil, ProviderNotFoundError(name) } return p, nil } diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 0000000..52a57a1 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,33 @@ +package store + +import ( + "errors" + "fmt" + "strings" +) + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") +) + +func NotFoundError(resource, id string) error { + if id == "" { + return fmt.Errorf("%w: %s", ErrNotFound, resource) + } + return fmt.Errorf("%w: %s %q", ErrNotFound, resource, id) +} + +func ConflictError(msg string) error { + return fmt.Errorf("%w: %s", ErrConflict, msg) +} + +func IsConstraintError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "UNIQUE constraint") || + strings.Contains(msg, "constraint failed") || + strings.Contains(msg, "constraint violation") +} diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index b1f2ae0..a1ee5a2 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -94,7 +94,7 @@ func (s *SQLiteStore) GetSandbox(ctx context.Context, id string) (*SandboxRecord ).Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, &sb.Metadata, &sb.OwnerID, &sb.VMID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, NotFoundError("sandbox", id) } return sb, err } @@ -130,7 +130,7 @@ func (s *SQLiteStore) UpdateSandboxState(ctx context.Context, id string, state s } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("sandbox %q not found", id) + return NotFoundError("sandbox", id) } return nil } @@ -145,7 +145,7 @@ func (s *SQLiteStore) UpdateSandboxExpiresAt(ctx context.Context, id string, exp } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("sandbox %q not found or already destroyed", id) + return NotFoundError("sandbox", id) } return nil } @@ -245,7 +245,7 @@ func (s *SQLiteStore) GetProviderConfig(ctx context.Context, name string) (*Prov SELECT name, config, enabled, updated_at FROM provider_configs WHERE name = ?`, name, ).Scan(&cfg.Name, &cfg.Config, &cfg.Enabled, &cfg.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("provider config %q not found", name) + return nil, NotFoundError("provider config", name) } return cfg, err } @@ -287,6 +287,9 @@ func (s *SQLiteStore) CreateTemplate(ctx context.Context, t *TemplateRecord) err t.MemoryMB, t.CPUCores, t.TTLSeconds, t.Env, t.Secrets, t.PoolSize, t.CreatedAt.UTC(), t.UpdatedAt.UTC(), ) + if IsConstraintError(err) { + return ConflictError("template already exists") + } return err } @@ -299,7 +302,7 @@ func (s *SQLiteStore) GetTemplate(ctx context.Context, name string) (*TemplateRe &t.MemoryMB, &t.CPUCores, &t.TTLSeconds, &t.Env, &t.Secrets, &t.PoolSize, &t.CreatedAt, &t.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("template %q not found", name) + return nil, NotFoundError("template", name) } return t, err } @@ -340,7 +343,7 @@ func (s *SQLiteStore) UpdateTemplate(ctx context.Context, t *TemplateRecord) err } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("template %q not found", t.Name) + return NotFoundError("template", t.Name) } return nil } @@ -352,7 +355,7 @@ func (s *SQLiteStore) DeleteTemplate(ctx context.Context, name string) error { } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("template %q not found", name) + return NotFoundError("template", name) } return nil } @@ -366,6 +369,9 @@ func (s *SQLiteStore) CreateEnvironmentSpec(ctx context.Context, spec *Environme spec.ID, spec.OwnerID, spec.Name, spec.BaseImage, spec.PythonPackages, spec.AptPackages, spec.PythonVersion, spec.CreatedAt.UTC(), spec.UpdatedAt.UTC(), ) + if IsConstraintError(err) { + return ConflictError("spec name already exists for this owner") + } return err } @@ -379,7 +385,7 @@ func (s *SQLiteStore) GetEnvironmentSpec(ctx context.Context, id string) (*Envir &spec.PythonVersion, &spec.CreatedAt, &spec.UpdatedAt, ) if err == sql.ErrNoRows { - return nil, fmt.Errorf("environment spec %q not found", id) + return nil, NotFoundError("environment spec", id) } return spec, err } @@ -422,7 +428,7 @@ func (s *SQLiteStore) UpdateEnvironmentSpec(ctx context.Context, spec *Environme } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("environment spec %q not found", spec.ID) + return NotFoundError("environment spec", spec.ID) } return nil } @@ -434,7 +440,7 @@ func (s *SQLiteStore) DeleteEnvironmentSpec(ctx context.Context, id string) erro } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("environment spec %q not found", id) + return NotFoundError("environment spec", id) } return nil } @@ -464,7 +470,7 @@ func (s *SQLiteStore) GetEnvironmentBuild(ctx context.Context, id string) (*Envi &build.DigestLocal, &build.Error, &build.CreatedAt, &finishedAt, &build.UpdatedAt, ) if err == sql.ErrNoRows { - return nil, fmt.Errorf("environment build %q not found", id) + return nil, NotFoundError("environment build", id) } if err != nil { return nil, err @@ -519,7 +525,7 @@ func (s *SQLiteStore) UpdateEnvironmentBuild(ctx context.Context, build *Environ } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("environment build %q not found", build.ID) + return NotFoundError("environment build", build.ID) } return nil } @@ -583,6 +589,9 @@ func (s *SQLiteStore) SaveRegistryConnection(ctx context.Context, conn *Registry conn.ID, conn.OwnerID, conn.Provider, conn.Username, conn.SecretRef, conn.IsDefault, time.Now().UTC(), time.Now().UTC(), ) + if IsConstraintError(err) { + return ConflictError("registry connection already exists") + } return err } @@ -594,7 +603,7 @@ func (s *SQLiteStore) GetRegistryConnection(ctx context.Context, id string) (*Re WHERE id = ?`, id, ).Scan(&conn.ID, &conn.OwnerID, &conn.Provider, &conn.Username, &conn.SecretRef, &conn.IsDefault, &conn.CreatedAt, &conn.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("registry connection %q not found", id) + return nil, NotFoundError("registry connection", id) } return conn, err } @@ -630,7 +639,7 @@ func (s *SQLiteStore) DeleteRegistryConnection(ctx context.Context, id string) e } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("registry connection %q not found", id) + return NotFoundError("registry connection", id) } return nil } diff --git a/internal/store/sqlite_test.go b/internal/store/sqlite_test.go index 57fc509..9b92329 100644 --- a/internal/store/sqlite_test.go +++ b/internal/store/sqlite_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "errors" "os" "path/filepath" "testing" @@ -247,6 +248,9 @@ func TestUpdateSandboxExpiresAt_Destroyed(t *testing.T) { if err == nil { t.Fatal("expected error extending destroyed sandbox") } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } } func TestUpdateSandboxExpiresAt_NotFound(t *testing.T) { @@ -255,6 +259,9 @@ func TestUpdateSandboxExpiresAt_NotFound(t *testing.T) { if err == nil { t.Fatal("expected error for nonexistent sandbox") } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } } func TestGetSandboxNotFound(t *testing.T) { @@ -263,6 +270,9 @@ func TestGetSandboxNotFound(t *testing.T) { if err == nil { t.Fatal("expected error for nonexistent sandbox") } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } } func TestEnvironmentSpecCRUD(t *testing.T) { @@ -285,6 +295,11 @@ func TestEnvironmentSpecCRUD(t *testing.T) { if err := s.CreateEnvironmentSpec(ctx, spec); err != nil { t.Fatalf("create spec: %v", err) } + conflicting := *spec + conflicting.ID = "envspec-conflict" + if err := s.CreateEnvironmentSpec(ctx, &conflicting); !errors.Is(err, ErrConflict) { + t.Fatalf("expected ErrConflict for duplicate owner/name, got %v", err) + } got, err := s.GetEnvironmentSpec(ctx, spec.ID) if err != nil { @@ -313,6 +328,8 @@ func TestEnvironmentSpecCRUD(t *testing.T) { } if _, err := s.GetEnvironmentSpec(ctx, spec.ID); err == nil { t.Fatal("expected get to fail after delete") + } else if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound after delete, got %v", err) } } From bbe82d63827994bca8ed11090899594fa565aac4 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 00:34:23 +0530 Subject: [PATCH 002/147] docs: add phase 1 foundation release notes --- docs/releases/phase-1-foundation-hardening.md | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 docs/releases/phase-1-foundation-hardening.md diff --git a/docs/releases/phase-1-foundation-hardening.md b/docs/releases/phase-1-foundation-hardening.md new file mode 100644 index 0000000..ed98ff0 --- /dev/null +++ b/docs/releases/phase-1-foundation-hardening.md @@ -0,0 +1,161 @@ +# Phase 1 Foundation Hardening Release Notes + +Date: 2026-05-08 +Branch: `feat/phase-1-foundation-hardening` +Commit: `194267e` + +## Summary + +Phase 1 focused on turning StacyVM's early provider/orchestrator foundation into a more production-ready base. The work improves error consistency, provider contracts, startup recovery, platform-aware conformance coverage, and local developer build reliability. + +This phase does not introduce a new user-facing sandbox feature. Instead, it strengthens the foundation that future phases will build on: predictable errors, safer reconciliation after restarts, clearer provider expectations, and stronger regression coverage. + +## What Changed + +### Provider Contract And Conformance + +- Added `docs/provider-contract.md` to document the required behavior for every sandbox provider. +- Added a reusable provider conformance test harness covering: + - spawn, status, and destroy lifecycle + - command execution success and non-zero exits + - streaming command output + - file write, read, stat, glob, move, chmod, and delete +- Wired conformance coverage for Mock, Docker, Custom, PRoot, and Firecracker providers. +- PRoot and Firecracker conformance tests are platform-gated so they skip locally unless the required runtime dependencies are available. + +### Typed Error Taxonomy + +- Added typed provider errors for: + - sandbox not found + - sandbox destroyed + - provider not found + - provider unavailable + - exec timeout + - resource limit +- Added typed store errors for: + - not found + - conflict +- Re-exported provider errors through the orchestrator package where API routes need stable domain-level matching. + +### API Error Handling + +- Added shared route error mapping in `internal/api/routes/errors.go`. +- Replaced string-matching error handling in sandbox, template, environment, and provider routes with typed error checks. +- Added response codes for timeout and resource-limit failures. +- API responses now map important failure classes consistently: + - `404` for missing resources and sandbox lifecycle misses + - `408` for exec timeout + - `429` for resource limits + - `503` for provider unavailability + +### Startup Reconciliation + +- Added `Manager.Reconcile(ctx)` to refresh persisted sandbox state from provider runtime state at server startup. +- Server startup now runs reconciliation before starting the manager reaper. +- Persisted sandboxes whose runtime no longer exists are marked `destroyed`. +- Persisted sandboxes whose provider is unavailable are marked `error`. +- Live provider runtimes can be restored into the manager's in-memory cache. + +### Docker Runtime Adoption + +- Docker sandboxes now include richer `stacyvm.*` labels for runtime discovery. +- Added Docker runtime inventory support through `ListRuntimeSandboxes`. +- Startup reconciliation can adopt StacyVM Docker containers that still exist but are missing from SQLite after a process restart. +- Docker missing-container cases now map to typed `ErrSandboxNotFound`. + +### Streaming Timeout Semantics + +- `Manager.ExecStream` now honors request-level timeout values. +- Streaming timeout paths emit an explicit stderr timeout chunk instead of silently closing. +- Docker, Custom, and Firecracker streaming paths now propagate timeout state more clearly. + +### macOS Build Reliability + +- The Linux-only `stacyvm-agent` entrypoint now has a Linux build tag. +- Added a non-Linux stub so `make build` and `make test` work on macOS while preserving the real Linux agent behavior. + +## Code Changes By Area + +### New Files + +- `CHANGELOG.md` +- `cmd/stacyvm-agent/main_unsupported.go` +- `docs/provider-contract.md` +- `internal/api/routes/errors.go` +- `internal/api/routes/templates_test.go` +- `internal/orchestrator/errors.go` +- `internal/providers/custom_conformance_test.go` +- `internal/providers/errors.go` +- `internal/providers/provider_conformance_test.go` +- `internal/store/errors.go` + +### Core Orchestrator + +- `internal/orchestrator/manager.go` + - Added startup reconciliation. + - Added provider runtime adoption. + - Added streaming timeout handling. +- `internal/orchestrator/manager_test.go` + - Added tests for reconciliation, runtime adoption, and streaming timeout behavior. +- `cmd/stacyvm/cmd_serve.go` + - Runs reconciliation during server startup. + +### Providers + +- `internal/providers/provider.go` + - Documented provider contract expectations. + - Added optional runtime inventory interfaces. +- `internal/providers/docker.go` + - Added StacyVM labels. + - Added runtime listing. + - Added typed not-found handling. + - Improved streaming timeout propagation. +- `internal/providers/custom.go` + - Added typed HTTP error mapping. + - Improved streaming timeout propagation. +- `internal/providers/firecracker.go` + - Added typed lifecycle behavior and streaming read deadlines. +- `internal/providers/proot.go` + - Added typed lifecycle, timeout, and resource-limit behavior. +- `internal/providers/mock.go` + - Added typed lifecycle and timeout behavior for tests. +- `internal/providers/registry.go` + - Uses typed provider-not-found errors. + +### Store + +- `internal/store/sqlite.go` + - Maps missing rows and constraint conflicts to typed store errors. +- `internal/store/sqlite_test.go` + - Adds coverage for typed store errors. + +### API Routes + +- `internal/api/routes/sandboxes.go` +- `internal/api/routes/templates.go` +- `internal/api/routes/environments.go` +- `internal/api/routes/providers.go` + +These routes now use shared typed error handling instead of string comparisons. + +## Verification + +The following checks passed: + +```sh +make test +make build +cd web && npm run build +``` + +Additional Docker provider conformance and runtime inventory checks passed with Docker daemon access. + +## Platform Notes + +- Firecracker conformance requires Linux, `/dev/kvm`, Firecracker, kernel, rootfs, and agent paths. +- PRoot conformance requires `proot` and a usable rootfs. +- The full Go integration suite uses `httptest`; local sandboxed runs need permission to bind local test sockets. + +## Impact + +Phase 1 leaves the codebase ready for Phase 2 work by making provider behavior explicit, testable, and recoverable. The next phase can focus on scalability and production operations without first untangling provider lifecycle ambiguity or inconsistent API failure behavior. From d17c4a0d09eaf223ada3f4b6f24bd92de90d62e8 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 00:45:36 +0530 Subject: [PATCH 003/147] feat: add operational readiness endpoints --- docs/api.md | 61 +++++++++++- internal/api/routes/swagger_types.go | 17 ++++ internal/api/routes/system.go | 126 ++++++++++++++++++++++-- internal/api/routes/system_test.go | 137 +++++++++++++++++++++++++++ internal/orchestrator/events.go | 17 ++++ 5 files changed, 346 insertions(+), 12 deletions(-) create mode 100644 internal/api/routes/system_test.go diff --git a/docs/api.md b/docs/api.md index 53b25e0..4a9884b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -479,6 +479,42 @@ GET /api/v1/health { "status": "ok", "version": "0.5.1", "uptime": "2h13m" } ``` +### Liveness + +``` +GET /api/v1/live +``` + +**Response** `200 OK`: +```json +{ "status": "alive", "version": "0.5.1", "uptime": "2h13m" } +``` + +Use this endpoint for process liveness checks. It only confirms that the API process is responding. + +### Readiness + +``` +GET /api/v1/ready +``` + +**Response** `200 OK`: +```json +{ + "status": "ready", + "version": "0.5.1", + "uptime": "2h13m", + "ready_providers": 1, + "total_providers": 2, + "providers": [ + { "name": "docker", "healthy": true, "default": true }, + { "name": "firecracker", "healthy": false, "default": false } + ] +} +``` + +**Response** `503 Service Unavailable` when no configured provider is healthy. + ### Metrics ``` @@ -488,10 +524,31 @@ GET /api/v1/metrics **Response** `200 OK`: ```json { + "uptime": "2h13m", "goroutines": 42, "memory_alloc": 17825792, - "active_sandboxes": 12, - "total_sandboxes": 138 + "memory_sys": 71303168, + "memory_heap_alloc": 17825792, + "gc_cycles": 8, + "sandboxes": { + "total": 138, + "active": 12, + "by_state": { "running": 12, "destroyed": 126 }, + "by_provider": { "docker": 90, "firecracker": 48 } + }, + "providers": { + "total": 2, + "healthy": 1, + "items": [ + { "name": "docker", "healthy": true, "default": true }, + { "name": "firecracker", "healthy": false, "default": false } + ] + }, + "events": { + "subscribers": 2, + "history_size": 1000, + "events_total": 2401 + } } ``` diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index 29fe6fb..c7a6649 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -17,6 +17,23 @@ type HealthResponse struct { Uptime string `json:"uptime" example:"2h30m15s"` } +// ProviderHealth is a provider readiness item. +type ProviderHealth struct { + Name string `json:"name" example:"docker"` + Healthy bool `json:"healthy" example:"true"` + Default bool `json:"default" example:"true"` +} + +// ReadinessResponse is the response from the readiness endpoint. +type ReadinessResponse struct { + Status string `json:"status" example:"ready"` + Version string `json:"version" example:"1.0.0"` + Uptime string `json:"uptime" example:"2h30m15s"` + Providers []ProviderHealth `json:"providers"` + ReadyProviders int `json:"ready_providers" example:"1"` + TotalProviders int `json:"total_providers" example:"2"` +} + // MetricsResponse is the response from the metrics endpoint. type MetricsResponse struct { SandboxesActive int `json:"sandboxes_active" example:"5"` diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index b266b20..81165d0 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -1,17 +1,18 @@ package routes import ( + "context" "encoding/json" "fmt" "net/http" "runtime" "time" - "github.com/go-chi/chi/v5" - "github.com/google/uuid" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" "github.com/StacyOs/stacyvm/internal/providers" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" ) type SystemRoutes struct { @@ -35,6 +36,8 @@ func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager func (s *SystemRoutes) Routes() chi.Router { r := chi.NewRouter() r.Get("/health", s.Health) + r.Get("/live", s.Live) + r.Get("/ready", s.Ready) r.Get("/metrics", s.Metrics) r.Get("/events", s.Events) return r @@ -57,6 +60,62 @@ func (s *SystemRoutes) Health(w http.ResponseWriter, r *http.Request) { }) } +// Live returns process liveness. +// +// @Summary Liveness check +// @Description Return whether the StacyVM API process is alive +// @Tags system +// @Produce json +// @Success 200 {object} HealthResponse +// @Security ApiKeyAuth +// @Router /live [get] +func (s *SystemRoutes) Live(w http.ResponseWriter, r *http.Request) { + httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ + "status": "alive", + "version": s.version, + "uptime": time.Since(s.startTime).String(), + }) +} + +// Ready returns dependency readiness. +// +// @Summary Readiness check +// @Description Return whether the API is ready to serve sandbox traffic +// @Tags system +// @Produce json +// @Success 200 {object} ReadinessResponse +// @Failure 503 {object} ReadinessResponse +// @Security ApiKeyAuth +// @Router /ready [get] +func (s *SystemRoutes) Ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + providers := s.providerHealth(ctx) + readyProviders := 0 + for _, provider := range providers { + if provider.Healthy { + readyProviders++ + } + } + + statusCode := http.StatusOK + status := "ready" + if len(providers) == 0 || readyProviders == 0 { + statusCode = http.StatusServiceUnavailable + status = "not_ready" + } + + httputil.WriteJSON(w, statusCode, map[string]interface{}{ + "status": status, + "version": s.version, + "uptime": time.Since(s.startTime).String(), + "providers": providers, + "ready_providers": readyProviders, + "total_providers": len(providers), + }) +} + // Metrics returns runtime metrics. // // @Summary Get metrics @@ -70,22 +129,69 @@ func (s *SystemRoutes) Metrics(w http.ResponseWriter, r *http.Request) { var mem runtime.MemStats runtime.ReadMemStats(&mem) - sandboxes, _ := s.manager.List(r.Context()) - active := 0 + sandboxes, err := s.manager.List(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + + byState := make(map[string]int) + byProvider := make(map[string]int) for _, sb := range sandboxes { - if sb.State == orchestrator.StateRunning { - active++ + byState[string(sb.State)]++ + byProvider[sb.Provider]++ + } + + providerHealth := s.providerHealth(r.Context()) + healthyProviders := 0 + for _, provider := range providerHealth { + if provider.Healthy { + healthyProviders++ } } + eventStats := s.events.Stats() httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ - "goroutines": runtime.NumGoroutine(), - "memory_alloc": mem.Alloc, - "active_sandboxes": active, - "total_sandboxes": len(sandboxes), + "uptime": time.Since(s.startTime).String(), + "goroutines": runtime.NumGoroutine(), + "memory_alloc": mem.Alloc, + "memory_sys": mem.Sys, + "memory_heap_alloc": mem.HeapAlloc, + "gc_cycles": mem.NumGC, + "sandboxes": map[string]interface{}{ + "total": len(sandboxes), + "active": byState[string(orchestrator.StateRunning)], + "by_state": byState, + "by_provider": byProvider, + }, + "providers": map[string]interface{}{ + "total": len(providerHealth), + "healthy": healthyProviders, + "items": providerHealth, + }, + "events": eventStats, }) } +func (s *SystemRoutes) providerHealth(ctx context.Context) []ProviderHealth { + names := s.registry.List() + out := make([]ProviderHealth, 0, len(names)) + defaultProvider := s.registry.Default() + for _, name := range names { + prov, err := s.registry.Get(name) + if err != nil { + out = append(out, ProviderHealth{Name: name, Healthy: false, Default: name == defaultProvider}) + continue + } + out = append(out, ProviderHealth{ + Name: name, + Healthy: prov.Healthy(ctx), + Default: name == defaultProvider, + }) + } + return out +} + // Events serves Server-Sent Events for real-time updates. // // @Summary Subscribe to events diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go new file mode 100644 index 0000000..7391a48 --- /dev/null +++ b/internal/api/routes/system_test.go @@ -0,0 +1,137 @@ +package routes + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/rs/zerolog" +) + +func setupSystemRoutes(t *testing.T, withProvider bool) (*SystemRoutes, *orchestrator.Manager) { + t.Helper() + + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + registry := providers.NewRegistry() + if withProvider { + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + } + + events := orchestrator.NewEventBus() + manager := orchestrator.NewManager(registry, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + + return NewSystemRoutes(registry, manager, events, "test-version"), manager +} + +func TestSystemRoutes_Live(t *testing.T) { + routes, _ := setupSystemRoutes(t, true) + req := httptest.NewRequest(http.MethodGet, "/live", nil) + w := httptest.NewRecorder() + + routes.Live(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + if body["status"] != "alive" { + t.Fatalf("status = %v, want alive", body["status"]) + } +} + +func TestSystemRoutes_Ready(t *testing.T) { + routes, _ := setupSystemRoutes(t, true) + req := httptest.NewRequest(http.MethodGet, "/ready", nil) + w := httptest.NewRecorder() + + routes.Ready(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + if body["status"] != "ready" { + t.Fatalf("status = %v, want ready", body["status"]) + } + if body["ready_providers"].(float64) != 1 { + t.Fatalf("ready providers = %v, want 1", body["ready_providers"]) + } +} + +func TestSystemRoutes_ReadyNoProviders(t *testing.T) { + routes, _ := setupSystemRoutes(t, false) + req := httptest.NewRequest(http.MethodGet, "/ready", nil) + w := httptest.NewRecorder() + + routes.Ready(w, req) + + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", w.Code, http.StatusServiceUnavailable) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + if body["status"] != "not_ready" { + t.Fatalf("status = %v, want not_ready", body["status"]) + } +} + +func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { + routes, manager := setupSystemRoutes(t, true) + if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { + t.Fatalf("spawn: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + w := httptest.NewRecorder() + + routes.Metrics(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + + sandboxes := body["sandboxes"].(map[string]interface{}) + if sandboxes["total"].(float64) != 1 { + t.Fatalf("sandbox total = %v, want 1", sandboxes["total"]) + } + providersBody := body["providers"].(map[string]interface{}) + if providersBody["healthy"].(float64) != 1 { + t.Fatalf("healthy providers = %v, want 1", providersBody["healthy"]) + } + if _, ok := body["events"].(map[string]interface{}); !ok { + t.Fatal("expected events metrics") + } +} + +func decodeSystemResponse(t *testing.T, w *httptest.ResponseRecorder, dst interface{}) { + t.Helper() + if err := json.Unmarshal(w.Body.Bytes(), dst); err != nil { + t.Fatalf("decode response: %v", err) + } +} diff --git a/internal/orchestrator/events.go b/internal/orchestrator/events.go index fbcce10..193fb2d 100644 --- a/internal/orchestrator/events.go +++ b/internal/orchestrator/events.go @@ -41,6 +41,12 @@ type EventBus struct { nextID int } +type EventBusStats struct { + Subscribers int `json:"subscribers"` + HistorySize int `json:"history_size"` + EventsTotal int `json:"events_total"` +} + func NewEventBus() *EventBus { return &EventBus{ subscribers: make(map[string]chan Event), @@ -113,3 +119,14 @@ func (eb *EventBus) History(limit int) []Event { copy(result, eb.history[start:]) return result } + +func (eb *EventBus) Stats() EventBusStats { + eb.mu.RLock() + defer eb.mu.RUnlock() + + return EventBusStats{ + Subscribers: len(eb.subscribers), + HistorySize: len(eb.history), + EventsTotal: eb.nextID, + } +} From 7280b386cc9e1730efb817573e7ca86555fac329 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 00:52:22 +0530 Subject: [PATCH 004/147] feat: add structured operation metrics --- docs/api.md | 35 ++++- internal/api/routes/prometheus.go | 91 +++++++++++++ internal/api/routes/system.go | 111 +++++++++++++--- internal/api/routes/system_test.go | 40 ++++++ internal/orchestrator/manager.go | 185 ++++++++++++++++++++++++-- internal/orchestrator/manager_test.go | 38 ++++++ internal/orchestrator/metrics.go | 134 +++++++++++++++++++ 7 files changed, 600 insertions(+), 34 deletions(-) create mode 100644 internal/api/routes/prometheus.go create mode 100644 internal/orchestrator/metrics.go diff --git a/docs/api.md b/docs/api.md index 4a9884b..cf7a929 100644 --- a/docs/api.md +++ b/docs/api.md @@ -548,11 +548,42 @@ GET /api/v1/metrics "subscribers": 2, "history_size": 1000, "events_total": 2401 - } + }, + "operations": [ + { + "operation": "exec", + "provider": "docker", + "success_total": 482, + "failure_total": 7, + "latency_count": 489, + "latency_total_ms": 39120, + "latency_min_ms": 3, + "latency_max_ms": 2500, + "latency_avg_ms": 80 + } + ] } ``` -For Prometheus-style metrics, scrape this endpoint and parse to your needs (a `/metrics` Prometheus exporter is on the roadmap). +### Prometheus metrics + +``` +GET /api/v1/metrics/prometheus +``` + +**Response** `200 OK`: +```text +# HELP stacyvm_uptime_seconds StacyVM API process uptime in seconds. +# TYPE stacyvm_uptime_seconds gauge +stacyvm_uptime_seconds 7980 +# HELP stacyvm_provider_healthy Provider health status where 1 is healthy and 0 is unhealthy. +# TYPE stacyvm_provider_healthy gauge +stacyvm_provider_healthy{provider="docker",default="true"} 1 +stacyvm_operation_success_total{operation="exec",provider="docker"} 482 +stacyvm_operation_failure_total{operation="exec",provider="docker"} 7 +``` + +Use this endpoint for Prometheus-compatible scraping of runtime, provider, sandbox, event, and operation metrics. --- diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go new file mode 100644 index 0000000..bc44525 --- /dev/null +++ b/internal/api/routes/prometheus.go @@ -0,0 +1,91 @@ +package routes + +import ( + "fmt" + "io" + "sort" + + "github.com/StacyOs/stacyvm/internal/orchestrator" +) + +func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { + writePrometheusHelp(w, "stacyvm_uptime_seconds", "StacyVM API process uptime in seconds.") + fmt.Fprintf(w, "stacyvm_uptime_seconds %d\n", int64(metrics.uptime.Seconds())) + + writePrometheusHelp(w, "stacyvm_runtime_goroutines", "Current number of goroutines.") + fmt.Fprintf(w, "stacyvm_runtime_goroutines %d\n", metrics.goroutines) + + writePrometheusHelp(w, "stacyvm_runtime_memory_alloc_bytes", "Current allocated memory in bytes.") + fmt.Fprintf(w, "stacyvm_runtime_memory_alloc_bytes %d\n", metrics.memoryAlloc) + + writePrometheusHelp(w, "stacyvm_runtime_memory_sys_bytes", "Total memory obtained from the OS in bytes.") + fmt.Fprintf(w, "stacyvm_runtime_memory_sys_bytes %d\n", metrics.memorySys) + + writePrometheusHelp(w, "stacyvm_runtime_gc_cycles_total", "Total completed GC cycles.") + fmt.Fprintf(w, "stacyvm_runtime_gc_cycles_total %d\n", metrics.gcCycles) + + writePrometheusHelp(w, "stacyvm_sandboxes_total", "Current sandbox records by state and provider.") + states := sortedKeys(metrics.sandboxesByState) + for _, state := range states { + fmt.Fprintf(w, "stacyvm_sandboxes_total{state=%q} %d\n", state, metrics.sandboxesByState[state]) + } + providers := sortedKeys(metrics.sandboxesByProvider) + for _, provider := range providers { + fmt.Fprintf(w, "stacyvm_sandboxes_by_provider_total{provider=%q} %d\n", provider, metrics.sandboxesByProvider[provider]) + } + + writePrometheusHelp(w, "stacyvm_provider_healthy", "Provider health status where 1 is healthy and 0 is unhealthy.") + for _, provider := range metrics.providerHealth { + healthy := 0 + if provider.Healthy { + healthy = 1 + } + fmt.Fprintf(w, "stacyvm_provider_healthy{provider=%q,default=%q} %d\n", provider.Name, boolLabel(provider.Default), healthy) + } + + writePrometheusHelp(w, "stacyvm_events_total", "Total events published by the in-process event bus.") + fmt.Fprintf(w, "stacyvm_events_total %d\n", metrics.eventStats.EventsTotal) + writePrometheusHelp(w, "stacyvm_event_subscribers", "Current event stream subscriber count.") + fmt.Fprintf(w, "stacyvm_event_subscribers %d\n", metrics.eventStats.Subscribers) + writePrometheusHelp(w, "stacyvm_event_history_size", "Current event history item count.") + fmt.Fprintf(w, "stacyvm_event_history_size %d\n", metrics.eventStats.HistorySize) + + writeOperationMetrics(w, metrics.operationMetrics) +} + +func writeOperationMetrics(w io.Writer, operationMetrics []orchestrator.OperationMetrics) { + writePrometheusHelp(w, "stacyvm_operation_success_total", "Total successful operations by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_failure_total", "Total failed operations by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_latency_milliseconds_sum", "Total operation latency in milliseconds by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_latency_milliseconds_count", "Total observed operation latency samples by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_latency_milliseconds_max", "Maximum observed operation latency in milliseconds by operation and provider.") + for _, metric := range operationMetrics { + labels := fmt.Sprintf("operation=%q,provider=%q", metric.Operation, metric.Provider) + fmt.Fprintf(w, "stacyvm_operation_success_total{%s} %d\n", labels, metric.SuccessTotal) + fmt.Fprintf(w, "stacyvm_operation_failure_total{%s} %d\n", labels, metric.FailureTotal) + fmt.Fprintf(w, "stacyvm_operation_latency_milliseconds_sum{%s} %d\n", labels, metric.LatencyTotalMS) + fmt.Fprintf(w, "stacyvm_operation_latency_milliseconds_count{%s} %d\n", labels, metric.LatencyCount) + fmt.Fprintf(w, "stacyvm_operation_latency_milliseconds_max{%s} %d\n", labels, metric.LatencyMaxMS) + } +} + +func writePrometheusHelp(w io.Writer, name, help string) { + fmt.Fprintf(w, "# HELP %s %s\n", name, help) + fmt.Fprintf(w, "# TYPE %s gauge\n", name) +} + +func sortedKeys(values map[string]int) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func boolLabel(value bool) string { + if value { + return "true" + } + return "false" +} diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index 81165d0..201bc88 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -1,6 +1,7 @@ package routes import ( + "bytes" "context" "encoding/json" "fmt" @@ -39,6 +40,7 @@ func (s *SystemRoutes) Routes() chi.Router { r.Get("/live", s.Live) r.Get("/ready", s.Ready) r.Get("/metrics", s.Metrics) + r.Get("/metrics/prometheus", s.PrometheusMetrics) r.Get("/events", s.Events) return r } @@ -126,15 +128,64 @@ func (s *SystemRoutes) Ready(w http.ResponseWriter, r *http.Request) { // @Security ApiKeyAuth // @Router /metrics [get] func (s *SystemRoutes) Metrics(w http.ResponseWriter, r *http.Request) { - var mem runtime.MemStats - runtime.ReadMemStats(&mem) + metrics, err := s.collectMetrics(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + + httputil.WriteJSON(w, http.StatusOK, metrics.toResponse()) +} - sandboxes, err := s.manager.List(r.Context()) +// PrometheusMetrics returns Prometheus-compatible operational metrics. +// +// @Summary Get Prometheus metrics +// @Description Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format +// @Tags system +// @Produce text/plain +// @Success 200 {string} string +// @Security ApiKeyAuth +// @Router /metrics/prometheus [get] +func (s *SystemRoutes) PrometheusMetrics(w http.ResponseWriter, r *http.Request) { + metrics, err := s.collectMetrics(r.Context()) if err != nil { writeRouteError(w, err) return } + var buf bytes.Buffer + writePrometheusMetrics(&buf, metrics) + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(buf.Bytes()) +} + +type systemMetricsSnapshot struct { + uptime time.Duration + goroutines int + memoryAlloc uint64 + memorySys uint64 + memoryHeapAlloc uint64 + gcCycles uint32 + sandboxTotal int + sandboxActive int + sandboxesByState map[string]int + sandboxesByProvider map[string]int + providerHealth []ProviderHealth + healthyProviders int + eventStats orchestrator.EventBusStats + operationMetrics []orchestrator.OperationMetrics +} + +func (s *SystemRoutes) collectMetrics(ctx context.Context) (systemMetricsSnapshot, error) { + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + + sandboxes, err := s.manager.List(ctx) + if err != nil { + return systemMetricsSnapshot{}, err + } + byState := make(map[string]int) byProvider := make(map[string]int) for _, sb := range sandboxes { @@ -142,7 +193,7 @@ func (s *SystemRoutes) Metrics(w http.ResponseWriter, r *http.Request) { byProvider[sb.Provider]++ } - providerHealth := s.providerHealth(r.Context()) + providerHealth := s.providerHealth(ctx) healthyProviders := 0 for _, provider := range providerHealth { if provider.Healthy { @@ -151,26 +202,46 @@ func (s *SystemRoutes) Metrics(w http.ResponseWriter, r *http.Request) { } eventStats := s.events.Stats() - httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ - "uptime": time.Since(s.startTime).String(), - "goroutines": runtime.NumGoroutine(), - "memory_alloc": mem.Alloc, - "memory_sys": mem.Sys, - "memory_heap_alloc": mem.HeapAlloc, - "gc_cycles": mem.NumGC, + return systemMetricsSnapshot{ + uptime: time.Since(s.startTime), + goroutines: runtime.NumGoroutine(), + memoryAlloc: mem.Alloc, + memorySys: mem.Sys, + memoryHeapAlloc: mem.HeapAlloc, + gcCycles: mem.NumGC, + sandboxTotal: len(sandboxes), + sandboxActive: byState[string(orchestrator.StateRunning)], + sandboxesByState: byState, + sandboxesByProvider: byProvider, + providerHealth: providerHealth, + healthyProviders: healthyProviders, + eventStats: eventStats, + operationMetrics: s.manager.OperationMetrics(), + }, nil +} + +func (m systemMetricsSnapshot) toResponse() map[string]interface{} { + return map[string]interface{}{ + "uptime": m.uptime.String(), + "goroutines": m.goroutines, + "memory_alloc": m.memoryAlloc, + "memory_sys": m.memorySys, + "memory_heap_alloc": m.memoryHeapAlloc, + "gc_cycles": m.gcCycles, "sandboxes": map[string]interface{}{ - "total": len(sandboxes), - "active": byState[string(orchestrator.StateRunning)], - "by_state": byState, - "by_provider": byProvider, + "total": m.sandboxTotal, + "active": m.sandboxActive, + "by_state": m.sandboxesByState, + "by_provider": m.sandboxesByProvider, }, "providers": map[string]interface{}{ - "total": len(providerHealth), - "healthy": healthyProviders, - "items": providerHealth, + "total": len(m.providerHealth), + "healthy": m.healthyProviders, + "items": m.providerHealth, }, - "events": eventStats, - }) + "events": m.eventStats, + "operations": m.operationMetrics, + } } func (s *SystemRoutes) providerHealth(ctx context.Context) []ProviderHealth { diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index 7391a48..13f1424 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "strings" "testing" "time" @@ -127,6 +128,45 @@ func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { if _, ok := body["events"].(map[string]interface{}); !ok { t.Fatal("expected events metrics") } + operations := body["operations"].([]interface{}) + if len(operations) == 0 { + t.Fatal("expected operation metrics") + } +} + +func TestSystemRoutes_PrometheusMetrics(t *testing.T) { + routes, manager := setupSystemRoutes(t, true) + sb, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if _, err := manager.Exec(context.Background(), sb.ID, orchestrator.ExecRequest{Command: "echo prometheus"}); err != nil { + t.Fatalf("exec: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/metrics/prometheus", nil) + w := httptest.NewRecorder() + + routes.PrometheusMetrics(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get("Content-Type"); !strings.Contains(got, "text/plain") { + t.Fatalf("content type = %q, want text/plain", got) + } + body := w.Body.String() + for _, want := range []string{ + "stacyvm_uptime_seconds", + "stacyvm_provider_healthy", + "stacyvm_operation_success_total", + `operation="spawn"`, + `operation="exec"`, + } { + if !strings.Contains(body, want) { + t.Fatalf("prometheus body missing %q:\n%s", want, body) + } + } } func decodeSystemResponse(t *testing.T, w *httptest.ResponseRecorder, dst interface{}) { diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index c18b407..3e46ba9 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -23,6 +23,7 @@ type Manager struct { store store.Store events *EventBus logger zerolog.Logger + metrics *MetricsRecorder mu sync.RWMutex sandboxes map[string]*Sandbox @@ -57,6 +58,7 @@ func NewManager(registry *providers.Registry, st store.Store, events *EventBus, store: st, events: events, logger: logger.With().Str("component", "manager").Logger(), + metrics: NewMetricsRecorder(), sandboxes: make(map[string]*Sandbox), defaultTTL: cfg.DefaultTTL, defaultImage: cfg.DefaultImage, @@ -270,11 +272,23 @@ func (m *Manager) reconcileProviderRuntimes(ctx context.Context, known map[strin } func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) { + start := time.Now() + metricsProvider := req.Provider + if metricsProvider == "" { + metricsProvider = m.registry.Default() + } + var metricsErr error + defer func() { + m.recordOperation(OperationSpawn, metricsProvider, time.Since(start), metricsErr) + }() + providerName := req.Provider prov, err := m.registry.Get(providerName) if err != nil { + metricsErr = err return nil, fmt.Errorf("getting provider: %w", err) } + metricsProvider = prov.Name() image := req.Image if image == "" { @@ -301,7 +315,9 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) // Pool mode: acquire a VM slot instead of spawning a new VM. if m.vmPoolMgr != nil { - return m.spawnPooled(ctx, prov, req, image, memMB, vcpus, ttl, now) + sb, err := m.spawnPooled(ctx, prov, req, image, memMB, vcpus, ttl, now) + metricsErr = err + return sb, err } sb := &Sandbox{ @@ -325,6 +341,7 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) Metadata: req.Metadata, }) if err != nil { + metricsErr = err return nil, fmt.Errorf("spawning sandbox: %w", err) } sb.ID = id @@ -348,6 +365,7 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) }); err != nil { // Best effort: destroy the sandbox if DB write fails prov.Destroy(ctx, id) + metricsErr = err return nil, fmt.Errorf("persisting sandbox: %w", err) } @@ -445,16 +463,26 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req } func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) (*ExecResult, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationExec, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return nil, err } + metricsProvider = sb.Provider execCtx := ctx var cancel context.CancelFunc if req.Timeout != "" { timeout, err := time.ParseDuration(req.Timeout) if err != nil { + metricsErr = err return nil, fmt.Errorf("parsing exec timeout: %w", err) } execCtx, cancel = context.WithTimeout(ctx, timeout) @@ -472,7 +500,7 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( workDir = "/workspace/" + sandboxID } - start := time.Now() + execStart := time.Now() result, err := prov.Exec(execCtx, m.resolveVMID(sb), providers.ExecOptions{ Command: req.Command, Args: req.Args, @@ -481,11 +509,13 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( }) if err != nil { if execCtx.Err() == context.DeadlineExceeded { - return nil, providers.ExecTimeoutError(sandboxID) + metricsErr = providers.ExecTimeoutError(sandboxID) + return nil, metricsErr } + metricsErr = err return nil, fmt.Errorf("exec: %w", err) } - duration := time.Since(start) + duration := time.Since(execStart) execResult := &ExecResult{ ExitCode: result.ExitCode, @@ -514,16 +544,28 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( } func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequest) (<-chan providers.StreamChunk, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + if metricsErr != nil { + m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), metricsErr) + } + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return nil, err } + metricsProvider = sb.Provider execCtx := ctx var cancel context.CancelFunc if req.Timeout != "" { timeout, err := time.ParseDuration(req.Timeout) if err != nil { + metricsErr = err return nil, fmt.Errorf("parsing exec timeout: %w", err) } execCtx, cancel = context.WithTimeout(ctx, timeout) @@ -545,12 +587,22 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ cancel() } if execCtx.Err() == context.DeadlineExceeded { - return nil, providers.ExecTimeoutError(sandboxID) + metricsErr = providers.ExecTimeoutError(sandboxID) + return nil, metricsErr } + metricsErr = err return nil, err } if cancel == nil { - return ch, nil + out := make(chan providers.StreamChunk, 64) + go func() { + defer close(out) + for chunk := range ch { + out <- chunk + } + m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), nil) + }() + return out, nil } out := make(chan providers.StreamChunk, 64) @@ -566,20 +618,31 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ } } if execCtx.Err() == context.DeadlineExceeded || timedOut { + metricsErr = providers.ExecTimeoutError(sandboxID) select { - case out <- providers.StreamChunk{Stream: "stderr", Data: providers.ExecTimeoutError(sandboxID).Error()}: + case out <- providers.StreamChunk{Stream: "stderr", Data: metricsErr.Error()}: case <-ctx.Done(): } } + m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), metricsErr) }() return out, nil } func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWriteRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileWrite, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return err } + metricsProvider = sb.Provider mode := req.Mode if mode == "" { @@ -588,6 +651,7 @@ func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWrite path := m.scopedPath(sb, req.Path) if err := prov.WriteFile(ctx, m.resolveVMID(sb), path, strings.NewReader(req.Content), mode); err != nil { + metricsErr = err return fmt.Errorf("writing file: %w", err) } @@ -599,19 +663,30 @@ func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWrite } func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ([]byte, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileRead, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return nil, err } + metricsProvider = sb.Provider rc, err := prov.ReadFile(ctx, m.resolveVMID(sb), m.scopedPath(sb, path)) if err != nil { + metricsErr = err return nil, fmt.Errorf("reading file: %w", err) } defer rc.Close() buf, err := io.ReadAll(rc) if err != nil { + metricsErr = err return nil, fmt.Errorf("reading file content: %w", err) } @@ -623,13 +698,23 @@ func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ( } func (m *Manager) ListFiles(ctx context.Context, sandboxID string, path string) ([]FileInfo, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileList, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return nil, err } + metricsProvider = sb.Provider pFiles, err := prov.ListFiles(ctx, m.resolveVMID(sb), m.scopedPath(sb, path)) if err != nil { + metricsErr = err return nil, err } @@ -647,45 +732,85 @@ func (m *Manager) ListFiles(ctx context.Context, sandboxID string, path string) } func (m *Manager) DeleteFile(ctx context.Context, sandboxID string, req FileDeleteRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileDelete, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return err } + metricsProvider = sb.Provider path := m.scopedPath(sb, req.Path) - return prov.DeleteFile(ctx, m.resolveVMID(sb), path, req.Recursive) + metricsErr = prov.DeleteFile(ctx, m.resolveVMID(sb), path, req.Recursive) + return metricsErr } func (m *Manager) MoveFile(ctx context.Context, sandboxID string, req FileMoveRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileMove, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return err } + metricsProvider = sb.Provider oldPath := m.scopedPath(sb, req.OldPath) newPath := m.scopedPath(sb, req.NewPath) - return prov.MoveFile(ctx, m.resolveVMID(sb), oldPath, newPath) + metricsErr = prov.MoveFile(ctx, m.resolveVMID(sb), oldPath, newPath) + return metricsErr } func (m *Manager) ChmodFile(ctx context.Context, sandboxID string, req FileChmodRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileChmod, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return err } + metricsProvider = sb.Provider path := m.scopedPath(sb, req.Path) - return prov.ChmodFile(ctx, m.resolveVMID(sb), path, req.Mode) + metricsErr = prov.ChmodFile(ctx, m.resolveVMID(sb), path, req.Mode) + return metricsErr } func (m *Manager) StatFile(ctx context.Context, sandboxID string, path string) (*FileInfo, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileStat, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return nil, err } + metricsProvider = sb.Provider scopedPath := m.scopedPath(sb, path) fi, err := prov.StatFile(ctx, m.resolveVMID(sb), scopedPath) if err != nil { + metricsErr = err return nil, err } return &FileInfo{ @@ -698,13 +823,24 @@ func (m *Manager) StatFile(ctx context.Context, sandboxID string, path string) ( } func (m *Manager) GlobFiles(ctx context.Context, sandboxID string, pattern string) ([]string, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileGlob, metricsProvider, time.Since(start), metricsErr) + }() + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err return nil, err } + metricsProvider = sb.Provider scopedPattern := m.scopedPath(sb, pattern) - return prov.GlobFiles(ctx, m.resolveVMID(sb), scopedPattern) + matches, err := prov.GlobFiles(ctx, m.resolveVMID(sb), scopedPattern) + metricsErr = err + return matches, err } // scopedPath prefixes a path with the sandbox workspace when running in pool mode. @@ -738,6 +874,17 @@ func (m *Manager) VMPoolStatus() *VMPoolStatus { return &status } +func (m *Manager) OperationMetrics() []OperationMetrics { + return m.metrics.Snapshot() +} + +func (m *Manager) recordOperation(operation, provider string, duration time.Duration, err error) { + if m.metrics == nil { + return + } + m.metrics.RecordOperation(operation, provider, duration, err) +} + // InitVMPool initializes the VM pool manager if pool mode is enabled. func (m *Manager) InitVMPool() { if !m.poolConfig.Enabled { @@ -871,13 +1018,24 @@ func (m *Manager) List(ctx context.Context) ([]*Sandbox, error) { } func (m *Manager) Destroy(ctx context.Context, id string) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationDestroy, metricsProvider, time.Since(start), metricsErr) + }() + // Check if this is a pooled sandbox. m.mu.RLock() sb := m.sandboxes[id] m.mu.RUnlock() + if sb != nil { + metricsProvider = sb.Provider + } if sb != nil && sb.VMID != "" && m.vmPoolMgr != nil { - return m.destroyPooled(ctx, id, sb) + metricsErr = m.destroyPooled(ctx, id, sb) + return metricsErr } prov, err := m.getProvider(id) @@ -889,6 +1047,9 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { m.mu.Unlock() return nil } + if sb != nil { + metricsProvider = sb.Provider + } if err := prov.Destroy(ctx, id); err != nil { // Debug-level: this is expected when VMs were killed externally (e.g. process restart). diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 0a33951..e13bb52 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -183,6 +183,44 @@ func TestManager_Exec(t *testing.T) { } } +func TestManager_OperationMetrics(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, err := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if _, err := m.Exec(ctx, sb.ID, ExecRequest{Command: "echo metrics"}); err != nil { + t.Fatalf("exec: %v", err) + } + if err := m.WriteFile(ctx, sb.ID, FileWriteRequest{Path: "/workspace/metrics.txt", Content: "ok"}); err != nil { + t.Fatalf("write: %v", err) + } + if err := m.Destroy(ctx, sb.ID); err != nil { + t.Fatalf("destroy: %v", err) + } + + metrics := m.OperationMetrics() + assertOperationMetric(t, metrics, OperationSpawn, "mock") + assertOperationMetric(t, metrics, OperationExec, "mock") + assertOperationMetric(t, metrics, OperationFileWrite, "mock") + assertOperationMetric(t, metrics, OperationDestroy, "mock") +} + +func assertOperationMetric(t *testing.T, metrics []OperationMetrics, operation, provider string) { + t.Helper() + for _, metric := range metrics { + if metric.Operation == operation && metric.Provider == provider { + if metric.SuccessTotal == 0 { + t.Fatalf("%s/%s success total = 0", operation, provider) + } + return + } + } + t.Fatalf("operation metric %s/%s not found in %+v", operation, provider, metrics) +} + func TestManager_ExecTimeout(t *testing.T) { m := setupManager(t) ctx := context.Background() diff --git a/internal/orchestrator/metrics.go b/internal/orchestrator/metrics.go new file mode 100644 index 0000000..3f74506 --- /dev/null +++ b/internal/orchestrator/metrics.go @@ -0,0 +1,134 @@ +package orchestrator + +import ( + "sort" + "sync" + "time" +) + +const ( + OperationSpawn = "spawn" + OperationExec = "exec" + OperationExecStream = "exec_stream" + OperationDestroy = "destroy" + OperationFileWrite = "file_write" + OperationFileRead = "file_read" + OperationFileList = "file_list" + OperationFileDelete = "file_delete" + OperationFileMove = "file_move" + OperationFileChmod = "file_chmod" + OperationFileStat = "file_stat" + OperationFileGlob = "file_glob" +) + +type OperationMetrics struct { + Operation string `json:"operation"` + Provider string `json:"provider"` + SuccessTotal uint64 `json:"success_total"` + FailureTotal uint64 `json:"failure_total"` + LatencyCount uint64 `json:"latency_count"` + LatencyTotalMS uint64 `json:"latency_total_ms"` + LatencyMinMS uint64 `json:"latency_min_ms"` + LatencyMaxMS uint64 `json:"latency_max_ms"` + LatencyAvgMS uint64 `json:"latency_avg_ms"` + LastError string `json:"last_error,omitempty"` + LastObservedUnix int64 `json:"last_observed_unix,omitempty"` +} + +type operationMetricKey struct { + operation string + provider string +} + +type operationMetricBucket struct { + successTotal uint64 + failureTotal uint64 + latencyCount uint64 + latencyTotalMS uint64 + latencyMinMS uint64 + latencyMaxMS uint64 + lastError string + lastObservedUnix int64 +} + +type MetricsRecorder struct { + mu sync.RWMutex + operations map[operationMetricKey]*operationMetricBucket +} + +func NewMetricsRecorder() *MetricsRecorder { + return &MetricsRecorder{operations: make(map[operationMetricKey]*operationMetricBucket)} +} + +func (r *MetricsRecorder) RecordOperation(operation, provider string, duration time.Duration, err error) { + if r == nil { + return + } + if provider == "" { + provider = "unknown" + } + latencyMS := uint64(duration.Milliseconds()) + key := operationMetricKey{operation: operation, provider: provider} + + r.mu.Lock() + defer r.mu.Unlock() + + bucket := r.operations[key] + if bucket == nil { + bucket = &operationMetricBucket{latencyMinMS: latencyMS} + r.operations[key] = bucket + } + if err != nil { + bucket.failureTotal++ + bucket.lastError = err.Error() + } else { + bucket.successTotal++ + } + bucket.latencyCount++ + bucket.latencyTotalMS += latencyMS + if latencyMS < bucket.latencyMinMS { + bucket.latencyMinMS = latencyMS + } + if latencyMS > bucket.latencyMaxMS { + bucket.latencyMaxMS = latencyMS + } + bucket.lastObservedUnix = time.Now().Unix() +} + +func (r *MetricsRecorder) Snapshot() []OperationMetrics { + if r == nil { + return nil + } + + r.mu.RLock() + defer r.mu.RUnlock() + + out := make([]OperationMetrics, 0, len(r.operations)) + for key, bucket := range r.operations { + avg := uint64(0) + if bucket.latencyCount > 0 { + avg = bucket.latencyTotalMS / bucket.latencyCount + } + out = append(out, OperationMetrics{ + Operation: key.operation, + Provider: key.provider, + SuccessTotal: bucket.successTotal, + FailureTotal: bucket.failureTotal, + LatencyCount: bucket.latencyCount, + LatencyTotalMS: bucket.latencyTotalMS, + LatencyMinMS: bucket.latencyMinMS, + LatencyMaxMS: bucket.latencyMaxMS, + LatencyAvgMS: avg, + LastError: bucket.lastError, + LastObservedUnix: bucket.lastObservedUnix, + }) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].Operation == out[j].Operation { + return out[i].Provider < out[j].Provider + } + return out[i].Operation < out[j].Operation + }) + return out +} From 3591b7f7dff57ec4d7e6a9f0e2f1cf78c661d1f5 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:02:37 +0530 Subject: [PATCH 005/147] feat: publish operational audit events --- docs/api.md | 16 ++-- internal/orchestrator/events.go | 14 ++++ internal/orchestrator/events_test.go | 3 + internal/orchestrator/manager.go | 106 ++++++++++++++++++++++++++ internal/orchestrator/manager_test.go | 31 ++++++++ 5 files changed, 164 insertions(+), 6 deletions(-) diff --git a/docs/api.md b/docs/api.md index cf7a929..73df11a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -596,16 +596,20 @@ GET /api/v1/events **Response** `200 OK` with `Content-Type: text/event-stream`. The server emits orchestrator events as Server-Sent Events: ``` -event: sandbox.spawned -data: {"id":"sb-a1b2c3d4","provider":"docker","image":"python:3.12"} +data: {"id":"evt-1","type":"sandbox.created","sandbox_id":"sb-a1b2c3d4","timestamp":"2026-05-08T10:30:00Z"} -event: sandbox.destroyed -data: {"id":"sb-a1b2c3d4","reason":"ttl_expired"} +data: {"id":"evt-2","type":"exec.timeout","sandbox_id":"sb-a1b2c3d4","timestamp":"2026-05-08T10:31:00Z","data":{"operation":"exec","provider":"docker","error":"exec timeout: sb-a1b2c3d4"}} -event: sandbox.exec -data: {"id":"sb-a1b2c3d4","command":"python3 main.py","exit_code":0} +data: {"id":"evt-3","type":"reconcile.action","sandbox_id":"sb-a1b2c3d4","timestamp":"2026-05-08T10:32:00Z","data":{"action":"adopted_runtime","provider":"docker","image":"python:3.12"}} ``` +Common event types include: + +- `sandbox.created`, `sandbox.running`, `sandbox.destroyed`, `sandbox.error` +- `exec.started`, `exec.completed`, `exec.failed`, `exec.timeout` +- `file.written`, `file.read` +- `operation.failed`, `resource.limit`, `provider.failed`, `reconcile.action` + Use any SSE client (`EventSource` in browsers, `httpx-sse` in Python, etc.) to consume. --- diff --git a/internal/orchestrator/events.go b/internal/orchestrator/events.go index 193fb2d..95864f2 100644 --- a/internal/orchestrator/events.go +++ b/internal/orchestrator/events.go @@ -2,6 +2,7 @@ package orchestrator import ( "encoding/json" + "strconv" "sync" "time" ) @@ -15,8 +16,14 @@ const ( EventSandboxError EventType = "sandbox.error" EventExecStarted EventType = "exec.started" EventExecCompleted EventType = "exec.completed" + EventExecFailed EventType = "exec.failed" + EventExecTimeout EventType = "exec.timeout" EventFileWritten EventType = "file.written" EventFileRead EventType = "file.read" + EventOperationFailed EventType = "operation.failed" + EventResourceLimit EventType = "resource.limit" + EventProviderFailed EventType = "provider.failed" + EventReconcileAction EventType = "reconcile.action" ) type Event struct { @@ -64,6 +71,9 @@ func (eb *EventBus) Publish(evt Event) { evt.Timestamp = time.Now() } eb.nextID++ + if evt.ID == "" { + evt.ID = stringID(eb.nextID) + } // Ring buffer: append or overwrite oldest if len(eb.history) < eb.historySize { @@ -81,6 +91,10 @@ func (eb *EventBus) Publish(evt Event) { } } +func stringID(id int) string { + return "evt-" + strconv.Itoa(id) +} + // Subscribe creates a new subscription and returns a channel + unsubscribe key. func (eb *EventBus) Subscribe(id string) <-chan Event { eb.mu.Lock() diff --git a/internal/orchestrator/events_test.go b/internal/orchestrator/events_test.go index fe45104..0e5571b 100644 --- a/internal/orchestrator/events_test.go +++ b/internal/orchestrator/events_test.go @@ -22,6 +22,9 @@ func TestEventBus_SubscribePublish(t *testing.T) { if evt.SandboxID != "sb-001" { t.Fatalf("expected sb-001, got %s", evt.SandboxID) } + if evt.ID == "" { + t.Fatal("expected event ID") + } case <-time.After(time.Second): t.Fatal("timeout waiting for event") } diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 3e46ba9..6072b71 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -139,6 +139,11 @@ func (m *Manager) Reconcile(ctx context.Context) error { prov, err := m.registry.Get(rec.Provider) if err != nil { m.logger.Warn().Err(err).Str("sandbox", rec.ID).Str("provider", rec.Provider).Msg("reconcile: provider unavailable") + m.publishOperationalEvent(EventProviderFailed, rec.ID, map[string]interface{}{ + "operation": "reconcile.status", + "provider": rec.Provider, + "error": err.Error(), + }) if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateError)); updateErr != nil { return fmt.Errorf("marking sandbox %s error: %w", rec.ID, updateErr) } @@ -155,9 +160,19 @@ func (m *Manager) Reconcile(ctx context.Context) error { delete(m.sandboxes, rec.ID) m.mu.Unlock() m.logger.Info().Str("sandbox", rec.ID).Msg("reconcile: stale sandbox marked destroyed") + m.publishOperationalEvent(EventReconcileAction, rec.ID, map[string]interface{}{ + "action": "marked_destroyed", + "provider": rec.Provider, + "reason": err.Error(), + }) continue } m.logger.Warn().Err(err).Str("sandbox", rec.ID).Msg("reconcile: provider status failed") + m.publishOperationalEvent(EventProviderFailed, rec.ID, map[string]interface{}{ + "operation": "reconcile.status", + "provider": rec.Provider, + "error": err.Error(), + }) if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateError)); updateErr != nil { return fmt.Errorf("marking sandbox %s error: %w", rec.ID, updateErr) } @@ -209,6 +224,11 @@ func (m *Manager) reconcileProviderRuntimes(ctx context.Context, known map[strin runtimes, err := lister.ListRuntimeSandboxes(ctx) if err != nil { m.logger.Warn().Err(err).Str("provider", name).Msg("reconcile: runtime inventory failed") + m.publishOperationalEvent(EventProviderFailed, "", map[string]interface{}{ + "operation": "reconcile.runtime_inventory", + "provider": name, + "error": err.Error(), + }) continue } for _, runtime := range runtimes { @@ -266,6 +286,11 @@ func (m *Manager) reconcileProviderRuntimes(ctx context.Context, known map[strin Str("sandbox", runtime.ID). Str("provider", runtime.Provider). Msg("reconcile: adopted provider runtime") + m.publishOperationalEvent(EventReconcileAction, runtime.ID, map[string]interface{}{ + "action": "adopted_runtime", + "provider": runtime.Provider, + "image": runtime.Image, + }) } } return nil @@ -286,6 +311,7 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) prov, err := m.registry.Get(providerName) if err != nil { metricsErr = err + m.publishOperationFailure(EventProviderFailed, "", OperationSpawn, metricsProvider, err) return nil, fmt.Errorf("getting provider: %w", err) } metricsProvider = prov.Name() @@ -317,6 +343,9 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) if m.vmPoolMgr != nil { sb, err := m.spawnPooled(ctx, prov, req, image, memMB, vcpus, ttl, now) metricsErr = err + if err != nil { + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + } return sb, err } @@ -342,6 +371,7 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) }) if err != nil { metricsErr = err + m.publishFailureForError("", OperationSpawn, metricsProvider, err) return nil, fmt.Errorf("spawning sandbox: %w", err) } sb.ID = id @@ -366,6 +396,7 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) // Best effort: destroy the sandbox if DB write fails prov.Destroy(ctx, id) metricsErr = err + m.publishOperationFailure(EventOperationFailed, id, OperationSpawn, metricsProvider, err) return nil, fmt.Errorf("persisting sandbox: %w", err) } @@ -424,6 +455,7 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req }) if err != nil { m.vmPoolMgr.Release(vmID, sandboxID) + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationSpawn, prov.Name(), err) return nil, fmt.Errorf("creating workspace: %w", err) } @@ -444,6 +476,7 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req UpdatedAt: now, }); err != nil { m.vmPoolMgr.Release(vmID, sandboxID) + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationSpawn, prov.Name(), err) return nil, fmt.Errorf("persisting sandbox: %w", err) } @@ -473,6 +506,7 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExec, metricsProvider, err) return nil, err } metricsProvider = sb.Provider @@ -483,6 +517,7 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( timeout, err := time.ParseDuration(req.Timeout) if err != nil { metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExec, metricsProvider, err) return nil, fmt.Errorf("parsing exec timeout: %w", err) } execCtx, cancel = context.WithTimeout(ctx, timeout) @@ -510,9 +545,11 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( if err != nil { if execCtx.Err() == context.DeadlineExceeded { metricsErr = providers.ExecTimeoutError(sandboxID) + m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExec, metricsProvider, metricsErr) return nil, metricsErr } metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExec, metricsProvider, err) return nil, fmt.Errorf("exec: %w", err) } duration := time.Since(execStart) @@ -556,6 +593,7 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, err) return nil, err } metricsProvider = sb.Provider @@ -566,6 +604,7 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ timeout, err := time.ParseDuration(req.Timeout) if err != nil { metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, err) return nil, fmt.Errorf("parsing exec timeout: %w", err) } execCtx, cancel = context.WithTimeout(ctx, timeout) @@ -588,9 +627,11 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ } if execCtx.Err() == context.DeadlineExceeded { metricsErr = providers.ExecTimeoutError(sandboxID) + m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExecStream, metricsProvider, metricsErr) return nil, metricsErr } metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, err) return nil, err } if cancel == nil { @@ -619,6 +660,7 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ } if execCtx.Err() == context.DeadlineExceeded || timedOut { metricsErr = providers.ExecTimeoutError(sandboxID) + m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExecStream, metricsProvider, metricsErr) select { case out <- providers.StreamChunk{Stream: "stderr", Data: metricsErr.Error()}: case <-ctx.Done(): @@ -640,6 +682,7 @@ func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWrite sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileWrite, metricsProvider, err) return err } metricsProvider = sb.Provider @@ -652,6 +695,7 @@ func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWrite path := m.scopedPath(sb, req.Path) if err := prov.WriteFile(ctx, m.resolveVMID(sb), path, strings.NewReader(req.Content), mode); err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileWrite, metricsProvider, err) return fmt.Errorf("writing file: %w", err) } @@ -673,6 +717,7 @@ func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ( sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) return nil, err } metricsProvider = sb.Provider @@ -680,6 +725,7 @@ func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ( rc, err := prov.ReadFile(ctx, m.resolveVMID(sb), m.scopedPath(sb, path)) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) return nil, fmt.Errorf("reading file: %w", err) } defer rc.Close() @@ -687,6 +733,7 @@ func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ( buf, err := io.ReadAll(rc) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) return nil, fmt.Errorf("reading file content: %w", err) } @@ -708,6 +755,7 @@ func (m *Manager) ListFiles(ctx context.Context, sandboxID string, path string) sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileList, metricsProvider, err) return nil, err } metricsProvider = sb.Provider @@ -715,6 +763,7 @@ func (m *Manager) ListFiles(ctx context.Context, sandboxID string, path string) pFiles, err := prov.ListFiles(ctx, m.resolveVMID(sb), m.scopedPath(sb, path)) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileList, metricsProvider, err) return nil, err } @@ -742,12 +791,16 @@ func (m *Manager) DeleteFile(ctx context.Context, sandboxID string, req FileDele sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileDelete, metricsProvider, err) return err } metricsProvider = sb.Provider path := m.scopedPath(sb, req.Path) metricsErr = prov.DeleteFile(ctx, m.resolveVMID(sb), path, req.Recursive) + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileDelete, metricsProvider, metricsErr) + } return metricsErr } @@ -762,6 +815,7 @@ func (m *Manager) MoveFile(ctx context.Context, sandboxID string, req FileMoveRe sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileMove, metricsProvider, err) return err } metricsProvider = sb.Provider @@ -769,6 +823,9 @@ func (m *Manager) MoveFile(ctx context.Context, sandboxID string, req FileMoveRe oldPath := m.scopedPath(sb, req.OldPath) newPath := m.scopedPath(sb, req.NewPath) metricsErr = prov.MoveFile(ctx, m.resolveVMID(sb), oldPath, newPath) + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileMove, metricsProvider, metricsErr) + } return metricsErr } @@ -783,12 +840,16 @@ func (m *Manager) ChmodFile(ctx context.Context, sandboxID string, req FileChmod sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileChmod, metricsProvider, err) return err } metricsProvider = sb.Provider path := m.scopedPath(sb, req.Path) metricsErr = prov.ChmodFile(ctx, m.resolveVMID(sb), path, req.Mode) + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileChmod, metricsProvider, metricsErr) + } return metricsErr } @@ -803,6 +864,7 @@ func (m *Manager) StatFile(ctx context.Context, sandboxID string, path string) ( sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileStat, metricsProvider, err) return nil, err } metricsProvider = sb.Provider @@ -811,6 +873,7 @@ func (m *Manager) StatFile(ctx context.Context, sandboxID string, path string) ( fi, err := prov.StatFile(ctx, m.resolveVMID(sb), scopedPath) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileStat, metricsProvider, err) return nil, err } return &FileInfo{ @@ -833,6 +896,7 @@ func (m *Manager) GlobFiles(ctx context.Context, sandboxID string, pattern strin sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileGlob, metricsProvider, err) return nil, err } metricsProvider = sb.Provider @@ -840,6 +904,9 @@ func (m *Manager) GlobFiles(ctx context.Context, sandboxID string, pattern strin scopedPattern := m.scopedPath(sb, pattern) matches, err := prov.GlobFiles(ctx, m.resolveVMID(sb), scopedPattern) metricsErr = err + if err != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileGlob, metricsProvider, err) + } return matches, err } @@ -885,6 +952,41 @@ func (m *Manager) recordOperation(operation, provider string, duration time.Dura m.metrics.RecordOperation(operation, provider, duration, err) } +func (m *Manager) publishFailureForError(sandboxID, operation, provider string, err error) { + if errors.Is(err, providers.ErrResourceLimit) { + m.publishOperationFailure(EventResourceLimit, sandboxID, operation, provider, err) + return + } + if errors.Is(err, providers.ErrProviderUnavailable) || errors.Is(err, providers.ErrProviderNotFound) { + m.publishOperationFailure(EventProviderFailed, sandboxID, operation, provider, err) + return + } + m.publishOperationFailure(EventOperationFailed, sandboxID, operation, provider, err) +} + +func (m *Manager) publishOperationFailure(eventType EventType, sandboxID, operation, provider string, err error) { + if err == nil { + return + } + m.publishOperationalEvent(eventType, sandboxID, map[string]interface{}{ + "operation": operation, + "provider": provider, + "error": err.Error(), + }) +} + +func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, data map[string]interface{}) { + if m.events == nil { + return + } + payload, _ := json.Marshal(data) + m.events.Publish(Event{ + Type: eventType, + SandboxID: sandboxID, + Data: payload, + }) +} + // InitVMPool initializes the VM pool manager if pool mode is enabled. func (m *Manager) InitVMPool() { if !m.poolConfig.Enabled { @@ -1035,6 +1137,9 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { if sb != nil && sb.VMID != "" && m.vmPoolMgr != nil { metricsErr = m.destroyPooled(ctx, id, sb) + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, id, OperationDestroy, metricsProvider, metricsErr) + } return metricsErr } @@ -1054,6 +1159,7 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { if err := prov.Destroy(ctx, id); err != nil { // Debug-level: this is expected when VMs were killed externally (e.g. process restart). m.logger.Debug().Err(err).Str("sandbox", id).Msg("provider destroy failed (VM may already be gone)") + m.publishOperationFailure(EventProviderFailed, id, OperationDestroy, metricsProvider, err) } m.store.UpdateSandboxState(ctx, id, string(StateDestroyed)) diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index e13bb52..420726b 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -109,6 +109,7 @@ func TestManager_ReconcileMarksMissingRuntimeDestroyed(t *testing.T) { if rec.State != string(StateDestroyed) { t.Fatalf("expected destroyed after reconcile, got %s", rec.State) } + assertEventType(t, m.events.History(10), EventReconcileAction) } type runtimeListerProvider struct { @@ -163,6 +164,7 @@ func TestManager_ReconcileAdoptsProviderRuntime(t *testing.T) { if rec.Provider != "mock" { t.Fatalf("expected mock provider, got %s", rec.Provider) } + assertEventType(t, m.events.History(10), EventReconcileAction) } func TestManager_Exec(t *testing.T) { @@ -234,6 +236,7 @@ func TestManager_ExecTimeout(t *testing.T) { if !errors.Is(err, ErrExecTimeout) { t.Fatalf("expected ErrExecTimeout, got %v", err) } + assertEventType(t, m.events.History(10), EventExecTimeout) } func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { @@ -259,6 +262,34 @@ func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { if !sawTimeout { t.Fatal("expected timeout error chunk") } + assertEventType(t, m.events.History(10), EventExecTimeout) +} + +func TestManager_PublishesOperationFailureEvent(t *testing.T) { + m := setupManager(t) + + if _, err := m.Exec(context.Background(), "sb-does-not-exist", ExecRequest{Command: "echo nope"}); err == nil { + t.Fatal("expected exec error") + } + + assertEventType(t, m.events.History(10), EventExecFailed) +} + +func assertEventType(t *testing.T, events []Event, eventType EventType) Event { + t.Helper() + for _, event := range events { + if event.Type == eventType { + if event.ID == "" { + t.Fatalf("event %s has empty ID", eventType) + } + if len(event.Data) == 0 { + t.Fatalf("event %s has empty data", eventType) + } + return event + } + } + t.Fatalf("event %s not found in %+v", eventType, events) + return Event{} } func TestManager_WriteAndReadFile(t *testing.T) { From 2984987680c0d98a670121e392476b62c2c7ed17 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:05:41 +0530 Subject: [PATCH 006/147] feat: add provider health detail --- docs/api.md | 50 ++++++++++++++-- internal/api/routes/prometheus.go | 6 ++ internal/api/routes/provider_health.go | 82 ++++++++++++++++++++++++++ internal/api/routes/providers.go | 45 +++++++++----- internal/api/routes/swagger_types.go | 11 +++- internal/api/routes/system.go | 17 +----- internal/api/routes/system_test.go | 8 +++ 7 files changed, 179 insertions(+), 40 deletions(-) create mode 100644 internal/api/routes/provider_health.go diff --git a/docs/api.md b/docs/api.md index 73df11a..305ccea 100644 --- a/docs/api.md +++ b/docs/api.md @@ -392,9 +392,24 @@ GET /api/v1/providers **Response** `200 OK`: ```json [ - { "name": "docker", "healthy": true, "default": true }, - { "name": "firecracker", "healthy": true, "default": false }, - { "name": "mock", "healthy": true, "default": false } + { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "exec_stream", "files", "console", "health", "runtime_inventory", "container"], + "runtime_count": 4 + }, + { + "name": "firecracker", + "healthy": false, + "default": false, + "latency_ms": 1, + "last_checked": "2026-05-08T10:30:00Z", + "error": "health check returned false", + "capabilities": ["spawn", "exec", "exec_stream", "files", "console", "health", "snapshots", "microvm", "vsock_agent"] + } ] ``` @@ -411,6 +426,15 @@ GET /api/v1/providers/{name} "healthy": true, "default": true, "sandbox_count": 12, + "health": { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "files", "runtime_inventory", "container"], + "runtime_count": 4 + }, "config": { "runtime": "runc", "network_mode": "stacyvm-network" } } ``` @@ -507,8 +531,24 @@ GET /api/v1/ready "ready_providers": 1, "total_providers": 2, "providers": [ - { "name": "docker", "healthy": true, "default": true }, - { "name": "firecracker", "healthy": false, "default": false } + { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "files", "runtime_inventory", "container"], + "runtime_count": 4 + }, + { + "name": "firecracker", + "healthy": false, + "default": false, + "latency_ms": 1, + "last_checked": "2026-05-08T10:30:00Z", + "error": "health check returned false", + "capabilities": ["spawn", "exec", "files", "snapshots", "microvm", "vsock_agent"] + } ] } ``` diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go index bc44525..2606f72 100644 --- a/internal/api/routes/prometheus.go +++ b/internal/api/routes/prometheus.go @@ -35,12 +35,18 @@ func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { } writePrometheusHelp(w, "stacyvm_provider_healthy", "Provider health status where 1 is healthy and 0 is unhealthy.") + writePrometheusHelp(w, "stacyvm_provider_health_latency_milliseconds", "Provider health check latency in milliseconds.") + writePrometheusHelp(w, "stacyvm_provider_runtime_sandboxes", "Runtime sandboxes discovered directly from providers.") for _, provider := range metrics.providerHealth { healthy := 0 if provider.Healthy { healthy = 1 } fmt.Fprintf(w, "stacyvm_provider_healthy{provider=%q,default=%q} %d\n", provider.Name, boolLabel(provider.Default), healthy) + fmt.Fprintf(w, "stacyvm_provider_health_latency_milliseconds{provider=%q} %d\n", provider.Name, provider.LatencyMS) + if provider.RuntimeCount != nil { + fmt.Fprintf(w, "stacyvm_provider_runtime_sandboxes{provider=%q} %d\n", provider.Name, *provider.RuntimeCount) + } } writePrometheusHelp(w, "stacyvm_events_total", "Total events published by the in-process event bus.") diff --git a/internal/api/routes/provider_health.go b/internal/api/routes/provider_health.go new file mode 100644 index 0000000..af188da --- /dev/null +++ b/internal/api/routes/provider_health.go @@ -0,0 +1,82 @@ +package routes + +import ( + "context" + "time" + + "github.com/StacyOs/stacyvm/internal/providers" +) + +func collectProviderHealth(ctx context.Context, registry *providers.Registry) []ProviderHealth { + names := registry.List() + out := make([]ProviderHealth, 0, len(names)) + defaultProvider := registry.Default() + + for _, name := range names { + start := time.Now() + checkedAt := time.Now().UTC() + item := ProviderHealth{ + Name: name, + Default: name == defaultProvider, + LastChecked: checkedAt.Format(time.RFC3339), + Capabilities: []string{"spawn", "exec", "exec_stream", "files", "console", "health"}, + } + + prov, err := registry.Get(name) + if err != nil { + item.Healthy = false + item.Error = err.Error() + item.LatencyMS = time.Since(start).Milliseconds() + out = append(out, item) + continue + } + + item.Healthy = prov.Healthy(ctx) + item.LatencyMS = time.Since(start).Milliseconds() + if !item.Healthy { + item.Error = "health check returned false" + } + item.Capabilities = append(item.Capabilities, providerCapabilities(prov)...) + + if lister, ok := prov.(providers.RuntimeSandboxLister); ok { + runtimes, err := lister.ListRuntimeSandboxes(ctx) + if err != nil { + if item.Error == "" { + item.Error = "runtime inventory: " + err.Error() + } + } else { + count := len(runtimes) + item.RuntimeCount = &count + } + } + + out = append(out, item) + } + + return out +} + +func providerCapabilities(prov providers.Provider) []string { + capabilities := make([]string, 0, 4) + if _, ok := prov.(providers.RuntimeSandboxLister); ok { + capabilities = append(capabilities, "runtime_inventory") + } + if _, ok := prov.(providers.SnapshotLister); ok { + capabilities = append(capabilities, "snapshots") + } + switch prov.(type) { + case *providers.FirecrackerProvider: + capabilities = append(capabilities, "microvm", "vsock_agent") + case *providers.DockerProvider: + capabilities = append(capabilities, "container") + case *providers.PRootProvider: + capabilities = append(capabilities, "userspace_isolation") + case *providers.CustomProvider: + capabilities = append(capabilities, "remote_http") + case *providers.E2BProvider: + capabilities = append(capabilities, "remote_e2b") + case *providers.MockProvider: + capabilities = append(capabilities, "test_provider") + } + return capabilities +} diff --git a/internal/api/routes/providers.go b/internal/api/routes/providers.go index 35b72e0..902c132 100644 --- a/internal/api/routes/providers.go +++ b/internal/api/routes/providers.go @@ -32,9 +32,14 @@ func (p *ProviderRoutes) Routes() chi.Router { // ProviderInfo is the summary info for a provider. type ProviderInfo struct { - Name string `json:"name" example:"firecracker"` - Healthy bool `json:"healthy" example:"true"` - Default bool `json:"default" example:"true"` + Name string `json:"name" example:"firecracker"` + Healthy bool `json:"healthy" example:"true"` + Default bool `json:"default" example:"true"` + LatencyMS int64 `json:"latency_ms" example:"3"` + LastChecked string `json:"last_checked" example:"2026-05-08T10:30:00Z"` + Error string `json:"error,omitempty" example:"health check returned false"` + Capabilities []string `json:"capabilities"` + RuntimeCount *int `json:"runtime_count,omitempty" example:"2"` } // ProviderDetail is the detailed info for a provider. @@ -43,6 +48,7 @@ type ProviderDetail struct { Healthy bool `json:"healthy" example:"true"` Default bool `json:"default" example:"true"` SandboxCount int `json:"sandbox_count" example:"3"` + Health ProviderHealth `json:"health"` Config map[string]string `json:"config"` } @@ -56,19 +62,18 @@ type ProviderDetail struct { // @Security ApiKeyAuth // @Router /providers [get] func (p *ProviderRoutes) List(w http.ResponseWriter, r *http.Request) { - names := p.registry.List() - infos := make([]ProviderInfo, 0, len(names)) - dflt := p.registry.Default() - - for _, name := range names { - prov, err := p.registry.Get(name) - if err != nil { - continue - } + health := collectProviderHealth(r.Context(), p.registry) + infos := make([]ProviderInfo, 0, len(health)) + for _, item := range health { infos = append(infos, ProviderInfo{ - Name: name, - Healthy: prov.Healthy(r.Context()), - Default: name == dflt, + Name: item.Name, + Healthy: item.Healthy, + Default: item.Default, + LatencyMS: item.LatencyMS, + LastChecked: item.LastChecked, + Error: item.Error, + Capabilities: item.Capabilities, + RuntimeCount: item.RuntimeCount, }) } @@ -149,12 +154,20 @@ func (p *ProviderRoutes) Detail(w http.ResponseWriter, r *http.Request) { if p.counter != nil { count = p.counter.CountByProvider(r.Context(), name) } + health := ProviderHealth{Name: name, Healthy: prov.Healthy(r.Context()), Default: name == dflt} + for _, item := range collectProviderHealth(r.Context(), p.registry) { + if item.Name == name { + health = item + break + } + } httputil.WriteJSON(w, http.StatusOK, ProviderDetail{ Name: name, - Healthy: prov.Healthy(r.Context()), + Healthy: health.Healthy, Default: name == dflt, SandboxCount: count, + Health: health, Config: cfg, }) } diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index c7a6649..57f87a5 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -19,9 +19,14 @@ type HealthResponse struct { // ProviderHealth is a provider readiness item. type ProviderHealth struct { - Name string `json:"name" example:"docker"` - Healthy bool `json:"healthy" example:"true"` - Default bool `json:"default" example:"true"` + Name string `json:"name" example:"docker"` + Healthy bool `json:"healthy" example:"true"` + Default bool `json:"default" example:"true"` + LatencyMS int64 `json:"latency_ms" example:"3"` + LastChecked string `json:"last_checked" example:"2026-05-08T10:30:00Z"` + Error string `json:"error,omitempty" example:"health check returned false"` + Capabilities []string `json:"capabilities" example:"spawn,exec,files"` + RuntimeCount *int `json:"runtime_count,omitempty" example:"2"` } // ReadinessResponse is the response from the readiness endpoint. diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index 201bc88..8166d6d 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -245,22 +245,7 @@ func (m systemMetricsSnapshot) toResponse() map[string]interface{} { } func (s *SystemRoutes) providerHealth(ctx context.Context) []ProviderHealth { - names := s.registry.List() - out := make([]ProviderHealth, 0, len(names)) - defaultProvider := s.registry.Default() - for _, name := range names { - prov, err := s.registry.Get(name) - if err != nil { - out = append(out, ProviderHealth{Name: name, Healthy: false, Default: name == defaultProvider}) - continue - } - out = append(out, ProviderHealth{ - Name: name, - Healthy: prov.Healthy(ctx), - Default: name == defaultProvider, - }) - } - return out + return collectProviderHealth(ctx, s.registry) } // Events serves Server-Sent Events for real-time updates. diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index 13f1424..33a132b 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -81,6 +81,13 @@ func TestSystemRoutes_Ready(t *testing.T) { if body["ready_providers"].(float64) != 1 { t.Fatalf("ready providers = %v, want 1", body["ready_providers"]) } + providersBody := body["providers"].([]interface{}) + firstProvider := providersBody[0].(map[string]interface{}) + for _, field := range []string{"latency_ms", "last_checked", "capabilities"} { + if _, ok := firstProvider[field]; !ok { + t.Fatalf("provider health missing %s: %#v", field, firstProvider) + } + } } func TestSystemRoutes_ReadyNoProviders(t *testing.T) { @@ -159,6 +166,7 @@ func TestSystemRoutes_PrometheusMetrics(t *testing.T) { for _, want := range []string{ "stacyvm_uptime_seconds", "stacyvm_provider_healthy", + "stacyvm_provider_health_latency_milliseconds", "stacyvm_operation_success_total", `operation="spawn"`, `operation="exec"`, From e57a4333edf99ff6506e5722a01fd2f379f64c8f Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:09:57 +0530 Subject: [PATCH 007/147] feat: add redacted diagnostics endpoint --- docs/api.md | 58 +++++++++++++++++++ internal/api/routes/swagger_types.go | 15 +++++ internal/api/routes/system.go | 87 +++++++++++++++++++++++++--- internal/api/routes/system_test.go | 32 +++++++++- internal/api/server.go | 2 +- 5 files changed, 185 insertions(+), 9 deletions(-) diff --git a/docs/api.md b/docs/api.md index 305ccea..5e4ca85 100644 --- a/docs/api.md +++ b/docs/api.md @@ -555,6 +555,64 @@ GET /api/v1/ready **Response** `503 Service Unavailable` when no configured provider is healthy. +### Diagnostics + +``` +GET /api/v1/diagnostics +``` + +**Response** `200 OK`: +```json +{ + "generated_at": "2026-05-08T10:30:00Z", + "build": { + "version": "0.5.1", + "goos": "linux", + "goarch": "amd64" + }, + "process": { + "uptime": "2h13m", + "goroutines": 42, + "memory": { + "alloc": 17825792, + "sys": 71303168, + "heap_alloc": 17825792, + "gc_cycles": 8 + } + }, + "store": { + "healthy": true, + "latency_ms": 1 + }, + "providers": [ + { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "files", "runtime_inventory", "container"], + "runtime_count": 4 + } + ], + "sandboxes": { + "total": 138, + "active": 12, + "by_state": { "running": 12, "destroyed": 126 }, + "by_provider": { "docker": 90, "firecracker": 48 } + }, + "events": { + "subscribers": 2, + "history_size": 1000, + "events_total": 2401 + }, + "operations": [], + "redactions": ["provider secrets", "registry credentials", "environment secrets", "API keys"] +} +``` + +Diagnostics are read-only and intentionally redacted. Use this endpoint for support bundles, incident debugging, and deployment sanity checks. + ### Metrics ``` diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index 57f87a5..a2af23c 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -1,5 +1,7 @@ package routes +import "github.com/StacyOs/stacyvm/internal/orchestrator" + // StatusResponse is a generic status response. type StatusResponse struct { Status string `json:"status" example:"destroyed"` @@ -39,6 +41,19 @@ type ReadinessResponse struct { TotalProviders int `json:"total_providers" example:"2"` } +// DiagnosticsResponse is the response from the diagnostics endpoint. +type DiagnosticsResponse struct { + GeneratedAt string `json:"generated_at" example:"2026-05-08T10:30:00Z"` + Build map[string]interface{} `json:"build"` + Process map[string]interface{} `json:"process"` + Store map[string]interface{} `json:"store"` + Providers []ProviderHealth `json:"providers"` + Sandboxes map[string]interface{} `json:"sandboxes"` + Events orchestrator.EventBusStats `json:"events"` + Operations []orchestrator.OperationMetrics `json:"operations"` + Redactions []string `json:"redactions"` +} + // MetricsResponse is the response from the metrics endpoint. type MetricsResponse struct { SandboxesActive int `json:"sandboxes_active" example:"5"` diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index 8166d6d..be3fb9a 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -12,6 +12,7 @@ import ( "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" "github.com/go-chi/chi/v5" "github.com/google/uuid" ) @@ -20,15 +21,17 @@ type SystemRoutes struct { registry *providers.Registry manager *orchestrator.Manager events *orchestrator.EventBus + store store.Store startTime time.Time version string } -func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, version string) *SystemRoutes { +func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, st store.Store, version string) *SystemRoutes { return &SystemRoutes{ registry: registry, manager: manager, events: events, + store: st, startTime: time.Now(), version: version, } @@ -39,6 +42,7 @@ func (s *SystemRoutes) Routes() chi.Router { r.Get("/health", s.Health) r.Get("/live", s.Live) r.Get("/ready", s.Ready) + r.Get("/diagnostics", s.Diagnostics) r.Get("/metrics", s.Metrics) r.Get("/metrics/prometheus", s.PrometheusMetrics) r.Get("/events", s.Events) @@ -118,6 +122,71 @@ func (s *SystemRoutes) Ready(w http.ResponseWriter, r *http.Request) { }) } +// Diagnostics returns redacted operational diagnostics. +// +// @Summary Get diagnostics +// @Description Return redacted build, store, provider, sandbox, event, and operation diagnostics +// @Tags system +// @Produce json +// @Success 200 {object} DiagnosticsResponse +// @Security ApiKeyAuth +// @Router /diagnostics [get] +func (s *SystemRoutes) Diagnostics(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + + metrics, err := s.collectMetrics(ctx) + if err != nil { + writeRouteError(w, err) + return + } + + storeStatus := map[string]interface{}{ + "healthy": false, + } + if s.store != nil { + start := time.Now() + if _, err := s.store.ListSandboxes(ctx); err != nil { + storeStatus["error"] = err.Error() + } else { + storeStatus["healthy"] = true + } + storeStatus["latency_ms"] = time.Since(start).Milliseconds() + } else { + storeStatus["error"] = "store unavailable" + } + + httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ + "generated_at": time.Now().UTC().Format(time.RFC3339), + "build": map[string]interface{}{ + "version": s.version, + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + }, + "process": map[string]interface{}{ + "uptime": metrics.uptime.String(), + "goroutines": metrics.goroutines, + "memory": map[string]interface{}{ + "alloc": metrics.memoryAlloc, + "sys": metrics.memorySys, + "heap_alloc": metrics.memoryHeapAlloc, + "gc_cycles": metrics.gcCycles, + }, + }, + "store": storeStatus, + "providers": metrics.providerHealth, + "sandboxes": metrics.sandboxSummary(), + "events": metrics.eventStats, + "operations": metrics.operationMetrics, + "redactions": []string{ + "provider secrets", + "registry credentials", + "environment secrets", + "API keys", + }, + }) +} + // Metrics returns runtime metrics. // // @Summary Get metrics @@ -228,12 +297,7 @@ func (m systemMetricsSnapshot) toResponse() map[string]interface{} { "memory_sys": m.memorySys, "memory_heap_alloc": m.memoryHeapAlloc, "gc_cycles": m.gcCycles, - "sandboxes": map[string]interface{}{ - "total": m.sandboxTotal, - "active": m.sandboxActive, - "by_state": m.sandboxesByState, - "by_provider": m.sandboxesByProvider, - }, + "sandboxes": m.sandboxSummary(), "providers": map[string]interface{}{ "total": len(m.providerHealth), "healthy": m.healthyProviders, @@ -244,6 +308,15 @@ func (m systemMetricsSnapshot) toResponse() map[string]interface{} { } } +func (m systemMetricsSnapshot) sandboxSummary() map[string]interface{} { + return map[string]interface{}{ + "total": m.sandboxTotal, + "active": m.sandboxActive, + "by_state": m.sandboxesByState, + "by_provider": m.sandboxesByProvider, + } +} + func (s *SystemRoutes) providerHealth(ctx context.Context) []ProviderHealth { return collectProviderHealth(ctx, s.registry) } diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index 33a132b..7473462 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -43,7 +43,7 @@ func setupSystemRoutes(t *testing.T, withProvider bool) (*SystemRoutes, *orchest DefaultVCPUs: 1, }) - return NewSystemRoutes(registry, manager, events, "test-version"), manager + return NewSystemRoutes(registry, manager, events, st, "test-version"), manager } func TestSystemRoutes_Live(t *testing.T) { @@ -107,6 +107,36 @@ func TestSystemRoutes_ReadyNoProviders(t *testing.T) { } } +func TestSystemRoutes_Diagnostics(t *testing.T) { + routes, manager := setupSystemRoutes(t, true) + if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { + t.Fatalf("spawn: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/diagnostics", nil) + w := httptest.NewRecorder() + + routes.Diagnostics(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + for _, field := range []string{"generated_at", "build", "process", "store", "providers", "sandboxes", "events", "operations", "redactions"} { + if _, ok := body[field]; !ok { + t.Fatalf("diagnostics missing %s: %#v", field, body) + } + } + storeBody := body["store"].(map[string]interface{}) + if storeBody["healthy"] != true { + t.Fatalf("store healthy = %v, want true", storeBody["healthy"]) + } + if strings.Contains(w.Body.String(), "X-API-Key") { + t.Fatal("diagnostics response leaked API key header name") + } +} + func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { routes, manager := setupSystemRoutes(t, true) if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { diff --git a/internal/api/server.go b/internal/api/server.go index ca41bd8..0f60707 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -79,7 +79,7 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr providerRoutes := routes.NewProviderRoutes(registry, manager) templateRoutes := routes.NewTemplateRoutes(templates, manager) snapshotRoutes := routes.NewSnapshotRoutes(registry) - systemRoutes := routes.NewSystemRoutes(registry, manager, events, cfg.Version) + systemRoutes := routes.NewSystemRoutes(registry, manager, events, st, cfg.Version) environmentRoutes := routes.NewEnvironmentRoutes(st, envBuild) r.Route("/api/v1", func(r chi.Router) { From aeb30f598340e52e5cb9e5a9a6f35d5c918bb023 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:13:48 +0530 Subject: [PATCH 008/147] feat: enforce operational limits --- README.md | 5 ++ cmd/stacyvm/cmd_serve.go | 10 +++ docs/api.md | 7 ++ internal/api/routes/swagger_types.go | 19 ++--- internal/api/routes/system.go | 1 + internal/api/routes/system_test.go | 2 +- internal/config/config.go | 52 ++++++++------ internal/orchestrator/manager.go | 97 ++++++++++++++++++++++---- internal/orchestrator/manager_test.go | 99 +++++++++++++++++++++++++-- internal/orchestrator/types.go | 16 +++++ 10 files changed, 257 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 68d7e4a..5ffa2fd 100644 --- a/README.md +++ b/README.md @@ -497,6 +497,11 @@ defaults: image: "alpine:latest" memory_mb: 1024 vcpus: 1 + max_ttl: "24h" + default_exec_timeout: "0s" # disabled unless set + max_exec_timeout: "10m" + max_sandboxes: 0 # 0 = unlimited + max_sandboxes_per_owner: 0 # 0 = unlimited auth: enabled: false diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index f409b3c..059c803 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -163,6 +163,9 @@ func runServe() error { // Manager ttl, _ := time.ParseDuration(cfg.Defaults.TTL) + maxTTL, _ := time.ParseDuration(cfg.Defaults.MaxTTL) + defaultExecTimeout, _ := time.ParseDuration(cfg.Defaults.DefaultExecTimeout) + maxExecTimeout, _ := time.ParseDuration(cfg.Defaults.MaxExecTimeout) mgr := orchestrator.NewManager(registry, st, events, logger, orchestrator.ManagerConfig{ DefaultTTL: ttl, DefaultImage: cfg.Defaults.Image, @@ -170,6 +173,13 @@ func runServe() error { DefaultVCPUs: cfg.Defaults.VCPUs, Pool: cfg.Pool, PreviewDomain: cfg.Server.PreviewDomain, + Limits: orchestrator.OperationalLimits{ + MaxSandboxes: cfg.Defaults.MaxSandboxes, + MaxSandboxesPerOwner: cfg.Defaults.MaxSandboxesPerOwner, + DefaultExecTimeout: defaultExecTimeout, + MaxExecTimeout: maxExecTimeout, + MaxTTL: maxTTL, + }, }) if err := mgr.Reconcile(context.Background()); err != nil { return err diff --git a/docs/api.md b/docs/api.md index 5e4ca85..036eff2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -584,6 +584,13 @@ GET /api/v1/diagnostics "healthy": true, "latency_ms": 1 }, + "limits": { + "max_sandboxes": 100, + "max_sandboxes_per_owner": 10, + "default_exec_timeout": "30s", + "max_exec_timeout": "10m0s", + "max_ttl": "24h0m0s" + }, "providers": [ { "name": "docker", diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index a2af23c..49c4527 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -43,15 +43,16 @@ type ReadinessResponse struct { // DiagnosticsResponse is the response from the diagnostics endpoint. type DiagnosticsResponse struct { - GeneratedAt string `json:"generated_at" example:"2026-05-08T10:30:00Z"` - Build map[string]interface{} `json:"build"` - Process map[string]interface{} `json:"process"` - Store map[string]interface{} `json:"store"` - Providers []ProviderHealth `json:"providers"` - Sandboxes map[string]interface{} `json:"sandboxes"` - Events orchestrator.EventBusStats `json:"events"` - Operations []orchestrator.OperationMetrics `json:"operations"` - Redactions []string `json:"redactions"` + GeneratedAt string `json:"generated_at" example:"2026-05-08T10:30:00Z"` + Build map[string]interface{} `json:"build"` + Process map[string]interface{} `json:"process"` + Store map[string]interface{} `json:"store"` + Limits orchestrator.OperationalLimitsInfo `json:"limits"` + Providers []ProviderHealth `json:"providers"` + Sandboxes map[string]interface{} `json:"sandboxes"` + Events orchestrator.EventBusStats `json:"events"` + Operations []orchestrator.OperationMetrics `json:"operations"` + Redactions []string `json:"redactions"` } // MetricsResponse is the response from the metrics endpoint. diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index be3fb9a..c4ce10c 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -174,6 +174,7 @@ func (s *SystemRoutes) Diagnostics(w http.ResponseWriter, r *http.Request) { }, }, "store": storeStatus, + "limits": s.manager.Limits(), "providers": metrics.providerHealth, "sandboxes": metrics.sandboxSummary(), "events": metrics.eventStats, diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index 7473462..3e7f6af 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -123,7 +123,7 @@ func TestSystemRoutes_Diagnostics(t *testing.T) { } var body map[string]interface{} decodeSystemResponse(t, w, &body) - for _, field := range []string{"generated_at", "build", "process", "store", "providers", "sandboxes", "events", "operations", "redactions"} { + for _, field := range []string{"generated_at", "build", "process", "store", "limits", "providers", "sandboxes", "events", "operations", "redactions"} { if _, ok := body[field]; !ok { t.Fatalf("diagnostics missing %s: %#v", field, body) } diff --git a/internal/config/config.go b/internal/config/config.go index bc033b4..570d932 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -63,20 +63,20 @@ type PRootConfig struct { } type DockerConfig struct { - Enabled bool `mapstructure:"enabled"` - Socket string `mapstructure:"socket"` - Runtime string `mapstructure:"runtime"` - DefaultImage string `mapstructure:"default_image"` - NetworkMode string `mapstructure:"network_mode"` - SeccompProfile string `mapstructure:"seccomp_profile"` - ReadOnlyRootfs bool `mapstructure:"read_only_rootfs"` - Memory string `mapstructure:"memory"` - CPUs string `mapstructure:"cpus"` - PidsLimit int64 `mapstructure:"pids_limit"` - User string `mapstructure:"user"` - DroppedCaps []string `mapstructure:"dropped_caps"` - AddedCaps []string `mapstructure:"added_caps"` - Tmpfs map[string]string `mapstructure:"tmpfs"` + Enabled bool `mapstructure:"enabled"` + Socket string `mapstructure:"socket"` + Runtime string `mapstructure:"runtime"` + DefaultImage string `mapstructure:"default_image"` + NetworkMode string `mapstructure:"network_mode"` + SeccompProfile string `mapstructure:"seccomp_profile"` + ReadOnlyRootfs bool `mapstructure:"read_only_rootfs"` + Memory string `mapstructure:"memory"` + CPUs string `mapstructure:"cpus"` + PidsLimit int64 `mapstructure:"pids_limit"` + User string `mapstructure:"user"` + DroppedCaps []string `mapstructure:"dropped_caps"` + AddedCaps []string `mapstructure:"added_caps"` + Tmpfs map[string]string `mapstructure:"tmpfs"` PoolSecurity PoolSecurityConfig `mapstructure:"pool_security"` } @@ -115,13 +115,18 @@ type FirecrackerConfig struct { } type DefaultsConfig struct { - TTL string `mapstructure:"ttl"` - Image string `mapstructure:"image"` - MemoryMB int `mapstructure:"memory_mb"` - VCPUs int `mapstructure:"vcpus"` - DiskSizeMB int `mapstructure:"disk_size_mb"` - PoolSize int `mapstructure:"pool_size"` - PoolTemplate string `mapstructure:"pool_template"` + TTL string `mapstructure:"ttl"` + Image string `mapstructure:"image"` + MemoryMB int `mapstructure:"memory_mb"` + VCPUs int `mapstructure:"vcpus"` + DiskSizeMB int `mapstructure:"disk_size_mb"` + PoolSize int `mapstructure:"pool_size"` + PoolTemplate string `mapstructure:"pool_template"` + MaxTTL string `mapstructure:"max_ttl"` + DefaultExecTimeout string `mapstructure:"default_exec_timeout"` + MaxExecTimeout string `mapstructure:"max_exec_timeout"` + MaxSandboxes int `mapstructure:"max_sandboxes"` + MaxSandboxesPerOwner int `mapstructure:"max_sandboxes_per_owner"` } type AuthConfig struct { @@ -195,6 +200,11 @@ func setDefaults(v *viper.Viper) { v.SetDefault("defaults.disk_size_mb", 1024) v.SetDefault("defaults.pool_size", 0) v.SetDefault("defaults.pool_template", "") + v.SetDefault("defaults.max_ttl", "24h") + v.SetDefault("defaults.default_exec_timeout", "0s") + v.SetDefault("defaults.max_exec_timeout", "10m") + v.SetDefault("defaults.max_sandboxes", 0) + v.SetDefault("defaults.max_sandboxes_per_owner", 0) v.SetDefault("auth.enabled", false) v.SetDefault("auth.api_key", "") diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 6072b71..c854d25 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -32,6 +32,7 @@ type Manager struct { defaultImage string defaultMemory int defaultVCPUs int + limits OperationalLimits vmPoolMgr *VMPoolManager poolConfig config.PoolConfig @@ -49,6 +50,7 @@ type ManagerConfig struct { DefaultVCPUs int Pool config.PoolConfig PreviewDomain string + Limits OperationalLimits } func NewManager(registry *providers.Registry, st store.Store, events *EventBus, logger zerolog.Logger, cfg ManagerConfig) *Manager { @@ -64,6 +66,7 @@ func NewManager(registry *providers.Registry, st store.Store, events *EventBus, defaultImage: cfg.DefaultImage, defaultMemory: cfg.DefaultMemory, defaultVCPUs: cfg.DefaultVCPUs, + limits: cfg.Limits, poolConfig: cfg.Pool, previewDomain: cfg.PreviewDomain, ctx: ctx, @@ -336,6 +339,11 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) } ttl = parsed } + if err := m.enforceSpawnLimits(ctx, req.OwnerID, ttl); err != nil { + metricsErr = err + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + return nil, err + } now := time.Now() @@ -513,13 +521,13 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( execCtx := ctx var cancel context.CancelFunc - if req.Timeout != "" { - timeout, err := time.ParseDuration(req.Timeout) - if err != nil { - metricsErr = err - m.publishOperationFailure(EventExecFailed, sandboxID, OperationExec, metricsProvider, err) - return nil, fmt.Errorf("parsing exec timeout: %w", err) - } + timeout, err := m.resolveExecTimeout(req.Timeout) + if err != nil { + metricsErr = err + m.publishFailureForError(sandboxID, OperationExec, metricsProvider, err) + return nil, err + } + if timeout > 0 { execCtx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } @@ -600,13 +608,13 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ execCtx := ctx var cancel context.CancelFunc - if req.Timeout != "" { - timeout, err := time.ParseDuration(req.Timeout) - if err != nil { - metricsErr = err - m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, err) - return nil, fmt.Errorf("parsing exec timeout: %w", err) - } + timeout, err := m.resolveExecTimeout(req.Timeout) + if err != nil { + metricsErr = err + m.publishFailureForError(sandboxID, OperationExecStream, metricsProvider, err) + return nil, err + } + if timeout > 0 { execCtx, cancel = context.WithTimeout(ctx, timeout) } @@ -945,6 +953,16 @@ func (m *Manager) OperationMetrics() []OperationMetrics { return m.metrics.Snapshot() } +func (m *Manager) Limits() OperationalLimitsInfo { + return OperationalLimitsInfo{ + MaxSandboxes: m.limits.MaxSandboxes, + MaxSandboxesPerOwner: m.limits.MaxSandboxesPerOwner, + DefaultExecTimeout: m.limits.DefaultExecTimeout.String(), + MaxExecTimeout: m.limits.MaxExecTimeout.String(), + MaxTTL: m.limits.MaxTTL.String(), + } +} + func (m *Manager) recordOperation(operation, provider string, duration time.Duration, err error) { if m.metrics == nil { return @@ -987,6 +1005,57 @@ func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, }) } +func (m *Manager) enforceSpawnLimits(ctx context.Context, ownerID string, ttl time.Duration) error { + if m.limits.MaxTTL > 0 && ttl > m.limits.MaxTTL { + return providers.ResourceLimitError(fmt.Sprintf("ttl %s exceeds max ttl %s", ttl, m.limits.MaxTTL)) + } + + if m.limits.MaxSandboxes <= 0 && (ownerID == "" || m.limits.MaxSandboxesPerOwner <= 0) { + return nil + } + + records, err := m.store.ListSandboxes(ctx) + if err != nil { + return fmt.Errorf("checking sandbox limits: %w", err) + } + total := 0 + ownerTotal := 0 + for _, rec := range records { + if SandboxState(rec.State) == StateDestroyed { + continue + } + total++ + if ownerID != "" && rec.OwnerID == ownerID { + ownerTotal++ + } + } + if m.limits.MaxSandboxes > 0 && total >= m.limits.MaxSandboxes { + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", m.limits.MaxSandboxes)) + } + if ownerID != "" && m.limits.MaxSandboxesPerOwner > 0 && ownerTotal >= m.limits.MaxSandboxesPerOwner { + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", m.limits.MaxSandboxesPerOwner)) + } + return nil +} + +func (m *Manager) resolveExecTimeout(raw string) (time.Duration, error) { + timeout := m.limits.DefaultExecTimeout + if raw != "" { + parsed, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("parsing exec timeout: %w", err) + } + timeout = parsed + } + if timeout < 0 { + return 0, providers.ResourceLimitError("exec timeout cannot be negative") + } + if m.limits.MaxExecTimeout > 0 && timeout > m.limits.MaxExecTimeout { + return 0, providers.ResourceLimitError(fmt.Sprintf("exec timeout %s exceeds max exec timeout %s", timeout, m.limits.MaxExecTimeout)) + } + return timeout, nil +} + // InitVMPool initializes the VM pool manager if pool mode is enabled. func (m *Manager) InitVMPool() { if !m.poolConfig.Enabled { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 420726b..1ad9727 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -14,6 +14,16 @@ import ( ) func setupManager(t *testing.T) *Manager { + t.Helper() + return setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) +} + +func setupManagerWithConfig(t *testing.T, cfg ManagerConfig) *Manager { t.Helper() dir := t.TempDir() st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) @@ -30,12 +40,7 @@ func setupManager(t *testing.T) *Manager { events := NewEventBus() logger := zerolog.Nop() - m := NewManager(reg, st, events, logger, ManagerConfig{ - DefaultTTL: 5 * time.Minute, - DefaultImage: "alpine:latest", - DefaultMemory: 512, - DefaultVCPUs: 1, - }) + m := NewManager(reg, st, events, logger, cfg) m.Start() t.Cleanup(func() { m.Stop() }) return m @@ -239,6 +244,88 @@ func TestManager_ExecTimeout(t *testing.T) { assertEventType(t, m.events.History(10), EventExecTimeout) } +func TestManager_MaxExecTimeoutLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxExecTimeout: 50 * time.Millisecond, + }, + }) + sb, _ := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + + _, err := m.Exec(context.Background(), sb.ID, ExecRequest{ + Command: "echo nope", + Timeout: "1s", + }) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected resource limit, got %v", err) + } + assertEventType(t, m.events.History(10), EventResourceLimit) +} + +func TestManager_SpawnLimits(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + MaxSandboxesPerOwner: 1, + MaxTTL: time.Hour, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-b"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected total resource limit, got %v", err) + } + assertEventType(t, m.events.History(10), EventResourceLimit) +} + +func TestManager_SpawnOwnerLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxesPerOwner: 1, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected owner resource limit, got %v", err) + } +} + +func TestManager_SpawnMaxTTLLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxTTL: time.Hour, + }, + }) + + _, err := m.Spawn(context.Background(), SpawnRequest{TTL: "2h"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected ttl resource limit, got %v", err) + } +} + func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { m := setupManager(t) ctx := context.Background() diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index 13a4cb3..2c9cbb7 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -97,3 +97,19 @@ type SandboxInfo struct { FileCount int `json:"file_count"` PreviewDomain string `json:"preview_domain,omitempty"` } + +type OperationalLimits struct { + MaxSandboxes int `json:"max_sandboxes"` + MaxSandboxesPerOwner int `json:"max_sandboxes_per_owner"` + DefaultExecTimeout time.Duration `json:"default_exec_timeout"` + MaxExecTimeout time.Duration `json:"max_exec_timeout"` + MaxTTL time.Duration `json:"max_ttl"` +} + +type OperationalLimitsInfo struct { + MaxSandboxes int `json:"max_sandboxes"` + MaxSandboxesPerOwner int `json:"max_sandboxes_per_owner"` + DefaultExecTimeout string `json:"default_exec_timeout"` + MaxExecTimeout string `json:"max_exec_timeout"` + MaxTTL string `json:"max_ttl"` +} From 1ca3ba6f21054b3069f2224151311426998e88d2 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:16:59 +0530 Subject: [PATCH 009/147] docs: add phase 2 observability release notes --- CHANGELOG.md | 28 +++ .../releases/phase-2-observability-and-ops.md | 183 ++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 docs/releases/phase-2-observability-and-ops.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b0eaaf8..0891f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## Phase 2 Observability And Ops - 2026-05-08 + +This checkpoint adds production operations surfaces for health checks, diagnostics, metrics, audit events, and runtime limits. + +### Added + +- Liveness endpoint at `/api/v1/live`. +- Readiness endpoint at `/api/v1/ready` with detailed provider health. +- Redacted diagnostics endpoint at `/api/v1/diagnostics`. +- Structured JSON operation metrics on `/api/v1/metrics`. +- Prometheus-compatible metrics endpoint at `/api/v1/metrics/prometheus`. +- Provider health detail with latency, last checked time, capabilities, error reason, and runtime inventory count when supported. +- Operational audit events for exec failures, exec timeouts, provider failures, resource limits, and reconciliation actions. +- Configurable operational limits for max TTL, default/max exec timeout, max sandboxes, and max sandboxes per owner. + +### Changed + +- `/api/v1/metrics` now includes sandbox state/provider breakdown, provider health, event bus stats, and operation metrics. +- `/api/v1/providers` and `/api/v1/providers/{name}` now expose richer provider health details. +- Diagnostics include store health, build/runtime data, sandbox counts, provider health, event stats, operation metrics, and explicit redaction categories. +- Manager-level spawn and exec flows now enforce configured operational limits centrally. + +### Verified + +- `make test` +- `make build` +- `cd web && npm run build` + ## Phase 1 Foundation Hardening - 2026-05-08 This checkpoint closes the Phase 1 reliability and production-readiness foundation. diff --git a/docs/releases/phase-2-observability-and-ops.md b/docs/releases/phase-2-observability-and-ops.md new file mode 100644 index 0000000..08fcb49 --- /dev/null +++ b/docs/releases/phase-2-observability-and-ops.md @@ -0,0 +1,183 @@ +# Phase 2 Observability And Ops Release Notes + +Date: 2026-05-08 +Branch: `phase-2-observability-and-ops` + +## Summary + +Phase 2 turns the Phase 1 foundation into an operable production surface. The API now exposes liveness, readiness, diagnostics, structured metrics, Prometheus scraping, richer provider health, operational audit events, and configurable runtime limits. + +The goal of this phase is to make StacyVM easier to run, debug, monitor, and safely scale before deeper multi-tenant and production deployment work. + +## What Changed + +### Liveness And Readiness + +- Added `/api/v1/live` for process liveness checks. +- Added `/api/v1/ready` for dependency readiness checks. +- Readiness now reports provider health instead of only returning a generic process status. + +### Structured Runtime Metrics + +- Added an in-process operation metrics recorder. +- Operations tracked include: + - spawn + - exec + - exec stream + - destroy + - file write, read, list, delete, move, chmod, stat, and glob +- Each operation tracks: + - success count + - failure count + - latency count + - total latency + - min, max, and average latency + - last error + - last observed timestamp +- `/api/v1/metrics` now includes sandbox, provider, event, process, runtime, and operation metrics. + +### Prometheus Metrics + +- Added `/api/v1/metrics/prometheus`. +- The Prometheus endpoint exposes: + - process uptime + - goroutines + - memory and GC metrics + - sandbox counts by state and provider + - provider health + - provider health latency + - provider runtime inventory counts + - event bus stats + - operation success/failure and latency counters + +### Operational Audit Events + +- Added event IDs for published events. +- Added operational event types: + - `exec.failed` + - `exec.timeout` + - `operation.failed` + - `resource.limit` + - `provider.failed` + - `reconcile.action` +- Manager paths now publish audit events for: + - exec failures and timeouts + - stream exec timeouts + - file operation failures + - spawn/provider/resource failures + - destroy provider failures + - reconciliation actions and provider inventory failures + +### Provider Health Detail + +- Provider health now includes: + - `latency_ms` + - `last_checked` + - `error` + - `capabilities` + - `runtime_count` when runtime inventory is supported +- Provider health detail is shared across: + - `/api/v1/ready` + - `/api/v1/metrics` + - `/api/v1/metrics/prometheus` + - `/api/v1/providers` + - `/api/v1/providers/{name}` + +### Redacted Diagnostics + +- Added `/api/v1/diagnostics`. +- Diagnostics include: + - generated timestamp + - version/build info + - GOOS/GOARCH + - uptime, goroutines, memory, and GC cycles + - store health and latency + - active operational limits + - detailed provider health + - sandbox counts by state/provider + - event bus stats + - operation metrics + - explicit redaction categories +- Diagnostics are read-only and intentionally avoid returning API keys, registry credentials, provider secrets, or environment secrets. + +### Operational Limits + +- Added configurable defaults: + - `defaults.max_ttl` + - `defaults.default_exec_timeout` + - `defaults.max_exec_timeout` + - `defaults.max_sandboxes` + - `defaults.max_sandboxes_per_owner` +- Manager now centrally enforces: + - max TTL + - max total active sandboxes + - max active sandboxes per owner + - default exec timeout + - max exec timeout +- Limit violations return typed resource-limit errors and publish `resource.limit` audit events. + +## Code Changes By Area + +### API Routes + +- `internal/api/routes/system.go` + - Added liveness, readiness, diagnostics, JSON metrics, and Prometheus metrics behavior. +- `internal/api/routes/provider_health.go` + - Added shared provider health detail collection. +- `internal/api/routes/prometheus.go` + - Added Prometheus text renderer. +- `internal/api/routes/providers.go` + - Added detailed health to provider list/detail responses. +- `internal/api/routes/system_test.go` + - Added coverage for readiness, diagnostics, metrics, and Prometheus output. + +### Orchestrator + +- `internal/orchestrator/metrics.go` + - Added operation metrics recorder. +- `internal/orchestrator/manager.go` + - Added metrics recording, audit event publishing, and operational limit enforcement. +- `internal/orchestrator/events.go` + - Added event IDs and operational event types. +- `internal/orchestrator/types.go` + - Added operational limit types. +- `internal/orchestrator/manager_test.go` + - Added tests for operation metrics, audit events, TTL limits, sandbox limits, owner limits, and exec timeout limits. + +### Config And Docs + +- `internal/config/config.go` + - Added default config fields for operational limits. +- `cmd/stacyvm/cmd_serve.go` + - Wires configured operational limits into the manager. +- `README.md` + - Documents new operational limit config. +- `docs/api.md` + - Documents liveness, readiness, diagnostics, metrics, Prometheus metrics, provider health detail, and operational event shape. +- `CHANGELOG.md` + - Adds this Phase 2 checkpoint entry. + +## Verification + +The following checks passed: + +```sh +make test +make build +cd web && npm run build +``` + +## Impact + +Phase 2 gives StacyVM the baseline visibility and guardrails needed to operate safely: + +- Operators can distinguish liveness from readiness. +- Dashboards can consume JSON or Prometheus metrics. +- Support/debug flows can use a redacted diagnostics endpoint. +- Provider health is actionable rather than a single boolean. +- Resource pressure and failure modes are visible through events. +- Runtime limits can prevent accidental overload before full multi-tenant quota systems arrive. + +## Next Phase Direction + +Phase 3 should focus on production scaling and multi-tenant control planes: persistent quotas, per-owner policy, rate limits, queueing/backpressure, distributed scheduler boundaries, and deployment/CI hardening. From a950decfffa8dcfb8f63fcc46ebb7d689ce1f51e Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:28:28 +0530 Subject: [PATCH 010/147] feat: add persistent owner quotas --- README.md | 6 + docs/api.md | 69 +++++++++++ internal/api/routes/quotas.go | 146 ++++++++++++++++++++++ internal/api/routes/quotas_test.go | 89 ++++++++++++++ internal/api/routes/swagger_types.go | 6 + internal/api/server.go | 2 + internal/orchestrator/manager.go | 166 ++++++++++++++++++++++++-- internal/orchestrator/manager_test.go | 45 +++++++ internal/orchestrator/types.go | 18 +++ internal/store/migrations.go | 13 ++ internal/store/sqlite.go | 65 ++++++++++ internal/store/sqlite_test.go | 43 +++++++ internal/store/store.go | 15 +++ 13 files changed, 673 insertions(+), 10 deletions(-) create mode 100644 internal/api/routes/quotas.go create mode 100644 internal/api/routes/quotas_test.go diff --git a/README.md b/README.md index 5ffa2fd..46c2d15 100644 --- a/README.md +++ b/README.md @@ -413,10 +413,16 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | `GET` | `/providers` | List configured providers | | `GET` | `/providers/{name}` | Provider details + sandbox count | | `POST` | `/providers/test` | Health-check all providers | +| `GET` | `/quotas` | List owner quota overrides | +| `PUT` | `/quotas/{ownerID}` | Create or update owner quota | +| `GET` | `/quotas/{ownerID}/usage` | Owner usage against effective quota | | `GET` | `/pool/status` | Pool VM and user counts | | `GET` | `/snapshots` | Available VM snapshots | | `GET` | `/health` | Health check | +| `GET` | `/ready` | Readiness check | +| `GET` | `/diagnostics` | Redacted operational diagnostics | | `GET` | `/metrics` | Runtime metrics (goroutines, alloc, sandbox counts) | +| `GET` | `/metrics/prometheus` | Prometheus-compatible metrics | | `GET` | `/events` | Server-sent events stream | Full schemas, request/response examples, and error codes: **[docs/api.md](docs/api.md)**. diff --git a/docs/api.md b/docs/api.md index 036eff2..f1b6d49 100644 --- a/docs/api.md +++ b/docs/api.md @@ -381,6 +381,75 @@ Optional override body: --- +## Quotas + +Owner quotas are persisted overrides for per-owner sandbox and runtime limits. They apply when requests include an owner via `X-User-ID` or `owner_id`. + +### List owner quotas + +``` +GET /api/v1/quotas +``` + +**Response** `200 OK`: +```json +[ + { + "owner_id": "team-a", + "max_sandboxes": 5, + "max_ttl": "2h0m0s", + "max_exec_timeout": "1m0s", + "created_at": "2026-05-08T10:30:00Z", + "updated_at": "2026-05-08T10:30:00Z" + } +] +``` + +### Save owner quota + +``` +PUT /api/v1/quotas/{ownerID} +``` + +**Request**: +```json +{ + "max_sandboxes": 5, + "max_ttl": "2h", + "max_exec_timeout": "1m" +} +``` + +**Response** `200 OK`: full owner quota object. + +### Get owner usage + +``` +GET /api/v1/quotas/{ownerID}/usage +``` + +**Response** `200 OK`: +```json +{ + "owner_id": "team-a", + "active_sandboxes": 3, + "max_sandboxes": 5, + "max_ttl": "2h0m0s", + "max_exec_timeout": "1m0s", + "quota_configured": true +} +``` + +### Delete owner quota + +``` +DELETE /api/v1/quotas/{ownerID} +``` + +**Response** `200 OK`: `{ "status": "deleted" }`. + +--- + ## Providers ### List providers diff --git a/internal/api/routes/quotas.go b/internal/api/routes/quotas.go new file mode 100644 index 0000000..6e6f383 --- /dev/null +++ b/internal/api/routes/quotas.go @@ -0,0 +1,146 @@ +package routes + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/go-chi/chi/v5" +) + +type quotaManager interface { + ListOwnerQuotas(ctx context.Context) ([]*orchestrator.OwnerQuota, error) + GetOwnerQuota(ctx context.Context, ownerID string) (*orchestrator.OwnerQuota, error) + SaveOwnerQuota(ctx context.Context, quota orchestrator.OwnerQuota) (*orchestrator.OwnerQuota, error) + DeleteOwnerQuota(ctx context.Context, ownerID string) error + OwnerUsage(ctx context.Context, ownerID string) (*orchestrator.OwnerUsage, error) +} + +type QuotaRoutes struct { + manager quotaManager +} + +func NewQuotaRoutes(manager quotaManager) *QuotaRoutes { + return &QuotaRoutes{manager: manager} +} + +func (q *QuotaRoutes) Routes() chi.Router { + r := chi.NewRouter() + r.Get("/", q.List) + r.Route("/{ownerID}", func(r chi.Router) { + r.Get("/", q.Get) + r.Put("/", q.Save) + r.Delete("/", q.Delete) + r.Get("/usage", q.Usage) + }) + return r +} + +// List returns all configured owner quotas. +// +// @Summary List owner quotas +// @Description Return all persisted owner quota overrides +// @Tags quotas +// @Produce json +// @Success 200 {array} orchestrator.OwnerQuota +// @Security ApiKeyAuth +// @Router /quotas [get] +func (q *QuotaRoutes) List(w http.ResponseWriter, r *http.Request) { + quotas, err := q.manager.ListOwnerQuotas(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + if quotas == nil { + quotas = []*orchestrator.OwnerQuota{} + } + httputil.WriteJSON(w, http.StatusOK, quotas) +} + +// Get returns one configured owner quota. +// +// @Summary Get owner quota +// @Description Return the persisted quota override for an owner +// @Tags quotas +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Success 200 {object} orchestrator.OwnerQuota +// @Failure 404 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /quotas/{ownerID} [get] +func (q *QuotaRoutes) Get(w http.ResponseWriter, r *http.Request) { + quota, err := q.manager.GetOwnerQuota(r.Context(), chi.URLParam(r, "ownerID")) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, quota) +} + +// Save creates or updates an owner quota. +// +// @Summary Save owner quota +// @Description Create or update quota overrides for an owner +// @Tags quotas +// @Accept json +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Param request body orchestrator.OwnerQuota true "Quota request" +// @Success 200 {object} orchestrator.OwnerQuota +// @Security ApiKeyAuth +// @Router /quotas/{ownerID} [put] +func (q *QuotaRoutes) Save(w http.ResponseWriter, r *http.Request) { + ownerID := chi.URLParam(r, "ownerID") + var quota orchestrator.OwnerQuota + if err := json.NewDecoder(r.Body).Decode("a); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + quota.OwnerID = ownerID + saved, err := q.manager.SaveOwnerQuota(r.Context(), quota) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, saved) +} + +// Delete removes an owner quota override. +// +// @Summary Delete owner quota +// @Description Delete the quota override for an owner +// @Tags quotas +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Success 200 {object} StatusResponse +// @Failure 404 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /quotas/{ownerID} [delete] +func (q *QuotaRoutes) Delete(w http.ResponseWriter, r *http.Request) { + if err := q.manager.DeleteOwnerQuota(r.Context(), chi.URLParam(r, "ownerID")); err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) +} + +// Usage returns current owner usage against quota. +// +// @Summary Get owner quota usage +// @Description Return active sandbox usage and effective quota for an owner +// @Tags quotas +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Success 200 {object} orchestrator.OwnerUsage +// @Security ApiKeyAuth +// @Router /quotas/{ownerID}/usage [get] +func (q *QuotaRoutes) Usage(w http.ResponseWriter, r *http.Request) { + usage, err := q.manager.OwnerUsage(r.Context(), chi.URLParam(r, "ownerID")) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, usage) +} diff --git a/internal/api/routes/quotas_test.go b/internal/api/routes/quotas_test.go new file mode 100644 index 0000000..97bae6c --- /dev/null +++ b/internal/api/routes/quotas_test.go @@ -0,0 +1,89 @@ +package routes + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog" +) + +func setupQuotaRouter(t *testing.T) (chi.Router, *orchestrator.Manager) { + t.Helper() + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := providers.NewMockProvider() + reg.Register(mock) + reg.SetDefault("mock") + + mgr := orchestrator.NewManager(reg, st, orchestrator.NewEventBus(), zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + + r := chi.NewRouter() + r.Mount("/api/v1/quotas", NewQuotaRoutes(mgr).Routes()) + return r, mgr +} + +func TestQuotaRoutes_SaveGetUsageDelete(t *testing.T) { + r, mgr := setupQuotaRouter(t) + + body := `{"max_sandboxes":1,"max_ttl":"30m","max_exec_timeout":"10s"}` + req := httptest.NewRequest(http.MethodPut, "/api/v1/quotas/owner-a", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("save status = %d: %s", w.Code, w.Body.String()) + } + + var quota orchestrator.OwnerQuota + if err := json.NewDecoder(w.Body).Decode("a); err != nil { + t.Fatalf("decode quota: %v", err) + } + if quota.OwnerID != "owner-a" || quota.MaxSandboxes != 1 { + t.Fatalf("unexpected quota: %+v", quota) + } + + if _, err := mgr.Spawn(req.Context(), orchestrator.SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/quotas/owner-a/usage", nil) + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("usage status = %d: %s", w.Code, w.Body.String()) + } + var usage orchestrator.OwnerUsage + if err := json.NewDecoder(w.Body).Decode(&usage); err != nil { + t.Fatalf("decode usage: %v", err) + } + if !usage.QuotaConfigured || usage.ActiveSandboxes != 1 || usage.MaxSandboxes != 1 { + t.Fatalf("unexpected usage: %+v", usage) + } + + req = httptest.NewRequest(http.MethodDelete, "/api/v1/quotas/owner-a", nil) + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("delete status = %d: %s", w.Code, w.Body.String()) + } +} diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index 49c4527..d2e0ce3 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -55,6 +55,12 @@ type DiagnosticsResponse struct { Redactions []string `json:"redactions"` } +// OwnerQuotaResponse is the response for owner quota configuration. +type OwnerQuotaResponse = orchestrator.OwnerQuota + +// OwnerUsageResponse is the response for owner quota usage. +type OwnerUsageResponse = orchestrator.OwnerUsage + // MetricsResponse is the response from the metrics endpoint. type MetricsResponse struct { SandboxesActive int `json:"sandboxes_active" example:"5"` diff --git a/internal/api/server.go b/internal/api/server.go index 0f60707..c321f6b 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -81,6 +81,7 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr snapshotRoutes := routes.NewSnapshotRoutes(registry) systemRoutes := routes.NewSystemRoutes(registry, manager, events, st, cfg.Version) environmentRoutes := routes.NewEnvironmentRoutes(st, envBuild) + quotaRoutes := routes.NewQuotaRoutes(manager) r.Route("/api/v1", func(r chi.Router) { r.Mount("/sandboxes", sandboxRoutes.Routes()) @@ -88,6 +89,7 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr r.Mount("/templates", templateRoutes.Routes()) r.Mount("/snapshots", snapshotRoutes.Routes()) r.Mount("/environments", environmentRoutes.Routes()) + r.Mount("/quotas", quotaRoutes.Routes()) r.Get("/pool/status", sandboxRoutes.VMPoolStatus) r.Mount("/", systemRoutes.Routes()) }) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index c854d25..16f49ae 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -521,7 +521,7 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( execCtx := ctx var cancel context.CancelFunc - timeout, err := m.resolveExecTimeout(req.Timeout) + timeout, err := m.resolveExecTimeout(req.Timeout, sb.OwnerID) if err != nil { metricsErr = err m.publishFailureForError(sandboxID, OperationExec, metricsProvider, err) @@ -608,7 +608,7 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ execCtx := ctx var cancel context.CancelFunc - timeout, err := m.resolveExecTimeout(req.Timeout) + timeout, err := m.resolveExecTimeout(req.Timeout, sb.OwnerID) if err != nil { metricsErr = err m.publishFailureForError(sandboxID, OperationExecStream, metricsProvider, err) @@ -963,6 +963,84 @@ func (m *Manager) Limits() OperationalLimitsInfo { } } +func (m *Manager) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuota, error) { + rec, err := m.store.GetOwnerQuota(ctx, ownerID) + if err != nil { + return nil, err + } + return ownerQuotaFromRecord(rec), nil +} + +func (m *Manager) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuota, error) { + records, err := m.store.ListOwnerQuotas(ctx) + if err != nil { + return nil, err + } + quotas := make([]*OwnerQuota, 0, len(records)) + for _, rec := range records { + quotas = append(quotas, ownerQuotaFromRecord(rec)) + } + return quotas, nil +} + +func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQuota, error) { + if quota.OwnerID == "" { + return nil, fmt.Errorf("owner_id is required") + } + maxTTL, err := parseOptionalDurationSeconds(quota.MaxTTL) + if err != nil { + return nil, fmt.Errorf("parsing max_ttl: %w", err) + } + maxExecTimeout, err := parseOptionalDurationSeconds(quota.MaxExecTimeout) + if err != nil { + return nil, fmt.Errorf("parsing max_exec_timeout: %w", err) + } + if quota.MaxSandboxes < 0 { + return nil, providers.ResourceLimitError("max_sandboxes cannot be negative") + } + rec := &store.OwnerQuotaRecord{ + OwnerID: quota.OwnerID, + MaxSandboxes: quota.MaxSandboxes, + MaxTTLSeconds: maxTTL, + MaxExecTimeoutSeconds: maxExecTimeout, + } + if err := m.store.SaveOwnerQuota(ctx, rec); err != nil { + return nil, err + } + return m.GetOwnerQuota(ctx, quota.OwnerID) +} + +func (m *Manager) DeleteOwnerQuota(ctx context.Context, ownerID string) error { + return m.store.DeleteOwnerQuota(ctx, ownerID) +} + +func (m *Manager) OwnerUsage(ctx context.Context, ownerID string) (*OwnerUsage, error) { + records, err := m.store.ListSandboxesByOwner(ctx, ownerID) + if err != nil { + return nil, err + } + usage := &OwnerUsage{ + OwnerID: ownerID, + ActiveSandboxes: len(records), + MaxSandboxes: m.limits.MaxSandboxesPerOwner, + MaxTTL: m.limits.MaxTTL.String(), + MaxExecTimeout: m.limits.MaxExecTimeout.String(), + } + if quota, err := m.store.GetOwnerQuota(ctx, ownerID); err == nil { + usage.QuotaConfigured = true + if quota.MaxSandboxes > 0 { + usage.MaxSandboxes = quota.MaxSandboxes + } + if quota.MaxTTLSeconds > 0 { + usage.MaxTTL = (time.Duration(quota.MaxTTLSeconds) * time.Second).String() + } + if quota.MaxExecTimeoutSeconds > 0 { + usage.MaxExecTimeout = (time.Duration(quota.MaxExecTimeoutSeconds) * time.Second).String() + } + } + return usage, nil +} + func (m *Manager) recordOperation(operation, provider string, duration time.Duration, err error) { if m.metrics == nil { return @@ -1006,11 +1084,12 @@ func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, } func (m *Manager) enforceSpawnLimits(ctx context.Context, ownerID string, ttl time.Duration) error { - if m.limits.MaxTTL > 0 && ttl > m.limits.MaxTTL { - return providers.ResourceLimitError(fmt.Sprintf("ttl %s exceeds max ttl %s", ttl, m.limits.MaxTTL)) + maxTTL, maxPerOwner := m.ownerLimitOverrides(ctx, ownerID) + if maxTTL > 0 && ttl > maxTTL { + return providers.ResourceLimitError(fmt.Sprintf("ttl %s exceeds max ttl %s", ttl, maxTTL)) } - if m.limits.MaxSandboxes <= 0 && (ownerID == "" || m.limits.MaxSandboxesPerOwner <= 0) { + if m.limits.MaxSandboxes <= 0 && (ownerID == "" || maxPerOwner <= 0) { return nil } @@ -1032,13 +1111,13 @@ func (m *Manager) enforceSpawnLimits(ctx context.Context, ownerID string, ttl ti if m.limits.MaxSandboxes > 0 && total >= m.limits.MaxSandboxes { return providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", m.limits.MaxSandboxes)) } - if ownerID != "" && m.limits.MaxSandboxesPerOwner > 0 && ownerTotal >= m.limits.MaxSandboxesPerOwner { - return providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", m.limits.MaxSandboxesPerOwner)) + if ownerID != "" && maxPerOwner > 0 && ownerTotal >= maxPerOwner { + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", maxPerOwner)) } return nil } -func (m *Manager) resolveExecTimeout(raw string) (time.Duration, error) { +func (m *Manager) resolveExecTimeout(raw string, ownerID string) (time.Duration, error) { timeout := m.limits.DefaultExecTimeout if raw != "" { parsed, err := time.ParseDuration(raw) @@ -1050,12 +1129,79 @@ func (m *Manager) resolveExecTimeout(raw string) (time.Duration, error) { if timeout < 0 { return 0, providers.ResourceLimitError("exec timeout cannot be negative") } - if m.limits.MaxExecTimeout > 0 && timeout > m.limits.MaxExecTimeout { - return 0, providers.ResourceLimitError(fmt.Sprintf("exec timeout %s exceeds max exec timeout %s", timeout, m.limits.MaxExecTimeout)) + maxExecTimeout := m.ownerMaxExecTimeout(context.Background(), ownerID) + if maxExecTimeout > 0 && timeout > maxExecTimeout { + return 0, providers.ResourceLimitError(fmt.Sprintf("exec timeout %s exceeds max exec timeout %s", timeout, maxExecTimeout)) } return timeout, nil } +func (m *Manager) ownerLimitOverrides(ctx context.Context, ownerID string) (time.Duration, int) { + maxTTL := m.limits.MaxTTL + maxPerOwner := m.limits.MaxSandboxesPerOwner + if ownerID == "" { + return maxTTL, maxPerOwner + } + quota, err := m.store.GetOwnerQuota(ctx, ownerID) + if err != nil { + return maxTTL, maxPerOwner + } + if quota.MaxTTLSeconds > 0 { + maxTTL = time.Duration(quota.MaxTTLSeconds) * time.Second + } + if quota.MaxSandboxes > 0 { + maxPerOwner = quota.MaxSandboxes + } + return maxTTL, maxPerOwner +} + +func (m *Manager) ownerMaxExecTimeout(ctx context.Context, ownerID string) time.Duration { + maxExecTimeout := m.limits.MaxExecTimeout + if ownerID == "" { + return maxExecTimeout + } + quota, err := m.store.GetOwnerQuota(ctx, ownerID) + if err != nil { + return maxExecTimeout + } + if quota.MaxExecTimeoutSeconds > 0 { + return time.Duration(quota.MaxExecTimeoutSeconds) * time.Second + } + return maxExecTimeout +} + +func ownerQuotaFromRecord(rec *store.OwnerQuotaRecord) *OwnerQuota { + return &OwnerQuota{ + OwnerID: rec.OwnerID, + MaxSandboxes: rec.MaxSandboxes, + MaxTTL: optionalSecondsString(rec.MaxTTLSeconds), + MaxExecTimeout: optionalSecondsString(rec.MaxExecTimeoutSeconds), + CreatedAt: rec.CreatedAt, + UpdatedAt: rec.UpdatedAt, + } +} + +func parseOptionalDurationSeconds(raw string) (int64, error) { + if raw == "" || raw == "0" || raw == "0s" { + return 0, nil + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, err + } + if d < 0 { + return 0, providers.ResourceLimitError("duration cannot be negative") + } + return int64(d.Seconds()), nil +} + +func optionalSecondsString(seconds int64) string { + if seconds <= 0 { + return "0s" + } + return (time.Duration(seconds) * time.Second).String() +} + // InitVMPool initializes the VM pool manager if pool mode is enabled. func (m *Manager) InitVMPool() { if !m.poolConfig.Enabled { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 1ad9727..06d24a1 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -309,6 +309,51 @@ func TestManager_SpawnOwnerLimit(t *testing.T) { } } +func TestManager_PersistentOwnerQuotaLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: "owner-quota", + MaxSandboxes: 1, + MaxTTL: "30m", + MaxExecTimeout: "2s", + }) + if err != nil { + t.Fatalf("save owner quota: %v", err) + } + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-quota", TTL: "10m"}); err != nil { + t.Fatalf("spawn: %v", err) + } + _, err = m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-quota", TTL: "10m"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected owner quota resource limit, got %v", err) + } + + usage, err := m.OwnerUsage(context.Background(), "owner-quota") + if err != nil { + t.Fatalf("owner usage: %v", err) + } + if !usage.QuotaConfigured || usage.ActiveSandboxes != 1 || usage.MaxSandboxes != 1 { + t.Fatalf("unexpected owner usage: %+v", usage) + } +} + +func TestManager_PersistentOwnerQuotaTTLLimit(t *testing.T) { + m := setupManager(t) + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{OwnerID: "owner-ttl", MaxTTL: "5m"}); err != nil { + t.Fatalf("save quota: %v", err) + } + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-ttl", TTL: "10m"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected ttl quota resource limit, got %v", err) + } +} + func TestManager_SpawnMaxTTLLimit(t *testing.T) { m := setupManagerWithConfig(t, ManagerConfig{ DefaultTTL: 5 * time.Minute, diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index 2c9cbb7..7a9275a 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -113,3 +113,21 @@ type OperationalLimitsInfo struct { MaxExecTimeout string `json:"max_exec_timeout"` MaxTTL string `json:"max_ttl"` } + +type OwnerQuota struct { + OwnerID string `json:"owner_id"` + MaxSandboxes int `json:"max_sandboxes"` + MaxTTL string `json:"max_ttl"` + MaxExecTimeout string `json:"max_exec_timeout"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type OwnerUsage struct { + OwnerID string `json:"owner_id"` + ActiveSandboxes int `json:"active_sandboxes"` + MaxSandboxes int `json:"max_sandboxes"` + MaxTTL string `json:"max_ttl"` + MaxExecTimeout string `json:"max_exec_timeout"` + QuotaConfigured bool `json:"quota_configured"` +} diff --git a/internal/store/migrations.go b/internal/store/migrations.go index 031ef50..1720e34 100644 --- a/internal/store/migrations.go +++ b/internal/store/migrations.go @@ -143,6 +143,19 @@ ALTER TABLE sandboxes ADD COLUMN owner_id TEXT NOT NULL DEFAULT ''; ALTER TABLE sandboxes ADD COLUMN vm_id TEXT NOT NULL DEFAULT ''; CREATE INDEX IF NOT EXISTS idx_sandboxes_owner ON sandboxes(owner_id); CREATE INDEX IF NOT EXISTS idx_sandboxes_vm ON sandboxes(vm_id); +`, + }, + { + version: 5, + sql: ` +CREATE TABLE IF NOT EXISTS owner_quotas ( + owner_id TEXT PRIMARY KEY, + max_sandboxes INTEGER NOT NULL DEFAULT 0, + max_ttl_seconds INTEGER NOT NULL DEFAULT 0, + max_exec_timeout_seconds INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) +); `, }, } diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index a1ee5a2..e3de639 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -205,6 +205,71 @@ func (s *SQLiteStore) CountSandboxesByVM(ctx context.Context, vmID string) (int, return count, err } +func (s *SQLiteStore) SaveOwnerQuota(ctx context.Context, quota *OwnerQuotaRecord) error { + now := time.Now().UTC() + if quota.CreatedAt.IsZero() { + quota.CreatedAt = now + } + quota.UpdatedAt = now + _, err := s.db.ExecContext(ctx, ` + INSERT INTO owner_quotas (owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(owner_id) DO UPDATE SET + max_sandboxes = excluded.max_sandboxes, + max_ttl_seconds = excluded.max_ttl_seconds, + max_exec_timeout_seconds = excluded.max_exec_timeout_seconds, + updated_at = excluded.updated_at`, + quota.OwnerID, quota.MaxSandboxes, quota.MaxTTLSeconds, quota.MaxExecTimeoutSeconds, + quota.CreatedAt.UTC(), quota.UpdatedAt.UTC(), + ) + return err +} + +func (s *SQLiteStore) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuotaRecord, error) { + quota := &OwnerQuotaRecord{} + err := s.db.QueryRowContext(ctx, ` + SELECT owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at + FROM owner_quotas WHERE owner_id = ?`, ownerID, + ).Scan("a.OwnerID, "a.MaxSandboxes, "a.MaxTTLSeconds, "a.MaxExecTimeoutSeconds, "a.CreatedAt, "a.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("owner_quota", ownerID) + } + return quota, err +} + +func (s *SQLiteStore) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuotaRecord, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at + FROM owner_quotas ORDER BY owner_id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var quotas []*OwnerQuotaRecord + for rows.Next() { + quota := &OwnerQuotaRecord{} + if err := rows.Scan("a.OwnerID, "a.MaxSandboxes, "a.MaxTTLSeconds, + "a.MaxExecTimeoutSeconds, "a.CreatedAt, "a.UpdatedAt); err != nil { + return nil, err + } + quotas = append(quotas, quota) + } + return quotas, rows.Err() +} + +func (s *SQLiteStore) DeleteOwnerQuota(ctx context.Context, ownerID string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM owner_quotas WHERE owner_id = ?`, ownerID) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("owner_quota", ownerID) + } + return nil +} + // --- Exec Logs --- func (s *SQLiteStore) CreateExecLog(ctx context.Context, log *ExecLogRecord) error { diff --git a/internal/store/sqlite_test.go b/internal/store/sqlite_test.go index 9b92329..af6e3dc 100644 --- a/internal/store/sqlite_test.go +++ b/internal/store/sqlite_test.go @@ -430,3 +430,46 @@ func TestEnvironmentBuildArtifactAndRegistryCRUD(t *testing.T) { t.Fatalf("delete registry connection: %v", err) } } + +func TestOwnerQuotaStore(t *testing.T) { + s := testStore(t) + ctx := context.Background() + + quota := &OwnerQuotaRecord{ + OwnerID: "owner-1", + MaxSandboxes: 3, + MaxTTLSeconds: 3600, + MaxExecTimeoutSeconds: 60, + } + if err := s.SaveOwnerQuota(ctx, quota); err != nil { + t.Fatalf("save quota: %v", err) + } + + got, err := s.GetOwnerQuota(ctx, "owner-1") + if err != nil { + t.Fatalf("get quota: %v", err) + } + if got.MaxSandboxes != 3 || got.MaxTTLSeconds != 3600 || got.MaxExecTimeoutSeconds != 60 { + t.Fatalf("unexpected quota: %+v", got) + } + + quota.MaxSandboxes = 5 + if err := s.SaveOwnerQuota(ctx, quota); err != nil { + t.Fatalf("update quota: %v", err) + } + + quotas, err := s.ListOwnerQuotas(ctx) + if err != nil { + t.Fatalf("list quotas: %v", err) + } + if len(quotas) != 1 || quotas[0].MaxSandboxes != 5 { + t.Fatalf("unexpected quotas: %+v", quotas) + } + + if err := s.DeleteOwnerQuota(ctx, "owner-1"); err != nil { + t.Fatalf("delete quota: %v", err) + } + if _, err := s.GetOwnerQuota(ctx, "owner-1"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound after delete, got %v", err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index d11bf37..4835ba0 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -104,6 +104,15 @@ type RegistryConnectionRecord struct { UpdatedAt time.Time } +type OwnerQuotaRecord struct { + OwnerID string + MaxSandboxes int + MaxTTLSeconds int64 + MaxExecTimeoutSeconds int64 + CreatedAt time.Time + UpdatedAt time.Time +} + // Store defines the persistence interface. type Store interface { // Sandbox CRUD @@ -117,6 +126,12 @@ type Store interface { ListSandboxesByOwner(ctx context.Context, ownerID string) ([]*SandboxRecord, error) CountSandboxesByVM(ctx context.Context, vmID string) (int, error) + // Owner quotas + SaveOwnerQuota(ctx context.Context, quota *OwnerQuotaRecord) error + GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuotaRecord, error) + ListOwnerQuotas(ctx context.Context) ([]*OwnerQuotaRecord, error) + DeleteOwnerQuota(ctx context.Context, ownerID string) error + // Exec logs CreateExecLog(ctx context.Context, log *ExecLogRecord) error ListExecLogs(ctx context.Context, sandboxID string) ([]*ExecLogRecord, error) From 418374eaf8664dcee97b2ea41f9005b570e63870 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 01:39:35 +0530 Subject: [PATCH 011/147] feat: add spawn backpressure queue --- README.md | 3 + cmd/stacyvm/cmd_serve.go | 4 + docs/api.md | 6 +- internal/api/routes/sandboxes.go | 3 +- internal/config/config.go | 6 ++ internal/orchestrator/events.go | 31 ++++--- internal/orchestrator/manager.go | 118 +++++++++++++++++++++++--- internal/orchestrator/manager_test.go | 82 ++++++++++++++++++ internal/orchestrator/types.go | 6 ++ 9 files changed, 233 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 46c2d15..7bec3df 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,9 @@ defaults: max_exec_timeout: "10m" max_sandboxes: 0 # 0 = unlimited max_sandboxes_per_owner: 0 # 0 = unlimited + spawn_overflow: "reject" # reject or queue when sandbox capacity is full + spawn_queue_timeout: "30s" + max_spawn_queue: 100 auth: enabled: false diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index 059c803..e7616f4 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -166,6 +166,7 @@ func runServe() error { maxTTL, _ := time.ParseDuration(cfg.Defaults.MaxTTL) defaultExecTimeout, _ := time.ParseDuration(cfg.Defaults.DefaultExecTimeout) maxExecTimeout, _ := time.ParseDuration(cfg.Defaults.MaxExecTimeout) + spawnQueueTimeout, _ := time.ParseDuration(cfg.Defaults.SpawnQueueTimeout) mgr := orchestrator.NewManager(registry, st, events, logger, orchestrator.ManagerConfig{ DefaultTTL: ttl, DefaultImage: cfg.Defaults.Image, @@ -179,6 +180,9 @@ func runServe() error { DefaultExecTimeout: defaultExecTimeout, MaxExecTimeout: maxExecTimeout, MaxTTL: maxTTL, + SpawnOverflow: cfg.Defaults.SpawnOverflow, + SpawnQueueTimeout: spawnQueueTimeout, + MaxSpawnQueue: cfg.Defaults.MaxSpawnQueue, }, }) if err := mgr.Reconcile(context.Background()); err != nil { diff --git a/docs/api.md b/docs/api.md index f1b6d49..da277e6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -658,7 +658,10 @@ GET /api/v1/diagnostics "max_sandboxes_per_owner": 10, "default_exec_timeout": "30s", "max_exec_timeout": "10m0s", - "max_ttl": "24h0m0s" + "max_ttl": "24h0m0s", + "spawn_overflow": "queue", + "spawn_queue_timeout": "30s", + "max_spawn_queue": 100 }, "providers": [ { @@ -783,6 +786,7 @@ Common event types include: - `exec.started`, `exec.completed`, `exec.failed`, `exec.timeout` - `file.written`, `file.read` - `operation.failed`, `resource.limit`, `provider.failed`, `reconcile.action` +- `spawn.queued`, `spawn.dequeued`, `spawn.queue_timeout` Use any SSE client (`EventSource` in browsers, `httpx-sse` in Python, etc.) to consume. diff --git a/internal/api/routes/sandboxes.go b/internal/api/routes/sandboxes.go index 382a150..4a626ee 100644 --- a/internal/api/routes/sandboxes.go +++ b/internal/api/routes/sandboxes.go @@ -55,6 +55,7 @@ func (s *SandboxRoutes) Routes() chi.Router { // @Param request body orchestrator.SpawnRequest true "Spawn request" // @Success 201 {object} orchestrator.Sandbox // @Failure 400 {object} httputil.APIError +// @Failure 429 {object} httputil.APIError // @Failure 500 {object} httputil.APIError // @Security ApiKeyAuth // @Router /sandboxes [post] @@ -72,7 +73,7 @@ func (s *SandboxRoutes) Create(w http.ResponseWriter, r *http.Request) { sb, err := s.manager.Spawn(r.Context(), req) if err != nil { - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } diff --git a/internal/config/config.go b/internal/config/config.go index 570d932..5f72e9d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -127,6 +127,9 @@ type DefaultsConfig struct { MaxExecTimeout string `mapstructure:"max_exec_timeout"` MaxSandboxes int `mapstructure:"max_sandboxes"` MaxSandboxesPerOwner int `mapstructure:"max_sandboxes_per_owner"` + SpawnOverflow string `mapstructure:"spawn_overflow"` + SpawnQueueTimeout string `mapstructure:"spawn_queue_timeout"` + MaxSpawnQueue int `mapstructure:"max_spawn_queue"` } type AuthConfig struct { @@ -205,6 +208,9 @@ func setDefaults(v *viper.Viper) { v.SetDefault("defaults.max_exec_timeout", "10m") v.SetDefault("defaults.max_sandboxes", 0) v.SetDefault("defaults.max_sandboxes_per_owner", 0) + v.SetDefault("defaults.spawn_overflow", "reject") + v.SetDefault("defaults.spawn_queue_timeout", "30s") + v.SetDefault("defaults.max_spawn_queue", 100) v.SetDefault("auth.enabled", false) v.SetDefault("auth.api_key", "") diff --git a/internal/orchestrator/events.go b/internal/orchestrator/events.go index 95864f2..03bc2e2 100644 --- a/internal/orchestrator/events.go +++ b/internal/orchestrator/events.go @@ -10,20 +10,23 @@ import ( type EventType string const ( - EventSandboxCreated EventType = "sandbox.created" - EventSandboxRunning EventType = "sandbox.running" - EventSandboxDestroyed EventType = "sandbox.destroyed" - EventSandboxError EventType = "sandbox.error" - EventExecStarted EventType = "exec.started" - EventExecCompleted EventType = "exec.completed" - EventExecFailed EventType = "exec.failed" - EventExecTimeout EventType = "exec.timeout" - EventFileWritten EventType = "file.written" - EventFileRead EventType = "file.read" - EventOperationFailed EventType = "operation.failed" - EventResourceLimit EventType = "resource.limit" - EventProviderFailed EventType = "provider.failed" - EventReconcileAction EventType = "reconcile.action" + EventSandboxCreated EventType = "sandbox.created" + EventSandboxRunning EventType = "sandbox.running" + EventSandboxDestroyed EventType = "sandbox.destroyed" + EventSandboxError EventType = "sandbox.error" + EventExecStarted EventType = "exec.started" + EventExecCompleted EventType = "exec.completed" + EventExecFailed EventType = "exec.failed" + EventExecTimeout EventType = "exec.timeout" + EventFileWritten EventType = "file.written" + EventFileRead EventType = "file.read" + EventOperationFailed EventType = "operation.failed" + EventResourceLimit EventType = "resource.limit" + EventProviderFailed EventType = "provider.failed" + EventReconcileAction EventType = "reconcile.action" + EventSpawnQueued EventType = "spawn.queued" + EventSpawnDequeued EventType = "spawn.dequeued" + EventSpawnQueueTimeout EventType = "spawn.queue_timeout" ) type Event struct { diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 16f49ae..a23b9dc 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -25,8 +25,11 @@ type Manager struct { logger zerolog.Logger metrics *MetricsRecorder - mu sync.RWMutex - sandboxes map[string]*Sandbox + mu sync.RWMutex + sandboxes map[string]*Sandbox + queueMu sync.Mutex + queueWaiters int + capacityCh chan struct{} defaultTTL time.Duration defaultImage string @@ -62,6 +65,7 @@ func NewManager(registry *providers.Registry, st store.Store, events *EventBus, logger: logger.With().Str("component", "manager").Logger(), metrics: NewMetricsRecorder(), sandboxes: make(map[string]*Sandbox), + capacityCh: make(chan struct{}), defaultTTL: cfg.DefaultTTL, defaultImage: cfg.DefaultImage, defaultMemory: cfg.DefaultMemory, @@ -84,6 +88,15 @@ func NewManager(registry *providers.Registry, st store.Store, events *EventBus, if m.defaultVCPUs == 0 { m.defaultVCPUs = 1 } + if m.limits.SpawnOverflow == "" { + m.limits.SpawnOverflow = "reject" + } + if m.limits.SpawnQueueTimeout == 0 { + m.limits.SpawnQueueTimeout = 30 * time.Second + } + if m.limits.MaxSpawnQueue == 0 { + m.limits.MaxSpawnQueue = 100 + } return m } @@ -339,7 +352,7 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) } ttl = parsed } - if err := m.enforceSpawnLimits(ctx, req.OwnerID, ttl); err != nil { + if err := m.waitForSpawnCapacity(ctx, req.OwnerID, ttl, metricsProvider); err != nil { metricsErr = err m.publishFailureForError("", OperationSpawn, metricsProvider, err) return nil, err @@ -960,6 +973,9 @@ func (m *Manager) Limits() OperationalLimitsInfo { DefaultExecTimeout: m.limits.DefaultExecTimeout.String(), MaxExecTimeout: m.limits.MaxExecTimeout.String(), MaxTTL: m.limits.MaxTTL.String(), + SpawnOverflow: m.limits.SpawnOverflow, + SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), + MaxSpawnQueue: m.limits.MaxSpawnQueue, } } @@ -1083,19 +1099,98 @@ func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, }) } -func (m *Manager) enforceSpawnLimits(ctx context.Context, ownerID string, ttl time.Duration) error { +func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl time.Duration, provider string) error { + queueable, err := m.checkSpawnLimits(ctx, ownerID, ttl) + if err == nil { + return nil + } + if !queueable || !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + return err + } + + m.queueMu.Lock() + if m.queueWaiters >= m.limits.MaxSpawnQueue { + m.queueMu.Unlock() + return providers.ResourceLimitError(fmt.Sprintf("spawn queue full (%d)", m.limits.MaxSpawnQueue)) + } + m.queueWaiters++ + depth := m.queueWaiters + capacityCh := m.capacityCh + m.queueMu.Unlock() + + m.publishOperationalEvent(EventSpawnQueued, "", map[string]interface{}{ + "operation": OperationSpawn, + "provider": provider, + "owner_id": ownerID, + "depth": depth, + }) + defer func() { + m.queueMu.Lock() + m.queueWaiters-- + m.queueMu.Unlock() + }() + + waitCtx := ctx + cancel := func() {} + if m.limits.SpawnQueueTimeout > 0 { + waitCtx, cancel = context.WithTimeout(ctx, m.limits.SpawnQueueTimeout) + } + defer cancel() + + for { + select { + case <-waitCtx.Done(): + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + err := providers.ResourceLimitError(fmt.Sprintf("spawn queue timeout after %s", m.limits.SpawnQueueTimeout)) + m.publishOperationalEvent(EventSpawnQueueTimeout, "", map[string]interface{}{ + "operation": OperationSpawn, + "provider": provider, + "owner_id": ownerID, + "error": err.Error(), + }) + return err + } + return waitCtx.Err() + case <-capacityCh: + queueable, err = m.checkSpawnLimits(ctx, ownerID, ttl) + if err == nil { + m.publishOperationalEvent(EventSpawnDequeued, "", map[string]interface{}{ + "operation": OperationSpawn, + "provider": provider, + "owner_id": ownerID, + }) + return nil + } + if !queueable { + return err + } + m.queueMu.Lock() + capacityCh = m.capacityCh + m.queueMu.Unlock() + } + } +} + +func (m *Manager) notifySpawnCapacity() { + m.queueMu.Lock() + close(m.capacityCh) + m.capacityCh = make(chan struct{}) + m.queueMu.Unlock() +} + +func (m *Manager) checkSpawnLimits(ctx context.Context, ownerID string, ttl time.Duration) (bool, error) { maxTTL, maxPerOwner := m.ownerLimitOverrides(ctx, ownerID) if maxTTL > 0 && ttl > maxTTL { - return providers.ResourceLimitError(fmt.Sprintf("ttl %s exceeds max ttl %s", ttl, maxTTL)) + return false, providers.ResourceLimitError(fmt.Sprintf("ttl %s exceeds max ttl %s", ttl, maxTTL)) } if m.limits.MaxSandboxes <= 0 && (ownerID == "" || maxPerOwner <= 0) { - return nil + return false, nil } records, err := m.store.ListSandboxes(ctx) if err != nil { - return fmt.Errorf("checking sandbox limits: %w", err) + return false, fmt.Errorf("checking sandbox limits: %w", err) } total := 0 ownerTotal := 0 @@ -1109,12 +1204,12 @@ func (m *Manager) enforceSpawnLimits(ctx context.Context, ownerID string, ttl ti } } if m.limits.MaxSandboxes > 0 && total >= m.limits.MaxSandboxes { - return providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", m.limits.MaxSandboxes)) + return true, providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", m.limits.MaxSandboxes)) } if ownerID != "" && maxPerOwner > 0 && ownerTotal >= maxPerOwner { - return providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", maxPerOwner)) + return true, providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", maxPerOwner)) } - return nil + return false, nil } func (m *Manager) resolveExecTimeout(raw string, ownerID string) (time.Duration, error) { @@ -1365,6 +1460,7 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { m.mu.Lock() delete(m.sandboxes, id) m.mu.Unlock() + m.notifySpawnCapacity() return nil } if sb != nil { @@ -1389,6 +1485,7 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { Type: EventSandboxDestroyed, SandboxID: id, }) + m.notifySpawnCapacity() m.logger.Info().Str("sandbox", id).Msg("sandbox destroyed") return nil @@ -1431,6 +1528,7 @@ func (m *Manager) destroyPooled(ctx context.Context, id string, sb *Sandbox) err m.mu.Unlock() m.events.Publish(Event{Type: EventSandboxDestroyed, SandboxID: id}) + m.notifySpawnCapacity() m.logger.Info().Str("sandbox", id).Str("vm_id", vmID).Msg("pooled sandbox destroyed") return nil } diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 06d24a1..efa32f4 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -371,6 +371,88 @@ func TestManager_SpawnMaxTTLLimit(t *testing.T) { } } +func TestManager_SpawnQueueWaitsForCapacity(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + SpawnOverflow: "queue", + SpawnQueueTimeout: 500 * time.Millisecond, + MaxSpawnQueue: 2, + }, + }) + ctx := context.Background() + + first, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-a"}) + if err != nil { + t.Fatalf("first spawn: %v", err) + } + + type spawnResult struct { + sb *Sandbox + err error + } + resultCh := make(chan spawnResult, 1) + go func() { + sb, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-b"}) + resultCh <- spawnResult{sb: sb, err: err} + }() + + select { + case result := <-resultCh: + t.Fatalf("second spawn returned before capacity opened: sb=%v err=%v", result.sb, result.err) + case <-time.After(25 * time.Millisecond): + } + + if err := m.Destroy(ctx, first.ID); err != nil { + t.Fatalf("destroy first: %v", err) + } + + select { + case result := <-resultCh: + if result.err != nil { + t.Fatalf("second spawn: %v", result.err) + } + if result.sb == nil || result.sb.OwnerID != "owner-b" { + t.Fatalf("unexpected second spawn: %+v", result.sb) + } + case <-time.After(time.Second): + t.Fatal("second spawn did not resume after capacity opened") + } + + events := m.events.History(20) + assertEventType(t, events, EventSpawnQueued) + assertEventType(t, events, EventSpawnDequeued) +} + +func TestManager_SpawnQueueTimesOut(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + SpawnOverflow: "queue", + SpawnQueueTimeout: 20 * time.Millisecond, + MaxSpawnQueue: 2, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("first spawn: %v", err) + } + + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-b"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected queue timeout resource limit, got %v", err) + } + assertEventType(t, m.events.History(20), EventSpawnQueueTimeout) +} + func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { m := setupManager(t) ctx := context.Background() diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index 7a9275a..dfca690 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -104,6 +104,9 @@ type OperationalLimits struct { DefaultExecTimeout time.Duration `json:"default_exec_timeout"` MaxExecTimeout time.Duration `json:"max_exec_timeout"` MaxTTL time.Duration `json:"max_ttl"` + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueTimeout time.Duration `json:"spawn_queue_timeout"` + MaxSpawnQueue int `json:"max_spawn_queue"` } type OperationalLimitsInfo struct { @@ -112,6 +115,9 @@ type OperationalLimitsInfo struct { DefaultExecTimeout string `json:"default_exec_timeout"` MaxExecTimeout string `json:"max_exec_timeout"` MaxTTL string `json:"max_ttl"` + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueTimeout string `json:"spawn_queue_timeout"` + MaxSpawnQueue int `json:"max_spawn_queue"` } type OwnerQuota struct { From e6524f1603bca0e181c5a79e1fb4aac69041f11f Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 02:06:13 +0530 Subject: [PATCH 012/147] feat: add API rate limiting --- README.md | 7 + cmd/stacyvm/cmd_serve.go | 7 + docs/api.md | 17 +++ internal/api/middleware/ratelimit.go | 157 ++++++++++++++++++++++ internal/api/middleware/ratelimit_test.go | 141 +++++++++++++++++++ internal/api/server.go | 11 +- internal/config/config.go | 13 ++ 7 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 internal/api/middleware/ratelimit.go create mode 100644 internal/api/middleware/ratelimit_test.go diff --git a/README.md b/README.md index 7bec3df..5898302 100644 --- a/README.md +++ b/README.md @@ -516,6 +516,12 @@ auth: enabled: false api_key: "" +rate_limit: + enabled: false + requests_per_minute: 120 + burst: 60 + key_by: "owner" # owner, api_key, or ip + database: path: "stacyvm.db" @@ -540,6 +546,7 @@ pool: STACYVM_SERVER_PORT=8080 STACYVM_PROVIDERS_DEFAULT=firecracker STACYVM_AUTH_API_KEY=sk-xyz123 +STACYVM_RATE_LIMIT_ENABLED=true STACYVM_LOGGING_LEVEL=debug ``` diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index e7616f4..4320839 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -8,6 +8,7 @@ import ( "time" "github.com/StacyOs/stacyvm/internal/api" + "github.com/StacyOs/stacyvm/internal/api/middleware" "github.com/StacyOs/stacyvm/internal/config" "github.com/StacyOs/stacyvm/internal/environments" "github.com/StacyOs/stacyvm/internal/orchestrator" @@ -215,6 +216,12 @@ func runServe() error { Addr: cfg.Server.Addr(), APIKey: cfg.Auth.APIKey, Version: version, + RateLimit: middleware.RateLimitConfig{ + Enabled: cfg.RateLimit.Enabled, + RequestsPerMinute: cfg.RateLimit.RequestsPerMinute, + Burst: cfg.RateLimit.Burst, + KeyBy: cfg.RateLimit.KeyBy, + }, }, registry, mgr, events, templates, pool, st, envBuilds, logger) // Graceful shutdown diff --git a/docs/api.md b/docs/api.md index da277e6..7fdd208 100644 --- a/docs/api.md +++ b/docs/api.md @@ -44,6 +44,22 @@ CORS is permissive by default (`*`). Lock it down via reverse proxy if you expos --- +## Rate limiting + +API rate limiting is optional and disabled by default. When `rate_limit.enabled` is true, StacyVM applies an in-memory token bucket to API routes. + +```yaml +rate_limit: + enabled: true + requests_per_minute: 120 + burst: 60 + key_by: owner # owner, api_key, or ip +``` + +The default `owner` mode uses `X-User-ID` when present, then falls back to `X-API-Key`, then client IP. Limited requests return `429 Too Many Requests` with `Retry-After`, `X-RateLimit-Limit`, and `X-RateLimit-Remaining` headers. + +--- + ## Conventions - **IDs.** Sandbox IDs look like `sb-a1b2c3d4`. Templates are addressed by `name`. @@ -71,6 +87,7 @@ Errors return a JSON body with HTTP status reflecting the failure class: | `401` | `unauthorized` | Bad / missing API key | | `404` | `not_found` | Sandbox / template / provider does not exist | | `409` | `conflict` | Template name already exists | +| `429` | `resource_limit` | Quota, capacity, or API rate limit exceeded | | `500` | `provider_error` | Provider failed (Docker, Firecracker, etc.) | | `503` | `unavailable` | Pool full with `overflow: reject` | diff --git a/internal/api/middleware/ratelimit.go b/internal/api/middleware/ratelimit.go new file mode 100644 index 0000000..275785f --- /dev/null +++ b/internal/api/middleware/ratelimit.go @@ -0,0 +1,157 @@ +package middleware + +import ( + "encoding/json" + "fmt" + "math" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +type RateLimitConfig struct { + Enabled bool + RequestsPerMinute int + Burst int + KeyBy string + Now func() time.Time +} + +type rateBucket struct { + tokens float64 + lastRefill time.Time + lastSeen time.Time +} + +type RateLimiter struct { + mu sync.Mutex + buckets map[string]*rateBucket + rate float64 + burst float64 + keyBy string + now func() time.Time + disabled bool +} + +func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { + if cfg.RequestsPerMinute < 0 { + cfg.RequestsPerMinute = 0 + } + if cfg.Burst <= 0 { + cfg.Burst = cfg.RequestsPerMinute + } + if cfg.Now == nil { + cfg.Now = time.Now + } + keyBy := strings.TrimSpace(strings.ToLower(cfg.KeyBy)) + if keyBy == "" { + keyBy = "owner" + } + return &RateLimiter{ + buckets: make(map[string]*rateBucket), + rate: float64(cfg.RequestsPerMinute) / 60.0, + burst: float64(cfg.Burst), + keyBy: keyBy, + now: cfg.Now, + disabled: !cfg.Enabled || cfg.RequestsPerMinute == 0, + } +} + +func RateLimit(cfg RateLimitConfig) func(http.Handler) http.Handler { + return NewRateLimiter(cfg).Middleware +} + +func (rl *RateLimiter) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if rl.disabled { + next.ServeHTTP(w, r) + return + } + + allowed, remaining, retryAfter := rl.allow(rl.key(r)) + w.Header().Set("X-RateLimit-Limit", strconv.Itoa(int(rl.burst))) + w.Header().Set("X-RateLimit-Remaining", strconv.Itoa(remaining)) + if !allowed { + seconds := int(math.Ceil(retryAfter.Seconds())) + if seconds < 1 { + seconds = 1 + } + w.Header().Set("Retry-After", strconv.Itoa(seconds)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _ = json.NewEncoder(w).Encode(map[string]string{ + "code": "RESOURCE_LIMIT", + "message": fmt.Sprintf("rate limit exceeded; retry after %ds", seconds), + }) + return + } + + next.ServeHTTP(w, r) + }) +} + +func (rl *RateLimiter) allow(key string) (bool, int, time.Duration) { + now := rl.now() + + rl.mu.Lock() + defer rl.mu.Unlock() + + bucket := rl.buckets[key] + if bucket == nil { + bucket = &rateBucket{tokens: rl.burst, lastRefill: now} + rl.buckets[key] = bucket + } + + elapsed := now.Sub(bucket.lastRefill).Seconds() + if elapsed > 0 { + bucket.tokens = math.Min(rl.burst, bucket.tokens+elapsed*rl.rate) + bucket.lastRefill = now + } + bucket.lastSeen = now + + if bucket.tokens >= 1 { + bucket.tokens-- + return true, int(math.Floor(bucket.tokens)), 0 + } + + needed := 1 - bucket.tokens + retryAfter := time.Duration(math.Ceil(needed/rl.rate)) * time.Second + return false, 0, retryAfter +} + +func (rl *RateLimiter) key(r *http.Request) string { + switch rl.keyBy { + case "api_key": + if apiKey := strings.TrimSpace(r.Header.Get("X-API-Key")); apiKey != "" { + return "api_key:" + apiKey + } + case "ip": + return "ip:" + clientIP(r) + default: + if ownerID := strings.TrimSpace(r.Header.Get("X-User-ID")); ownerID != "" { + return "owner:" + ownerID + } + if apiKey := strings.TrimSpace(r.Header.Get("X-API-Key")); apiKey != "" { + return "api_key:" + apiKey + } + } + return "ip:" + clientIP(r) +} + +func clientIP(r *http.Request) string { + if forwardedFor := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); forwardedFor != "" { + parts := strings.Split(forwardedFor, ",") + return strings.TrimSpace(parts[0]) + } + if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); realIP != "" { + return realIP + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + return host + } + return r.RemoteAddr +} diff --git a/internal/api/middleware/ratelimit_test.go b/internal/api/middleware/ratelimit_test.go new file mode 100644 index 0000000..c709498 --- /dev/null +++ b/internal/api/middleware/ratelimit_test.go @@ -0,0 +1,141 @@ +package middleware + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestRateLimitByOwner(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + KeyBy: "owner", + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "owner-a") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("first owner-a status = %d", w.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "owner-a") + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusTooManyRequests { + t.Fatalf("second owner-a status = %d", w.Code) + } + if w.Header().Get("Retry-After") == "" { + t.Fatal("expected Retry-After header") + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "owner-b") + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("owner-b status = %d", w.Code) + } +} + +func TestRateLimitRefills(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.10:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("first status = %d", w.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.10:5000" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusTooManyRequests { + t.Fatalf("second status = %d", w.Code) + } + + now = now.Add(time.Second) + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.10:5000" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("refilled status = %d", w.Code) + } +} + +func TestRateLimitDisabled(t *testing.T) { + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: false, + RequestsPerMinute: 1, + Burst: 1, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + for i := 0; i < 3; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("request %d status = %d", i, w.Code) + } + } +} + +func TestRateLimitErrorBody(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + for i := 0; i < 2; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.20:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if i == 1 { + var body map[string]string + if err := json.NewDecoder(w.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body["code"] != "RESOURCE_LIMIT" { + t.Fatalf("unexpected body: %+v", body) + } + } + } +} diff --git a/internal/api/server.go b/internal/api/server.go index c321f6b..d60bafa 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -19,9 +19,10 @@ import ( ) type ServerConfig struct { - Addr string - APIKey string - Version string + Addr string + APIKey string + Version string + RateLimit middleware.RateLimitConfig } type Server struct { @@ -74,6 +75,10 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr }) }) + if cfg.RateLimit.Enabled { + r.Use(middleware.RateLimit(cfg.RateLimit)) + } + // Routes sandboxRoutes := routes.NewSandboxRoutes(manager) providerRoutes := routes.NewProviderRoutes(registry, manager) diff --git a/internal/config/config.go b/internal/config/config.go index 5f72e9d..17e2c7d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,6 +15,7 @@ type Config struct { Providers ProvidersConfig `mapstructure:"providers"` Defaults DefaultsConfig `mapstructure:"defaults"` Auth AuthConfig `mapstructure:"auth"` + RateLimit RateLimitConfig `mapstructure:"rate_limit"` Database DatabaseConfig `mapstructure:"database"` Logging LoggingConfig `mapstructure:"logging"` Pool PoolConfig `mapstructure:"pool"` @@ -137,6 +138,13 @@ type AuthConfig struct { APIKey string `mapstructure:"api_key"` } +type RateLimitConfig struct { + Enabled bool `mapstructure:"enabled"` + RequestsPerMinute int `mapstructure:"requests_per_minute"` + Burst int `mapstructure:"burst"` + KeyBy string `mapstructure:"key_by"` +} + type DatabaseConfig struct { Path string `mapstructure:"path"` } @@ -215,6 +223,11 @@ func setDefaults(v *viper.Viper) { v.SetDefault("auth.enabled", false) v.SetDefault("auth.api_key", "") + v.SetDefault("rate_limit.enabled", false) + v.SetDefault("rate_limit.requests_per_minute", 120) + v.SetDefault("rate_limit.burst", 60) + v.SetDefault("rate_limit.key_by", "owner") + v.SetDefault("database.path", "stacyvm.db") v.SetDefault("logging.level", "info") From 42fe84f5f1e3d89e361a9934a7c63107814dd427 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 06:31:01 +0530 Subject: [PATCH 013/147] feat: expose scheduling observability --- docs/api.md | 32 ++++++++++++ internal/api/middleware/ratelimit.go | 59 ++++++++++++++++++----- internal/api/middleware/ratelimit_test.go | 30 ++++++++++++ internal/api/routes/prometheus.go | 12 +++++ internal/api/routes/swagger_types.go | 7 ++- internal/api/routes/system.go | 24 ++++++++- internal/api/routes/system_test.go | 10 +++- internal/api/server.go | 7 +-- internal/orchestrator/manager.go | 11 +++++ internal/orchestrator/manager_test.go | 22 +++++++++ internal/orchestrator/types.go | 7 +++ 11 files changed, 202 insertions(+), 19 deletions(-) diff --git a/docs/api.md b/docs/api.md index 7fdd208..15b9a62 100644 --- a/docs/api.md +++ b/docs/api.md @@ -680,6 +680,21 @@ GET /api/v1/diagnostics "spawn_queue_timeout": "30s", "max_spawn_queue": 100 }, + "scheduler": { + "spawn_overflow": "queue", + "spawn_queue_depth": 3, + "max_spawn_queue": 100, + "spawn_queue_timeout": "30s" + }, + "rate_limit": { + "enabled": true, + "requests_per_minute": 120, + "burst": 60, + "key_by": "owner", + "active_buckets": 14, + "allowed_total": 9132, + "limited_total": 27 + }, "providers": [ { "name": "docker", @@ -743,6 +758,21 @@ GET /api/v1/metrics "history_size": 1000, "events_total": 2401 }, + "scheduler": { + "spawn_overflow": "queue", + "spawn_queue_depth": 3, + "max_spawn_queue": 100, + "spawn_queue_timeout": "30s" + }, + "rate_limit": { + "enabled": true, + "requests_per_minute": 120, + "burst": 60, + "key_by": "owner", + "active_buckets": 14, + "allowed_total": 9132, + "limited_total": 27 + }, "operations": [ { "operation": "exec", @@ -773,6 +803,8 @@ stacyvm_uptime_seconds 7980 # HELP stacyvm_provider_healthy Provider health status where 1 is healthy and 0 is unhealthy. # TYPE stacyvm_provider_healthy gauge stacyvm_provider_healthy{provider="docker",default="true"} 1 +stacyvm_spawn_queue_depth 3 +stacyvm_rate_limit_blocked_total 27 stacyvm_operation_success_total{operation="exec",provider="docker"} 482 stacyvm_operation_failure_total{operation="exec",provider="docker"} 7 ``` diff --git a/internal/api/middleware/ratelimit.go b/internal/api/middleware/ratelimit.go index 275785f..ecd881f 100644 --- a/internal/api/middleware/ratelimit.go +++ b/internal/api/middleware/ratelimit.go @@ -27,13 +27,26 @@ type rateBucket struct { } type RateLimiter struct { - mu sync.Mutex - buckets map[string]*rateBucket - rate float64 - burst float64 - keyBy string - now func() time.Time - disabled bool + mu sync.Mutex + buckets map[string]*rateBucket + rate float64 + requestsPerMinute int + burst float64 + keyBy string + now func() time.Time + disabled bool + allowedTotal uint64 + limitedTotal uint64 +} + +type RateLimitStats struct { + Enabled bool `json:"enabled"` + RequestsPerMinute int `json:"requests_per_minute"` + Burst int `json:"burst"` + KeyBy string `json:"key_by"` + ActiveBuckets int `json:"active_buckets"` + AllowedTotal uint64 `json:"allowed_total"` + LimitedTotal uint64 `json:"limited_total"` } func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { @@ -51,12 +64,13 @@ func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { keyBy = "owner" } return &RateLimiter{ - buckets: make(map[string]*rateBucket), - rate: float64(cfg.RequestsPerMinute) / 60.0, - burst: float64(cfg.Burst), - keyBy: keyBy, - now: cfg.Now, - disabled: !cfg.Enabled || cfg.RequestsPerMinute == 0, + buckets: make(map[string]*rateBucket), + rate: float64(cfg.RequestsPerMinute) / 60.0, + requestsPerMinute: cfg.RequestsPerMinute, + burst: float64(cfg.Burst), + keyBy: keyBy, + now: cfg.Now, + disabled: !cfg.Enabled || cfg.RequestsPerMinute == 0, } } @@ -114,14 +128,33 @@ func (rl *RateLimiter) allow(key string) (bool, int, time.Duration) { if bucket.tokens >= 1 { bucket.tokens-- + rl.allowedTotal++ return true, int(math.Floor(bucket.tokens)), 0 } needed := 1 - bucket.tokens retryAfter := time.Duration(math.Ceil(needed/rl.rate)) * time.Second + rl.limitedTotal++ return false, 0, retryAfter } +func (rl *RateLimiter) Stats() RateLimitStats { + if rl == nil { + return RateLimitStats{} + } + rl.mu.Lock() + defer rl.mu.Unlock() + return RateLimitStats{ + Enabled: !rl.disabled, + RequestsPerMinute: rl.requestsPerMinute, + Burst: int(rl.burst), + KeyBy: rl.keyBy, + ActiveBuckets: len(rl.buckets), + AllowedTotal: rl.allowedTotal, + LimitedTotal: rl.limitedTotal, + } +} + func (rl *RateLimiter) key(r *http.Request) string { switch rl.keyBy { case "api_key": diff --git a/internal/api/middleware/ratelimit_test.go b/internal/api/middleware/ratelimit_test.go index c709498..4ec82a5 100644 --- a/internal/api/middleware/ratelimit_test.go +++ b/internal/api/middleware/ratelimit_test.go @@ -139,3 +139,33 @@ func TestRateLimitErrorBody(t *testing.T) { } } } + +func TestRateLimitStats(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + KeyBy: "ip", + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + for i := 0; i < 2; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.30:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + } + + stats := limiter.Stats() + if !stats.Enabled || stats.RequestsPerMinute != 60 || stats.Burst != 1 || stats.KeyBy != "ip" { + t.Fatalf("unexpected config stats: %+v", stats) + } + if stats.ActiveBuckets != 1 || stats.AllowedTotal != 1 || stats.LimitedTotal != 1 { + t.Fatalf("unexpected counters: %+v", stats) + } +} diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go index 2606f72..4dde173 100644 --- a/internal/api/routes/prometheus.go +++ b/internal/api/routes/prometheus.go @@ -56,6 +56,18 @@ func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { writePrometheusHelp(w, "stacyvm_event_history_size", "Current event history item count.") fmt.Fprintf(w, "stacyvm_event_history_size %d\n", metrics.eventStats.HistorySize) + writePrometheusHelp(w, "stacyvm_spawn_queue_depth", "Current number of spawn requests waiting for capacity.") + fmt.Fprintf(w, "stacyvm_spawn_queue_depth %d\n", metrics.schedulerStatus.SpawnQueueDepth) + writePrometheusHelp(w, "stacyvm_spawn_queue_capacity", "Configured maximum number of queued spawn requests.") + fmt.Fprintf(w, "stacyvm_spawn_queue_capacity %d\n", metrics.schedulerStatus.MaxSpawnQueue) + + writePrometheusHelp(w, "stacyvm_rate_limit_allowed_total", "Total API requests allowed by the in-process rate limiter.") + fmt.Fprintf(w, "stacyvm_rate_limit_allowed_total %d\n", metrics.rateLimitStats.AllowedTotal) + writePrometheusHelp(w, "stacyvm_rate_limit_blocked_total", "Total API requests blocked by the in-process rate limiter.") + fmt.Fprintf(w, "stacyvm_rate_limit_blocked_total %d\n", metrics.rateLimitStats.LimitedTotal) + writePrometheusHelp(w, "stacyvm_rate_limit_active_buckets", "Current number of active rate-limit buckets.") + fmt.Fprintf(w, "stacyvm_rate_limit_active_buckets %d\n", metrics.rateLimitStats.ActiveBuckets) + writeOperationMetrics(w, metrics.operationMetrics) } diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index d2e0ce3..d543b1a 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -1,6 +1,9 @@ package routes -import "github.com/StacyOs/stacyvm/internal/orchestrator" +import ( + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/orchestrator" +) // StatusResponse is a generic status response. type StatusResponse struct { @@ -52,6 +55,8 @@ type DiagnosticsResponse struct { Sandboxes map[string]interface{} `json:"sandboxes"` Events orchestrator.EventBusStats `json:"events"` Operations []orchestrator.OperationMetrics `json:"operations"` + Scheduler orchestrator.SchedulerStatus `json:"scheduler"` + RateLimit middleware.RateLimitStats `json:"rate_limit"` Redactions []string `json:"redactions"` } diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index c4ce10c..c80d81e 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -9,6 +9,7 @@ import ( "runtime" "time" + "github.com/StacyOs/stacyvm/internal/api/middleware" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" "github.com/StacyOs/stacyvm/internal/providers" @@ -24,9 +25,14 @@ type SystemRoutes struct { store store.Store startTime time.Time version string + limiter *middleware.RateLimiter } -func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, st store.Store, version string) *SystemRoutes { +func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, st store.Store, version string, limiter ...*middleware.RateLimiter) *SystemRoutes { + var rateLimiter *middleware.RateLimiter + if len(limiter) > 0 { + rateLimiter = limiter[0] + } return &SystemRoutes{ registry: registry, manager: manager, @@ -34,6 +40,7 @@ func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager store: st, startTime: time.Now(), version: version, + limiter: rateLimiter, } } @@ -175,6 +182,8 @@ func (s *SystemRoutes) Diagnostics(w http.ResponseWriter, r *http.Request) { }, "store": storeStatus, "limits": s.manager.Limits(), + "scheduler": s.manager.SchedulerStatus(), + "rate_limit": s.rateLimitStats(), "providers": metrics.providerHealth, "sandboxes": metrics.sandboxSummary(), "events": metrics.eventStats, @@ -245,6 +254,8 @@ type systemMetricsSnapshot struct { healthyProviders int eventStats orchestrator.EventBusStats operationMetrics []orchestrator.OperationMetrics + schedulerStatus orchestrator.SchedulerStatus + rateLimitStats middleware.RateLimitStats } func (s *SystemRoutes) collectMetrics(ctx context.Context) (systemMetricsSnapshot, error) { @@ -287,6 +298,8 @@ func (s *SystemRoutes) collectMetrics(ctx context.Context) (systemMetricsSnapsho healthyProviders: healthyProviders, eventStats: eventStats, operationMetrics: s.manager.OperationMetrics(), + schedulerStatus: s.manager.SchedulerStatus(), + rateLimitStats: s.rateLimitStats(), }, nil } @@ -306,6 +319,8 @@ func (m systemMetricsSnapshot) toResponse() map[string]interface{} { }, "events": m.eventStats, "operations": m.operationMetrics, + "scheduler": m.schedulerStatus, + "rate_limit": m.rateLimitStats, } } @@ -322,6 +337,13 @@ func (s *SystemRoutes) providerHealth(ctx context.Context) []ProviderHealth { return collectProviderHealth(ctx, s.registry) } +func (s *SystemRoutes) rateLimitStats() middleware.RateLimitStats { + if s.limiter == nil { + return middleware.RateLimitStats{} + } + return s.limiter.Stats() +} + // Events serves Server-Sent Events for real-time updates. // // @Summary Subscribe to events diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index 3e7f6af..f05313f 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -123,7 +123,7 @@ func TestSystemRoutes_Diagnostics(t *testing.T) { } var body map[string]interface{} decodeSystemResponse(t, w, &body) - for _, field := range []string{"generated_at", "build", "process", "store", "limits", "providers", "sandboxes", "events", "operations", "redactions"} { + for _, field := range []string{"generated_at", "build", "process", "store", "limits", "scheduler", "rate_limit", "providers", "sandboxes", "events", "operations", "redactions"} { if _, ok := body[field]; !ok { t.Fatalf("diagnostics missing %s: %#v", field, body) } @@ -165,6 +165,12 @@ func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { if _, ok := body["events"].(map[string]interface{}); !ok { t.Fatal("expected events metrics") } + if _, ok := body["scheduler"].(map[string]interface{}); !ok { + t.Fatal("expected scheduler metrics") + } + if _, ok := body["rate_limit"].(map[string]interface{}); !ok { + t.Fatal("expected rate limit metrics") + } operations := body["operations"].([]interface{}) if len(operations) == 0 { t.Fatal("expected operation metrics") @@ -197,6 +203,8 @@ func TestSystemRoutes_PrometheusMetrics(t *testing.T) { "stacyvm_uptime_seconds", "stacyvm_provider_healthy", "stacyvm_provider_health_latency_milliseconds", + "stacyvm_spawn_queue_depth", + "stacyvm_rate_limit_allowed_total", "stacyvm_operation_success_total", `operation="spawn"`, `operation="exec"`, diff --git a/internal/api/server.go b/internal/api/server.go index d60bafa..a7d9485 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -75,8 +75,10 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr }) }) + var rateLimiter *middleware.RateLimiter if cfg.RateLimit.Enabled { - r.Use(middleware.RateLimit(cfg.RateLimit)) + rateLimiter = middleware.NewRateLimiter(cfg.RateLimit) + r.Use(rateLimiter.Middleware) } // Routes @@ -84,10 +86,9 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr providerRoutes := routes.NewProviderRoutes(registry, manager) templateRoutes := routes.NewTemplateRoutes(templates, manager) snapshotRoutes := routes.NewSnapshotRoutes(registry) - systemRoutes := routes.NewSystemRoutes(registry, manager, events, st, cfg.Version) + systemRoutes := routes.NewSystemRoutes(registry, manager, events, st, cfg.Version, rateLimiter) environmentRoutes := routes.NewEnvironmentRoutes(st, envBuild) quotaRoutes := routes.NewQuotaRoutes(manager) - r.Route("/api/v1", func(r chi.Router) { r.Mount("/sandboxes", sandboxRoutes.Routes()) r.Mount("/providers", providerRoutes.Routes()) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index a23b9dc..edff330 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -979,6 +979,17 @@ func (m *Manager) Limits() OperationalLimitsInfo { } } +func (m *Manager) SchedulerStatus() SchedulerStatus { + m.queueMu.Lock() + defer m.queueMu.Unlock() + return SchedulerStatus{ + SpawnOverflow: m.limits.SpawnOverflow, + SpawnQueueDepth: m.queueWaiters, + MaxSpawnQueue: m.limits.MaxSpawnQueue, + SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), + } +} + func (m *Manager) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuota, error) { rec, err := m.store.GetOwnerQuota(ctx, ownerID) if err != nil { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index efa32f4..2706947 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -453,6 +453,28 @@ func TestManager_SpawnQueueTimesOut(t *testing.T) { assertEventType(t, m.events.History(20), EventSpawnQueueTimeout) } +func TestManager_SchedulerStatus(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + SpawnOverflow: "queue", + SpawnQueueTimeout: 10 * time.Second, + MaxSpawnQueue: 7, + }, + }) + + status := m.SchedulerStatus() + if status.SpawnOverflow != "queue" || status.MaxSpawnQueue != 7 || status.SpawnQueueTimeout != "10s" { + t.Fatalf("unexpected scheduler status: %+v", status) + } + if status.SpawnQueueDepth != 0 { + t.Fatalf("queue depth = %d, want 0", status.SpawnQueueDepth) + } +} + func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { m := setupManager(t) ctx := context.Background() diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index dfca690..b69b3e9 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -120,6 +120,13 @@ type OperationalLimitsInfo struct { MaxSpawnQueue int `json:"max_spawn_queue"` } +type SchedulerStatus struct { + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueDepth int `json:"spawn_queue_depth"` + MaxSpawnQueue int `json:"max_spawn_queue"` + SpawnQueueTimeout string `json:"spawn_queue_timeout"` +} + type OwnerQuota struct { OwnerID string `json:"owner_id"` MaxSandboxes int `json:"max_sandboxes"` From 1a7fea0ba6ff33006568e20bc513bc08cdddb068 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 06:34:00 +0530 Subject: [PATCH 014/147] refactor: model spawn admission decisions --- docs/api.md | 6 ++- internal/orchestrator/manager.go | 68 +++++++++++++++++++------ internal/orchestrator/manager_test.go | 73 ++++++++++++++++++++++++++- internal/orchestrator/types.go | 12 +++++ 4 files changed, 141 insertions(+), 18 deletions(-) diff --git a/docs/api.md b/docs/api.md index 15b9a62..2bc2454 100644 --- a/docs/api.md +++ b/docs/api.md @@ -684,7 +684,8 @@ GET /api/v1/diagnostics "spawn_overflow": "queue", "spawn_queue_depth": 3, "max_spawn_queue": 100, - "spawn_queue_timeout": "30s" + "spawn_queue_timeout": "30s", + "admission_control": "single_node" }, "rate_limit": { "enabled": true, @@ -762,7 +763,8 @@ GET /api/v1/metrics "spawn_overflow": "queue", "spawn_queue_depth": 3, "max_spawn_queue": 100, - "spawn_queue_timeout": "30s" + "spawn_queue_timeout": "30s", + "admission_control": "single_node" }, "rate_limit": { "enabled": true, diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index edff330..4237c90 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -987,6 +987,7 @@ func (m *Manager) SchedulerStatus() SchedulerStatus { SpawnQueueDepth: m.queueWaiters, MaxSpawnQueue: m.limits.MaxSpawnQueue, SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), + AdmissionControl: "single_node", } } @@ -1111,12 +1112,15 @@ func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, } func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl time.Duration, provider string) error { - queueable, err := m.checkSpawnLimits(ctx, ownerID, ttl) - if err == nil { + decision, err := m.EvaluateSpawnAdmission(ctx, ownerID, ttl) + if err != nil { + return err + } + if decision.Allowed { return nil } - if !queueable || !strings.EqualFold(m.limits.SpawnOverflow, "queue") { - return err + if !decision.Queueable || !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + return spawnAdmissionError(decision) } m.queueMu.Lock() @@ -1163,8 +1167,11 @@ func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl } return waitCtx.Err() case <-capacityCh: - queueable, err = m.checkSpawnLimits(ctx, ownerID, ttl) - if err == nil { + decision, err = m.EvaluateSpawnAdmission(ctx, ownerID, ttl) + if err != nil { + return err + } + if decision.Allowed { m.publishOperationalEvent(EventSpawnDequeued, "", map[string]interface{}{ "operation": OperationSpawn, "provider": provider, @@ -1172,8 +1179,8 @@ func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl }) return nil } - if !queueable { - return err + if !decision.Queueable { + return spawnAdmissionError(decision) } m.queueMu.Lock() capacityCh = m.capacityCh @@ -1189,19 +1196,29 @@ func (m *Manager) notifySpawnCapacity() { m.queueMu.Unlock() } -func (m *Manager) checkSpawnLimits(ctx context.Context, ownerID string, ttl time.Duration) (bool, error) { +func (m *Manager) EvaluateSpawnAdmission(ctx context.Context, ownerID string, ttl time.Duration) (SpawnAdmissionDecision, error) { maxTTL, maxPerOwner := m.ownerLimitOverrides(ctx, ownerID) + decision := SpawnAdmissionDecision{ + Allowed: true, + MaxSandboxes: m.limits.MaxSandboxes, + MaxOwnerSandboxes: maxPerOwner, + } + if maxTTL > 0 { + decision.MaxTTL = maxTTL.String() + } if maxTTL > 0 && ttl > maxTTL { - return false, providers.ResourceLimitError(fmt.Sprintf("ttl %s exceeds max ttl %s", ttl, maxTTL)) + decision.Allowed = false + decision.Reason = "max_ttl" + return decision, nil } if m.limits.MaxSandboxes <= 0 && (ownerID == "" || maxPerOwner <= 0) { - return false, nil + return decision, nil } records, err := m.store.ListSandboxes(ctx) if err != nil { - return false, fmt.Errorf("checking sandbox limits: %w", err) + return SpawnAdmissionDecision{}, fmt.Errorf("checking sandbox limits: %w", err) } total := 0 ownerTotal := 0 @@ -1214,13 +1231,34 @@ func (m *Manager) checkSpawnLimits(ctx context.Context, ownerID string, ttl time ownerTotal++ } } + decision.ActiveSandboxes = total + decision.ActiveOwnerSandboxes = ownerTotal if m.limits.MaxSandboxes > 0 && total >= m.limits.MaxSandboxes { - return true, providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", m.limits.MaxSandboxes)) + decision.Allowed = false + decision.Queueable = true + decision.Reason = "max_sandboxes" + return decision, nil } if ownerID != "" && maxPerOwner > 0 && ownerTotal >= maxPerOwner { - return true, providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", maxPerOwner)) + decision.Allowed = false + decision.Queueable = true + decision.Reason = "max_sandboxes_per_owner" + return decision, nil + } + return decision, nil +} + +func spawnAdmissionError(decision SpawnAdmissionDecision) error { + switch decision.Reason { + case "max_ttl": + return providers.ResourceLimitError(fmt.Sprintf("ttl exceeds max ttl %s", decision.MaxTTL)) + case "max_sandboxes": + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", decision.MaxSandboxes)) + case "max_sandboxes_per_owner": + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", decision.MaxOwnerSandboxes)) + default: + return providers.ResourceLimitError("spawn admission denied") } - return false, nil } func (m *Manager) resolveExecTimeout(raw string, ownerID string) (time.Duration, error) { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 2706947..4d2dd80 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -371,6 +371,77 @@ func TestManager_SpawnMaxTTLLimit(t *testing.T) { } } +func TestManager_EvaluateSpawnAdmissionAllowsWhenUnderLimits(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 2, + MaxSandboxesPerOwner: 2, + MaxTTL: time.Hour, + }, + }) + + decision, err := m.EvaluateSpawnAdmission(context.Background(), "owner-a", 30*time.Minute) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if !decision.Allowed || decision.Queueable || decision.Reason != "" { + t.Fatalf("unexpected admission decision: %+v", decision) + } + if decision.MaxSandboxes != 2 || decision.MaxOwnerSandboxes != 2 || decision.MaxTTL != "1h0m0s" { + t.Fatalf("unexpected admission limits: %+v", decision) + } +} + +func TestManager_EvaluateSpawnAdmissionDeniesQueueableCapacity(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + decision, err := m.EvaluateSpawnAdmission(context.Background(), "owner-b", 5*time.Minute) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if decision.Allowed || !decision.Queueable || decision.Reason != "max_sandboxes" { + t.Fatalf("unexpected admission decision: %+v", decision) + } + if decision.ActiveSandboxes != 1 || decision.MaxSandboxes != 1 { + t.Fatalf("unexpected admission counts: %+v", decision) + } +} + +func TestManager_EvaluateSpawnAdmissionDeniesNonQueueableTTL(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxTTL: time.Hour, + }, + }) + + decision, err := m.EvaluateSpawnAdmission(context.Background(), "owner-a", 2*time.Hour) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if decision.Allowed || decision.Queueable || decision.Reason != "max_ttl" { + t.Fatalf("unexpected admission decision: %+v", decision) + } +} + func TestManager_SpawnQueueWaitsForCapacity(t *testing.T) { m := setupManagerWithConfig(t, ManagerConfig{ DefaultTTL: 5 * time.Minute, @@ -467,7 +538,7 @@ func TestManager_SchedulerStatus(t *testing.T) { }) status := m.SchedulerStatus() - if status.SpawnOverflow != "queue" || status.MaxSpawnQueue != 7 || status.SpawnQueueTimeout != "10s" { + if status.SpawnOverflow != "queue" || status.MaxSpawnQueue != 7 || status.SpawnQueueTimeout != "10s" || status.AdmissionControl != "single_node" { t.Fatalf("unexpected scheduler status: %+v", status) } if status.SpawnQueueDepth != 0 { diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index b69b3e9..0e69338 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -125,6 +125,18 @@ type SchedulerStatus struct { SpawnQueueDepth int `json:"spawn_queue_depth"` MaxSpawnQueue int `json:"max_spawn_queue"` SpawnQueueTimeout string `json:"spawn_queue_timeout"` + AdmissionControl string `json:"admission_control"` +} + +type SpawnAdmissionDecision struct { + Allowed bool `json:"allowed"` + Queueable bool `json:"queueable"` + Reason string `json:"reason,omitempty"` + ActiveSandboxes int `json:"active_sandboxes"` + MaxSandboxes int `json:"max_sandboxes"` + ActiveOwnerSandboxes int `json:"active_owner_sandboxes,omitempty"` + MaxOwnerSandboxes int `json:"max_owner_sandboxes,omitempty"` + MaxTTL string `json:"max_ttl,omitempty"` } type OwnerQuota struct { From da095ff186c5e5879db324bdde7325645f476420 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 06:45:00 +0530 Subject: [PATCH 015/147] fix: prune inactive rate limit buckets --- README.md | 2 ++ cmd/stacyvm/cmd_serve.go | 4 +++ docs/api.md | 12 +++++-- internal/api/middleware/ratelimit.go | 38 ++++++++++++++++++++++ internal/api/middleware/ratelimit_test.go | 39 +++++++++++++++++++++++ internal/api/routes/prometheus.go | 2 ++ internal/config/config.go | 4 +++ 7 files changed, 99 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5898302..52df9f8 100644 --- a/README.md +++ b/README.md @@ -521,6 +521,8 @@ rate_limit: requests_per_minute: 120 burst: 60 key_by: "owner" # owner, api_key, or ip + bucket_ttl: "15m" + cleanup_interval: "1m" database: path: "stacyvm.db" diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index 4320839..5573745 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -212,6 +212,8 @@ func runServe() error { } // Server + rateLimitBucketTTL, _ := time.ParseDuration(cfg.RateLimit.BucketTTL) + rateLimitCleanupInterval, _ := time.ParseDuration(cfg.RateLimit.CleanupInterval) srv := api.NewServer(api.ServerConfig{ Addr: cfg.Server.Addr(), APIKey: cfg.Auth.APIKey, @@ -221,6 +223,8 @@ func runServe() error { RequestsPerMinute: cfg.RateLimit.RequestsPerMinute, Burst: cfg.RateLimit.Burst, KeyBy: cfg.RateLimit.KeyBy, + BucketTTL: rateLimitBucketTTL, + CleanupInterval: rateLimitCleanupInterval, }, }, registry, mgr, events, templates, pool, st, envBuilds, logger) diff --git a/docs/api.md b/docs/api.md index 2bc2454..cf9dfc1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -54,6 +54,8 @@ rate_limit: requests_per_minute: 120 burst: 60 key_by: owner # owner, api_key, or ip + bucket_ttl: 15m + cleanup_interval: 1m ``` The default `owner` mode uses `X-User-ID` when present, then falls back to `X-API-Key`, then client IP. Limited requests return `429 Too Many Requests` with `Retry-After`, `X-RateLimit-Limit`, and `X-RateLimit-Remaining` headers. @@ -694,7 +696,10 @@ GET /api/v1/diagnostics "key_by": "owner", "active_buckets": 14, "allowed_total": 9132, - "limited_total": 27 + "limited_total": 27, + "evicted_total": 4, + "bucket_ttl": "15m0s", + "cleanup_interval": "1m0s" }, "providers": [ { @@ -773,7 +778,10 @@ GET /api/v1/metrics "key_by": "owner", "active_buckets": 14, "allowed_total": 9132, - "limited_total": 27 + "limited_total": 27, + "evicted_total": 4, + "bucket_ttl": "15m0s", + "cleanup_interval": "1m0s" }, "operations": [ { diff --git a/internal/api/middleware/ratelimit.go b/internal/api/middleware/ratelimit.go index ecd881f..af76511 100644 --- a/internal/api/middleware/ratelimit.go +++ b/internal/api/middleware/ratelimit.go @@ -17,6 +17,8 @@ type RateLimitConfig struct { RequestsPerMinute int Burst int KeyBy string + BucketTTL time.Duration + CleanupInterval time.Duration Now func() time.Time } @@ -37,6 +39,10 @@ type RateLimiter struct { disabled bool allowedTotal uint64 limitedTotal uint64 + evictedTotal uint64 + bucketTTL time.Duration + cleanupInterval time.Duration + lastCleanup time.Time } type RateLimitStats struct { @@ -47,6 +53,9 @@ type RateLimitStats struct { ActiveBuckets int `json:"active_buckets"` AllowedTotal uint64 `json:"allowed_total"` LimitedTotal uint64 `json:"limited_total"` + EvictedTotal uint64 `json:"evicted_total"` + BucketTTL string `json:"bucket_ttl"` + CleanupInterval string `json:"cleanup_interval"` } func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { @@ -59,6 +68,12 @@ func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { if cfg.Now == nil { cfg.Now = time.Now } + if cfg.BucketTTL == 0 { + cfg.BucketTTL = 15 * time.Minute + } + if cfg.CleanupInterval == 0 { + cfg.CleanupInterval = time.Minute + } keyBy := strings.TrimSpace(strings.ToLower(cfg.KeyBy)) if keyBy == "" { keyBy = "owner" @@ -71,6 +86,8 @@ func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { keyBy: keyBy, now: cfg.Now, disabled: !cfg.Enabled || cfg.RequestsPerMinute == 0, + bucketTTL: cfg.BucketTTL, + cleanupInterval: cfg.CleanupInterval, } } @@ -113,6 +130,8 @@ func (rl *RateLimiter) allow(key string) (bool, int, time.Duration) { rl.mu.Lock() defer rl.mu.Unlock() + rl.cleanupExpiredLocked(now) + bucket := rl.buckets[key] if bucket == nil { bucket = &rateBucket{tokens: rl.burst, lastRefill: now} @@ -152,6 +171,25 @@ func (rl *RateLimiter) Stats() RateLimitStats { ActiveBuckets: len(rl.buckets), AllowedTotal: rl.allowedTotal, LimitedTotal: rl.limitedTotal, + EvictedTotal: rl.evictedTotal, + BucketTTL: rl.bucketTTL.String(), + CleanupInterval: rl.cleanupInterval.String(), + } +} + +func (rl *RateLimiter) cleanupExpiredLocked(now time.Time) { + if rl.bucketTTL <= 0 || rl.cleanupInterval <= 0 { + return + } + if !rl.lastCleanup.IsZero() && now.Sub(rl.lastCleanup) < rl.cleanupInterval { + return + } + rl.lastCleanup = now + for key, bucket := range rl.buckets { + if now.Sub(bucket.lastSeen) > rl.bucketTTL { + delete(rl.buckets, key) + rl.evictedTotal++ + } } } diff --git a/internal/api/middleware/ratelimit_test.go b/internal/api/middleware/ratelimit_test.go index 4ec82a5..82f9330 100644 --- a/internal/api/middleware/ratelimit_test.go +++ b/internal/api/middleware/ratelimit_test.go @@ -168,4 +168,43 @@ func TestRateLimitStats(t *testing.T) { if stats.ActiveBuckets != 1 || stats.AllowedTotal != 1 || stats.LimitedTotal != 1 { t.Fatalf("unexpected counters: %+v", stats) } + if stats.BucketTTL == "" || stats.CleanupInterval == "" { + t.Fatalf("expected cleanup settings in stats: %+v", stats) + } +} + +func TestRateLimitEvictsInactiveBuckets(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + KeyBy: "ip", + BucketTTL: time.Minute, + CleanupInterval: time.Second, + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.40:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if limiter.Stats().ActiveBuckets != 1 { + t.Fatalf("expected one active bucket: %+v", limiter.Stats()) + } + + now = now.Add(2 * time.Minute) + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.41:5000" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + stats := limiter.Stats() + if stats.ActiveBuckets != 1 || stats.EvictedTotal != 1 { + t.Fatalf("unexpected cleanup stats: %+v", stats) + } } diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go index 4dde173..66b8f6e 100644 --- a/internal/api/routes/prometheus.go +++ b/internal/api/routes/prometheus.go @@ -65,6 +65,8 @@ func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { fmt.Fprintf(w, "stacyvm_rate_limit_allowed_total %d\n", metrics.rateLimitStats.AllowedTotal) writePrometheusHelp(w, "stacyvm_rate_limit_blocked_total", "Total API requests blocked by the in-process rate limiter.") fmt.Fprintf(w, "stacyvm_rate_limit_blocked_total %d\n", metrics.rateLimitStats.LimitedTotal) + writePrometheusHelp(w, "stacyvm_rate_limit_evicted_buckets_total", "Total inactive rate-limit buckets evicted from memory.") + fmt.Fprintf(w, "stacyvm_rate_limit_evicted_buckets_total %d\n", metrics.rateLimitStats.EvictedTotal) writePrometheusHelp(w, "stacyvm_rate_limit_active_buckets", "Current number of active rate-limit buckets.") fmt.Fprintf(w, "stacyvm_rate_limit_active_buckets %d\n", metrics.rateLimitStats.ActiveBuckets) diff --git a/internal/config/config.go b/internal/config/config.go index 17e2c7d..2b8efc8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -143,6 +143,8 @@ type RateLimitConfig struct { RequestsPerMinute int `mapstructure:"requests_per_minute"` Burst int `mapstructure:"burst"` KeyBy string `mapstructure:"key_by"` + BucketTTL string `mapstructure:"bucket_ttl"` + CleanupInterval string `mapstructure:"cleanup_interval"` } type DatabaseConfig struct { @@ -227,6 +229,8 @@ func setDefaults(v *viper.Viper) { v.SetDefault("rate_limit.requests_per_minute", 120) v.SetDefault("rate_limit.burst", 60) v.SetDefault("rate_limit.key_by", "owner") + v.SetDefault("rate_limit.bucket_ttl", "15m") + v.SetDefault("rate_limit.cleanup_interval", "1m") v.SetDefault("database.path", "stacyvm.db") From aed7730c2f8b5bb33dfffe055209360c62a4dbbe Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:06:15 +0530 Subject: [PATCH 016/147] fix: serialize spawn admission --- internal/orchestrator/manager.go | 27 +++++++- internal/orchestrator/manager_test.go | 93 +++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 4237c90..6918582 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -27,6 +27,7 @@ type Manager struct { mu sync.RWMutex sandboxes map[string]*Sandbox + admissionMu sync.Mutex queueMu sync.Mutex queueWaiters int capacityCh chan struct{} @@ -352,11 +353,12 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) } ttl = parsed } - if err := m.waitForSpawnCapacity(ctx, req.OwnerID, ttl, metricsProvider); err != nil { + if err := m.acquireSpawnAdmission(ctx, req.OwnerID, ttl, metricsProvider); err != nil { metricsErr = err m.publishFailureForError("", OperationSpawn, metricsProvider, err) return nil, err } + defer m.admissionMu.Unlock() now := time.Now() @@ -1111,6 +1113,29 @@ func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, }) } +func (m *Manager) acquireSpawnAdmission(ctx context.Context, ownerID string, ttl time.Duration, provider string) error { + for { + if err := m.waitForSpawnCapacity(ctx, ownerID, ttl, provider); err != nil { + return err + } + + m.admissionMu.Lock() + decision, err := m.EvaluateSpawnAdmission(ctx, ownerID, ttl) + if err != nil { + m.admissionMu.Unlock() + return err + } + if decision.Allowed { + return nil + } + m.admissionMu.Unlock() + + if !decision.Queueable || !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + return spawnAdmissionError(decision) + } + } +} + func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl time.Duration, provider string) error { decision, err := m.EvaluateSpawnAdmission(ctx, ownerID, ttl) if err != nil { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 4d2dd80..85daafb 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -5,6 +5,7 @@ import ( "errors" "path/filepath" "strings" + "sync" "testing" "time" @@ -46,6 +47,23 @@ func setupManagerWithConfig(t *testing.T, cfg ManagerConfig) *Manager { return m } +type slowSpawnProvider struct { + providers.Provider + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (p *slowSpawnProvider) Spawn(ctx context.Context, opts providers.SpawnOptions) (string, error) { + p.once.Do(func() { close(p.entered) }) + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-p.release: + } + return p.Provider.Spawn(ctx, opts) +} + func TestManager_SpawnAndGet(t *testing.T) { m := setupManager(t) ctx := context.Background() @@ -442,6 +460,81 @@ func TestManager_EvaluateSpawnAdmissionDeniesNonQueueableTTL(t *testing.T) { } } +func TestManager_SpawnAdmissionSerializesConcurrentCreates(t *testing.T) { + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + base := providers.NewMockProvider() + slow := &slowSpawnProvider{ + Provider: base, + entered: make(chan struct{}), + release: make(chan struct{}), + } + reg := providers.NewRegistry() + reg.Register(slow) + if err := reg.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + + m := NewManager(reg, st, NewEventBus(), zerolog.Nop(), ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + }, + }) + m.Start() + t.Cleanup(func() { m.Stop() }) + + firstCh := make(chan error, 1) + go func() { + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}) + firstCh <- err + }() + + select { + case <-slow.entered: + case <-time.After(time.Second): + t.Fatal("first spawn did not enter provider") + } + + secondCh := make(chan error, 1) + go func() { + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-b"}) + secondCh <- err + }() + + select { + case err := <-secondCh: + t.Fatalf("second spawn completed before first persisted: %v", err) + case <-time.After(25 * time.Millisecond): + } + + close(slow.release) + if err := <-firstCh; err != nil { + t.Fatalf("first spawn: %v", err) + } + + err = <-secondCh + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected second spawn resource limit, got %v", err) + } + + list, err := m.List(context.Background()) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(list) != 1 { + t.Fatalf("expected one persisted sandbox, got %d", len(list)) + } +} + func TestManager_SpawnQueueWaitsForCapacity(t *testing.T) { m := setupManagerWithConfig(t, ManagerConfig{ DefaultTTL: 5 * time.Minute, From 6d1250a411e84de2361156cdc26a0aeb34cda2ef Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:14:41 +0530 Subject: [PATCH 017/147] fix: validate phase three config --- internal/config/config.go | 72 +++++++++++++++++++++++++ internal/config/config_test.go | 97 ++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 internal/config/config_test.go diff --git a/internal/config/config.go b/internal/config/config.go index 2b8efc8..51a14d4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/spf13/viper" ) @@ -311,6 +312,77 @@ func Load() (*Config, error) { if err := v.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("unmarshaling config: %w", err) } + if err := cfg.Validate(); err != nil { + return nil, err + } return &cfg, nil } + +func (c *Config) Validate() error { + durationFields := map[string]string{ + "defaults.ttl": c.Defaults.TTL, + "defaults.max_ttl": c.Defaults.MaxTTL, + "defaults.default_exec_timeout": c.Defaults.DefaultExecTimeout, + "defaults.max_exec_timeout": c.Defaults.MaxExecTimeout, + "defaults.spawn_queue_timeout": c.Defaults.SpawnQueueTimeout, + "rate_limit.bucket_ttl": c.RateLimit.BucketTTL, + "rate_limit.cleanup_interval": c.RateLimit.CleanupInterval, + "providers.custom.timeout": c.Providers.Custom.Timeout, + "providers.proot.default_timeout": c.Providers.PRoot.DefaultTimeout, + } + for name, value := range durationFields { + if err := validateDuration(name, value); err != nil { + return err + } + } + + if c.Defaults.MaxSandboxes < 0 { + return fmt.Errorf("defaults.max_sandboxes cannot be negative") + } + if c.Defaults.MaxSandboxesPerOwner < 0 { + return fmt.Errorf("defaults.max_sandboxes_per_owner cannot be negative") + } + if c.Defaults.MaxSpawnQueue < 0 { + return fmt.Errorf("defaults.max_spawn_queue cannot be negative") + } + if c.RateLimit.RequestsPerMinute < 0 { + return fmt.Errorf("rate_limit.requests_per_minute cannot be negative") + } + if c.RateLimit.Burst < 0 { + return fmt.Errorf("rate_limit.burst cannot be negative") + } + if !isOneOf(c.Defaults.SpawnOverflow, "", "reject", "queue") { + return fmt.Errorf("defaults.spawn_overflow must be reject or queue") + } + if !isOneOf(c.RateLimit.KeyBy, "", "owner", "api_key", "ip") { + return fmt.Errorf("rate_limit.key_by must be owner, api_key, or ip") + } + if !isOneOf(c.Pool.Overflow, "", "reject", "queue") { + return fmt.Errorf("pool.overflow must be reject or queue") + } + return nil +} + +func validateDuration(name, value string) error { + if value == "" { + return nil + } + d, err := time.ParseDuration(value) + if err != nil { + return fmt.Errorf("%s must be a valid duration: %w", name, err) + } + if d < 0 { + return fmt.Errorf("%s cannot be negative", name) + } + return nil +} + +func isOneOf(value string, allowed ...string) bool { + for _, candidate := range allowed { + if value == candidate { + return true + } + } + return false +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..23d644c --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,97 @@ +package config + +import ( + "os" + "strings" + "testing" +) + +func TestLoadRejectsInvalidDuration(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + spawn_queue_timeout: "soon" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected invalid duration error") + } + if !strings.Contains(err.Error(), "defaults.spawn_queue_timeout") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadRejectsInvalidEnums(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + spawn_overflow: "stall" +rate_limit: + key_by: "cookie" +pool: + overflow: "stall" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected invalid enum error") + } + if !strings.Contains(err.Error(), "defaults.spawn_overflow") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadRejectsNegativeLimits(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + max_spawn_queue: -1 +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected negative limit error") + } + if !strings.Contains(err.Error(), "defaults.max_spawn_queue") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadAcceptsPhaseThreeConfig(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + spawn_overflow: "queue" + spawn_queue_timeout: "45s" + max_spawn_queue: 25 +rate_limit: + enabled: true + requests_per_minute: 240 + burst: 80 + key_by: "api_key" + bucket_ttl: "30m" + cleanup_interval: "2m" +pool: + overflow: "queue" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load() + if err != nil { + t.Fatalf("load config: %v", err) + } + if cfg.Defaults.SpawnOverflow != "queue" || cfg.Defaults.SpawnQueueTimeout != "45s" || cfg.Defaults.MaxSpawnQueue != 25 { + t.Fatalf("unexpected defaults config: %+v", cfg.Defaults) + } + if !cfg.RateLimit.Enabled || cfg.RateLimit.KeyBy != "api_key" || cfg.RateLimit.BucketTTL != "30m" { + t.Fatalf("unexpected rate limit config: %+v", cfg.RateLimit) + } +} From 0b82c6da9d2e78919ed4759ed9d477535852d727 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:18:32 +0530 Subject: [PATCH 018/147] fix: validate owner quota inputs --- docs/api.md | 4 +++ internal/api/routes/errors.go | 2 ++ internal/api/routes/quotas_test.go | 37 +++++++++++++++++++ internal/orchestrator/errors.go | 12 ++++++- internal/orchestrator/manager.go | 52 +++++++++++++++++++++++---- internal/orchestrator/manager_test.go | 30 ++++++++++++++++ 6 files changed, 130 insertions(+), 7 deletions(-) diff --git a/docs/api.md b/docs/api.md index cf9dfc1..94a9283 100644 --- a/docs/api.md +++ b/docs/api.md @@ -404,6 +404,8 @@ Optional override body: Owner quotas are persisted overrides for per-owner sandbox and runtime limits. They apply when requests include an owner via `X-User-ID` or `owner_id`. +Owner IDs are trimmed and must be 128 characters or fewer. They cannot contain whitespace, control characters, or path separators. Quota durations must use whole-second Go duration strings; use `0s` or omit a duration to inherit the global default. + ### List owner quotas ``` @@ -441,6 +443,8 @@ PUT /api/v1/quotas/{ownerID} **Response** `200 OK`: full owner quota object. +Invalid owner IDs, negative sandbox counts, malformed durations, sub-second durations, and fractional-second durations return `400 Bad Request`. + ### Get owner usage ``` diff --git a/internal/api/routes/errors.go b/internal/api/routes/errors.go index 4992017..3d5b65a 100644 --- a/internal/api/routes/errors.go +++ b/internal/api/routes/errors.go @@ -11,6 +11,8 @@ import ( func writeRouteError(w http.ResponseWriter, err error) { switch { + case errors.Is(err, orchestrator.ErrInvalidInput): + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, err.Error()) case errors.Is(err, orchestrator.ErrSandboxNotFound), errors.Is(err, orchestrator.ErrSandboxDestroyed), errors.Is(err, orchestrator.ErrProviderNotFound), diff --git a/internal/api/routes/quotas_test.go b/internal/api/routes/quotas_test.go index 97bae6c..3266102 100644 --- a/internal/api/routes/quotas_test.go +++ b/internal/api/routes/quotas_test.go @@ -87,3 +87,40 @@ func TestQuotaRoutes_SaveGetUsageDelete(t *testing.T) { t.Fatalf("delete status = %d: %s", w.Code, w.Body.String()) } } + +func TestQuotaRoutes_InvalidQuotaReturnsBadRequest(t *testing.T) { + r, _ := setupQuotaRouter(t) + + tests := []struct { + name string + path string + body string + }{ + { + name: "bad duration", + path: "/api/v1/quotas/owner-a", + body: `{"max_ttl":"500ms"}`, + }, + { + name: "negative sandboxes", + path: "/api/v1/quotas/owner-a", + body: `{"max_sandboxes":-1}`, + }, + { + name: "bad owner", + path: "/api/v1/quotas/owner%20a", + body: `{"max_sandboxes":1}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, tt.path, bytes.NewBufferString(tt.body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusBadRequest, w.Body.String()) + } + }) + } +} diff --git a/internal/orchestrator/errors.go b/internal/orchestrator/errors.go index 1c5c73b..6562a78 100644 --- a/internal/orchestrator/errors.go +++ b/internal/orchestrator/errors.go @@ -1,8 +1,14 @@ package orchestrator -import "github.com/StacyOs/stacyvm/internal/providers" +import ( + "errors" + "fmt" + + "github.com/StacyOs/stacyvm/internal/providers" +) var ( + ErrInvalidInput = errors.New("invalid input") ErrSandboxNotFound = providers.ErrSandboxNotFound ErrSandboxDestroyed = providers.ErrSandboxDestroyed ErrProviderNotFound = providers.ErrProviderNotFound @@ -10,3 +16,7 @@ var ( ErrExecTimeout = providers.ErrExecTimeout ErrResourceLimit = providers.ErrResourceLimit ) + +func InvalidInputError(message string) error { + return fmt.Errorf("%w: %s", ErrInvalidInput, message) +} diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 6918582..4deddf1 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -994,6 +994,10 @@ func (m *Manager) SchedulerStatus() SchedulerStatus { } func (m *Manager) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuota, error) { + ownerID, err := normalizeOwnerID(ownerID) + if err != nil { + return nil, err + } rec, err := m.store.GetOwnerQuota(ctx, ownerID) if err != nil { return nil, err @@ -1014,19 +1018,21 @@ func (m *Manager) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuota, error) { } func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQuota, error) { - if quota.OwnerID == "" { - return nil, fmt.Errorf("owner_id is required") + ownerID, err := normalizeOwnerID(quota.OwnerID) + if err != nil { + return nil, err } + quota.OwnerID = ownerID maxTTL, err := parseOptionalDurationSeconds(quota.MaxTTL) if err != nil { - return nil, fmt.Errorf("parsing max_ttl: %w", err) + return nil, InvalidInputError(fmt.Sprintf("parsing max_ttl: %v", err)) } maxExecTimeout, err := parseOptionalDurationSeconds(quota.MaxExecTimeout) if err != nil { - return nil, fmt.Errorf("parsing max_exec_timeout: %w", err) + return nil, InvalidInputError(fmt.Sprintf("parsing max_exec_timeout: %v", err)) } if quota.MaxSandboxes < 0 { - return nil, providers.ResourceLimitError("max_sandboxes cannot be negative") + return nil, InvalidInputError("max_sandboxes cannot be negative") } rec := &store.OwnerQuotaRecord{ OwnerID: quota.OwnerID, @@ -1041,10 +1047,18 @@ func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQ } func (m *Manager) DeleteOwnerQuota(ctx context.Context, ownerID string) error { + ownerID, err := normalizeOwnerID(ownerID) + if err != nil { + return err + } return m.store.DeleteOwnerQuota(ctx, ownerID) } func (m *Manager) OwnerUsage(ctx context.Context, ownerID string) (*OwnerUsage, error) { + ownerID, err := normalizeOwnerID(ownerID) + if err != nil { + return nil, err + } records, err := m.store.ListSandboxesByOwner(ctx, ownerID) if err != nil { return nil, err @@ -1351,6 +1365,7 @@ func ownerQuotaFromRecord(rec *store.OwnerQuotaRecord) *OwnerQuota { } func parseOptionalDurationSeconds(raw string) (int64, error) { + raw = strings.TrimSpace(raw) if raw == "" || raw == "0" || raw == "0s" { return 0, nil } @@ -1359,11 +1374,36 @@ func parseOptionalDurationSeconds(raw string) (int64, error) { return 0, err } if d < 0 { - return 0, providers.ResourceLimitError("duration cannot be negative") + return 0, fmt.Errorf("duration cannot be negative") + } + if d > 0 && d < time.Second { + return 0, fmt.Errorf("duration must be at least 1s") + } + if d%time.Second != 0 { + return 0, fmt.Errorf("duration must use whole seconds") } return int64(d.Seconds()), nil } +func normalizeOwnerID(ownerID string) (string, error) { + ownerID = strings.TrimSpace(ownerID) + if ownerID == "" { + return "", InvalidInputError("owner_id is required") + } + if len(ownerID) > 128 { + return "", InvalidInputError("owner_id must be 128 characters or fewer") + } + if strings.ContainsAny(ownerID, `/\`) { + return "", InvalidInputError("owner_id cannot contain path separators") + } + for _, r := range ownerID { + if r <= 31 || r == 127 || r == ' ' || r == '\t' || r == '\n' || r == '\r' { + return "", InvalidInputError("owner_id cannot contain whitespace or control characters") + } + } + return ownerID, nil +} + func optionalSecondsString(seconds int64) string { if seconds <= 0 { return "0s" diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 85daafb..06e81e9 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -372,6 +372,36 @@ func TestManager_PersistentOwnerQuotaTTLLimit(t *testing.T) { } } +func TestManager_OwnerQuotaValidation(t *testing.T) { + m := setupManager(t) + + tests := []OwnerQuota{ + {OwnerID: " ", MaxSandboxes: 1}, + {OwnerID: "owner/a", MaxSandboxes: 1}, + {OwnerID: "owner a", MaxSandboxes: 1}, + {OwnerID: "owner-a", MaxSandboxes: -1}, + {OwnerID: "owner-a", MaxTTL: "500ms"}, + {OwnerID: "owner-a", MaxExecTimeout: "1.5s"}, + } + for _, quota := range tests { + if _, err := m.SaveOwnerQuota(context.Background(), quota); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected invalid input for %+v, got %v", quota, err) + } + } + + saved, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: " owner-trimmed ", + MaxSandboxes: 2, + MaxTTL: "10s", + }) + if err != nil { + t.Fatalf("save trimmed owner quota: %v", err) + } + if saved.OwnerID != "owner-trimmed" || saved.MaxTTL != "10s" { + t.Fatalf("unexpected saved quota: %+v", saved) + } +} + func TestManager_SpawnMaxTTLLimit(t *testing.T) { m := setupManagerWithConfig(t, ManagerConfig{ DefaultTTL: 5 * time.Minute, From c3ea763c727da97539e603d01d5b085a68cd500e Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:21:48 +0530 Subject: [PATCH 019/147] feat: audit owner quota changes --- docs/api.md | 1 + internal/orchestrator/events.go | 2 ++ internal/orchestrator/manager.go | 20 ++++++++++++++++++-- internal/orchestrator/manager_test.go | 13 +++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 94a9283..c929024 100644 --- a/docs/api.md +++ b/docs/api.md @@ -850,6 +850,7 @@ Common event types include: - `file.written`, `file.read` - `operation.failed`, `resource.limit`, `provider.failed`, `reconcile.action` - `spawn.queued`, `spawn.dequeued`, `spawn.queue_timeout` +- `quota.saved`, `quota.deleted` Use any SSE client (`EventSource` in browsers, `httpx-sse` in Python, etc.) to consume. diff --git a/internal/orchestrator/events.go b/internal/orchestrator/events.go index 03bc2e2..1248ae5 100644 --- a/internal/orchestrator/events.go +++ b/internal/orchestrator/events.go @@ -27,6 +27,8 @@ const ( EventSpawnQueued EventType = "spawn.queued" EventSpawnDequeued EventType = "spawn.dequeued" EventSpawnQueueTimeout EventType = "spawn.queue_timeout" + EventQuotaSaved EventType = "quota.saved" + EventQuotaDeleted EventType = "quota.deleted" ) type Event struct { diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 4deddf1..0846612 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -1043,7 +1043,17 @@ func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQ if err := m.store.SaveOwnerQuota(ctx, rec); err != nil { return nil, err } - return m.GetOwnerQuota(ctx, quota.OwnerID) + saved, err := m.GetOwnerQuota(ctx, quota.OwnerID) + if err != nil { + return nil, err + } + m.publishOperationalEvent(EventQuotaSaved, "", map[string]interface{}{ + "owner_id": saved.OwnerID, + "max_sandboxes": saved.MaxSandboxes, + "max_ttl": saved.MaxTTL, + "max_exec_timeout": saved.MaxExecTimeout, + }) + return saved, nil } func (m *Manager) DeleteOwnerQuota(ctx context.Context, ownerID string) error { @@ -1051,7 +1061,13 @@ func (m *Manager) DeleteOwnerQuota(ctx context.Context, ownerID string) error { if err != nil { return err } - return m.store.DeleteOwnerQuota(ctx, ownerID) + if err := m.store.DeleteOwnerQuota(ctx, ownerID); err != nil { + return err + } + m.publishOperationalEvent(EventQuotaDeleted, "", map[string]interface{}{ + "owner_id": ownerID, + }) + return nil } func (m *Manager) OwnerUsage(ctx context.Context, ownerID string) (*OwnerUsage, error) { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 06e81e9..aec234d 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -359,6 +359,19 @@ func TestManager_PersistentOwnerQuotaLimit(t *testing.T) { if !usage.QuotaConfigured || usage.ActiveSandboxes != 1 || usage.MaxSandboxes != 1 { t.Fatalf("unexpected owner usage: %+v", usage) } + assertEventType(t, m.events.History(20), EventQuotaSaved) +} + +func TestManager_OwnerQuotaDeletePublishesEvent(t *testing.T) { + m := setupManager(t) + + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{OwnerID: "owner-delete", MaxSandboxes: 1}); err != nil { + t.Fatalf("save quota: %v", err) + } + if err := m.DeleteOwnerQuota(context.Background(), "owner-delete"); err != nil { + t.Fatalf("delete quota: %v", err) + } + assertEventType(t, m.events.History(20), EventQuotaDeleted) } func TestManager_PersistentOwnerQuotaTTLLimit(t *testing.T) { From ef1ea4f4e134680356a76ba4b8014a06a795e41d Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:25:37 +0530 Subject: [PATCH 020/147] feat: expose quota summary metrics --- docs/api.md | 13 +++++++++++ internal/api/routes/prometheus.go | 7 ++++++ internal/api/routes/swagger_types.go | 1 + internal/api/routes/system.go | 8 +++++++ internal/api/routes/system_test.go | 32 ++++++++++++++++++++++++++- internal/orchestrator/manager.go | 20 +++++++++++++++++ internal/orchestrator/manager_test.go | 26 ++++++++++++++++++++++ internal/orchestrator/types.go | 7 ++++++ 8 files changed, 113 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index c929024..d8a974b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -693,6 +693,12 @@ GET /api/v1/diagnostics "spawn_queue_timeout": "30s", "admission_control": "single_node" }, + "quotas": { + "total": 8, + "with_max_sandboxes": 6, + "with_max_ttl": 4, + "with_max_exec_timeout": 3 + }, "rate_limit": { "enabled": true, "requests_per_minute": 120, @@ -775,6 +781,12 @@ GET /api/v1/metrics "spawn_queue_timeout": "30s", "admission_control": "single_node" }, + "quotas": { + "total": 8, + "with_max_sandboxes": 6, + "with_max_ttl": 4, + "with_max_exec_timeout": 3 + }, "rate_limit": { "enabled": true, "requests_per_minute": 120, @@ -818,6 +830,7 @@ stacyvm_uptime_seconds 7980 # TYPE stacyvm_provider_healthy gauge stacyvm_provider_healthy{provider="docker",default="true"} 1 stacyvm_spawn_queue_depth 3 +stacyvm_owner_quotas_total 8 stacyvm_rate_limit_blocked_total 27 stacyvm_operation_success_total{operation="exec",provider="docker"} 482 stacyvm_operation_failure_total{operation="exec",provider="docker"} 7 diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go index 66b8f6e..41b5053 100644 --- a/internal/api/routes/prometheus.go +++ b/internal/api/routes/prometheus.go @@ -61,6 +61,13 @@ func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { writePrometheusHelp(w, "stacyvm_spawn_queue_capacity", "Configured maximum number of queued spawn requests.") fmt.Fprintf(w, "stacyvm_spawn_queue_capacity %d\n", metrics.schedulerStatus.MaxSpawnQueue) + writePrometheusHelp(w, "stacyvm_owner_quotas_total", "Total configured owner quota policies.") + fmt.Fprintf(w, "stacyvm_owner_quotas_total %d\n", metrics.quotaSummary.Total) + writePrometheusHelp(w, "stacyvm_owner_quota_overrides_total", "Total configured owner quota overrides by override type.") + fmt.Fprintf(w, "stacyvm_owner_quota_overrides_total{type=%q} %d\n", "max_sandboxes", metrics.quotaSummary.WithMaxSandboxes) + fmt.Fprintf(w, "stacyvm_owner_quota_overrides_total{type=%q} %d\n", "max_ttl", metrics.quotaSummary.WithMaxTTL) + fmt.Fprintf(w, "stacyvm_owner_quota_overrides_total{type=%q} %d\n", "max_exec_timeout", metrics.quotaSummary.WithMaxExecTimeout) + writePrometheusHelp(w, "stacyvm_rate_limit_allowed_total", "Total API requests allowed by the in-process rate limiter.") fmt.Fprintf(w, "stacyvm_rate_limit_allowed_total %d\n", metrics.rateLimitStats.AllowedTotal) writePrometheusHelp(w, "stacyvm_rate_limit_blocked_total", "Total API requests blocked by the in-process rate limiter.") diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index d543b1a..8fd4973 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -56,6 +56,7 @@ type DiagnosticsResponse struct { Events orchestrator.EventBusStats `json:"events"` Operations []orchestrator.OperationMetrics `json:"operations"` Scheduler orchestrator.SchedulerStatus `json:"scheduler"` + Quotas orchestrator.QuotaSummary `json:"quotas"` RateLimit middleware.RateLimitStats `json:"rate_limit"` Redactions []string `json:"redactions"` } diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index c80d81e..f233592 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -183,6 +183,7 @@ func (s *SystemRoutes) Diagnostics(w http.ResponseWriter, r *http.Request) { "store": storeStatus, "limits": s.manager.Limits(), "scheduler": s.manager.SchedulerStatus(), + "quotas": metrics.quotaSummary, "rate_limit": s.rateLimitStats(), "providers": metrics.providerHealth, "sandboxes": metrics.sandboxSummary(), @@ -255,6 +256,7 @@ type systemMetricsSnapshot struct { eventStats orchestrator.EventBusStats operationMetrics []orchestrator.OperationMetrics schedulerStatus orchestrator.SchedulerStatus + quotaSummary orchestrator.QuotaSummary rateLimitStats middleware.RateLimitStats } @@ -282,6 +284,10 @@ func (s *SystemRoutes) collectMetrics(ctx context.Context) (systemMetricsSnapsho } } eventStats := s.events.Stats() + quotaSummary, err := s.manager.QuotaSummary(ctx) + if err != nil { + return systemMetricsSnapshot{}, err + } return systemMetricsSnapshot{ uptime: time.Since(s.startTime), @@ -299,6 +305,7 @@ func (s *SystemRoutes) collectMetrics(ctx context.Context) (systemMetricsSnapsho eventStats: eventStats, operationMetrics: s.manager.OperationMetrics(), schedulerStatus: s.manager.SchedulerStatus(), + quotaSummary: quotaSummary, rateLimitStats: s.rateLimitStats(), }, nil } @@ -320,6 +327,7 @@ func (m systemMetricsSnapshot) toResponse() map[string]interface{} { "events": m.eventStats, "operations": m.operationMetrics, "scheduler": m.schedulerStatus, + "quotas": m.quotaSummary, "rate_limit": m.rateLimitStats, } } diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index f05313f..57868bc 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -109,6 +109,14 @@ func TestSystemRoutes_ReadyNoProviders(t *testing.T) { func TestSystemRoutes_Diagnostics(t *testing.T) { routes, manager := setupSystemRoutes(t, true) + if _, err := manager.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "team-a", + MaxSandboxes: 3, + MaxTTL: "30m", + MaxExecTimeout: "10s", + }); err != nil { + t.Fatalf("save quota: %v", err) + } if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { t.Fatalf("spawn: %v", err) } @@ -123,11 +131,15 @@ func TestSystemRoutes_Diagnostics(t *testing.T) { } var body map[string]interface{} decodeSystemResponse(t, w, &body) - for _, field := range []string{"generated_at", "build", "process", "store", "limits", "scheduler", "rate_limit", "providers", "sandboxes", "events", "operations", "redactions"} { + for _, field := range []string{"generated_at", "build", "process", "store", "limits", "scheduler", "quotas", "rate_limit", "providers", "sandboxes", "events", "operations", "redactions"} { if _, ok := body[field]; !ok { t.Fatalf("diagnostics missing %s: %#v", field, body) } } + quotas := body["quotas"].(map[string]interface{}) + if quotas["total"].(float64) != 1 || quotas["with_max_sandboxes"].(float64) != 1 { + t.Fatalf("unexpected quota summary: %#v", quotas) + } storeBody := body["store"].(map[string]interface{}) if storeBody["healthy"] != true { t.Fatalf("store healthy = %v, want true", storeBody["healthy"]) @@ -139,6 +151,12 @@ func TestSystemRoutes_Diagnostics(t *testing.T) { func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { routes, manager := setupSystemRoutes(t, true) + if _, err := manager.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "team-a", + MaxSandboxes: 2, + }); err != nil { + t.Fatalf("save quota: %v", err) + } if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { t.Fatalf("spawn: %v", err) } @@ -168,6 +186,10 @@ func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { if _, ok := body["scheduler"].(map[string]interface{}); !ok { t.Fatal("expected scheduler metrics") } + quotas := body["quotas"].(map[string]interface{}) + if quotas["total"].(float64) != 1 { + t.Fatalf("unexpected quota metrics: %#v", quotas) + } if _, ok := body["rate_limit"].(map[string]interface{}); !ok { t.Fatal("expected rate limit metrics") } @@ -179,6 +201,12 @@ func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { func TestSystemRoutes_PrometheusMetrics(t *testing.T) { routes, manager := setupSystemRoutes(t, true) + if _, err := manager.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "team-a", + MaxSandboxes: 2, + }); err != nil { + t.Fatalf("save quota: %v", err) + } sb, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}) if err != nil { t.Fatalf("spawn: %v", err) @@ -204,6 +232,8 @@ func TestSystemRoutes_PrometheusMetrics(t *testing.T) { "stacyvm_provider_healthy", "stacyvm_provider_health_latency_milliseconds", "stacyvm_spawn_queue_depth", + "stacyvm_owner_quotas_total", + `type="max_sandboxes"`, "stacyvm_rate_limit_allowed_total", "stacyvm_operation_success_total", `operation="spawn"`, diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 0846612..a913ce5 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -1017,6 +1017,26 @@ func (m *Manager) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuota, error) { return quotas, nil } +func (m *Manager) QuotaSummary(ctx context.Context) (QuotaSummary, error) { + records, err := m.store.ListOwnerQuotas(ctx) + if err != nil { + return QuotaSummary{}, err + } + summary := QuotaSummary{Total: len(records)} + for _, quota := range records { + if quota.MaxSandboxes > 0 { + summary.WithMaxSandboxes++ + } + if quota.MaxTTLSeconds > 0 { + summary.WithMaxTTL++ + } + if quota.MaxExecTimeoutSeconds > 0 { + summary.WithMaxExecTimeout++ + } + } + return summary, nil +} + func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQuota, error) { ownerID, err := normalizeOwnerID(quota.OwnerID) if err != nil { diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index aec234d..9f22950 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -374,6 +374,32 @@ func TestManager_OwnerQuotaDeletePublishesEvent(t *testing.T) { assertEventType(t, m.events.History(20), EventQuotaDeleted) } +func TestManager_QuotaSummary(t *testing.T) { + m := setupManager(t) + + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: "owner-a", + MaxSandboxes: 2, + MaxTTL: "30s", + }); err != nil { + t.Fatalf("save owner-a quota: %v", err) + } + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: "owner-b", + MaxExecTimeout: "5s", + }); err != nil { + t.Fatalf("save owner-b quota: %v", err) + } + + summary, err := m.QuotaSummary(context.Background()) + if err != nil { + t.Fatalf("quota summary: %v", err) + } + if summary.Total != 2 || summary.WithMaxSandboxes != 1 || summary.WithMaxTTL != 1 || summary.WithMaxExecTimeout != 1 { + t.Fatalf("unexpected quota summary: %+v", summary) + } +} + func TestManager_PersistentOwnerQuotaTTLLimit(t *testing.T) { m := setupManager(t) if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{OwnerID: "owner-ttl", MaxTTL: "5m"}); err != nil { diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index 0e69338..7cbfa0a 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -156,3 +156,10 @@ type OwnerUsage struct { MaxExecTimeout string `json:"max_exec_timeout"` QuotaConfigured bool `json:"quota_configured"` } + +type QuotaSummary struct { + Total int `json:"total"` + WithMaxSandboxes int `json:"with_max_sandboxes"` + WithMaxTTL int `json:"with_max_ttl"` + WithMaxExecTimeout int `json:"with_max_exec_timeout"` +} From 828ff6bb4dd0a079dd679f7d221da724a21b10d3 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:28:23 +0530 Subject: [PATCH 021/147] fix: validate sandbox owner identity --- README.md | 2 ++ docs/api.md | 2 ++ internal/api/routes/sandboxes_test.go | 15 +++++++++++++++ internal/orchestrator/manager.go | 15 +++++++++++++++ internal/orchestrator/manager_test.go | 18 ++++++++++++++++++ 5 files changed, 52 insertions(+) diff --git a/README.md b/README.md index 52df9f8..51f0e1c 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,8 @@ client = Client("http://localhost:7423", user_id="alice@example.com") const client = new Client({ baseUrl: "http://localhost:7423", userId: "alice@example.com" }); ``` +User IDs are trimmed by the server. They must be 128 characters or fewer and cannot contain whitespace, control characters, or path separators. + Hardening knobs (Docker provider): ```yaml diff --git a/docs/api.md b/docs/api.md index d8a974b..67d2cb2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -42,6 +42,8 @@ curl -H 'X-API-Key: sk-xyz123' \ CORS is permissive by default (`*`). Lock it down via reverse proxy if you expose StacyVM to the open internet. +`X-User-ID` is trimmed when present. It must be 128 characters or fewer and cannot contain whitespace, control characters, or path separators. + --- ## Rate limiting diff --git a/internal/api/routes/sandboxes_test.go b/internal/api/routes/sandboxes_test.go index d72e547..acb6bcc 100644 --- a/internal/api/routes/sandboxes_test.go +++ b/internal/api/routes/sandboxes_test.go @@ -521,6 +521,21 @@ func TestCreateSandbox_WithOwnerID(t *testing.T) { } } +func TestCreateSandbox_InvalidOwnerID(t *testing.T) { + r, _ := setupTestRouter(t) + + body := `{"image":"alpine:latest"}` + req := httptest.NewRequest("POST", "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-User-ID", "alice smith") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String()) + } +} + func TestWriteAndReadFile(t *testing.T) { r, _ := setupTestRouter(t) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index a913ce5..9063b77 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -353,6 +353,13 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) } ttl = parsed } + ownerID, err := normalizeOptionalOwnerID(req.OwnerID) + if err != nil { + metricsErr = err + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + return nil, err + } + req.OwnerID = ownerID if err := m.acquireSpawnAdmission(ctx, req.OwnerID, ttl, metricsProvider); err != nil { metricsErr = err m.publishFailureForError("", OperationSpawn, metricsProvider, err) @@ -1440,6 +1447,14 @@ func normalizeOwnerID(ownerID string) (string, error) { return ownerID, nil } +func normalizeOptionalOwnerID(ownerID string) (string, error) { + ownerID = strings.TrimSpace(ownerID) + if ownerID == "" { + return "", nil + } + return normalizeOwnerID(ownerID) +} + func optionalSecondsString(seconds int64) string { if seconds <= 0 { return "0s" diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 9f22950..97ddf5a 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -327,6 +327,24 @@ func TestManager_SpawnOwnerLimit(t *testing.T) { } } +func TestManager_SpawnOwnerIDValidation(t *testing.T) { + m := setupManager(t) + + sb, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: " owner-trimmed "}) + if err != nil { + t.Fatalf("spawn trimmed owner: %v", err) + } + if sb.OwnerID != "owner-trimmed" { + t.Fatalf("owner_id = %q, want owner-trimmed", sb.OwnerID) + } + + for _, ownerID := range []string{"owner/a", "owner a", strings.Repeat("a", 129)} { + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: ownerID}); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected invalid owner for %q, got %v", ownerID, err) + } + } +} + func TestManager_PersistentOwnerQuotaLimit(t *testing.T) { m := setupManagerWithConfig(t, ManagerConfig{ DefaultTTL: 5 * time.Minute, From 088c6c78f659be9cca356c8d3d6378a834b1aeaf Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:31:25 +0530 Subject: [PATCH 022/147] fix: hash rate limit bucket keys --- docs/api.md | 2 ++ internal/api/middleware/ratelimit.go | 17 +++++++--- internal/api/middleware/ratelimit_test.go | 38 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docs/api.md b/docs/api.md index 67d2cb2..b77d414 100644 --- a/docs/api.md +++ b/docs/api.md @@ -62,6 +62,8 @@ rate_limit: The default `owner` mode uses `X-User-ID` when present, then falls back to `X-API-Key`, then client IP. Limited requests return `429 Too Many Requests` with `Retry-After`, `X-RateLimit-Limit`, and `X-RateLimit-Remaining` headers. +Rate-limit buckets store hashed identity keys internally; raw owner IDs, API keys, and IP addresses are not exposed in diagnostics or metrics. + --- ## Conventions diff --git a/internal/api/middleware/ratelimit.go b/internal/api/middleware/ratelimit.go index af76511..336c5a0 100644 --- a/internal/api/middleware/ratelimit.go +++ b/internal/api/middleware/ratelimit.go @@ -1,6 +1,8 @@ package middleware import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "math" @@ -197,19 +199,24 @@ func (rl *RateLimiter) key(r *http.Request) string { switch rl.keyBy { case "api_key": if apiKey := strings.TrimSpace(r.Header.Get("X-API-Key")); apiKey != "" { - return "api_key:" + apiKey + return bucketKey("api_key", apiKey) } case "ip": - return "ip:" + clientIP(r) + return bucketKey("ip", clientIP(r)) default: if ownerID := strings.TrimSpace(r.Header.Get("X-User-ID")); ownerID != "" { - return "owner:" + ownerID + return bucketKey("owner", ownerID) } if apiKey := strings.TrimSpace(r.Header.Get("X-API-Key")); apiKey != "" { - return "api_key:" + apiKey + return bucketKey("api_key", apiKey) } } - return "ip:" + clientIP(r) + return bucketKey("ip", clientIP(r)) +} + +func bucketKey(kind, value string) string { + sum := sha256.Sum256([]byte(kind + ":" + value)) + return kind + ":" + hex.EncodeToString(sum[:]) } func clientIP(r *http.Request) string { diff --git a/internal/api/middleware/ratelimit_test.go b/internal/api/middleware/ratelimit_test.go index 82f9330..1c571ff 100644 --- a/internal/api/middleware/ratelimit_test.go +++ b/internal/api/middleware/ratelimit_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "time" ) @@ -208,3 +209,40 @@ func TestRateLimitEvictsInactiveBuckets(t *testing.T) { t.Fatalf("unexpected cleanup stats: %+v", stats) } } + +func TestRateLimitBucketKeysDoNotStoreRawIdentity(t *testing.T) { + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 10, + KeyBy: "owner", + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "sensitive-owner") + req.Header.Set("X-API-Key", "sk-sensitive") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.77:5000" + limiter.keyBy = "ip" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + limiter.mu.Lock() + defer limiter.mu.Unlock() + for key := range limiter.buckets { + if strings.Contains(key, "sensitive-owner") || strings.Contains(key, "sk-sensitive") || strings.Contains(key, "203.0.113.77") { + t.Fatalf("bucket key contains raw identity: %q", key) + } + parts := strings.Split(key, ":") + if len(parts) != 2 || len(parts[1]) != 64 { + t.Fatalf("bucket key is not typed sha256 form: %q", key) + } + } +} From 5106a775476c8c39b8ab1a5dccda8e7c5daa459d Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:35:34 +0530 Subject: [PATCH 023/147] feat: expose spawn queue wait metrics --- docs/api.md | 25 ++++++++++- internal/api/routes/prometheus.go | 12 +++++ internal/api/routes/system_test.go | 2 + internal/orchestrator/manager.go | 64 ++++++++++++++++++++++++--- internal/orchestrator/manager_test.go | 14 ++++++ internal/orchestrator/types.go | 20 ++++++--- 6 files changed, 125 insertions(+), 12 deletions(-) diff --git a/docs/api.md b/docs/api.md index b77d414..78d963e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -695,7 +695,17 @@ GET /api/v1/diagnostics "spawn_queue_depth": 3, "max_spawn_queue": 100, "spawn_queue_timeout": "30s", - "admission_control": "single_node" + "admission_control": "single_node", + "spawn_queued_total": 18, + "spawn_dequeued_total": 16, + "spawn_queue_timeouts": 2, + "spawn_queue_wait_count": 18, + "spawn_queue_wait_total": "1m42s", + "spawn_queue_wait_max": "12s", + "spawn_queue_wait_avg": "5.666s", + "spawn_queue_wait_total_ms": 102000, + "spawn_queue_wait_max_ms": 12000, + "spawn_queue_wait_avg_ms": 5666 }, "quotas": { "total": 8, @@ -783,7 +793,17 @@ GET /api/v1/metrics "spawn_queue_depth": 3, "max_spawn_queue": 100, "spawn_queue_timeout": "30s", - "admission_control": "single_node" + "admission_control": "single_node", + "spawn_queued_total": 18, + "spawn_dequeued_total": 16, + "spawn_queue_timeouts": 2, + "spawn_queue_wait_count": 18, + "spawn_queue_wait_total": "1m42s", + "spawn_queue_wait_max": "12s", + "spawn_queue_wait_avg": "5.666s", + "spawn_queue_wait_total_ms": 102000, + "spawn_queue_wait_max_ms": 12000, + "spawn_queue_wait_avg_ms": 5666 }, "quotas": { "total": 8, @@ -834,6 +854,7 @@ stacyvm_uptime_seconds 7980 # TYPE stacyvm_provider_healthy gauge stacyvm_provider_healthy{provider="docker",default="true"} 1 stacyvm_spawn_queue_depth 3 +stacyvm_spawn_queue_wait_milliseconds_count 18 stacyvm_owner_quotas_total 8 stacyvm_rate_limit_blocked_total 27 stacyvm_operation_success_total{operation="exec",provider="docker"} 482 diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go index 41b5053..dd675a4 100644 --- a/internal/api/routes/prometheus.go +++ b/internal/api/routes/prometheus.go @@ -60,6 +60,18 @@ func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { fmt.Fprintf(w, "stacyvm_spawn_queue_depth %d\n", metrics.schedulerStatus.SpawnQueueDepth) writePrometheusHelp(w, "stacyvm_spawn_queue_capacity", "Configured maximum number of queued spawn requests.") fmt.Fprintf(w, "stacyvm_spawn_queue_capacity %d\n", metrics.schedulerStatus.MaxSpawnQueue) + writePrometheusHelp(w, "stacyvm_spawn_queue_enqueued_total", "Total spawn requests admitted into the capacity wait queue.") + fmt.Fprintf(w, "stacyvm_spawn_queue_enqueued_total %d\n", metrics.schedulerStatus.SpawnQueuedTotal) + writePrometheusHelp(w, "stacyvm_spawn_queue_dequeued_total", "Total spawn requests released from the capacity wait queue.") + fmt.Fprintf(w, "stacyvm_spawn_queue_dequeued_total %d\n", metrics.schedulerStatus.SpawnDequeuedTotal) + writePrometheusHelp(w, "stacyvm_spawn_queue_timeout_total", "Total spawn requests that timed out while waiting in the capacity queue.") + fmt.Fprintf(w, "stacyvm_spawn_queue_timeout_total %d\n", metrics.schedulerStatus.SpawnQueueTimeouts) + writePrometheusHelp(w, "stacyvm_spawn_queue_wait_milliseconds_sum", "Total observed spawn queue wait time in milliseconds.") + fmt.Fprintf(w, "stacyvm_spawn_queue_wait_milliseconds_sum %d\n", metrics.schedulerStatus.SpawnQueueWaitTotalMS) + writePrometheusHelp(w, "stacyvm_spawn_queue_wait_milliseconds_count", "Total observed spawn queue wait samples.") + fmt.Fprintf(w, "stacyvm_spawn_queue_wait_milliseconds_count %d\n", metrics.schedulerStatus.SpawnQueueWaitCount) + writePrometheusHelp(w, "stacyvm_spawn_queue_wait_milliseconds_max", "Maximum observed spawn queue wait time in milliseconds.") + fmt.Fprintf(w, "stacyvm_spawn_queue_wait_milliseconds_max %d\n", metrics.schedulerStatus.SpawnQueueWaitMaxMS) writePrometheusHelp(w, "stacyvm_owner_quotas_total", "Total configured owner quota policies.") fmt.Fprintf(w, "stacyvm_owner_quotas_total %d\n", metrics.quotaSummary.Total) diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go index 57868bc..4cceb86 100644 --- a/internal/api/routes/system_test.go +++ b/internal/api/routes/system_test.go @@ -232,6 +232,8 @@ func TestSystemRoutes_PrometheusMetrics(t *testing.T) { "stacyvm_provider_healthy", "stacyvm_provider_health_latency_milliseconds", "stacyvm_spawn_queue_depth", + "stacyvm_spawn_queue_enqueued_total", + "stacyvm_spawn_queue_wait_milliseconds_count", "stacyvm_owner_quotas_total", `type="max_sandboxes"`, "stacyvm_rate_limit_allowed_total", diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 9063b77..64ea367 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -31,6 +31,7 @@ type Manager struct { queueMu sync.Mutex queueWaiters int capacityCh chan struct{} + queueStats spawnQueueStats defaultTTL time.Duration defaultImage string @@ -47,6 +48,15 @@ type Manager struct { cancel context.CancelFunc } +type spawnQueueStats struct { + queuedTotal uint64 + dequeuedTotal uint64 + timeoutTotal uint64 + waitCount uint64 + waitTotal time.Duration + waitMax time.Duration +} + type ManagerConfig struct { DefaultTTL time.Duration DefaultImage string @@ -991,12 +1001,26 @@ func (m *Manager) Limits() OperationalLimitsInfo { func (m *Manager) SchedulerStatus() SchedulerStatus { m.queueMu.Lock() defer m.queueMu.Unlock() + waitAvg := time.Duration(0) + if m.queueStats.waitCount > 0 { + waitAvg = time.Duration(int64(m.queueStats.waitTotal) / int64(m.queueStats.waitCount)) + } return SchedulerStatus{ - SpawnOverflow: m.limits.SpawnOverflow, - SpawnQueueDepth: m.queueWaiters, - MaxSpawnQueue: m.limits.MaxSpawnQueue, - SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), - AdmissionControl: "single_node", + SpawnOverflow: m.limits.SpawnOverflow, + SpawnQueueDepth: m.queueWaiters, + MaxSpawnQueue: m.limits.MaxSpawnQueue, + SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), + AdmissionControl: "single_node", + SpawnQueuedTotal: m.queueStats.queuedTotal, + SpawnDequeuedTotal: m.queueStats.dequeuedTotal, + SpawnQueueTimeouts: m.queueStats.timeoutTotal, + SpawnQueueWaitCount: m.queueStats.waitCount, + SpawnQueueWaitTotal: m.queueStats.waitTotal.String(), + SpawnQueueWaitMax: m.queueStats.waitMax.String(), + SpawnQueueWaitAvg: waitAvg.String(), + SpawnQueueWaitTotalMS: m.queueStats.waitTotal.Milliseconds(), + SpawnQueueWaitMaxMS: m.queueStats.waitMax.Milliseconds(), + SpawnQueueWaitAvgMS: waitAvg.Milliseconds(), } } @@ -1211,9 +1235,11 @@ func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl return providers.ResourceLimitError(fmt.Sprintf("spawn queue full (%d)", m.limits.MaxSpawnQueue)) } m.queueWaiters++ + m.queueStats.queuedTotal++ depth := m.queueWaiters capacityCh := m.capacityCh m.queueMu.Unlock() + queuedAt := time.Now() m.publishOperationalEvent(EventSpawnQueued, "", map[string]interface{}{ "operation": OperationSpawn, @@ -1239,11 +1265,14 @@ func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl case <-waitCtx.Done(): if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { err := providers.ResourceLimitError(fmt.Sprintf("spawn queue timeout after %s", m.limits.SpawnQueueTimeout)) + waitDuration := time.Since(queuedAt) + m.recordSpawnQueueTimeout(waitDuration) m.publishOperationalEvent(EventSpawnQueueTimeout, "", map[string]interface{}{ "operation": OperationSpawn, "provider": provider, "owner_id": ownerID, "error": err.Error(), + "wait_ms": waitDuration.Milliseconds(), }) return err } @@ -1254,10 +1283,13 @@ func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl return err } if decision.Allowed { + waitDuration := time.Since(queuedAt) + m.recordSpawnDequeued(waitDuration) m.publishOperationalEvent(EventSpawnDequeued, "", map[string]interface{}{ "operation": OperationSpawn, "provider": provider, "owner_id": ownerID, + "wait_ms": waitDuration.Milliseconds(), }) return nil } @@ -1271,6 +1303,28 @@ func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl } } +func (m *Manager) recordSpawnDequeued(waitDuration time.Duration) { + m.queueMu.Lock() + defer m.queueMu.Unlock() + m.queueStats.dequeuedTotal++ + m.queueStats.waitCount++ + m.queueStats.waitTotal += waitDuration + if waitDuration > m.queueStats.waitMax { + m.queueStats.waitMax = waitDuration + } +} + +func (m *Manager) recordSpawnQueueTimeout(waitDuration time.Duration) { + m.queueMu.Lock() + defer m.queueMu.Unlock() + m.queueStats.timeoutTotal++ + m.queueStats.waitCount++ + m.queueStats.waitTotal += waitDuration + if waitDuration > m.queueStats.waitMax { + m.queueStats.waitMax = waitDuration + } +} + func (m *Manager) notifySpawnCapacity() { m.queueMu.Lock() close(m.capacityCh) diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 97ddf5a..e159a03 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -677,6 +677,13 @@ func TestManager_SpawnQueueWaitsForCapacity(t *testing.T) { events := m.events.History(20) assertEventType(t, events, EventSpawnQueued) assertEventType(t, events, EventSpawnDequeued) + status := m.SchedulerStatus() + if status.SpawnQueuedTotal != 1 || status.SpawnDequeuedTotal != 1 || status.SpawnQueueWaitCount != 1 { + t.Fatalf("unexpected queue status: %+v", status) + } + if status.SpawnQueueWaitTotalMS <= 0 || status.SpawnQueueWaitMaxMS <= 0 || status.SpawnQueueWaitAvgMS <= 0 { + t.Fatalf("expected positive queue wait metrics: %+v", status) + } } func TestManager_SpawnQueueTimesOut(t *testing.T) { @@ -702,6 +709,10 @@ func TestManager_SpawnQueueTimesOut(t *testing.T) { t.Fatalf("expected queue timeout resource limit, got %v", err) } assertEventType(t, m.events.History(20), EventSpawnQueueTimeout) + status := m.SchedulerStatus() + if status.SpawnQueuedTotal != 1 || status.SpawnQueueTimeouts != 1 || status.SpawnQueueWaitCount != 1 { + t.Fatalf("unexpected timeout queue status: %+v", status) + } } func TestManager_SchedulerStatus(t *testing.T) { @@ -724,6 +735,9 @@ func TestManager_SchedulerStatus(t *testing.T) { if status.SpawnQueueDepth != 0 { t.Fatalf("queue depth = %d, want 0", status.SpawnQueueDepth) } + if status.SpawnQueuedTotal != 0 || status.SpawnQueueWaitTotal != "0s" || status.SpawnQueueWaitAvg != "0s" { + t.Fatalf("unexpected empty queue metrics: %+v", status) + } } func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index 7cbfa0a..8b2fae9 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -121,11 +121,21 @@ type OperationalLimitsInfo struct { } type SchedulerStatus struct { - SpawnOverflow string `json:"spawn_overflow"` - SpawnQueueDepth int `json:"spawn_queue_depth"` - MaxSpawnQueue int `json:"max_spawn_queue"` - SpawnQueueTimeout string `json:"spawn_queue_timeout"` - AdmissionControl string `json:"admission_control"` + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueDepth int `json:"spawn_queue_depth"` + MaxSpawnQueue int `json:"max_spawn_queue"` + SpawnQueueTimeout string `json:"spawn_queue_timeout"` + AdmissionControl string `json:"admission_control"` + SpawnQueuedTotal uint64 `json:"spawn_queued_total"` + SpawnDequeuedTotal uint64 `json:"spawn_dequeued_total"` + SpawnQueueTimeouts uint64 `json:"spawn_queue_timeouts"` + SpawnQueueWaitCount uint64 `json:"spawn_queue_wait_count"` + SpawnQueueWaitTotal string `json:"spawn_queue_wait_total"` + SpawnQueueWaitMax string `json:"spawn_queue_wait_max"` + SpawnQueueWaitAvg string `json:"spawn_queue_wait_avg"` + SpawnQueueWaitTotalMS int64 `json:"spawn_queue_wait_total_ms"` + SpawnQueueWaitMaxMS int64 `json:"spawn_queue_wait_max_ms"` + SpawnQueueWaitAvgMS int64 `json:"spawn_queue_wait_avg_ms"` } type SpawnAdmissionDecision struct { From 06bb8a847cad5ee4e3d60d3240415a76608258a5 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:41:34 +0530 Subject: [PATCH 024/147] fix: wake spawn queue on quota changes --- internal/orchestrator/manager.go | 2 + internal/orchestrator/manager_test.go | 58 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 64ea367..5c702e4 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -1104,6 +1104,7 @@ func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQ "max_ttl": saved.MaxTTL, "max_exec_timeout": saved.MaxExecTimeout, }) + m.notifySpawnCapacity() return saved, nil } @@ -1118,6 +1119,7 @@ func (m *Manager) DeleteOwnerQuota(ctx context.Context, ownerID string) error { m.publishOperationalEvent(EventQuotaDeleted, "", map[string]interface{}{ "owner_id": ownerID, }) + m.notifySpawnCapacity() return nil } diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index e159a03..9effe58 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -686,6 +686,64 @@ func TestManager_SpawnQueueWaitsForCapacity(t *testing.T) { } } +func TestManager_SpawnQueueResumesWhenQuotaChanges(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + SpawnOverflow: "queue", + SpawnQueueTimeout: 500 * time.Millisecond, + MaxSpawnQueue: 2, + }, + }) + ctx := context.Background() + if _, err := m.SaveOwnerQuota(ctx, OwnerQuota{OwnerID: "owner-a", MaxSandboxes: 1}); err != nil { + t.Fatalf("save initial quota: %v", err) + } + if _, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("first spawn: %v", err) + } + + type spawnResult struct { + sb *Sandbox + err error + } + resultCh := make(chan spawnResult, 1) + go func() { + sb, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-a"}) + resultCh <- spawnResult{sb: sb, err: err} + }() + + select { + case result := <-resultCh: + t.Fatalf("second spawn returned before quota changed: sb=%v err=%v", result.sb, result.err) + case <-time.After(25 * time.Millisecond): + } + + if _, err := m.SaveOwnerQuota(ctx, OwnerQuota{OwnerID: "owner-a", MaxSandboxes: 2}); err != nil { + t.Fatalf("increase quota: %v", err) + } + + select { + case result := <-resultCh: + if result.err != nil { + t.Fatalf("second spawn: %v", result.err) + } + if result.sb == nil || result.sb.OwnerID != "owner-a" { + t.Fatalf("unexpected second spawn: %+v", result.sb) + } + case <-time.After(time.Second): + t.Fatal("second spawn did not resume after quota changed") + } + + status := m.SchedulerStatus() + if status.SpawnQueuedTotal != 1 || status.SpawnDequeuedTotal != 1 { + t.Fatalf("unexpected queue status: %+v", status) + } +} + func TestManager_SpawnQueueTimesOut(t *testing.T) { m := setupManagerWithConfig(t, ManagerConfig{ DefaultTTL: 5 * time.Minute, From e44689cd0164c4de0f80446d2c0357ace132e2e9 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:45:32 +0530 Subject: [PATCH 025/147] feat: add quota summary endpoint --- README.md | 1 + docs/api.md | 18 +++++++++++++++ internal/api/routes/quotas.go | 20 +++++++++++++++++ internal/api/routes/quotas_test.go | 33 ++++++++++++++++++++++++++++ internal/api/routes/swagger_types.go | 3 +++ 5 files changed, 75 insertions(+) diff --git a/README.md b/README.md index 51f0e1c..341c5cd 100644 --- a/README.md +++ b/README.md @@ -416,6 +416,7 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | `GET` | `/providers/{name}` | Provider details + sandbox count | | `POST` | `/providers/test` | Health-check all providers | | `GET` | `/quotas` | List owner quota overrides | +| `GET` | `/quotas/summary` | Redacted owner quota policy counts | | `PUT` | `/quotas/{ownerID}` | Create or update owner quota | | `GET` | `/quotas/{ownerID}/usage` | Owner usage against effective quota | | `GET` | `/pool/status` | Pool VM and user counts | diff --git a/docs/api.md b/docs/api.md index 78d963e..5b59664 100644 --- a/docs/api.md +++ b/docs/api.md @@ -430,6 +430,24 @@ GET /api/v1/quotas ] ``` +### Get quota summary + +``` +GET /api/v1/quotas/summary +``` + +Returns redacted policy coverage counts without exposing owner IDs. + +**Response** `200 OK`: +```json +{ + "total": 2, + "with_max_sandboxes": 1, + "with_max_ttl": 1, + "with_max_exec_timeout": 1 +} +``` + ### Save owner quota ``` diff --git a/internal/api/routes/quotas.go b/internal/api/routes/quotas.go index 6e6f383..fc90a88 100644 --- a/internal/api/routes/quotas.go +++ b/internal/api/routes/quotas.go @@ -12,6 +12,7 @@ import ( type quotaManager interface { ListOwnerQuotas(ctx context.Context) ([]*orchestrator.OwnerQuota, error) + QuotaSummary(ctx context.Context) (orchestrator.QuotaSummary, error) GetOwnerQuota(ctx context.Context, ownerID string) (*orchestrator.OwnerQuota, error) SaveOwnerQuota(ctx context.Context, quota orchestrator.OwnerQuota) (*orchestrator.OwnerQuota, error) DeleteOwnerQuota(ctx context.Context, ownerID string) error @@ -29,6 +30,7 @@ func NewQuotaRoutes(manager quotaManager) *QuotaRoutes { func (q *QuotaRoutes) Routes() chi.Router { r := chi.NewRouter() r.Get("/", q.List) + r.Get("/summary", q.Summary) r.Route("/{ownerID}", func(r chi.Router) { r.Get("/", q.Get) r.Put("/", q.Save) @@ -59,6 +61,24 @@ func (q *QuotaRoutes) List(w http.ResponseWriter, r *http.Request) { httputil.WriteJSON(w, http.StatusOK, quotas) } +// Summary returns redacted quota coverage counts. +// +// @Summary Get quota summary +// @Description Return non-identifying counts for persisted owner quota overrides +// @Tags quotas +// @Produce json +// @Success 200 {object} orchestrator.QuotaSummary +// @Security ApiKeyAuth +// @Router /quotas/summary [get] +func (q *QuotaRoutes) Summary(w http.ResponseWriter, r *http.Request) { + summary, err := q.manager.QuotaSummary(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, summary) +} + // Get returns one configured owner quota. // // @Summary Get owner quota diff --git a/internal/api/routes/quotas_test.go b/internal/api/routes/quotas_test.go index 3266102..489cd8a 100644 --- a/internal/api/routes/quotas_test.go +++ b/internal/api/routes/quotas_test.go @@ -88,6 +88,39 @@ func TestQuotaRoutes_SaveGetUsageDelete(t *testing.T) { } } +func TestQuotaRoutes_Summary(t *testing.T) { + r, _ := setupQuotaRouter(t) + + quotas := map[string]string{ + "owner-a": `{"max_sandboxes":1,"max_ttl":"30s"}`, + "owner-b": `{"max_exec_timeout":"10s"}`, + } + for ownerID, body := range quotas { + req := httptest.NewRequest(http.MethodPut, "/api/v1/quotas/"+ownerID, bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("save %s status = %d: %s", ownerID, w.Code, w.Body.String()) + } + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/quotas/summary", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("summary status = %d: %s", w.Code, w.Body.String()) + } + + var summary orchestrator.QuotaSummary + if err := json.NewDecoder(w.Body).Decode(&summary); err != nil { + t.Fatalf("decode summary: %v", err) + } + if summary.Total != 2 || summary.WithMaxSandboxes != 1 || summary.WithMaxTTL != 1 || summary.WithMaxExecTimeout != 1 { + t.Fatalf("unexpected summary: %+v", summary) + } +} + func TestQuotaRoutes_InvalidQuotaReturnsBadRequest(t *testing.T) { r, _ := setupQuotaRouter(t) diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index 8fd4973..9a1b755 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -67,6 +67,9 @@ type OwnerQuotaResponse = orchestrator.OwnerQuota // OwnerUsageResponse is the response for owner quota usage. type OwnerUsageResponse = orchestrator.OwnerUsage +// QuotaSummaryResponse is the response for redacted quota coverage counts. +type QuotaSummaryResponse = orchestrator.QuotaSummary + // MetricsResponse is the response from the metrics endpoint. type MetricsResponse struct { SandboxesActive int `json:"sandboxes_active" example:"5"` From d0aaf52b4a66e50808cb1a3405e164506b5df319 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:54:50 +0530 Subject: [PATCH 026/147] fix: distinguish stream cancellation from timeout --- internal/orchestrator/manager.go | 9 ++- internal/orchestrator/manager_test.go | 89 +++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 5c702e4..5a4e654 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -690,21 +690,24 @@ func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequ go func() { defer close(out) defer cancel() - timedOut := false + interrupted := false for chunk := range ch { select { case out <- chunk: case <-execCtx.Done(): - timedOut = true + interrupted = true } } - if execCtx.Err() == context.DeadlineExceeded || timedOut { + if errors.Is(execCtx.Err(), context.DeadlineExceeded) { metricsErr = providers.ExecTimeoutError(sandboxID) m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExecStream, metricsProvider, metricsErr) select { case out <- providers.StreamChunk{Stream: "stderr", Data: metricsErr.Error()}: case <-ctx.Done(): } + } else if interrupted && execCtx.Err() != nil { + metricsErr = execCtx.Err() + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, metricsErr) } m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), metricsErr) }() diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 9effe58..65398b8 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -64,6 +64,35 @@ func (p *slowSpawnProvider) Spawn(ctx context.Context, opts providers.SpawnOptio return p.Provider.Spawn(ctx, opts) } +type cancellableStreamProvider struct { + providers.Provider + started chan struct{} + filled chan struct{} + once sync.Once + fill sync.Once +} + +func (p *cancellableStreamProvider) ExecStream(ctx context.Context, sandboxID string, opts providers.ExecOptions) (<-chan providers.StreamChunk, error) { + ch := make(chan providers.StreamChunk, 64) + go func() { + defer close(ch) + p.once.Do(func() { close(p.started) }) + sent := 0 + for { + select { + case <-ctx.Done(): + return + case ch <- providers.StreamChunk{Stream: "stdout", Data: "streaming\n"}: + sent++ + if sent >= 128 { + p.fill.Do(func() { close(p.filled) }) + } + } + } + }() + return ch, nil +} + func TestManager_SpawnAndGet(t *testing.T) { m := setupManager(t) ctx := context.Background() @@ -824,6 +853,66 @@ func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { assertEventType(t, m.events.History(10), EventExecTimeout) } +func TestManager_ExecStreamCancellationDoesNotEmitTimeout(t *testing.T) { + m := setupManager(t) + base, err := m.registry.Get("mock") + if err != nil { + t.Fatalf("get mock provider: %v", err) + } + streamProvider := &cancellableStreamProvider{ + Provider: base, + started: make(chan struct{}), + filled: make(chan struct{}), + } + m.registry.Register(streamProvider) + + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + ch, err := m.ExecStream(ctx, sb.ID, ExecRequest{Command: "stream forever"}) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + + select { + case <-streamProvider.filled: + case <-time.After(time.Second): + t.Fatal("stream provider did not fill the stream buffers") + } + deadline := time.After(time.Second) + for len(ch) < cap(ch) { + select { + case <-deadline: + t.Fatalf("stream output buffer length = %d, want %d", len(ch), cap(ch)) + default: + time.Sleep(time.Millisecond) + } + } + cancel() + time.Sleep(100 * time.Millisecond) + + drained := make(chan struct{}) + go func() { + defer close(drained) + for range ch { + } + }() + select { + case <-drained: + case <-time.After(time.Second): + t.Fatal("stream did not close after cancellation") + } + + for _, event := range m.events.History(20) { + if event.Type == EventExecTimeout { + t.Fatalf("unexpected timeout event after cancellation: %+v", event) + } + } +} + func TestManager_PublishesOperationFailureEvent(t *testing.T) { m := setupManager(t) From ee746b61b75f02eecb3dbcb09fdfe1d6d9c2d0a8 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 07:57:43 +0530 Subject: [PATCH 027/147] fix: map stream exec limit errors --- internal/api/routes/sandboxes.go | 2 +- internal/api/routes/sandboxes_test.go | 34 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/api/routes/sandboxes.go b/internal/api/routes/sandboxes.go index 4a626ee..1c6312d 100644 --- a/internal/api/routes/sandboxes.go +++ b/internal/api/routes/sandboxes.go @@ -233,7 +233,7 @@ func (s *SandboxRoutes) Exec(w http.ResponseWriter, r *http.Request) { func (s *SandboxRoutes) execStream(w http.ResponseWriter, r *http.Request, id string, req orchestrator.ExecRequest) { ch, err := s.manager.ExecStream(r.Context(), id, req) if err != nil { - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } diff --git a/internal/api/routes/sandboxes_test.go b/internal/api/routes/sandboxes_test.go index acb6bcc..8c45524 100644 --- a/internal/api/routes/sandboxes_test.go +++ b/internal/api/routes/sandboxes_test.go @@ -2,6 +2,7 @@ package routes import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -143,6 +144,39 @@ func TestExecInSandbox_Timeout(t *testing.T) { } } +func TestExecStream_MaxTimeoutLimit(t *testing.T) { + r, mgr := setupTestRouter(t) + if _, err := mgr.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "owner-a", + MaxExecTimeout: "1s", + }); err != nil { + t.Fatalf("save owner quota: %v", err) + } + + body := `{"image":"alpine:latest","owner_id":"owner-a"}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusCreated { + t.Fatalf("create status = %d: %s", w.Code, w.Body.String()) + } + var sb orchestrator.Sandbox + if err := json.NewDecoder(w.Body).Decode(&sb); err != nil { + t.Fatalf("decode sandbox: %v", err) + } + + execBody := `{"command":"echo hello","timeout":"2s","stream":true}` + req = httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/"+sb.ID+"/exec", bytes.NewBufferString(execBody)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusTooManyRequests { + t.Fatalf("expected 429, got %d: %s", w.Code, w.Body.String()) + } +} + func TestDestroyAndGet404(t *testing.T) { r, _ := setupTestRouter(t) From f44669c41ea986f61db5ee6b50284af4c5f20f93 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 08:01:45 +0530 Subject: [PATCH 028/147] feat: add spawn admission preflight --- README.md | 1 + docs/api.md | 26 ++++++ internal/api/routes/sandboxes.go | 32 ++++++++ internal/api/routes/sandboxes_test.go | 109 ++++++++++++++++++++++++-- internal/orchestrator/manager.go | 25 ++++++ 5 files changed, 187 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 341c5cd..fbe3fc1 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,7 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | Method | Endpoint | Description | |---|---|---| | `POST` | `/sandboxes` | Spawn a sandbox | +| `POST` | `/sandboxes/admission` | Preflight quota and scheduler admission | | `GET` | `/sandboxes` | List active sandboxes | | `DELETE` | `/sandboxes` | Prune expired sandboxes | | `GET` | `/sandboxes/{id}` | Get sandbox details | diff --git a/docs/api.md b/docs/api.md index 5b59664..605d46f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -135,6 +135,32 @@ POST /api/v1/sandboxes } ``` +### Evaluate spawn admission + +``` +POST /api/v1/sandboxes/admission +``` + +Preflight a spawn request against current quota and scheduler limits without creating a sandbox. `X-User-ID` overrides `owner_id`, matching the spawn endpoint. + +**Request body**: same shape as `POST /api/v1/sandboxes`. + +**Response** `200 OK`: +```json +{ + "allowed": false, + "queueable": true, + "reason": "max_sandboxes", + "active_sandboxes": 100, + "max_sandboxes": 100, + "active_owner_sandboxes": 2, + "max_owner_sandboxes": 10, + "max_ttl": "24h0m0s" +} +``` + +`queueable` reflects the configured spawn overflow mode. Capacity denials are queueable only when `defaults.spawn_overflow` is `queue`; TTL denials are never queueable. + ### List sandboxes ``` diff --git a/internal/api/routes/sandboxes.go b/internal/api/routes/sandboxes.go index 1c6312d..0a99251 100644 --- a/internal/api/routes/sandboxes.go +++ b/internal/api/routes/sandboxes.go @@ -26,6 +26,7 @@ func (s *SandboxRoutes) Routes() chi.Router { r.Post("/", s.Create) r.Get("/", s.List) r.Delete("/", s.Prune) + r.Post("/admission", s.Admission) r.Route("/{sandboxID}", func(r chi.Router) { r.Get("/", s.Get) r.Delete("/", s.Destroy) @@ -80,6 +81,37 @@ func (s *SandboxRoutes) Create(w http.ResponseWriter, r *http.Request) { httputil.WriteJSON(w, http.StatusCreated, sb) } +// Admission evaluates whether a spawn request would be admitted. +// +// @Summary Evaluate spawn admission +// @Description Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits +// @Tags sandboxes +// @Accept json +// @Produce json +// @Param request body orchestrator.SpawnRequest true "Spawn request" +// @Success 200 {object} orchestrator.SpawnAdmissionDecision +// @Failure 400 {object} httputil.APIError +// @Failure 500 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /sandboxes/admission [post] +func (s *SandboxRoutes) Admission(w http.ResponseWriter, r *http.Request) { + var req orchestrator.SpawnRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + if userID := r.Header.Get("X-User-ID"); userID != "" { + req.OwnerID = userID + } + + decision, err := s.manager.EvaluateSpawnRequestAdmission(r.Context(), req) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, decision) +} + // List lists all active sandboxes. // // @Summary List sandboxes diff --git a/internal/api/routes/sandboxes_test.go b/internal/api/routes/sandboxes_test.go index 8c45524..64a9d1b 100644 --- a/internal/api/routes/sandboxes_test.go +++ b/internal/api/routes/sandboxes_test.go @@ -18,6 +18,16 @@ import ( ) func setupTestRouter(t *testing.T) (chi.Router, *orchestrator.Manager) { + t.Helper() + return setupTestRouterWithConfig(t, orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) +} + +func setupTestRouterWithConfig(t *testing.T, cfg orchestrator.ManagerConfig) (chi.Router, *orchestrator.Manager) { t.Helper() dir := t.TempDir() st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) @@ -34,12 +44,7 @@ func setupTestRouter(t *testing.T) (chi.Router, *orchestrator.Manager) { events := orchestrator.NewEventBus() logger := zerolog.Nop() - mgr := orchestrator.NewManager(reg, st, events, logger, orchestrator.ManagerConfig{ - DefaultTTL: 5 * time.Minute, - DefaultImage: "alpine:latest", - DefaultMemory: 512, - DefaultVCPUs: 1, - }) + mgr := orchestrator.NewManager(reg, st, events, logger, cfg) mgr.Start() t.Cleanup(func() { mgr.Stop() }) @@ -72,6 +77,98 @@ func TestCreateSandbox(t *testing.T) { } } +func TestSpawnAdmissionRoute(t *testing.T) { + r, _ := setupTestRouterWithConfig(t, orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: orchestrator.OperationalLimits{ + MaxSandboxes: 1, + SpawnOverflow: "queue", + SpawnQueueTimeout: time.Second, + MaxSpawnQueue: 2, + }, + }) + + body := `{"image":"alpine:latest","owner_id":"owner-a"}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusCreated { + t.Fatalf("create status = %d: %s", w.Code, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/admission", bytes.NewBufferString(`{"ttl":"1m","owner_id":"owner-b"}`)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("admission status = %d: %s", w.Code, w.Body.String()) + } + + var decision orchestrator.SpawnAdmissionDecision + if err := json.NewDecoder(w.Body).Decode(&decision); err != nil { + t.Fatalf("decode admission: %v", err) + } + if decision.Allowed || !decision.Queueable || decision.Reason != "max_sandboxes" { + t.Fatalf("unexpected admission decision: %+v", decision) + } + if decision.ActiveSandboxes != 1 || decision.MaxSandboxes != 1 { + t.Fatalf("unexpected admission counts: %+v", decision) + } +} + +func TestSpawnAdmissionRouteRejectModeNotQueueable(t *testing.T) { + r, _ := setupTestRouterWithConfig(t, orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: orchestrator.OperationalLimits{ + MaxSandboxes: 1, + }, + }) + + body := `{"image":"alpine:latest","owner_id":"owner-a"}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusCreated { + t.Fatalf("create status = %d: %s", w.Code, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/admission", bytes.NewBufferString(`{"owner_id":"owner-b"}`)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("admission status = %d: %s", w.Code, w.Body.String()) + } + + var decision orchestrator.SpawnAdmissionDecision + if err := json.NewDecoder(w.Body).Decode(&decision); err != nil { + t.Fatalf("decode admission: %v", err) + } + if decision.Allowed || decision.Queueable || decision.Reason != "max_sandboxes" { + t.Fatalf("unexpected admission decision: %+v", decision) + } +} + +func TestSpawnAdmissionRouteInvalidTTL(t *testing.T) { + r, _ := setupTestRouter(t) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/admission", bytes.NewBufferString(`{"ttl":"not-a-duration"}`)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String()) + } +} + func TestListSandboxes(t *testing.T) { r, _ := setupTestRouter(t) diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 5a4e654..83e174c 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -1389,6 +1389,31 @@ func (m *Manager) EvaluateSpawnAdmission(ctx context.Context, ownerID string, tt return decision, nil } +// EvaluateSpawnRequestAdmission evaluates a spawn request against the current +// quota and scheduler limits without creating provider resources. +func (m *Manager) EvaluateSpawnRequestAdmission(ctx context.Context, req SpawnRequest) (SpawnAdmissionDecision, error) { + ttl := m.defaultTTL + if req.TTL != "" { + parsed, err := time.ParseDuration(req.TTL) + if err != nil { + return SpawnAdmissionDecision{}, fmt.Errorf("%w: parsing TTL: %v", ErrInvalidInput, err) + } + ttl = parsed + } + ownerID, err := normalizeOptionalOwnerID(req.OwnerID) + if err != nil { + return SpawnAdmissionDecision{}, err + } + decision, err := m.EvaluateSpawnAdmission(ctx, ownerID, ttl) + if err != nil { + return SpawnAdmissionDecision{}, err + } + if decision.Queueable && !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + decision.Queueable = false + } + return decision, nil +} + func spawnAdmissionError(decision SpawnAdmissionDecision) error { switch decision.Reason { case "max_ttl": From a8c5e195d5b0d3d02682163e65852f353dc17be7 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 08:15:43 +0530 Subject: [PATCH 029/147] docs: close phase three release --- CHANGELOG.md | 30 + docs/docs.go | 1617 +++++++++++++---- .../releases/phase-3-quotas-and-scheduling.md | 152 ++ docs/swagger.json | 1617 +++++++++++++---- docs/swagger.yaml | 752 +++++++- sdk/js/README.md | 13 + sdk/js/src/client.ts | 40 + sdk/js/src/index.ts | 2 + sdk/js/src/types.ts | 34 + sdk/python/README.md | 13 + sdk/python/stacyvm/__init__.py | 10 +- sdk/python/stacyvm/async_client.py | 42 +- sdk/python/stacyvm/client.py | 43 +- sdk/python/stacyvm/models.py | 24 + 14 files changed, 3639 insertions(+), 750 deletions(-) create mode 100644 docs/releases/phase-3-quotas-and-scheduling.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0891f42..3765365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## Phase 3 Quotas And Scheduling - 2026-05-08 + +This checkpoint adds the first production multi-tenant control plane: persisted owner quotas, API rate limiting, spawn backpressure, scheduler visibility, admission preflight, and SDK helpers. + +### Added + +- Persisted owner quota policies for max sandboxes, max TTL, and max exec timeout. +- Owner quota APIs, including usage and redacted summary endpoints. +- Spawn admission decisions and `POST /api/v1/sandboxes/admission`. +- Configurable spawn overflow queue with queue timeout and maximum queue depth. +- Optional API rate limiting by owner, API key, or IP address. +- Scheduler, quota, and rate-limit metrics in JSON diagnostics/metrics and Prometheus output. +- TypeScript and Python SDK helpers for admission preflight and quota summary. + +### Changed + +- Spawn admission is serialized to avoid concurrent over-admission. +- Queued spawns wake when capacity opens or owner quotas change. +- Rate-limit bucket keys are hashed before storage. +- Streaming exec cancellation is no longer reported as a timeout. +- Streaming exec preflight errors now use the same API error mapping as non-streaming exec. +- OpenAPI docs were regenerated for the Phase 3 API surface. + +### Verified + +- `go test ./internal/api/routes ./internal/orchestrator` +- `make build` +- `cd web && npm run build` +- `make test` + ## Phase 2 Observability And Ops - 2026-05-08 This checkpoint adds production operations surfaces for health checks, diagnostics, metrics, audit events, and runtime limits. diff --git a/docs/docs.go b/docs/docs.go index c76277f..48eeeb3 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -15,6 +15,31 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/diagnostics": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return redacted build, store, provider, sandbox, event, and operation diagnostics", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Get diagnostics", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.DiagnosticsResponse" + } + } + } + } + }, "/events": { "get": { "security": [ @@ -34,7 +59,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Event" } } } @@ -65,6 +90,31 @@ const docTemplate = `{ } } }, + "/live": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the StacyVM API process is alive", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Liveness check", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.HealthResponse" + } + } + } + } + }, "/metrics": { "get": { "security": [ @@ -90,6 +140,31 @@ const docTemplate = `{ } } }, + "/metrics/prometheus": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format", + "produces": [ + "text/plain" + ], + "tags": [ + "system" + ], + "summary": "Get Prometheus metrics", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, "/providers": { "get": { "security": [ @@ -180,52 +255,111 @@ const docTemplate = `{ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes": { + "/quotas": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all active sandboxes", + "description": "Return all persisted owner quota overrides", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "List sandboxes", + "summary": "List owner quotas", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } + } + } + } + }, + "/quotas/summary": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return non-identifying counts for persisted owner quota overrides", + "produces": [ + "application/json" + ], + "tags": [ + "quotas" + ], + "summary": "Get quota summary", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + } + } + } + } + }, + "/quotas/{ownerID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return the persisted quota override for an owner", + "produces": [ + "application/json" + ], + "tags": [ + "quotas" + ], + "summary": "Get owner quota", + "parameters": [ + { + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } }, - "500": { - "description": "Internal Server Error", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } }, - "post": { + "put": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Spawn a new sandbox with the given configuration", + "description": "Create or update quota overrides for an owner", "consumes": [ "application/json" ], @@ -233,37 +367,32 @@ const docTemplate = `{ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Create a sandbox", + "summary": "Save owner quota", "parameters": [ { - "description": "Spawn request", + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + }, + { + "description": "Quota request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } ], "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } } @@ -274,50 +403,59 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Destroy all expired sandboxes and return the count", + "description": "Delete the quota override for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" + ], + "summary": "Delete owner quota", + "parameters": [ + { + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + } ], - "summary": "Prune sandboxes", "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.PruneResponse" + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, - "500": { - "description": "Internal Server Error", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}": { + "/quotas/{ownerID}/usage": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a sandbox by its ID", + "description": "Return active sandbox usage and effective quota for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Get a sandbox", + "summary": "Get owner quota usage", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true } @@ -326,76 +464,83 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage" } - }, - "404": { - "description": "Not Found", + } + } + } + }, + "/ready": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the API is ready to serve sandbox traffic", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Readiness check", + "responses": { + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" } }, - "500": { - "description": "Internal Server Error", + "503": { + "description": "Service Unavailable", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" } } } - }, - "delete": { + } + }, + "/sandboxes": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Destroy a sandbox and release its resources", + "description": "Return all active sandboxes", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Destroy a sandbox", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - } - ], + "summary": "List sandboxes", "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/exec": { + }, "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Run a command inside a sandbox. Set stream=true for streaming output.", + "description": "Spawn a new sandbox with the given configuration", "consumes": [ "application/json" ], @@ -405,153 +550,141 @@ const docTemplate = `{ "tags": [ "sandboxes" ], - "summary": "Execute a command", + "summary": "Create a sandbox", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "description": "Exec request", + "description": "Spawn request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" } } ], "responses": { - "200": { - "description": "OK", + "201": { + "description": "Created", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "429": { + "description": "Too Many Requests", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/exec/ws": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Open a WebSocket connection to execute a command with streaming output", + "description": "Destroy all expired sandboxes and return the count", + "produces": [ + "application/json" + ], "tags": [ "sandboxes" ], - "summary": "Execute via WebSocket", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - } - ], + "summary": "Prune sandboxes", "responses": { - "101": { - "description": "WebSocket upgrade" + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.PruneResponse" + } }, - "400": { - "description": "Bad request" + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } } } } }, - "/sandboxes/{sandboxID}/files": { - "get": { + "/sandboxes/admission": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Read file content from a sandbox", + "description": "Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits", + "consumes": [ + "application/json" + ], "produces": [ - "application/octet-stream" + "application/json" ], "tags": [ "sandboxes" ], - "summary": "Read a file", + "summary": "Evaluate spawn admission", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "File path inside the sandbox", - "name": "path", - "in": "query", - "required": true + "description": "Spawn request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + } } ], "responses": { "200": { "description": "OK", "schema": { - "type": "file" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "post": { + } + }, + "/sandboxes/{sandboxID}": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Write content to a file inside a sandbox", - "consumes": [ - "application/json" - ], + "description": "Return a sandbox by its ID", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Write a file", + "summary": "Get a sandbox", "parameters": [ { "type": "string", @@ -559,60 +692,43 @@ const docTemplate = `{ "name": "sandboxID", "in": "path", "required": true - }, - { - "description": "File write request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest" - } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/files/list": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "List files in a directory inside a sandbox", + "description": "Destroy a sandbox and release its resources", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "List files", + "summary": "Destroy a sandbox", "parameters": [ { "type": "string", @@ -620,54 +736,48 @@ const docTemplate = `{ "name": "sandboxID", "in": "path", "required": true - }, - { - "type": "string", - "description": "Directory path (default: /)", - "name": "path", - "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo" - } + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}/logs": { - "get": { + "/sandboxes/{sandboxID}/exec": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Retrieve console log lines from a sandbox", + "description": "Run a command inside a sandbox. Set stream=true for streaming output.", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Get console logs", + "summary": "Execute a command", "parameters": [ { "type": "string", @@ -677,77 +787,76 @@ const docTemplate = `{ "required": true }, { - "type": "integer", - "description": "Number of lines to retrieve (default: 100)", - "name": "lines", - "in": "query" + "description": "Exec request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest" + } } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates": { + "/sandboxes/{sandboxID}/exec/ws": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all registered templates", - "produces": [ - "application/json" - ], + "description": "Open a WebSocket connection to execute a command with streaming output", "tags": [ - "templates" + "sandboxes" + ], + "summary": "Execute via WebSocket", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + } ], - "summary": "List templates", "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - } + "101": { + "description": "WebSocket upgrade" }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } + "400": { + "description": "Bad request" } } - }, + } + }, + "/sandboxes/{sandboxID}/extend": { "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Register a new sandbox template", + "description": "Add additional time to a sandbox's expiration", "consumes": [ "application/json" ], @@ -755,100 +864,125 @@ const docTemplate = `{ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Create a template", + "summary": "Extend sandbox TTL", "parameters": [ { - "description": "Template definition", + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "description": "TTL extension", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "type": "object", + "properties": { + "ttl": { + "type": "string" + } + } } } ], "responses": { - "201": { - "description": "Created", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, - "409": { - "description": "Conflict", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates/{name}": { + "/sandboxes/{sandboxID}/files": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a template by its name", + "description": "Read file content from a sandbox", "produces": [ - "application/json" + "application/octet-stream" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Get a template", + "summary": "Read a file", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true + }, + { + "type": "string", + "description": "File path inside the sandbox", + "name": "path", + "in": "query", + "required": true } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } }, - "put": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Update an existing template by name", + "description": "Write content to a file inside a sandbox", "consumes": [ "application/json" ], @@ -856,24 +990,24 @@ const docTemplate = `{ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Update a template", + "summary": "Write a file", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true }, { - "description": "Updated template", + "description": "File write request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest" } } ], @@ -881,69 +1015,404 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "delete": { + } + }, + "/sandboxes/{sandboxID}/files/list": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Delete a template by name", + "description": "List files in a directory inside a sandbox", "produces": [ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Delete a template", + "summary": "List files", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true + }, + { + "type": "string", + "description": "Directory path (default: /)", + "name": "path", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" - } + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/logs": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Retrieve console log lines from a sandbox", + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Get console logs", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Number of lines to retrieve (default: 100)", + "name": "lines", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/snapshots": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all pre-built VM snapshots available for fast restore", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "List snapshots", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary" + } + } + } + } + } + }, + "/templates": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all registered templates", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "List templates", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Register a new sandbox template", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Create a template", + "parameters": [ + { + "description": "Template definition", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/templates/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return a template by its name", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Get a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Update an existing template by name", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Update a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "Updated template", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete a template by name", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Delete a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.StatusResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } @@ -988,19 +1457,19 @@ const docTemplate = `{ "201": { "description": "Created", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } @@ -1008,18 +1477,53 @@ const docTemplate = `{ } }, "definitions": { - "github_com_stacyvm-dev_stacyvm_internal_httputil.APIError": { + "github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats": { + "type": "object", + "properties": { + "active_buckets": { + "type": "integer" + }, + "allowed_total": { + "type": "integer" + }, + "bucket_ttl": { + "type": "string" + }, + "burst": { + "type": "integer" + }, + "cleanup_interval": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "evicted_total": { + "type": "integer" + }, + "key_by": { + "type": "string" + }, + "limited_total": { + "type": "integer" + }, + "requests_per_minute": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_httputil.APIError": { "type": "object", "properties": { "code": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode" }, "message": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode": { + "github_com_StacyOs_stacyvm_internal_httputil.ErrorCode": { "type": "string", "enum": [ "NOT_FOUND", @@ -1027,7 +1531,9 @@ const docTemplate = `{ "INTERNAL_ERROR", "UNAUTHORIZED", "CONFLICT", - "UNAVAILABLE" + "UNAVAILABLE", + "TIMEOUT", + "RESOURCE_LIMIT" ], "x-enum-varnames": [ "CodeNotFound", @@ -1035,10 +1541,12 @@ const docTemplate = `{ "CodeInternal", "CodeUnauth", "CodeConflict", - "CodeUnavailable" + "CodeUnavailable", + "CodeTimeout", + "CodeResourceLimit" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Event": { "type": "object", "properties": { "data": { @@ -1056,115 +1564,281 @@ const docTemplate = `{ "timestamp": { "type": "string" }, - "type": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType" + "type": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventType" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats": { + "type": "object", + "properties": { + "events_total": { + "type": "integer" + }, + "history_size": { + "type": "integer" + }, + "subscribers": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventType": { + "type": "string", + "enum": [ + "sandbox.created", + "sandbox.running", + "sandbox.destroyed", + "sandbox.error", + "exec.started", + "exec.completed", + "exec.failed", + "exec.timeout", + "file.written", + "file.read", + "operation.failed", + "resource.limit", + "provider.failed", + "reconcile.action", + "spawn.queued", + "spawn.dequeued", + "spawn.queue_timeout", + "quota.saved", + "quota.deleted" + ], + "x-enum-varnames": [ + "EventSandboxCreated", + "EventSandboxRunning", + "EventSandboxDestroyed", + "EventSandboxError", + "EventExecStarted", + "EventExecCompleted", + "EventExecFailed", + "EventExecTimeout", + "EventFileWritten", + "EventFileRead", + "EventOperationFailed", + "EventResourceLimit", + "EventProviderFailed", + "EventReconcileAction", + "EventSpawnQueued", + "EventSpawnDequeued", + "EventSpawnQueueTimeout", + "EventQuotaSaved", + "EventQuotaDeleted" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest": { + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "type": "string" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "stream": { + "type": "boolean" + }, + "timeout": { + "type": "string" + }, + "workdir": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult": { + "type": "object", + "properties": { + "duration": { + "type": "string" + }, + "exit_code": { + "type": "integer" + }, + "stderr": { + "type": "string" + }, + "stdout": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo": { + "type": "object", + "properties": { + "is_dir": { + "type": "boolean" + }, + "mod_time": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "path": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics": { + "type": "object", + "properties": { + "failure_total": { + "type": "integer" + }, + "last_error": { + "type": "string" + }, + "last_observed_unix": { + "type": "integer" + }, + "latency_avg_ms": { + "type": "integer" + }, + "latency_count": { + "type": "integer" + }, + "latency_max_ms": { + "type": "integer" + }, + "latency_min_ms": { + "type": "integer" + }, + "latency_total_ms": { + "type": "integer" + }, + "operation": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "success_total": { + "type": "integer" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType": { - "type": "string", - "enum": [ - "sandbox.created", - "sandbox.running", - "sandbox.destroyed", - "sandbox.error", - "exec.started", - "exec.completed", - "file.written", - "file.read" - ], - "x-enum-varnames": [ - "EventSandboxCreated", - "EventSandboxRunning", - "EventSandboxDestroyed", - "EventSandboxError", - "EventExecStarted", - "EventExecCompleted", - "EventFileWritten", - "EventFileRead" - ] - }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo": { "type": "object", "properties": { - "args": { - "type": "array", - "items": { - "type": "string" - } + "default_exec_timeout": { + "type": "string" }, - "command": { + "max_exec_timeout": { "type": "string" }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "max_sandboxes": { + "type": "integer" }, - "stream": { - "type": "boolean" + "max_sandboxes_per_owner": { + "type": "integer" }, - "timeout": { + "max_spawn_queue": { + "type": "integer" + }, + "max_ttl": { "type": "string" }, - "workdir": { + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_timeout": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota": { "type": "object", "properties": { - "duration": { + "created_at": { "type": "string" }, - "exit_code": { + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { "type": "integer" }, - "stderr": { + "max_ttl": { "type": "string" }, - "stdout": { + "owner_id": { + "type": "string" + }, + "updated_at": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage": { "type": "object", "properties": { - "is_dir": { - "type": "boolean" + "active_sandboxes": { + "type": "integer" }, - "mod_time": { + "max_exec_timeout": { "type": "string" }, - "mode": { + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { "type": "string" }, - "path": { + "owner_id": { "type": "string" }, - "size": { - "type": "integer" + "quota_configured": { + "type": "boolean" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary": { "type": "object", "properties": { - "content": { - "type": "string" + "total": { + "type": "integer" }, - "mode": { - "type": "string" + "with_max_exec_timeout": { + "type": "integer" }, - "path": { - "type": "string" + "with_max_sandboxes": { + "type": "integer" + }, + "with_max_ttl": { + "type": "integer" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox": { "type": "object", "properties": { "created_at": { @@ -1188,18 +1862,27 @@ const docTemplate = `{ "type": "string" } }, + "owner_id": { + "type": "string" + }, + "preview_domain": { + "type": "string" + }, "provider": { "type": "string" }, "state": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState" }, "vcpus": { "type": "integer" + }, + "vm_id": { + "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState": { "type": "string", "enum": [ "creating", @@ -1216,7 +1899,57 @@ const docTemplate = `{ "StateError" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus": { + "type": "object", + "properties": { + "admission_control": { + "type": "string" + }, + "max_spawn_queue": { + "type": "integer" + }, + "spawn_dequeued_total": { + "type": "integer" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_depth": { + "type": "integer" + }, + "spawn_queue_timeout": { + "type": "string" + }, + "spawn_queue_timeouts": { + "type": "integer" + }, + "spawn_queue_wait_avg": { + "type": "string" + }, + "spawn_queue_wait_avg_ms": { + "type": "integer" + }, + "spawn_queue_wait_count": { + "type": "integer" + }, + "spawn_queue_wait_max": { + "type": "string" + }, + "spawn_queue_wait_max_ms": { + "type": "integer" + }, + "spawn_queue_wait_total": { + "type": "string" + }, + "spawn_queue_wait_total_ms": { + "type": "integer" + }, + "spawn_queued_total": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig": { "type": "object", "properties": { "inject_at": { @@ -1227,7 +1960,36 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision": { + "type": "object", + "properties": { + "active_owner_sandboxes": { + "type": "integer" + }, + "active_sandboxes": { + "type": "integer" + }, + "allowed": { + "type": "boolean" + }, + "max_owner_sandboxes": { + "type": "integer" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "queueable": { + "type": "boolean" + }, + "reason": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest": { "type": "object", "properties": { "image": { @@ -1242,6 +2004,9 @@ const docTemplate = `{ "type": "string" } }, + "owner_id": { + "type": "string" + }, "provider": { "type": "string" }, @@ -1256,7 +2021,7 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Template": { "type": "object", "properties": { "allowed_hosts": { @@ -1295,7 +2060,7 @@ const docTemplate = `{ "secrets": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig" } }, "setup": { @@ -1315,6 +2080,78 @@ const docTemplate = `{ } } }, + "github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "image": { + "type": "string" + }, + "provider": { + "type": "string" + } + } + }, + "internal_api_routes.DiagnosticsResponse": { + "type": "object", + "properties": { + "build": { + "type": "object", + "additionalProperties": true + }, + "events": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats" + }, + "generated_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "limits": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics" + } + }, + "process": { + "type": "object", + "additionalProperties": true + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "quotas": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + }, + "rate_limit": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats" + }, + "redactions": { + "type": "array", + "items": { + "type": "string" + } + }, + "sandboxes": { + "type": "object", + "additionalProperties": true + }, + "scheduler": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus" + }, + "store": { + "type": "object", + "additionalProperties": true + } + } + }, "internal_api_routes.HealthResponse": { "type": "object", "properties": { @@ -1370,6 +2207,9 @@ const docTemplate = `{ "type": "boolean", "example": true }, + "health": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + }, "healthy": { "type": "boolean", "example": true @@ -1384,20 +2224,86 @@ const docTemplate = `{ } } }, + "internal_api_routes.ProviderHealth": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "spawn", + "exec", + "files" + ] + }, + "default": { + "type": "boolean", + "example": true + }, + "error": { + "type": "string", + "example": "health check returned false" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, + "name": { + "type": "string", + "example": "docker" + }, + "runtime_count": { + "type": "integer", + "example": 2 + } + } + }, "internal_api_routes.ProviderInfo": { "type": "object", "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, "default": { "type": "boolean", "example": true }, + "error": { + "type": "string", + "example": "health check returned false" + }, "healthy": { "type": "boolean", "example": true }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, "name": { "type": "string", "example": "firecracker" + }, + "runtime_count": { + "type": "integer", + "example": 2 } } }, @@ -1410,6 +2316,37 @@ const docTemplate = `{ } } }, + "internal_api_routes.ReadinessResponse": { + "type": "object", + "properties": { + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "ready_providers": { + "type": "integer", + "example": 1 + }, + "status": { + "type": "string", + "example": "ready" + }, + "total_providers": { + "type": "integer", + "example": 2 + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + }, + "version": { + "type": "string", + "example": "1.0.0" + } + } + }, "internal_api_routes.StatusResponse": { "type": "object", "properties": { diff --git a/docs/releases/phase-3-quotas-and-scheduling.md b/docs/releases/phase-3-quotas-and-scheduling.md new file mode 100644 index 0000000..3cc2b03 --- /dev/null +++ b/docs/releases/phase-3-quotas-and-scheduling.md @@ -0,0 +1,152 @@ +# Phase 3 Quotas And Scheduling Release Notes + +Date: 2026-05-08 +Branch: `phase-3-quotas-and-scheduling` + +## Summary + +Phase 3 adds the first production multi-tenant control plane for StacyVM. The server now supports persisted owner quota policies, API rate limiting, spawn backpressure, scheduler visibility, quota audit events, admission preflight, and SDK helpers for the new quota and admission surfaces. + +The goal of this phase is to make StacyVM safer under shared usage and load: operators can define per-owner limits, clients can understand whether work will run or queue, and dashboards can observe queue pressure and quota coverage. + +## What Changed + +### Persistent Owner Quotas + +- Added persisted owner quota policies backed by SQLite. +- Quotas can override: + - max active sandboxes per owner + - max sandbox TTL + - max exec timeout +- Added owner quota APIs: + - `GET /api/v1/quotas` + - `GET /api/v1/quotas/summary` + - `GET /api/v1/quotas/{ownerID}` + - `PUT /api/v1/quotas/{ownerID}` + - `DELETE /api/v1/quotas/{ownerID}` + - `GET /api/v1/quotas/{ownerID}/usage` +- Owner IDs are normalized and validated before quota use. +- Quota saves and deletes emit audit events. + +### Spawn Admission And Backpressure + +- Added serialized spawn admission checks to prevent concurrent over-admission. +- Added configurable spawn overflow behavior: + - `reject` + - `queue` +- Added configurable queue controls: + - `defaults.spawn_queue_timeout` + - `defaults.max_spawn_queue` +- Queued spawn requests resume when capacity opens or owner quota changes. +- Spawn queue timeouts return typed resource-limit errors. +- Added `POST /api/v1/sandboxes/admission` for preflight admission checks without creating provider resources. + +### API Rate Limiting + +- Added optional in-memory API rate limiting. +- Supported rate-limit keys: + - owner + - API key + - IP address +- Rate-limit buckets use hashed keys so raw identifiers are not exposed in memory snapshots or metrics. +- Inactive rate-limit buckets are pruned on a configurable interval. + +### Scheduler And Quota Observability + +- Diagnostics and metrics now include scheduler state, queue depth, queue totals, queue timeouts, wait totals, wait max, and wait averages. +- Prometheus now exposes spawn queue gauges/counters and quota summary metrics. +- Added redacted quota summary counts for operators without exposing owner IDs. + +### Streaming Timeout Semantics + +- Streaming exec deadline expiry still emits `exec.timeout` and a timeout stderr chunk. +- Caller cancellation is no longer mislabeled as an exec timeout. +- Pre-stream exec limit errors now use the central API error mapper, so streaming and non-streaming exec return consistent status codes. + +### SDK Support + +- TypeScript SDK: + - Added `client.admission(...)`. + - Added `client.quotaSummary()`. + - Added `SpawnAdmissionDecision` and `QuotaSummary` types. + - Added `owner_id` on `SpawnOptions`. +- Python SDK: + - Added `Client.admission(...)` and `AsyncClient.admission(...)`. + - Added `Client.quota_summary()` and `AsyncClient.quota_summary()`. + - Added `SpawnAdmissionDecision` and `QuotaSummary` models. + - Added `owner_id` spawn parameter. + +## Code Changes By Area + +### API Routes + +- `internal/api/routes/quotas.go` + - Added owner quota CRUD, owner usage, and redacted summary routes. +- `internal/api/routes/sandboxes.go` + - Added spawn admission preflight. + - Aligned streaming exec preflight error mapping with non-streaming exec. +- `internal/api/routes/system.go` + - Added scheduler, quota, and rate-limit data to diagnostics and metrics. +- `internal/api/routes/prometheus.go` + - Added scheduler queue, quota summary, and rate-limit metrics. + +### Orchestrator + +- `internal/orchestrator/manager.go` + - Added quota enforcement, quota summary, owner usage, spawn admission decisions, queue wait/resume behavior, and refined stream timeout handling. +- `internal/orchestrator/types.go` + - Added quota, owner usage, scheduler status, and admission decision types. +- `internal/orchestrator/events.go` + - Added spawn queue and quota audit events. + +### Store And Config + +- `internal/store/migrations.go` + - Added `owner_quotas` persistence. +- `internal/store/sqlite.go` + - Added owner quota CRUD. +- `internal/config/config.go` + - Added spawn queue and API rate-limit configuration. + +### SDKs And Docs + +- `sdk/js/src/client.ts` and `sdk/js/src/types.ts` + - Added quota summary and admission helpers/types. +- `sdk/python/stacyvm/client.py`, `sdk/python/stacyvm/async_client.py`, and `sdk/python/stacyvm/models.py` + - Added quota summary and admission helpers/models. +- `docs/api.md`, `docs/swagger.yaml`, `docs/swagger.json`, and `docs/docs.go` + - Documented and regenerated the Phase 3 API surface. + +## Verification + +The following checks passed during Phase 3 closeout: + +```sh +go test ./internal/api/routes ./internal/orchestrator +make build +cd web && npm run build +make test +``` + +Full `make test` requires local socket-binding permission for `httptest` integration servers in this sandboxed environment. + +## Platform Notes + +- Docker daemon validation remains host-gated when the local sandbox cannot access Docker. +- Firecracker conformance remains Linux/KVM-gated. +- PRoot conformance remains gated on a real `proot` binary and usable rootfs. + +## Impact + +Phase 3 makes StacyVM meaningfully safer for shared usage: + +- Operators can assign persistent per-owner policy. +- Clients can preflight work before spawning provider resources. +- Burst load can queue instead of failing immediately. +- Queue pressure is observable in JSON and Prometheus metrics. +- API rate limiting protects the control plane. +- SDKs expose the new control-plane helpers directly. + +## Next Phase Direction + +Phase 4 should focus on distributed production operation: durable distributed scheduling semantics, deployment/CI hardening, runtime conformance on real platform hosts, and deeper admin workflows for quota policy management. diff --git a/docs/swagger.json b/docs/swagger.json index fb171e5..84dbddf 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -9,6 +9,31 @@ "host": "localhost:7423", "basePath": "/api/v1", "paths": { + "/diagnostics": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return redacted build, store, provider, sandbox, event, and operation diagnostics", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Get diagnostics", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.DiagnosticsResponse" + } + } + } + } + }, "/events": { "get": { "security": [ @@ -28,7 +53,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Event" } } } @@ -59,6 +84,31 @@ } } }, + "/live": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the StacyVM API process is alive", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Liveness check", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.HealthResponse" + } + } + } + } + }, "/metrics": { "get": { "security": [ @@ -84,6 +134,31 @@ } } }, + "/metrics/prometheus": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format", + "produces": [ + "text/plain" + ], + "tags": [ + "system" + ], + "summary": "Get Prometheus metrics", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, "/providers": { "get": { "security": [ @@ -174,52 +249,111 @@ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes": { + "/quotas": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all active sandboxes", + "description": "Return all persisted owner quota overrides", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "List sandboxes", + "summary": "List owner quotas", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } + } + } + } + }, + "/quotas/summary": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return non-identifying counts for persisted owner quota overrides", + "produces": [ + "application/json" + ], + "tags": [ + "quotas" + ], + "summary": "Get quota summary", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + } + } + } + } + }, + "/quotas/{ownerID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return the persisted quota override for an owner", + "produces": [ + "application/json" + ], + "tags": [ + "quotas" + ], + "summary": "Get owner quota", + "parameters": [ + { + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } }, - "500": { - "description": "Internal Server Error", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } }, - "post": { + "put": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Spawn a new sandbox with the given configuration", + "description": "Create or update quota overrides for an owner", "consumes": [ "application/json" ], @@ -227,37 +361,32 @@ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Create a sandbox", + "summary": "Save owner quota", "parameters": [ { - "description": "Spawn request", + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + }, + { + "description": "Quota request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } ], "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } } @@ -268,50 +397,59 @@ "ApiKeyAuth": [] } ], - "description": "Destroy all expired sandboxes and return the count", + "description": "Delete the quota override for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" + ], + "summary": "Delete owner quota", + "parameters": [ + { + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + } ], - "summary": "Prune sandboxes", "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.PruneResponse" + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, - "500": { - "description": "Internal Server Error", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}": { + "/quotas/{ownerID}/usage": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a sandbox by its ID", + "description": "Return active sandbox usage and effective quota for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Get a sandbox", + "summary": "Get owner quota usage", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true } @@ -320,76 +458,83 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage" } - }, - "404": { - "description": "Not Found", + } + } + } + }, + "/ready": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the API is ready to serve sandbox traffic", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Readiness check", + "responses": { + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" } }, - "500": { - "description": "Internal Server Error", + "503": { + "description": "Service Unavailable", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" } } } - }, - "delete": { + } + }, + "/sandboxes": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Destroy a sandbox and release its resources", + "description": "Return all active sandboxes", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Destroy a sandbox", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - } - ], + "summary": "List sandboxes", "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/exec": { + }, "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Run a command inside a sandbox. Set stream=true for streaming output.", + "description": "Spawn a new sandbox with the given configuration", "consumes": [ "application/json" ], @@ -399,153 +544,141 @@ "tags": [ "sandboxes" ], - "summary": "Execute a command", + "summary": "Create a sandbox", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "description": "Exec request", + "description": "Spawn request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" } } ], "responses": { - "200": { - "description": "OK", + "201": { + "description": "Created", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "429": { + "description": "Too Many Requests", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/exec/ws": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Open a WebSocket connection to execute a command with streaming output", + "description": "Destroy all expired sandboxes and return the count", + "produces": [ + "application/json" + ], "tags": [ "sandboxes" ], - "summary": "Execute via WebSocket", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - } - ], + "summary": "Prune sandboxes", "responses": { - "101": { - "description": "WebSocket upgrade" + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.PruneResponse" + } }, - "400": { - "description": "Bad request" + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } } } } }, - "/sandboxes/{sandboxID}/files": { - "get": { + "/sandboxes/admission": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Read file content from a sandbox", + "description": "Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits", + "consumes": [ + "application/json" + ], "produces": [ - "application/octet-stream" + "application/json" ], "tags": [ "sandboxes" ], - "summary": "Read a file", + "summary": "Evaluate spawn admission", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "File path inside the sandbox", - "name": "path", - "in": "query", - "required": true + "description": "Spawn request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + } } ], "responses": { "200": { "description": "OK", "schema": { - "type": "file" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "post": { + } + }, + "/sandboxes/{sandboxID}": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Write content to a file inside a sandbox", - "consumes": [ - "application/json" - ], + "description": "Return a sandbox by its ID", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Write a file", + "summary": "Get a sandbox", "parameters": [ { "type": "string", @@ -553,60 +686,43 @@ "name": "sandboxID", "in": "path", "required": true - }, - { - "description": "File write request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest" - } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/files/list": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "List files in a directory inside a sandbox", + "description": "Destroy a sandbox and release its resources", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "List files", + "summary": "Destroy a sandbox", "parameters": [ { "type": "string", @@ -614,54 +730,48 @@ "name": "sandboxID", "in": "path", "required": true - }, - { - "type": "string", - "description": "Directory path (default: /)", - "name": "path", - "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo" - } + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}/logs": { - "get": { + "/sandboxes/{sandboxID}/exec": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Retrieve console log lines from a sandbox", + "description": "Run a command inside a sandbox. Set stream=true for streaming output.", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Get console logs", + "summary": "Execute a command", "parameters": [ { "type": "string", @@ -671,77 +781,76 @@ "required": true }, { - "type": "integer", - "description": "Number of lines to retrieve (default: 100)", - "name": "lines", - "in": "query" + "description": "Exec request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest" + } } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates": { + "/sandboxes/{sandboxID}/exec/ws": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all registered templates", - "produces": [ - "application/json" - ], + "description": "Open a WebSocket connection to execute a command with streaming output", "tags": [ - "templates" + "sandboxes" + ], + "summary": "Execute via WebSocket", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + } ], - "summary": "List templates", "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - } + "101": { + "description": "WebSocket upgrade" }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } + "400": { + "description": "Bad request" } } - }, + } + }, + "/sandboxes/{sandboxID}/extend": { "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Register a new sandbox template", + "description": "Add additional time to a sandbox's expiration", "consumes": [ "application/json" ], @@ -749,100 +858,125 @@ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Create a template", + "summary": "Extend sandbox TTL", "parameters": [ { - "description": "Template definition", + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "description": "TTL extension", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "type": "object", + "properties": { + "ttl": { + "type": "string" + } + } } } ], "responses": { - "201": { - "description": "Created", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, - "409": { - "description": "Conflict", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates/{name}": { + "/sandboxes/{sandboxID}/files": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a template by its name", + "description": "Read file content from a sandbox", "produces": [ - "application/json" + "application/octet-stream" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Get a template", + "summary": "Read a file", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true + }, + { + "type": "string", + "description": "File path inside the sandbox", + "name": "path", + "in": "query", + "required": true } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } }, - "put": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Update an existing template by name", + "description": "Write content to a file inside a sandbox", "consumes": [ "application/json" ], @@ -850,24 +984,24 @@ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Update a template", + "summary": "Write a file", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true }, { - "description": "Updated template", + "description": "File write request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest" } } ], @@ -875,69 +1009,404 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "delete": { + } + }, + "/sandboxes/{sandboxID}/files/list": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Delete a template by name", + "description": "List files in a directory inside a sandbox", "produces": [ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Delete a template", + "summary": "List files", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true + }, + { + "type": "string", + "description": "Directory path (default: /)", + "name": "path", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" - } + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/logs": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Retrieve console log lines from a sandbox", + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Get console logs", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Number of lines to retrieve (default: 100)", + "name": "lines", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/snapshots": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all pre-built VM snapshots available for fast restore", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "List snapshots", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary" + } + } + } + } + } + }, + "/templates": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all registered templates", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "List templates", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Register a new sandbox template", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Create a template", + "parameters": [ + { + "description": "Template definition", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/templates/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return a template by its name", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Get a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Update an existing template by name", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Update a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "Updated template", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete a template by name", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Delete a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.StatusResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } @@ -982,19 +1451,19 @@ "201": { "description": "Created", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } @@ -1002,18 +1471,53 @@ } }, "definitions": { - "github_com_stacyvm-dev_stacyvm_internal_httputil.APIError": { + "github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats": { + "type": "object", + "properties": { + "active_buckets": { + "type": "integer" + }, + "allowed_total": { + "type": "integer" + }, + "bucket_ttl": { + "type": "string" + }, + "burst": { + "type": "integer" + }, + "cleanup_interval": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "evicted_total": { + "type": "integer" + }, + "key_by": { + "type": "string" + }, + "limited_total": { + "type": "integer" + }, + "requests_per_minute": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_httputil.APIError": { "type": "object", "properties": { "code": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode" }, "message": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode": { + "github_com_StacyOs_stacyvm_internal_httputil.ErrorCode": { "type": "string", "enum": [ "NOT_FOUND", @@ -1021,7 +1525,9 @@ "INTERNAL_ERROR", "UNAUTHORIZED", "CONFLICT", - "UNAVAILABLE" + "UNAVAILABLE", + "TIMEOUT", + "RESOURCE_LIMIT" ], "x-enum-varnames": [ "CodeNotFound", @@ -1029,10 +1535,12 @@ "CodeInternal", "CodeUnauth", "CodeConflict", - "CodeUnavailable" + "CodeUnavailable", + "CodeTimeout", + "CodeResourceLimit" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Event": { "type": "object", "properties": { "data": { @@ -1050,115 +1558,281 @@ "timestamp": { "type": "string" }, - "type": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType" + "type": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventType" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats": { + "type": "object", + "properties": { + "events_total": { + "type": "integer" + }, + "history_size": { + "type": "integer" + }, + "subscribers": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventType": { + "type": "string", + "enum": [ + "sandbox.created", + "sandbox.running", + "sandbox.destroyed", + "sandbox.error", + "exec.started", + "exec.completed", + "exec.failed", + "exec.timeout", + "file.written", + "file.read", + "operation.failed", + "resource.limit", + "provider.failed", + "reconcile.action", + "spawn.queued", + "spawn.dequeued", + "spawn.queue_timeout", + "quota.saved", + "quota.deleted" + ], + "x-enum-varnames": [ + "EventSandboxCreated", + "EventSandboxRunning", + "EventSandboxDestroyed", + "EventSandboxError", + "EventExecStarted", + "EventExecCompleted", + "EventExecFailed", + "EventExecTimeout", + "EventFileWritten", + "EventFileRead", + "EventOperationFailed", + "EventResourceLimit", + "EventProviderFailed", + "EventReconcileAction", + "EventSpawnQueued", + "EventSpawnDequeued", + "EventSpawnQueueTimeout", + "EventQuotaSaved", + "EventQuotaDeleted" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest": { + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "type": "string" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "stream": { + "type": "boolean" + }, + "timeout": { + "type": "string" + }, + "workdir": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult": { + "type": "object", + "properties": { + "duration": { + "type": "string" + }, + "exit_code": { + "type": "integer" + }, + "stderr": { + "type": "string" + }, + "stdout": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo": { + "type": "object", + "properties": { + "is_dir": { + "type": "boolean" + }, + "mod_time": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "path": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics": { + "type": "object", + "properties": { + "failure_total": { + "type": "integer" + }, + "last_error": { + "type": "string" + }, + "last_observed_unix": { + "type": "integer" + }, + "latency_avg_ms": { + "type": "integer" + }, + "latency_count": { + "type": "integer" + }, + "latency_max_ms": { + "type": "integer" + }, + "latency_min_ms": { + "type": "integer" + }, + "latency_total_ms": { + "type": "integer" + }, + "operation": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "success_total": { + "type": "integer" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType": { - "type": "string", - "enum": [ - "sandbox.created", - "sandbox.running", - "sandbox.destroyed", - "sandbox.error", - "exec.started", - "exec.completed", - "file.written", - "file.read" - ], - "x-enum-varnames": [ - "EventSandboxCreated", - "EventSandboxRunning", - "EventSandboxDestroyed", - "EventSandboxError", - "EventExecStarted", - "EventExecCompleted", - "EventFileWritten", - "EventFileRead" - ] - }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo": { "type": "object", "properties": { - "args": { - "type": "array", - "items": { - "type": "string" - } + "default_exec_timeout": { + "type": "string" }, - "command": { + "max_exec_timeout": { "type": "string" }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "max_sandboxes": { + "type": "integer" }, - "stream": { - "type": "boolean" + "max_sandboxes_per_owner": { + "type": "integer" }, - "timeout": { + "max_spawn_queue": { + "type": "integer" + }, + "max_ttl": { "type": "string" }, - "workdir": { + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_timeout": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota": { "type": "object", "properties": { - "duration": { + "created_at": { "type": "string" }, - "exit_code": { + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { "type": "integer" }, - "stderr": { + "max_ttl": { "type": "string" }, - "stdout": { + "owner_id": { + "type": "string" + }, + "updated_at": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage": { "type": "object", "properties": { - "is_dir": { - "type": "boolean" + "active_sandboxes": { + "type": "integer" }, - "mod_time": { + "max_exec_timeout": { "type": "string" }, - "mode": { + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { "type": "string" }, - "path": { + "owner_id": { "type": "string" }, - "size": { - "type": "integer" + "quota_configured": { + "type": "boolean" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary": { "type": "object", "properties": { - "content": { - "type": "string" + "total": { + "type": "integer" }, - "mode": { - "type": "string" + "with_max_exec_timeout": { + "type": "integer" }, - "path": { - "type": "string" + "with_max_sandboxes": { + "type": "integer" + }, + "with_max_ttl": { + "type": "integer" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox": { "type": "object", "properties": { "created_at": { @@ -1182,18 +1856,27 @@ "type": "string" } }, + "owner_id": { + "type": "string" + }, + "preview_domain": { + "type": "string" + }, "provider": { "type": "string" }, "state": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState" }, "vcpus": { "type": "integer" + }, + "vm_id": { + "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState": { "type": "string", "enum": [ "creating", @@ -1210,7 +1893,57 @@ "StateError" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus": { + "type": "object", + "properties": { + "admission_control": { + "type": "string" + }, + "max_spawn_queue": { + "type": "integer" + }, + "spawn_dequeued_total": { + "type": "integer" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_depth": { + "type": "integer" + }, + "spawn_queue_timeout": { + "type": "string" + }, + "spawn_queue_timeouts": { + "type": "integer" + }, + "spawn_queue_wait_avg": { + "type": "string" + }, + "spawn_queue_wait_avg_ms": { + "type": "integer" + }, + "spawn_queue_wait_count": { + "type": "integer" + }, + "spawn_queue_wait_max": { + "type": "string" + }, + "spawn_queue_wait_max_ms": { + "type": "integer" + }, + "spawn_queue_wait_total": { + "type": "string" + }, + "spawn_queue_wait_total_ms": { + "type": "integer" + }, + "spawn_queued_total": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig": { "type": "object", "properties": { "inject_at": { @@ -1221,7 +1954,36 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision": { + "type": "object", + "properties": { + "active_owner_sandboxes": { + "type": "integer" + }, + "active_sandboxes": { + "type": "integer" + }, + "allowed": { + "type": "boolean" + }, + "max_owner_sandboxes": { + "type": "integer" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "queueable": { + "type": "boolean" + }, + "reason": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest": { "type": "object", "properties": { "image": { @@ -1236,6 +1998,9 @@ "type": "string" } }, + "owner_id": { + "type": "string" + }, "provider": { "type": "string" }, @@ -1250,7 +2015,7 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Template": { "type": "object", "properties": { "allowed_hosts": { @@ -1289,7 +2054,7 @@ "secrets": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig" } }, "setup": { @@ -1309,6 +2074,78 @@ } } }, + "github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "image": { + "type": "string" + }, + "provider": { + "type": "string" + } + } + }, + "internal_api_routes.DiagnosticsResponse": { + "type": "object", + "properties": { + "build": { + "type": "object", + "additionalProperties": true + }, + "events": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats" + }, + "generated_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "limits": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics" + } + }, + "process": { + "type": "object", + "additionalProperties": true + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "quotas": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + }, + "rate_limit": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats" + }, + "redactions": { + "type": "array", + "items": { + "type": "string" + } + }, + "sandboxes": { + "type": "object", + "additionalProperties": true + }, + "scheduler": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus" + }, + "store": { + "type": "object", + "additionalProperties": true + } + } + }, "internal_api_routes.HealthResponse": { "type": "object", "properties": { @@ -1364,6 +2201,9 @@ "type": "boolean", "example": true }, + "health": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + }, "healthy": { "type": "boolean", "example": true @@ -1378,20 +2218,86 @@ } } }, + "internal_api_routes.ProviderHealth": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "spawn", + "exec", + "files" + ] + }, + "default": { + "type": "boolean", + "example": true + }, + "error": { + "type": "string", + "example": "health check returned false" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, + "name": { + "type": "string", + "example": "docker" + }, + "runtime_count": { + "type": "integer", + "example": 2 + } + } + }, "internal_api_routes.ProviderInfo": { "type": "object", "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, "default": { "type": "boolean", "example": true }, + "error": { + "type": "string", + "example": "health check returned false" + }, "healthy": { "type": "boolean", "example": true }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, "name": { "type": "string", "example": "firecracker" + }, + "runtime_count": { + "type": "integer", + "example": 2 } } }, @@ -1404,6 +2310,37 @@ } } }, + "internal_api_routes.ReadinessResponse": { + "type": "object", + "properties": { + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "ready_providers": { + "type": "integer", + "example": 1 + }, + "status": { + "type": "string", + "example": "ready" + }, + "total_providers": { + "type": "integer", + "example": 2 + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + }, + "version": { + "type": "string", + "example": "1.0.0" + } + } + }, "internal_api_routes.StatusResponse": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index bb86c2a..d7d7782 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,13 +1,36 @@ basePath: /api/v1 definitions: - github_com_stacyvm-dev_stacyvm_internal_httputil.APIError: + github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats: + properties: + active_buckets: + type: integer + allowed_total: + type: integer + bucket_ttl: + type: string + burst: + type: integer + cleanup_interval: + type: string + enabled: + type: boolean + evicted_total: + type: integer + key_by: + type: string + limited_total: + type: integer + requests_per_minute: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_httputil.APIError: properties: code: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode' message: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode: + github_com_StacyOs_stacyvm_internal_httputil.ErrorCode: enum: - NOT_FOUND - BAD_REQUEST @@ -15,6 +38,8 @@ definitions: - UNAUTHORIZED - CONFLICT - UNAVAILABLE + - TIMEOUT + - RESOURCE_LIMIT type: string x-enum-varnames: - CodeNotFound @@ -23,7 +48,9 @@ definitions: - CodeUnauth - CodeConflict - CodeUnavailable - github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event: + - CodeTimeout + - CodeResourceLimit + github_com_StacyOs_stacyvm_internal_orchestrator.Event: properties: data: items: @@ -36,9 +63,18 @@ definitions: timestamp: type: string type: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventType' type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType: + github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats: + properties: + events_total: + type: integer + history_size: + type: integer + subscribers: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.EventType: enum: - sandbox.created - sandbox.running @@ -46,8 +82,19 @@ definitions: - sandbox.error - exec.started - exec.completed + - exec.failed + - exec.timeout - file.written - file.read + - operation.failed + - resource.limit + - provider.failed + - reconcile.action + - spawn.queued + - spawn.dequeued + - spawn.queue_timeout + - quota.saved + - quota.deleted type: string x-enum-varnames: - EventSandboxCreated @@ -56,9 +103,20 @@ definitions: - EventSandboxError - EventExecStarted - EventExecCompleted + - EventExecFailed + - EventExecTimeout - EventFileWritten - EventFileRead - github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest: + - EventOperationFailed + - EventResourceLimit + - EventProviderFailed + - EventReconcileAction + - EventSpawnQueued + - EventSpawnDequeued + - EventSpawnQueueTimeout + - EventQuotaSaved + - EventQuotaDeleted + github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest: properties: args: items: @@ -77,7 +135,7 @@ definitions: workdir: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult: + github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult: properties: duration: type: string @@ -88,7 +146,7 @@ definitions: stdout: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo: + github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo: properties: is_dir: type: boolean @@ -101,7 +159,7 @@ definitions: size: type: integer type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest: + github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest: properties: content: type: string @@ -110,7 +168,92 @@ definitions: path: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox: + github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics: + properties: + failure_total: + type: integer + last_error: + type: string + last_observed_unix: + type: integer + latency_avg_ms: + type: integer + latency_count: + type: integer + latency_max_ms: + type: integer + latency_min_ms: + type: integer + latency_total_ms: + type: integer + operation: + type: string + provider: + type: string + success_total: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo: + properties: + default_exec_timeout: + type: string + max_exec_timeout: + type: string + max_sandboxes: + type: integer + max_sandboxes_per_owner: + type: integer + max_spawn_queue: + type: integer + max_ttl: + type: string + spawn_overflow: + type: string + spawn_queue_timeout: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota: + properties: + created_at: + type: string + max_exec_timeout: + type: string + max_sandboxes: + type: integer + max_ttl: + type: string + owner_id: + type: string + updated_at: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage: + properties: + active_sandboxes: + type: integer + max_exec_timeout: + type: string + max_sandboxes: + type: integer + max_ttl: + type: string + owner_id: + type: string + quota_configured: + type: boolean + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary: + properties: + total: + type: integer + with_max_exec_timeout: + type: integer + with_max_sandboxes: + type: integer + with_max_ttl: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox: properties: created_at: type: string @@ -126,14 +269,20 @@ definitions: additionalProperties: type: string type: object + owner_id: + type: string + preview_domain: + type: string provider: type: string state: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState' vcpus: type: integer + vm_id: + type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState: + github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState: enum: - creating - running @@ -147,14 +296,66 @@ definitions: - StateIdle - StateDestroyed - StateError - github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig: + github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus: + properties: + admission_control: + type: string + max_spawn_queue: + type: integer + spawn_dequeued_total: + type: integer + spawn_overflow: + type: string + spawn_queue_depth: + type: integer + spawn_queue_timeout: + type: string + spawn_queue_timeouts: + type: integer + spawn_queue_wait_avg: + type: string + spawn_queue_wait_avg_ms: + type: integer + spawn_queue_wait_count: + type: integer + spawn_queue_wait_max: + type: string + spawn_queue_wait_max_ms: + type: integer + spawn_queue_wait_total: + type: string + spawn_queue_wait_total_ms: + type: integer + spawn_queued_total: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig: properties: inject_at: type: string name: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest: + github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision: + properties: + active_owner_sandboxes: + type: integer + active_sandboxes: + type: integer + allowed: + type: boolean + max_owner_sandboxes: + type: integer + max_sandboxes: + type: integer + max_ttl: + type: string + queueable: + type: boolean + reason: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest: properties: image: type: string @@ -164,6 +365,8 @@ definitions: additionalProperties: type: string type: object + owner_id: + type: string provider: type: string template: @@ -173,7 +376,7 @@ definitions: vcpus: type: integer type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template: + github_com_StacyOs_stacyvm_internal_orchestrator.Template: properties: allowed_hosts: items: @@ -199,7 +402,7 @@ definitions: type: integer secrets: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig' type: array setup: items: @@ -212,6 +415,55 @@ definitions: version: type: integer type: object + github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary: + properties: + created_at: + type: string + image: + type: string + provider: + type: string + type: object + internal_api_routes.DiagnosticsResponse: + properties: + build: + additionalProperties: true + type: object + events: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats' + generated_at: + example: "2026-05-08T10:30:00Z" + type: string + limits: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo' + operations: + items: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics' + type: array + process: + additionalProperties: true + type: object + providers: + items: + $ref: '#/definitions/internal_api_routes.ProviderHealth' + type: array + quotas: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary' + rate_limit: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats' + redactions: + items: + type: string + type: array + sandboxes: + additionalProperties: true + type: object + scheduler: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus' + store: + additionalProperties: true + type: object + type: object internal_api_routes.HealthResponse: properties: status: @@ -251,6 +503,8 @@ definitions: default: example: true type: boolean + health: + $ref: '#/definitions/internal_api_routes.ProviderHealth' healthy: example: true type: boolean @@ -261,17 +515,65 @@ definitions: example: 3 type: integer type: object + internal_api_routes.ProviderHealth: + properties: + capabilities: + example: + - spawn + - exec + - files + items: + type: string + type: array + default: + example: true + type: boolean + error: + example: health check returned false + type: string + healthy: + example: true + type: boolean + last_checked: + example: "2026-05-08T10:30:00Z" + type: string + latency_ms: + example: 3 + type: integer + name: + example: docker + type: string + runtime_count: + example: 2 + type: integer + type: object internal_api_routes.ProviderInfo: properties: + capabilities: + items: + type: string + type: array default: example: true type: boolean + error: + example: health check returned false + type: string healthy: example: true type: boolean + last_checked: + example: "2026-05-08T10:30:00Z" + type: string + latency_ms: + example: 3 + type: integer name: example: firecracker type: string + runtime_count: + example: 2 + type: integer type: object internal_api_routes.PruneResponse: properties: @@ -279,6 +581,28 @@ definitions: example: 3 type: integer type: object + internal_api_routes.ReadinessResponse: + properties: + providers: + items: + $ref: '#/definitions/internal_api_routes.ProviderHealth' + type: array + ready_providers: + example: 1 + type: integer + status: + example: ready + type: string + total_providers: + example: 2 + type: integer + uptime: + example: 2h30m15s + type: string + version: + example: 1.0.0 + type: string + type: object internal_api_routes.StatusResponse: properties: status: @@ -301,6 +625,22 @@ info: title: StacyVM API version: "1.0" paths: + /diagnostics: + get: + description: Return redacted build, store, provider, sandbox, event, and operation + diagnostics + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.DiagnosticsResponse' + security: + - ApiKeyAuth: [] + summary: Get diagnostics + tags: + - system /events: get: description: Open an SSE stream for real-time sandbox and system events @@ -310,7 +650,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Event' security: - ApiKeyAuth: [] summary: Subscribe to events @@ -331,6 +671,21 @@ paths: summary: Health check tags: - system + /live: + get: + description: Return whether the StacyVM API process is alive + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.HealthResponse' + security: + - ApiKeyAuth: [] + summary: Liveness check + tags: + - system /metrics: get: description: Return runtime metrics including sandbox count, goroutines, and @@ -347,6 +702,22 @@ paths: summary: Get metrics tags: - system + /metrics/prometheus: + get: + description: Return runtime, provider, sandbox, event, and operation metrics + in Prometheus text format + produces: + - text/plain + responses: + "200": + description: OK + schema: + type: string + security: + - ApiKeyAuth: [] + summary: Get Prometheus metrics + tags: + - system /providers: get: description: Return all registered providers with health status @@ -383,7 +754,7 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get provider details @@ -406,6 +777,155 @@ paths: summary: Test providers tags: - providers + /quotas: + get: + description: Return all persisted owner quota overrides + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + type: array + security: + - ApiKeyAuth: [] + summary: List owner quotas + tags: + - quotas + /quotas/{ownerID}: + delete: + description: Delete the quota override for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.StatusResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Delete owner quota + tags: + - quotas + get: + description: Return the persisted quota override for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Get owner quota + tags: + - quotas + put: + consumes: + - application/json + description: Create or update quota overrides for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + - description: Quota request + in: body + name: request + required: true + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + security: + - ApiKeyAuth: [] + summary: Save owner quota + tags: + - quotas + /quotas/{ownerID}/usage: + get: + description: Return active sandbox usage and effective quota for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage' + security: + - ApiKeyAuth: [] + summary: Get owner quota usage + tags: + - quotas + /quotas/summary: + get: + description: Return non-identifying counts for persisted owner quota overrides + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary' + security: + - ApiKeyAuth: [] + summary: Get quota summary + tags: + - quotas + /ready: + get: + description: Return whether the API is ready to serve sandbox traffic + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.ReadinessResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_routes.ReadinessResponse' + security: + - ApiKeyAuth: [] + summary: Readiness check + tags: + - system /sandboxes: delete: description: Destroy all expired sandboxes and return the count @@ -419,7 +939,7 @@ paths: "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Prune sandboxes @@ -434,12 +954,12 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' type: array "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: List sandboxes @@ -455,22 +975,26 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest' produces: - application/json responses: "201": description: Created schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "429": + description: Too Many Requests + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Create a sandbox @@ -495,11 +1019,11 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Destroy a sandbox @@ -519,15 +1043,15 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get a sandbox @@ -549,22 +1073,22 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Execute a command @@ -590,6 +1114,50 @@ paths: summary: Execute via WebSocket tags: - sandboxes + /sandboxes/{sandboxID}/extend: + post: + consumes: + - application/json + description: Add additional time to a sandbox's expiration + parameters: + - description: Sandbox ID + in: path + name: sandboxID + required: true + type: string + - description: TTL extension + in: body + name: request + required: true + schema: + properties: + ttl: + type: string + type: object + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' + "400": + description: Bad Request + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Extend sandbox TTL + tags: + - sandboxes /sandboxes/{sandboxID}/files: get: description: Read file content from a sandbox @@ -614,15 +1182,15 @@ paths: "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Read a file @@ -643,7 +1211,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest' produces: - application/json responses: @@ -654,15 +1222,15 @@ paths: "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Write a file @@ -688,16 +1256,16 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo' type: array "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: List files @@ -728,16 +1296,66 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get console logs tags: - sandboxes + /sandboxes/admission: + post: + consumes: + - application/json + description: Return whether a spawn request would be allowed, queued, or denied + by quota and scheduler limits + parameters: + - description: Spawn request + in: body + name: request + required: true + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision' + "400": + description: Bad Request + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Evaluate spawn admission + tags: + - sandboxes + /snapshots: + get: + description: Return all pre-built VM snapshots available for fast restore + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary' + type: array + security: + - ApiKeyAuth: [] + summary: List snapshots + tags: + - snapshots /templates: get: description: Return all registered templates @@ -748,12 +1366,12 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' type: array "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: List templates @@ -769,26 +1387,26 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' produces: - application/json responses: "201": description: Created schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "409": description: Conflict schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Create a template @@ -813,11 +1431,11 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Delete a template @@ -837,15 +1455,15 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get a template @@ -866,26 +1484,26 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Update a template @@ -914,15 +1532,15 @@ paths: "201": description: Created schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Spawn from template diff --git a/sdk/js/README.md b/sdk/js/README.md index 9a18e9b..d806f13 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -112,8 +112,18 @@ All fields on `SpawnOptions` are optional. Server defaults apply when fields are | `memory_mb` | `number` | RAM in MB | | `vcpus` | `number` | Virtual CPUs | | `ttl` | `string` | Auto-destroy after this duration | +| `owner_id` | `string` | Owner ID for per-owner quotas when no `userId` header is set | | `metadata` | `Record` | Free-form labels | +Preflight quota and scheduler admission without creating a sandbox: + +```typescript +const decision = await client.admission({ image: "python:3.12", ttl: "1h" }); +if (!decision.allowed && decision.queueable) { + console.log(`Request would queue because ${decision.reason}`); +} +``` + --- ## Executing commands @@ -305,6 +315,7 @@ await client.health(); // { status: "ok", version: "0.5.1", uptime: "2h13 await client.list(); // SandboxInfo[] — all active sandboxes await client.providers(); // [{ name: "docker", healthy: true, default: true }, ...] await client.poolStatus(); // pool VM and user counts +await client.quotaSummary(); // redacted owner quota policy counts await client.prune(); // returns count of expired sandboxes destroyed ``` @@ -371,6 +382,8 @@ import { ProviderInfo, HealthInfo, VMPoolStatus, + SpawnAdmissionDecision, + QuotaSummary, ForgevmClientOptions, } from "stacyvm"; ``` diff --git a/sdk/js/src/client.ts b/sdk/js/src/client.ts index 81b92e8..8f094e8 100644 --- a/sdk/js/src/client.ts +++ b/sdk/js/src/client.ts @@ -14,7 +14,9 @@ import type { ForgevmClientOptions, HealthInfo, ProviderInfo, + QuotaSummary, SandboxInfo, + SpawnAdmissionDecision, SpawnOptions, VMPoolStatus, } from "./types.js"; @@ -173,6 +175,7 @@ export class Client { if (opts?.memory_mb !== undefined) body["memory_mb"] = opts.memory_mb; if (opts?.vcpus !== undefined) body["vcpus"] = opts.vcpus; if (opts?.ttl) body["ttl"] = opts.ttl; + if (opts?.owner_id) body["owner_id"] = opts.owner_id; if (opts?.metadata) body["metadata"] = opts.metadata; const response = await this._fetch("/api/v1/sandboxes", { @@ -186,6 +189,31 @@ export class Client { return new Sandbox(this._baseUrl, this._headers, this._timeout, data); } + /** + * Preflight a spawn request against quota and scheduler limits. + * + * @param opts - Sandbox configuration to evaluate. + * @returns Admission decision without creating a sandbox. + */ + async admission(opts?: SpawnOptions): Promise { + const body: Record = {}; + if (opts?.image) body["image"] = opts.image; + if (opts?.provider) body["provider"] = opts.provider; + if (opts?.memory_mb !== undefined) body["memory_mb"] = opts.memory_mb; + if (opts?.vcpus !== undefined) body["vcpus"] = opts.vcpus; + if (opts?.ttl) body["ttl"] = opts.ttl; + if (opts?.owner_id) body["owner_id"] = opts.owner_id; + if (opts?.metadata) body["metadata"] = opts.metadata; + + const response = await this._fetch("/api/v1/sandboxes/admission", { + method: "POST", + body: JSON.stringify(body), + }); + + await handleResponse(response); + return (await response.json()) as SpawnAdmissionDecision; + } + /** * Retrieve an existing sandbox by its ID. * @@ -319,6 +347,18 @@ export class Client { return (await response.json()) as VMPoolStatus; } + /** + * Get redacted quota policy coverage counts. + */ + async quotaSummary(): Promise { + const response = await this._fetch("/api/v1/quotas/summary", { + method: "GET", + }); + + await handleResponse(response); + return (await response.json()) as QuotaSummary; + } + // ----------------------------------------------------------------------- // Convenience patterns // ----------------------------------------------------------------------- diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 6452266..8e422ef 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -46,7 +46,9 @@ export type { TemplateConfig, TemplateSpawnOverrides, ProviderInfo, + QuotaSummary, HealthInfo, + SpawnAdmissionDecision, StreamChunk, FileInfo, ForgevmClientOptions, diff --git a/sdk/js/src/types.ts b/sdk/js/src/types.ts index 07bcbba..45d5a99 100644 --- a/sdk/js/src/types.ts +++ b/sdk/js/src/types.ts @@ -57,10 +57,34 @@ export interface SpawnOptions { vcpus?: number; /** Time-to-live duration string. */ ttl?: string; + /** Owner ID used for per-owner quotas when no X-User-ID header is set. */ + owner_id?: string; /** Arbitrary key-value metadata. */ metadata?: Record; } +/** + * Admission result for a spawn preflight request. + */ +export interface SpawnAdmissionDecision { + /** Whether the request can be spawned immediately. */ + allowed: boolean; + /** Whether the request can wait in the spawn queue. */ + queueable: boolean; + /** Denial reason, when allowed is false. */ + reason?: string; + /** Current active sandbox count. */ + active_sandboxes: number; + /** Configured global sandbox limit. */ + max_sandboxes: number; + /** Current active sandbox count for the owner. */ + active_owner_sandboxes?: number; + /** Effective sandbox limit for the owner. */ + max_owner_sandboxes?: number; + /** Effective maximum TTL for the request. */ + max_ttl?: string; +} + /** * Full information about a sandbox as returned by list / get endpoints. */ @@ -287,6 +311,16 @@ export interface VMPoolStatus { max_users_per_vm: number; } +/** + * Redacted quota policy coverage counts. + */ +export interface QuotaSummary { + total: number; + with_max_sandboxes: number; + with_max_ttl: number; + with_max_exec_timeout: number; +} + // --------------------------------------------------------------------------- // API error envelope // --------------------------------------------------------------------------- diff --git a/sdk/python/README.md b/sdk/python/README.md index 7914255..ac02619 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -97,6 +97,7 @@ sandbox = client.spawn( memory_mb=1024, vcpus=2, ttl="1h", # "30s", "5m", "2h" — Go duration syntax + owner_id="team-a", # optional per-owner quota identity metadata={"user": "alice", "task": "data-analysis"}, ) ``` @@ -110,6 +111,7 @@ All parameters are optional. Server defaults apply when omitted. | `memory_mb` | `int \| None` | RAM in MB | | `vcpus` | `int \| None` | Virtual CPUs | | `ttl` | `str \| None` | Auto-destroy after this duration | +| `owner_id` | `str \| None` | Owner ID for per-owner quotas when no `user_id` header is set | | `template` | `str \| None` | Spawn from a server-side template by name | | `metadata` | `dict[str, str] \| None` | Free-form labels | @@ -119,6 +121,14 @@ Spawn from a template directly: sandbox = client.spawn_template("python-dev") ``` +Preflight quota and scheduler admission without creating a sandbox: + +```python +decision = client.admission(image="python:3.12", ttl="1h") +if not decision.allowed and decision.queueable: + print(f"Request would queue because {decision.reason}") +``` + --- ## Executing commands @@ -349,6 +359,7 @@ Behavioural notes: client.health() # {"status": "ok", "version": "0.5.1", "uptime": "2h13m"} client.list() # list[SandboxInfo] — all active sandboxes client.pool_status() # pool VM and user counts +client.quota_summary() # QuotaSummary — redacted owner quota policy counts client.prune() # int — count of expired sandboxes destroyed ``` @@ -401,7 +412,9 @@ from stacyvm import ( TemplateManager, # Models ExecResult, + QuotaSummary, SandboxInfo, + SpawnAdmissionDecision, StreamChunk, # Exceptions ForgevmError, diff --git a/sdk/python/stacyvm/__init__.py b/sdk/python/stacyvm/__init__.py index 9ae21c1..1081d1a 100644 --- a/sdk/python/stacyvm/__init__.py +++ b/sdk/python/stacyvm/__init__.py @@ -4,7 +4,13 @@ from stacyvm.sandbox import Sandbox from stacyvm.async_client import AsyncClient from stacyvm.async_sandbox import AsyncSandbox -from stacyvm.models import ExecResult, SandboxInfo, Template +from stacyvm.models import ( + ExecResult, + QuotaSummary, + SandboxInfo, + SpawnAdmissionDecision, + Template, +) from stacyvm.exceptions import ( ForgevmError, SandboxNotFound, @@ -19,7 +25,9 @@ "AsyncClient", "AsyncSandbox", "ExecResult", + "QuotaSummary", "SandboxInfo", + "SpawnAdmissionDecision", "Template", "ForgevmError", "SandboxNotFound", diff --git a/sdk/python/stacyvm/async_client.py b/sdk/python/stacyvm/async_client.py index e1d672d..dd7bfcc 100644 --- a/sdk/python/stacyvm/async_client.py +++ b/sdk/python/stacyvm/async_client.py @@ -6,7 +6,7 @@ from stacyvm.async_sandbox import AsyncSandbox from stacyvm.exceptions import ConnectionError, handle_response -from stacyvm.models import SandboxInfo +from stacyvm.models import QuotaSummary, SandboxInfo, SpawnAdmissionDecision class AsyncClient: @@ -46,6 +46,7 @@ async def spawn( memory_mb: int | None = None, vcpus: int | None = None, ttl: str | None = None, + owner_id: str | None = None, template: str | None = None, metadata: dict[str, str] | None = None, ) -> AsyncSandbox: @@ -59,6 +60,8 @@ async def spawn( body["vcpus"] = vcpus if ttl: body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id if template: body["template"] = template if metadata: @@ -73,6 +76,37 @@ async def spawn( data = resp.json() return AsyncSandbox(self._http, data["id"], info=data) + async def admission( + self, + image: str | None = None, + provider: str | None = None, + memory_mb: int | None = None, + vcpus: int | None = None, + ttl: str | None = None, + owner_id: str | None = None, + metadata: dict[str, str] | None = None, + ) -> SpawnAdmissionDecision: + """Preflight a spawn request without creating a sandbox.""" + body: dict = {} + if image: + body["image"] = image + if provider: + body["provider"] = provider + if memory_mb: + body["memory_mb"] = memory_mb + if vcpus: + body["vcpus"] = vcpus + if ttl: + body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id + if metadata: + body["metadata"] = metadata + + resp = await self._http.post("/api/v1/sandboxes/admission", json=body) + handle_response(resp) + return SpawnAdmissionDecision(**resp.json()) + async def spawn_template(self, template_name: str) -> AsyncSandbox: """Spawn a sandbox from a saved template.""" try: @@ -122,6 +156,12 @@ async def pool_status(self) -> dict: handle_response(resp) return resp.json() + async def quota_summary(self) -> QuotaSummary: + """Get redacted quota policy coverage counts.""" + resp = await self._http.get("/api/v1/quotas/summary") + handle_response(resp) + return QuotaSummary(**resp.json()) + async def health(self) -> dict: """Check server health.""" resp = await self._http.get("/api/v1/health") diff --git a/sdk/python/stacyvm/client.py b/sdk/python/stacyvm/client.py index 064dd2a..24e65cf 100644 --- a/sdk/python/stacyvm/client.py +++ b/sdk/python/stacyvm/client.py @@ -5,7 +5,7 @@ import httpx from stacyvm.exceptions import ConnectionError, handle_response -from stacyvm.models import SandboxInfo +from stacyvm.models import QuotaSummary, SandboxInfo, SpawnAdmissionDecision from stacyvm.sandbox import Sandbox @@ -46,6 +46,7 @@ def spawn( memory_mb: int | None = None, vcpus: int | None = None, ttl: str | None = None, + owner_id: str | None = None, template: str | None = None, metadata: dict[str, str] | None = None, ) -> Sandbox: @@ -59,6 +60,8 @@ def spawn( body["vcpus"] = vcpus if ttl: body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id if template: body["template"] = template if metadata: @@ -73,6 +76,38 @@ def spawn( data = resp.json() return Sandbox(self._http, data["id"], info=data) + def admission( + self, + image: str | None = None, + provider: str | None = None, + memory_mb: int | None = None, + vcpus: int | None = None, + ttl: str | None = None, + owner_id: str | None = None, + metadata: dict[str, str] | None = None, + ) -> SpawnAdmissionDecision: + """Preflight a spawn request without creating a sandbox.""" + body: dict = {} + if image: + body["image"] = image + if provider: + body["provider"] = provider + if memory_mb: + body["memory_mb"] = memory_mb + if vcpus: + body["vcpus"] = vcpus + if ttl: + body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id + if metadata: + body["metadata"] = metadata + + resp = self._http.post("/api/v1/sandboxes/admission", json=body) + handle_response(resp) + data = resp.json() + return SpawnAdmissionDecision(**data) + def get(self, sandbox_id: str) -> Sandbox: """Get an existing sandbox by ID.""" resp = self._http.get(f"/api/v1/sandboxes/{sandbox_id}") @@ -122,6 +157,12 @@ def pool_status(self) -> dict: handle_response(resp) return resp.json() + def quota_summary(self) -> QuotaSummary: + """Get redacted quota policy coverage counts.""" + resp = self._http.get("/api/v1/quotas/summary") + handle_response(resp) + return QuotaSummary(**resp.json()) + def health(self) -> dict: """Check server health.""" resp = self._http.get("/api/v1/health") diff --git a/sdk/python/stacyvm/models.py b/sdk/python/stacyvm/models.py index 6c70abd..f438386 100644 --- a/sdk/python/stacyvm/models.py +++ b/sdk/python/stacyvm/models.py @@ -31,6 +31,30 @@ class SandboxInfo: preview_domain: str = "localhost" +@dataclass +class SpawnAdmissionDecision: + """Admission result for a spawn preflight request.""" + + allowed: bool + queueable: bool + reason: str = "" + active_sandboxes: int = 0 + max_sandboxes: int = 0 + active_owner_sandboxes: int = 0 + max_owner_sandboxes: int = 0 + max_ttl: str = "" + + +@dataclass +class QuotaSummary: + """Redacted quota policy coverage counts.""" + + total: int = 0 + with_max_sandboxes: int = 0 + with_max_ttl: int = 0 + with_max_exec_timeout: int = 0 + + @dataclass class Template: """Sandbox template configuration.""" From bea5493dfe87005aa3e768b4057633596ca8435e Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 08:20:42 +0530 Subject: [PATCH 030/147] fix: support string sdk client config --- sdk/js/README.md | 1 + sdk/js/src/client.ts | 7 +++++-- sdk/js/src/index.ts | 1 + sdk/js/src/types.ts | 5 +++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/js/README.md b/sdk/js/README.md index d806f13..9a80078 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -384,6 +384,7 @@ import { VMPoolStatus, SpawnAdmissionDecision, QuotaSummary, + ForgevmClientConfig, ForgevmClientOptions, } from "stacyvm"; ``` diff --git a/sdk/js/src/client.ts b/sdk/js/src/client.ts index 8f094e8..7797622 100644 --- a/sdk/js/src/client.ts +++ b/sdk/js/src/client.ts @@ -11,6 +11,7 @@ import { import { Sandbox } from "./sandbox.js"; import { TemplateManager } from "./templates.js"; import type { + ForgevmClientConfig, ForgevmClientOptions, HealthInfo, ProviderInfo, @@ -114,8 +115,10 @@ export class Client { * }); * ``` */ - constructor(options?: ForgevmClientOptions) { - const opts = options ?? {}; + constructor(options?: ForgevmClientConfig) { + const opts: ForgevmClientOptions = typeof options === "string" + ? { baseUrl: options } + : options ?? {}; if (opts.baseUrl) { // Strip trailing slash for consistent URL construction. diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 8e422ef..f1ffe58 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -51,6 +51,7 @@ export type { SpawnAdmissionDecision, StreamChunk, FileInfo, + ForgevmClientConfig, ForgevmClientOptions, ApiErrorBody, } from "./types.js"; diff --git a/sdk/js/src/types.ts b/sdk/js/src/types.ts index 45d5a99..0f3fdd4 100644 --- a/sdk/js/src/types.ts +++ b/sdk/js/src/types.ts @@ -300,6 +300,11 @@ export interface ForgevmClientOptions { timeout?: number; } +/** + * Client constructor input: either a full base URL string or an options object. + */ +export type ForgevmClientConfig = string | ForgevmClientOptions; + /** * VM pool status information. */ From 209af80bd8be74b675ce6f1c7b0a7032f8e55d0f Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 08:32:05 +0530 Subject: [PATCH 031/147] ci: add core verification workflow --- .github/workflows/ci.yml | 111 +++++++++++++++++++++++++++++++++++++++ scripts/check-swagger.sh | 28 ++++++++++ 2 files changed, 139 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100755 scripts/check-swagger.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4a874ed --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,111 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + - 'phase-*' + - 'feat/*' + +permissions: + contents: read + +jobs: + go: + name: Go tests and build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run Go tests + run: make test + + - name: Build CLI + run: make build + + swagger: + name: Swagger drift + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Check generated Swagger docs + run: scripts/check-swagger.sh + + web: + name: Web build + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: web/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build web app + run: npm run build + + sdk-js: + name: TypeScript SDK build + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/js + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build TypeScript SDK + run: bun run build + + sdk-python: + name: Python SDK import check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python SDK + run: python -m pip install -e sdk/python + + - name: Compile Python SDK + run: python -m compileall sdk/python/stacyvm + + - name: Import Python SDK + run: python -c "import stacyvm; print(stacyvm.__version__)" diff --git a/scripts/check-swagger.sh b/scripts/check-swagger.sh new file mode 100755 index 0000000..edf6be9 --- /dev/null +++ b/scripts/check-swagger.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_ROOT="$(mktemp -d)" +TMP_DIR="$TMP_ROOT/docs" +cleanup() { + rm -rf "$TMP_ROOT" +} +trap cleanup EXIT + +cd "$ROOT" + +go run github.com/swaggo/swag/cmd/swag@v1.16.4 init \ + -g internal/api/server.go \ + -o "$TMP_DIR" \ + --parseDependency + +for file in docs.go swagger.json swagger.yaml; do + if ! diff -u "docs/$file" "$TMP_DIR/$file"; then + echo + echo "Swagger docs are stale. Regenerate with:" + echo " go run github.com/swaggo/swag/cmd/swag@v1.16.4 init -g internal/api/server.go -o docs --parseDependency" + exit 1 + fi +done + +echo "Swagger docs are up to date." From 289623b51e1662882b38fffaafe813654b765293 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 08:39:32 +0530 Subject: [PATCH 032/147] ci: stabilize swagger drift check --- scripts/check-swagger.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/check-swagger.sh b/scripts/check-swagger.sh index edf6be9..d5f403c 100755 --- a/scripts/check-swagger.sh +++ b/scripts/check-swagger.sh @@ -11,6 +11,8 @@ trap cleanup EXIT cd "$ROOT" +go mod download + go run github.com/swaggo/swag/cmd/swag@v1.16.4 init \ -g internal/api/server.go \ -o "$TMP_DIR" \ From 0f83e75488d53da9016bddb457f4d17a65234e29 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 08:48:41 +0530 Subject: [PATCH 033/147] docs: add production deployment guide --- CHANGELOG.md | 27 ++++ README.md | 8 ++ deploy/.env.example | 7 ++ deploy/docker-compose.yml | 47 +++++++ deploy/stacyvm.env.example | 7 ++ deploy/stacyvm.production.yaml | 99 +++++++++++++++ deploy/stacyvm.service | 29 +++++ docs/deployment.md | 117 ++++++++++++++++++ .../releases/phase-4-production-deployment.md | 106 ++++++++++++++++ 9 files changed, 447 insertions(+) create mode 100644 deploy/.env.example create mode 100644 deploy/docker-compose.yml create mode 100644 deploy/stacyvm.env.example create mode 100644 deploy/stacyvm.production.yaml create mode 100644 deploy/stacyvm.service create mode 100644 docs/deployment.md create mode 100644 docs/releases/phase-4-production-deployment.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3765365..94d25fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## Phase 4 Production Deployment - 2026-05-08 + +This checkpoint adds the first production deployment and verification surface for Phase 4: GitHub Actions CI, deployment templates, and an operator runbook. + +### Added + +- GitHub Actions workflow for Go tests/build, Swagger drift, web build, TypeScript SDK build, and Python SDK import checks. +- Production Docker Compose template with StacyVM and Traefik for live previews. +- Production baseline config with auth, rate limiting, sandbox caps, queueing, JSON logs, and persistent SQLite state. +- systemd unit and environment template for binary-based Linux installs. +- Deployment guide covering host requirements, health probes, Prometheus metrics, reverse proxy setup, backups, upgrades, and provider notes. +- Phase 4 release notes under `docs/releases/phase-4-production-deployment.md`. + +### Changed + +- Swagger drift checks now download Go modules before invoking `swag`, which makes cold CI runners reliable. +- README navigation now links to the production deployment guide. + +### Verified + +- `docker compose --env-file deploy/.env.example -f deploy/docker-compose.yml config` +- YAML parsing for deployment templates +- `git diff --check` +- `go test ./...` +- `cd web && npm run build` +- `scripts/check-swagger.sh` + ## Phase 3 Quotas And Scheduling - 2026-05-08 This checkpoint adds the first production multi-tenant control plane: persisted owner quotas, API rate limiting, spawn backpressure, scheduler visibility, admission preflight, and SDK helpers. diff --git a/README.md b/README.md index fbe3fc1..7185516 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Self-hosted. Single binary. Python & TypeScript SDKs. MIT licensed. No cloud Providers • Live Preview • Pool Mode • + Deployment • API Reference • Contributing

@@ -56,6 +57,7 @@ Self-hosted. Single binary. Python & TypeScript SDKs. MIT licensed. No cloud - [Providers, pool, system](#providers-pool-system) - [CLI](#cli) - [Configuration](#configuration) +- [Production deployment](#production-deployment) - [Templates](#templates-1) - [Security defaults](#security-defaults) - [Architecture](#architecture) @@ -558,6 +560,12 @@ STACYVM_LOGGING_LEVEL=debug --- +## Production deployment + +Use [docs/deployment.md](docs/deployment.md) for production setup guidance, including Docker Compose and systemd templates, auth and rate-limit defaults, health/readiness probes, Prometheus scraping, backup steps, and provider-specific rollout notes. The reusable templates live under [`deploy/`](deploy/). + +--- + ## Templates Templates are pre-baked sandbox specs stored server-side. Define once, spawn many times. diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..f9bc17b --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,7 @@ +STACYVM_IMAGE=ghcr.io/stacyos/stacyvm:latest +STACYVM_HOST_PORT=7423 +STACYVM_API_KEY=change-me-generate-at-least-32-bytes +STACYVM_PREVIEW_DOMAIN=localhost +STACYVM_LOG_LEVEL=info +STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db +STACYVM_DOCKER_SOCKET=/var/run/docker.sock diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..e0aca62 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,47 @@ +services: + stacyvm: + image: ${STACYVM_IMAGE:-ghcr.io/stacyos/stacyvm:latest} + restart: unless-stopped + working_dir: /etc/stacyvm + ports: + - "${STACYVM_HOST_PORT:-7423}:7423" + volumes: + - stacyvm-data:/var/lib/stacyvm + - ${STACYVM_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock + - ./stacyvm.production.yaml:/etc/stacyvm/stacyvm.yaml:ro + environment: + STACYVM_AUTH_API_KEY: ${STACYVM_API_KEY:?set STACYVM_API_KEY} + STACYVM_DATABASE_PATH: ${STACYVM_DATABASE_PATH:-/var/lib/stacyvm/stacyvm.db} + STACYVM_LOGGING_LEVEL: ${STACYVM_LOG_LEVEL:-info} + STACYVM_SERVER_PREVIEW_DOMAIN: ${STACYVM_PREVIEW_DOMAIN:-localhost} + STACYVM_PROVIDERS_DOCKER_NETWORK_MODE: stacyvm-network + networks: + - stacyvm-network + healthcheck: + test: ["CMD-SHELL", "wget -qO- --header=\"X-API-Key: $${STACYVM_AUTH_API_KEY}\" http://127.0.0.1:7423/api/v1/live >/dev/null"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + + traefik: + image: traefik:v3.6 + restart: unless-stopped + command: + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + ports: + - "80:80" + volumes: + - ${STACYVM_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock:ro + networks: + - stacyvm-network + +volumes: + stacyvm-data: + +networks: + stacyvm-network: + name: stacyvm-network + driver: bridge diff --git a/deploy/stacyvm.env.example b/deploy/stacyvm.env.example new file mode 100644 index 0000000..80e386a --- /dev/null +++ b/deploy/stacyvm.env.example @@ -0,0 +1,7 @@ +STACYVM_AUTH_API_KEY=change-me-generate-at-least-32-bytes +STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db +STACYVM_LOGGING_LEVEL=info +STACYVM_LOGGING_FORMAT=json +STACYVM_SERVER_PREVIEW_DOMAIN=localhost +STACYVM_PROVIDERS_DEFAULT=docker +STACYVM_PROVIDERS_DOCKER_NETWORK_MODE=bridge diff --git a/deploy/stacyvm.production.yaml b/deploy/stacyvm.production.yaml new file mode 100644 index 0000000..73fada3 --- /dev/null +++ b/deploy/stacyvm.production.yaml @@ -0,0 +1,99 @@ +server: + host: "0.0.0.0" + port: 7423 + preview_domain: "localhost" + +providers: + default: "docker" + + docker: + enabled: true + socket: "unix:///var/run/docker.sock" + runtime: "runc" + default_image: "alpine:latest" + network_mode: "stacyvm-network" + seccomp_profile: "default" + read_only_rootfs: false + memory: "512m" + cpus: "1" + pids_limit: 256 + dropped_caps: ["ALL"] + added_caps: [] + pool_security: + per_user_uid: false + pid_namespace: false + workspace_permissions: true + hidepid: false + + firecracker: + enabled: false + firecracker_path: "/usr/local/bin/firecracker" + kernel_path: "/var/lib/stacyvm/vmlinux.bin" + agent_path: "/usr/local/bin/stacyvm-agent" + data_dir: "/var/lib/stacyvm" + + e2b: + enabled: false + api_key: "" + base_url: "https://api.e2b.dev" + + custom: + enabled: false + name: "custom" + base_url: "" + api_key: "" + timeout: "60s" + + proot: + enabled: false + rootfs_path: "/var/lib/stacyvm/rootfs" + proot_binary: "proot" + workspace_base: "/var/lib/stacyvm/workspaces" + default_timeout: "60s" + max_sandboxes: 10 + max_memory_mb: 512 + max_disk_mb: 1024 + languages: ["python3", "node", "bash"] + +defaults: + ttl: "30m" + image: "alpine:latest" + memory_mb: 1024 + vcpus: 1 + disk_size_mb: 1024 + max_ttl: "24h" + default_exec_timeout: "30s" + max_exec_timeout: "10m" + max_sandboxes: 100 + max_sandboxes_per_owner: 10 + spawn_overflow: "queue" + spawn_queue_timeout: "30s" + max_spawn_queue: 100 + +auth: + enabled: true + api_key: "change-me-generate-at-least-32-bytes" + +rate_limit: + enabled: true + requests_per_minute: 120 + burst: 60 + key_by: "api_key" + bucket_ttl: "15m" + cleanup_interval: "1m" + +database: + path: "/var/lib/stacyvm/stacyvm.db" + +logging: + level: "info" + format: "json" + +pool: + enabled: false + max_vms: 10 + max_users_per_vm: 5 + image: "alpine:latest" + memory_mb: 2048 + vcpus: 2 + overflow: "reject" diff --git a/deploy/stacyvm.service b/deploy/stacyvm.service new file mode 100644 index 0000000..9fda38f --- /dev/null +++ b/deploy/stacyvm.service @@ -0,0 +1,29 @@ +[Unit] +Description=StacyVM sandbox API +Documentation=https://github.com/StacyOS/stacyvm +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +User=stacyvm +Group=stacyvm +SupplementaryGroups=docker +WorkingDirectory=/etc/stacyvm +EnvironmentFile=-/etc/stacyvm/stacyvm.env +Environment=STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db +ExecStart=/usr/local/bin/stacyvm serve +Restart=on-failure +RestartSec=5s +TimeoutStopSec=30s + +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/stacyvm +ReadOnlyPaths=/etc/stacyvm + +[Install] +WantedBy=multi-user.target diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..a693018 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,117 @@ +# Production Deployment + +This guide covers a single-node StacyVM deployment suitable for an internal service, staging, or a small production installation. The default production path uses the Docker provider because it works on the broadest set of hosts; Firecracker and PRoot require extra host setup and should be validated on the target platform before rollout. + +## Requirements + +- Linux host with Docker installed when using the Docker provider. +- A persistent data directory, normally `/var/lib/stacyvm`. +- A generated API key with at least 32 bytes of entropy. +- TLS and public ingress handled by a reverse proxy or load balancer in front of StacyVM. +- Health checks wired to the API endpoints listed below. + +StacyVM reads config from `./stacyvm.yaml`, then `~/.stacyvm/config.yaml`, then `STACYVM_` environment variables. In production, prefer a checked-in baseline config plus environment variables for secrets and environment-specific values. + +## Health and Metrics + +Use these endpoints for load balancers and monitors: + +| Endpoint | Purpose | +|---|---| +| `GET /api/v1/live` | Process liveness. Use this for simple restart checks. | +| `GET /api/v1/ready` | Readiness. Use this before routing traffic after deploys. | +| `GET /api/v1/health` | Dependency and provider health summary. | +| `GET /api/v1/metrics/prometheus` | Prometheus metrics scrape endpoint. | + +Authenticated deployments should send `X-API-Key: ` to protected API endpoints. Keep health probes scoped to your private network if they bypass auth at an upstream proxy. + +## Docker Compose + +The files in `deploy/` provide a production-oriented Compose starting point: + +- `deploy/docker-compose.yml` starts StacyVM and Traefik for live previews. +- `deploy/stacyvm.production.yaml` enables auth, rate limiting, sandbox caps, queueing, JSON logs, and persistent SQLite state. +- `deploy/.env.example` lists the environment variables expected by the Compose file. +- `deploy/stacyvm.env.example` is the systemd environment file template. + +```bash +cd deploy +cp .env.example .env +# Edit .env and replace STACYVM_API_KEY before starting. +docker compose up -d +docker compose logs -f stacyvm +``` + +For local image testing before a registry image exists: + +```bash +docker build -t stacyvm:local .. +STACYVM_IMAGE=stacyvm:local docker compose up -d +``` + +## systemd + +Use `deploy/stacyvm.service` when running the binary directly on a Linux host. + +```bash +sudo useradd --system --home /var/lib/stacyvm --shell /usr/sbin/nologin stacyvm +sudo usermod -aG docker stacyvm +sudo install -d -o stacyvm -g stacyvm /var/lib/stacyvm +sudo install -d -m 0750 /etc/stacyvm +sudo install -o root -g stacyvm -m 0640 deploy/stacyvm.production.yaml /etc/stacyvm/stacyvm.yaml +sudo install -o root -g stacyvm -m 0640 deploy/stacyvm.env.example /etc/stacyvm/stacyvm.env +sudo install -m 0755 bin/stacyvm /usr/local/bin/stacyvm +sudo install -m 0755 bin/stacyvm-agent /usr/local/bin/stacyvm-agent +sudo install -m 0644 deploy/stacyvm.service /etc/systemd/system/stacyvm.service +``` + +Edit `/etc/stacyvm/stacyvm.env` and set a real `STACYVM_AUTH_API_KEY`. Then enable the service: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now stacyvm +sudo systemctl status stacyvm +``` + +The included unit uses `WorkingDirectory=/etc/stacyvm` so StacyVM can load `/etc/stacyvm/stacyvm.yaml` through its current `./stacyvm.yaml` lookup path while keeping persistent database state in `/var/lib/stacyvm`. + +## Reverse Proxy + +Terminate TLS before StacyVM. A typical proxy should: + +- Forward API traffic to `http://127.0.0.1:7423`. +- Preserve `X-API-Key` headers. +- Route live preview hostnames such as `3000-sb-.` to Traefik when using Docker live previews. +- Restrict admin and metrics endpoints to trusted networks. + +Set `server.preview_domain` or `STACYVM_SERVER_PREVIEW_DOMAIN` to the domain that resolves preview subdomains to your proxy. + +## Backups + +The default store is SQLite at `/var/lib/stacyvm/stacyvm.db`. For a consistent backup: + +```bash +sudo systemctl stop stacyvm +sudo cp /var/lib/stacyvm/stacyvm.db /backup/stacyvm.db +sudo cp /var/lib/stacyvm/stacyvm.db-wal /backup/ 2>/dev/null || true +sudo cp /var/lib/stacyvm/stacyvm.db-shm /backup/ 2>/dev/null || true +sudo systemctl start stacyvm +``` + +If you run with Docker Compose, stop the service or snapshot the backing volume with your volume provider's backup tooling. + +## Upgrades + +1. Check the release notes for config or API changes. +2. Back up `/var/lib/stacyvm/stacyvm.db`. +3. Replace the binary or update `STACYVM_IMAGE`. +4. Restart the service. +5. Confirm `GET /api/v1/ready` succeeds before routing traffic. + +## Provider Notes + +Docker is the safest default for broad deployment compatibility. For stronger isolation, run Docker with gVisor (`runtime: "runsc"`) or Kata after validating the runtime on the host. + +Firecracker requires Linux/KVM, a kernel image, rootfs images, networking setup, and the `stacyvm-agent` binary available to the runtime. Keep Firecracker disabled in shared templates until a host conformance check passes. + +PRoot requires a real rootfs with the binaries your sandboxes need. Use it for restricted environments where Docker and KVM are unavailable, and validate memory/disk limits against the host because PRoot enforcement is not equivalent to VM isolation. diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md new file mode 100644 index 0000000..9081bca --- /dev/null +++ b/docs/releases/phase-4-production-deployment.md @@ -0,0 +1,106 @@ +# Phase 4 Production Deployment Release Notes + +Date: 2026-05-08 +Branch: `phase-4-production-deployment` + +## Summary + +Phase 4 starts turning the production control-plane work from earlier phases into repeatable shipping and deployment workflows. This checkpoint adds GitHub Actions CI coverage plus operator-facing deployment templates and runbooks for single-node production installations. + +The goal of this phase is to make StacyVM easier to validate, release, and run outside a developer laptop while keeping host-specific runtime conformance explicit. + +## What Changed + +### Continuous Integration + +- Added a GitHub Actions workflow for core project verification. +- CI now validates: + - Go tests across the repository. + - CLI build. + - Swagger/OpenAPI drift. + - Web dashboard production build. + - TypeScript SDK build. + - Python SDK package install, compile, and import. +- Stabilized the Swagger drift check for cold CI runners by downloading Go modules before invoking `swag`. + +### Production Deployment Templates + +- Added `deploy/docker-compose.yml` for a production-oriented Docker provider deployment with Traefik live-preview routing. +- Added `deploy/stacyvm.production.yaml` with production defaults for: + - API auth. + - API rate limiting. + - sandbox caps and queue backpressure. + - JSON logging. + - persistent SQLite state. + - Docker as the default provider. +- Added Compose and systemd environment templates: + - `deploy/.env.example` + - `deploy/stacyvm.env.example` +- Added `deploy/stacyvm.service` for binary-based Linux/systemd installs. + +### Deployment Runbook + +- Added `docs/deployment.md` covering: + - host requirements. + - Docker Compose deployment. + - systemd deployment. + - health, readiness, liveness, and Prometheus endpoints. + - reverse proxy expectations. + - SQLite backup and restore basics. + - upgrade procedure. + - Docker, Firecracker, and PRoot provider notes. +- Linked the deployment guide from the README. + +## Code Changes By Area + +### CI + +- `.github/workflows/ci.yml` + - Adds repository verification jobs for Go, Swagger, web, and SDKs. +- `scripts/check-swagger.sh` + - Downloads modules before generating docs in a temporary workspace. + +### Deployment + +- `deploy/docker-compose.yml` + - Adds a reusable production Compose template. +- `deploy/stacyvm.production.yaml` + - Adds a production baseline config. +- `deploy/stacyvm.service` + - Adds a systemd unit for running the StacyVM binary. +- `deploy/.env.example` and `deploy/stacyvm.env.example` + - Add environment templates for Compose and systemd. + +### Docs + +- `docs/deployment.md` + - Adds the deployment guide and operator runbook. +- `README.md` + - Links the deployment guide from navigation and configuration docs. +- `CHANGELOG.md` + - Adds this Phase 4 checkpoint entry. + +## Verification + +The following checks passed during this checkpoint: + +```sh +docker compose --env-file deploy/.env.example -f deploy/docker-compose.yml config +ruby -e 'require "yaml"; YAML.load_file("deploy/docker-compose.yml"); YAML.load_file("deploy/stacyvm.production.yaml")' +git diff --check +go test ./... +cd web && npm run build +scripts/check-swagger.sh +``` + +GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swagger drift check stabilization. + +## Platform Notes + +- Docker Compose validation does not require daemon access, but runtime sandbox conformance still requires Docker daemon access on the host. +- Firecracker remains Linux/KVM-gated and should be rolled out only after host conformance checks pass. +- PRoot remains gated on a real `proot` binary and a rootfs with the expected sandbox tooling. + +## Next Phase 4 Direction + +Remaining Phase 4 work should focus on release automation, container publishing, deployment smoke tests, and a clearer production conformance matrix for Docker, gVisor/Kata, Firecracker, and PRoot hosts. From 892d3760c0b9ccca45e4b40093d13a222266e863 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 09:04:41 +0530 Subject: [PATCH 034/147] ci: add release publishing workflow --- .dockerignore | 22 +++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 148 ++++++++++++++++++ CHANGELOG.md | 7 + Dockerfile | 10 +- Makefile | 16 +- README.md | 2 + .../releases/phase-4-production-deployment.md | 21 +++ docs/releasing.md | 81 ++++++++++ 9 files changed, 300 insertions(+), 10 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/release.yml create mode 100644 docs/releasing.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a6e4f69 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +.git +.github +.codex +dist +bin +stacyvm +stacyvm-agent +checksums.txt + +web/node_modules +web/dist +sdk/js/node_modules + +images/evm/contracts/lib +images/evm/frontend/node_modules +images/evm/frontend/.next + +*.log +*.db +*.db-shm +*.db-wal +.env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a874ed..df83a3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: go: name: Go tests and build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e875138 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,148 @@ +name: Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Release version or image tag, for example v0.4.0' + required: true + type: string + publish_image: + description: 'Publish the container image to GHCR' + required: true + default: true + type: boolean + create_release: + description: 'Create a GitHub release with binary artifacts' + required: true + default: false + type: boolean + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + IMAGE_NAME: ghcr.io/stacyos/stacyvm + +permissions: + contents: write + packages: write + +jobs: + release-artifacts: + name: Build release artifacts + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Resolve release version + id: version + shell: bash + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT" + else + echo "value=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + fi + + - name: Build binaries + run: make release-build-all VERSION="${{ steps.version.outputs.value }}" + + - name: Upload release artifacts + uses: actions/upload-artifact@v4 + with: + name: stacyvm-release-${{ steps.version.outputs.value }} + path: dist/* + if-no-files-found: error + + github-release: + name: Create GitHub release + needs: release-artifacts + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') || inputs.create_release == true + steps: + - name: Download release artifacts + uses: actions/download-artifact@v4 + with: + path: release-assets + + - name: Resolve release version + id: version + shell: bash + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT" + else + echo "value=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + fi + + - name: Create release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ steps.version.outputs.value }}" release-assets/**/* \ + --title "${{ steps.version.outputs.value }}" \ + --generate-notes + + container-image: + name: Publish container image + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') || inputs.publish_image == true + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve image version + id: version + shell: bash + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT" + else + echo "value=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ steps.version.outputs.value }} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + + - name: Build and publish image + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + VERSION=${{ steps.version.outputs.value }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 94d25fb..4e6df98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,17 @@ This checkpoint adds the first production deployment and verification surface fo - Production baseline config with auth, rate limiting, sandbox caps, queueing, JSON logs, and persistent SQLite state. - systemd unit and environment template for binary-based Linux installs. - Deployment guide covering host requirements, health probes, Prometheus metrics, reverse proxy setup, backups, upgrades, and provider notes. +- Release workflow for GitHub releases and GHCR container image publishing. +- Release runbook documenting tags, manual dispatch, binary artifacts, image tags, and preflight checks. +- `.dockerignore` for smaller and safer Docker build contexts. - Phase 4 release notes under `docs/releases/phase-4-production-deployment.md`. ### Changed - Swagger drift checks now download Go modules before invoking `swag`, which makes cold CI runners reliable. +- CI opts into Node 24-based JavaScript actions to address the GitHub Actions Node 20 deprecation warning. +- Docker image builds now accept an explicit `VERSION` build argument and BuildKit target platform args. +- Release artifacts now build into `dist/` with checksums instead of the repository root. - README navigation now links to the production deployment guide. ### Verified @@ -26,6 +32,7 @@ This checkpoint adds the first production deployment and verification surface fo - `go test ./...` - `cd web && npm run build` - `scripts/check-swagger.sh` +- `make release-build-all VERSION=phase-4-test` ## Phase 3 Quotas And Scheduling - 2026-05-08 diff --git a/Dockerfile b/Dockerfile index 306fcdc..cda445c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,15 @@ -FROM golang:1.25-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(git describe --tags --always 2>/dev/null || echo dev)" -o /stacyvm ./cmd/stacyvm -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o /stacyvm-agent ./cmd/stacyvm-agent +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -ldflags="-s -w -X main.version=$VERSION" -o /stacyvm ./cmd/stacyvm +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -ldflags="-s -w" -o /stacyvm-agent ./cmd/stacyvm-agent FROM alpine:3.20 diff --git a/Makefile b/Makefile index 4a4eac6..9651da1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: build build-agent build-android build-agent-arm64 test lint clean serve dev release-build release-build-all VERSION ?= $(shell git describe --tags --always 2>/dev/null || echo dev) +DIST_DIR ?= dist # Build the server/CLI binary build: @@ -40,7 +41,7 @@ serve: build # Clean build artifacts clean: rm -f stacyvm stacyvm-agent - rm -rf bin/ web/dist/ + rm -rf bin/ web/dist/ $(DIST_DIR)/ # Run go vet lint: @@ -48,12 +49,13 @@ lint: # Build static release binaries + checksums (amd64 only) release-build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o stacyvm-linux-amd64 ./cmd/stacyvm - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o stacyvm-agent-linux-amd64 ./cmd/stacyvm-agent - sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 > checksums.txt + mkdir -p $(DIST_DIR) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(DIST_DIR)/stacyvm-linux-amd64 ./cmd/stacyvm + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(DIST_DIR)/stacyvm-agent-linux-amd64 ./cmd/stacyvm-agent + cd $(DIST_DIR) && sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 > checksums.txt # Build release binaries for all architectures (amd64 + arm64) release-build-all: release-build - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o stacyvm-linux-arm64 ./cmd/stacyvm - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o stacyvm-agent-linux-arm64 ./cmd/stacyvm-agent - sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 stacyvm-linux-arm64 stacyvm-agent-linux-arm64 > checksums.txt + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(DIST_DIR)/stacyvm-linux-arm64 ./cmd/stacyvm + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o $(DIST_DIR)/stacyvm-agent-linux-arm64 ./cmd/stacyvm-agent + cd $(DIST_DIR) && sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 stacyvm-linux-arm64 stacyvm-agent-linux-arm64 > checksums.txt diff --git a/README.md b/README.md index 7185516..e3b333a 100644 --- a/README.md +++ b/README.md @@ -564,6 +564,8 @@ STACYVM_LOGGING_LEVEL=debug Use [docs/deployment.md](docs/deployment.md) for production setup guidance, including Docker Compose and systemd templates, auth and rate-limit defaults, health/readiness probes, Prometheus scraping, backup steps, and provider-specific rollout notes. The reusable templates live under [`deploy/`](deploy/). +Release automation and GHCR publishing are documented in [docs/releasing.md](docs/releasing.md). + --- ## Templates diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md index 9081bca..debb34d 100644 --- a/docs/releases/phase-4-production-deployment.md +++ b/docs/releases/phase-4-production-deployment.md @@ -38,6 +38,15 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - `deploy/stacyvm.env.example` - Added `deploy/stacyvm.service` for binary-based Linux/systemd installs. +### Release Automation + +- Added a release workflow for tag-driven and manually-dispatched releases. +- Release automation builds static Linux binary artifacts for `amd64` and `arm64`. +- Release automation publishes multi-arch container images to `ghcr.io/stacyos/stacyvm`. +- Docker image builds now accept an explicit `VERSION` build argument. +- Release artifacts now build into `dist/` instead of the repository root. +- Added `.dockerignore` to keep local build outputs and dependency directories out of release image contexts. + ### Deployment Runbook - Added `docs/deployment.md` covering: @@ -57,6 +66,9 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - `.github/workflows/ci.yml` - Adds repository verification jobs for Go, Swagger, web, and SDKs. + - Opts into Node 24-based JavaScript actions to address the GitHub Actions Node 20 deprecation warning. +- `.github/workflows/release.yml` + - Adds binary and container image release automation. - `scripts/check-swagger.sh` - Downloads modules before generating docs in a temporary workspace. @@ -70,11 +82,19 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - Adds a systemd unit for running the StacyVM binary. - `deploy/.env.example` and `deploy/stacyvm.env.example` - Add environment templates for Compose and systemd. +- `Dockerfile` + - Adds BuildKit platform args and explicit version injection for release image publishing. +- `Makefile` + - Moves release artifacts into `dist/` and keeps checksums with the artifacts. +- `.dockerignore` + - Excludes build outputs, local dependency directories, and local state files from Docker build contexts. ### Docs - `docs/deployment.md` - Adds the deployment guide and operator runbook. +- `docs/releasing.md` + - Adds release workflow and GHCR publishing instructions. - `README.md` - Links the deployment guide from navigation and configuration docs. - `CHANGELOG.md` @@ -91,6 +111,7 @@ git diff --check go test ./... cd web && npm run build scripts/check-swagger.sh +make release-build-all VERSION=phase-4-test ``` GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swagger drift check stabilization. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..a85146b --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,81 @@ +# Releasing StacyVM + +StacyVM releases publish two deliverables: + +- Static Linux binaries for `stacyvm` and `stacyvm-agent` under the GitHub release. +- A multi-arch container image at `ghcr.io/stacyos/stacyvm`. + +## Release Workflow + +The release workflow lives at `.github/workflows/release.yml`. + +It runs automatically for tags that match `v*`: + +```bash +git tag v0.4.0 +git push origin v0.4.0 +``` + +It can also be started manually from GitHub Actions with: + +- `version`: release version or image tag, for example `v0.4.0`. +- `publish_image`: whether to publish the GHCR image. +- `create_release`: whether to create a GitHub release with binary artifacts. + +Tag-triggered releases always build binaries, create the GitHub release, and publish the container image. + +## Binary Artifacts + +Local release artifacts can be built with: + +```bash +make release-build-all VERSION=v0.4.0 +``` + +The command writes artifacts to `dist/`: + +- `stacyvm-linux-amd64` +- `stacyvm-agent-linux-amd64` +- `stacyvm-linux-arm64` +- `stacyvm-agent-linux-arm64` +- `checksums.txt` + +## Container Image + +The release workflow publishes: + +- `ghcr.io/stacyos/stacyvm:` +- `ghcr.io/stacyos/stacyvm:latest` for `v*` tag releases + +The Dockerfile accepts a `VERSION` build argument and uses BuildKit target platform args so the release workflow can publish `linux/amd64` and `linux/arm64` images from one workflow. + +To test the image locally before publishing: + +```bash +docker build --build-arg VERSION=dev -t stacyvm:dev . +docker run --rm stacyvm:dev version +``` + +## Preflight Checklist + +Before tagging: + +```bash +make test +make build +cd web && npm run build +scripts/check-swagger.sh +make release-build-all VERSION=v0.4.0 +``` + +For Phase 4, also confirm the production deployment templates still render: + +```bash +docker compose --env-file deploy/.env.example -f deploy/docker-compose.yml config +``` + +## Notes + +- Do not store release secrets in `stacyvm.production.yaml`; pass them through environment variables. +- Keep release notes in `docs/releases/` up to date before creating a GitHub release. +- Platform conformance for Docker, gVisor/Kata, Firecracker, and PRoot remains host-gated and should be reported separately from generic build health. From 1a6d142d718bf0b2df5eaef742f23e0e764b5022 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 09:14:12 +0530 Subject: [PATCH 035/147] ops: add deployment smoke checks --- CHANGELOG.md | 3 + README.md | 2 +- cmd/stacyvm/cmd_serve.go | 5 + docs/deployment.md | 8 + .../releases/phase-4-production-deployment.md | 10 + docs/runtime-conformance.md | 188 ++++++++++++++++++ scripts/smoke-deployment.sh | 55 +++++ 7 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 docs/runtime-conformance.md create mode 100755 scripts/smoke-deployment.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6df98..04c6ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ This checkpoint adds the first production deployment and verification surface fo - Release workflow for GitHub releases and GHCR container image publishing. - Release runbook documenting tags, manual dispatch, binary artifacts, image tags, and preflight checks. - `.dockerignore` for smaller and safer Docker build contexts. +- Deployment smoke script for live, health, readiness, and Prometheus probes. +- Runtime conformance matrix for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. - Phase 4 release notes under `docs/releases/phase-4-production-deployment.md`. ### Changed @@ -22,6 +24,7 @@ This checkpoint adds the first production deployment and verification surface fo - CI opts into Node 24-based JavaScript actions to address the GitHub Actions Node 20 deprecation warning. - Docker image builds now accept an explicit `VERSION` build argument and BuildKit target platform args. - Release artifacts now build into `dist/` with checksums instead of the repository root. +- `stacyvm serve` now registers the mock provider when `providers.mock.enabled` is true. - README navigation now links to the production deployment guide. ### Verified diff --git a/README.md b/README.md index e3b333a..e5b9f62 100644 --- a/README.md +++ b/README.md @@ -562,7 +562,7 @@ STACYVM_LOGGING_LEVEL=debug ## Production deployment -Use [docs/deployment.md](docs/deployment.md) for production setup guidance, including Docker Compose and systemd templates, auth and rate-limit defaults, health/readiness probes, Prometheus scraping, backup steps, and provider-specific rollout notes. The reusable templates live under [`deploy/`](deploy/). +Use [docs/deployment.md](docs/deployment.md) for production setup guidance, including Docker Compose and systemd templates, auth and rate-limit defaults, health/readiness probes, Prometheus scraping, backup steps, and provider-specific rollout notes. Runtime signoff expectations live in [docs/runtime-conformance.md](docs/runtime-conformance.md). The reusable templates live under [`deploy/`](deploy/). Release automation and GHCR publishing are documented in [docs/releasing.md](docs/releasing.md). diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index 5573745..5f3d1f7 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -58,6 +58,11 @@ func runServe() error { // Provider registry registry := providers.NewRegistry() + if cfg.Providers.Mock.Enabled { + mock := providers.NewMockProvider() + registry.Register(mock) + logger.Info().Msg("mock provider registered") + } if cfg.Providers.Firecracker.Enabled { fc := providers.NewFirecrackerProvider(providers.FirecrackerProviderConfig{ FirecrackerPath: cfg.Providers.Firecracker.FirecrackerPath, diff --git a/docs/deployment.md b/docs/deployment.md index a693018..5085698 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -25,6 +25,12 @@ Use these endpoints for load balancers and monitors: Authenticated deployments should send `X-API-Key: ` to protected API endpoints. Keep health probes scoped to your private network if they bypass auth at an upstream proxy. +After a deploy, run the smoke script: + +```bash +STACYVM_SMOKE_URL=https://stacyvm.example.com STACYVM_API_KEY=sk-live scripts/smoke-deployment.sh +``` + ## Docker Compose The files in `deploy/` provide a production-oriented Compose starting point: @@ -115,3 +121,5 @@ Docker is the safest default for broad deployment compatibility. For stronger is Firecracker requires Linux/KVM, a kernel image, rootfs images, networking setup, and the `stacyvm-agent` binary available to the runtime. Keep Firecracker disabled in shared templates until a host conformance check passes. PRoot requires a real rootfs with the binaries your sandboxes need. Use it for restricted environments where Docker and KVM are unavailable, and validate memory/disk limits against the host because PRoot enforcement is not equivalent to VM isolation. + +Use [runtime-conformance.md](runtime-conformance.md) as the signoff checklist for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md index debb34d..f9be3e4 100644 --- a/docs/releases/phase-4-production-deployment.md +++ b/docs/releases/phase-4-production-deployment.md @@ -59,6 +59,9 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - upgrade procedure. - Docker, Firecracker, and PRoot provider notes. - Linked the deployment guide from the README. +- Added `scripts/smoke-deployment.sh` for liveness, health, readiness, and Prometheus deployment probes. +- Added `docs/runtime-conformance.md` with host requirements and signoff checks for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. +- Registered the mock provider in `stacyvm serve` when `providers.mock.enabled` is set, giving operators and CI a no-Docker smoke path. ## Code Changes By Area @@ -71,6 +74,8 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - Adds binary and container image release automation. - `scripts/check-swagger.sh` - Downloads modules before generating docs in a temporary workspace. +- `cmd/stacyvm/cmd_serve.go` + - Registers the mock provider when enabled in config. ### Deployment @@ -88,6 +93,8 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - Moves release artifacts into `dist/` and keeps checksums with the artifacts. - `.dockerignore` - Excludes build outputs, local dependency directories, and local state files from Docker build contexts. +- `scripts/smoke-deployment.sh` + - Adds a portable post-deploy smoke test for live, health, readiness, and Prometheus metrics endpoints. ### Docs @@ -95,6 +102,8 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - Adds the deployment guide and operator runbook. - `docs/releasing.md` - Adds release workflow and GHCR publishing instructions. +- `docs/runtime-conformance.md` + - Adds provider/runtime production signoff expectations. - `README.md` - Links the deployment guide from navigation and configuration docs. - `CHANGELOG.md` @@ -112,6 +121,7 @@ go test ./... cd web && npm run build scripts/check-swagger.sh make release-build-all VERSION=phase-4-test +scripts/smoke-deployment.sh http://127.0.0.1:7423 ``` GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swagger drift check stabilization. diff --git a/docs/runtime-conformance.md b/docs/runtime-conformance.md new file mode 100644 index 0000000..ea398cb --- /dev/null +++ b/docs/runtime-conformance.md @@ -0,0 +1,188 @@ +# Runtime Conformance Matrix + +This matrix describes what operators should validate before treating a StacyVM runtime provider as production-ready on a host class. The shared provider contract is documented in `docs/provider-contract.md`; this guide focuses on deployment conformance. + +## Summary + +| Runtime | Host requirement | Production status | Required validation | +|---|---|---|---| +| Docker with `runc` | Docker daemon and socket access | Default broad-compatibility path | Provider health, lifecycle, exec, files, live preview, reconciliation | +| Docker with gVisor `runsc` | Docker daemon plus installed `runsc` runtime | Stronger container isolation | Same as Docker plus runtime selection and syscall compatibility | +| Docker with Kata | Docker daemon plus installed Kata runtime and virtualization support | VM-backed container isolation | Same as Docker plus nested virtualization/runtime availability | +| Firecracker | Linux, `/dev/kvm`, Firecracker binary, kernel, rootfs, networking, `stacyvm-agent` | Highest-isolation target | Full lifecycle and file/exec conformance on real Linux/KVM host | +| PRoot | `proot` binary, rootfs with expected tools, writable workspace base | Restricted-host fallback | Lifecycle, exec, files, limits, and rootfs language/tool availability | +| E2B | E2B API key and network access | Hybrid/cloud burst option | API reachability, lifecycle, exec, files, and failure mapping | +| Custom | Reachable provider HTTP service | Bring-your-own runtime | Contract conformance against the custom backend | + +## Baseline Checks + +Run these checks for every runtime: + +```bash +make test +scripts/smoke-deployment.sh http://127.0.0.1:7423 "$STACYVM_API_KEY" +curl -fsS -H "X-API-Key: $STACYVM_API_KEY" http://127.0.0.1:7423/api/v1/providers +curl -fsS -H "X-API-Key: $STACYVM_API_KEY" http://127.0.0.1:7423/api/v1/ready +``` + +For a deployed service, use `STACYVM_SMOKE_URL` instead of positional arguments: + +```bash +STACYVM_SMOKE_URL=https://stacyvm.example.com STACYVM_API_KEY=sk-live scripts/smoke-deployment.sh +``` + +## Docker + +Required host state: + +- Docker daemon is running. +- StacyVM can access the configured Docker socket. +- The sandbox network exists when `providers.docker.network_mode` is a named network. +- Traefik or another reverse proxy can reach sandbox containers for live preview. + +Recommended validation: + +```bash +docker info +docker network inspect stacyvm-network +STACYVM_PROVIDERS_DEFAULT=docker make test +``` + +Runtime behavior to verify: + +- `GET /api/v1/providers/docker` reports healthy. +- Spawn an `alpine:latest` sandbox. +- Execute `echo ok`. +- Write, read, list, move, chmod, stat, glob, and delete a file. +- Destroy the sandbox. +- Restart StacyVM and confirm orphaned StacyVM containers reconcile correctly. + +## Docker gVisor + +Required host state: + +- Docker daemon is running. +- `runsc` is installed and registered as a Docker runtime. +- StacyVM config sets `providers.docker.runtime: "runsc"`. + +Recommended validation: + +```bash +docker info | grep -A5 Runtimes +docker run --rm --runtime=runsc alpine:latest echo ok +``` + +Runtime behavior to verify: + +- Docker provider health remains healthy with `runtime=runsc`. +- Basic spawn, exec, file operations, destroy, and live preview still pass. +- Workloads that need unusual syscalls are tested explicitly because gVisor changes syscall behavior. + +## Docker Kata + +Required host state: + +- Kata runtime is installed and registered with Docker. +- Host supports the virtualization mode required by the Kata installation. +- StacyVM config sets `providers.docker.runtime` to the registered Kata runtime name. + +Recommended validation: + +```bash +docker info | grep -A5 Runtimes +docker run --rm --runtime=kata-runtime alpine:latest echo ok +``` + +Runtime behavior to verify: + +- Docker provider health remains healthy with the Kata runtime. +- Spawn, exec, file operations, destroy, and live preview pass. +- Cold-start latency and memory overhead are measured against operator SLOs. + +## Firecracker + +Required host state: + +- Linux host with `/dev/kvm` available. +- Firecracker binary installed and executable. +- Kernel image exists at `providers.firecracker.kernel_path`. +- Rootfs image exists for the requested sandbox image or template. +- `stacyvm-agent` is available at `providers.firecracker.agent_path`. +- Networking setup permits guest communication. + +Recommended validation: + +```bash +test -e /dev/kvm +firecracker --version +test -f /var/lib/stacyvm/vmlinux.bin +test -x /usr/local/bin/stacyvm-agent +``` + +Runtime behavior to verify: + +- `GET /api/v1/providers/firecracker` reports healthy. +- Full provider conformance passes on the Linux/KVM host. +- Snapshot restore paths work for prepared rootfs images. +- Destroy cleans up processes, sockets, tap devices, and temporary runtime files. +- Reconciliation correctly handles stale persisted sandboxes after a StacyVM restart. + +## PRoot + +Required host state: + +- `proot` binary is installed. +- Rootfs exists at `providers.proot.rootfs_path`. +- Workspace base is writable by the StacyVM process. +- Rootfs contains the languages and binaries advertised by `providers.proot.languages`. + +Recommended validation: + +```bash +proot --version +test -d /var/lib/stacyvm/rootfs +test -w /var/lib/stacyvm/workspaces +``` + +Runtime behavior to verify: + +- `GET /api/v1/providers/proot` reports healthy. +- Basic lifecycle, exec, and file operations pass against the real rootfs. +- Configured memory and disk caps are understood as operational controls, not VM-grade isolation. +- Rootfs language availability matches templates and SDK examples. + +## E2B And Custom Providers + +Required host state: + +- Outbound network access to the provider. +- API keys configured through environment variables or a secret manager. +- Provider-specific base URL configured. + +Runtime behavior to verify: + +- Provider health returns actionable errors when credentials or network are wrong. +- Lifecycle, exec, streaming exec, files, and destroy match `docs/provider-contract.md`. +- Provider errors map to typed StacyVM errors instead of leaking backend-specific response bodies. + +## Signoff Template + +Use this checklist before marking a runtime production-ready: + +```text +Runtime: +Host OS/kernel: +StacyVM version: +Config file: +Provider health endpoint: +Smoke script result: +Lifecycle conformance: +Exec conformance: +File conformance: +Streaming conformance: +Live preview: +Restart reconciliation: +Known host caveats: +Owner/signoff: +Date: +``` diff --git a/scripts/smoke-deployment.sh b/scripts/smoke-deployment.sh new file mode 100755 index 0000000..a58f054 --- /dev/null +++ b/scripts/smoke-deployment.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${STACYVM_SMOKE_URL:-${1:-http://127.0.0.1:7423}}" +API_KEY="${STACYVM_API_KEY:-${2:-}}" +TIMEOUT_SECONDS="${STACYVM_SMOKE_TIMEOUT:-5}" + +BASE_URL="${BASE_URL%/}" + +headers=() +if [[ -n "$API_KEY" ]]; then + headers=(-H "X-API-Key: $API_KEY") +fi + +curl_base=(curl --silent --show-error --fail --max-time "$TIMEOUT_SECONDS" "${headers[@]}") + +probe_json() { + local path="$1" + local expected="$2" + local url="$BASE_URL$path" + + printf 'Checking %s ... ' "$url" + local body + body="$("${curl_base[@]}" "$url")" + if [[ "$body" != *"$expected"* ]]; then + printf 'failed\n' + printf 'Expected response to contain: %s\n' "$expected" >&2 + printf 'Response:\n%s\n' "$body" >&2 + return 1 + fi + printf 'ok\n' +} + +probe_metrics() { + local path="/api/v1/metrics/prometheus" + local url="$BASE_URL$path" + + printf 'Checking %s ... ' "$url" + local body + body="$("${curl_base[@]}" "$url")" + if [[ "$body" != *"stacyvm_uptime_seconds"* ]]; then + printf 'failed\n' + printf 'Expected Prometheus metrics to contain stacyvm_uptime_seconds.\n' >&2 + printf 'Response:\n%s\n' "$body" >&2 + return 1 + fi + printf 'ok\n' +} + +probe_json "/api/v1/live" '"status":"alive"' +probe_json "/api/v1/health" '"status":"ok"' +probe_json "/api/v1/ready" '"status":"ready"' +probe_metrics + +printf 'StacyVM deployment smoke checks passed for %s\n' "$BASE_URL" From c62fcd226ab27796ab61426cd890bd3c32546de3 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 09:23:29 +0530 Subject: [PATCH 036/147] ci: verify deployment smoke path --- .github/workflows/ci.yml | 19 ++++++++ CHANGELOG.md | 1 + .../releases/phase-4-production-deployment.md | 4 ++ scripts/ci-smoke-deployment.sh | 44 +++++++++++++++++++ 4 files changed, 68 insertions(+) create mode 100755 scripts/ci-smoke-deployment.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df83a3a..ba424fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,25 @@ jobs: - name: Check generated Swagger docs run: scripts/check-swagger.sh + deployment-smoke: + name: Deployment smoke + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build CLI + run: make build + + - name: Run mock-provider deployment smoke + run: scripts/ci-smoke-deployment.sh + web: name: Web build runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 04c6ef5..9eee8cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This checkpoint adds the first production deployment and verification surface fo - Release runbook documenting tags, manual dispatch, binary artifacts, image tags, and preflight checks. - `.dockerignore` for smaller and safer Docker build contexts. - Deployment smoke script for live, health, readiness, and Prometheus probes. +- CI deployment smoke job using the mock provider. - Runtime conformance matrix for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. - Phase 4 release notes under `docs/releases/phase-4-production-deployment.md`. diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md index f9be3e4..98f9e06 100644 --- a/docs/releases/phase-4-production-deployment.md +++ b/docs/releases/phase-4-production-deployment.md @@ -70,10 +70,13 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - `.github/workflows/ci.yml` - Adds repository verification jobs for Go, Swagger, web, and SDKs. - Opts into Node 24-based JavaScript actions to address the GitHub Actions Node 20 deprecation warning. + - Runs a mock-provider deployment smoke job against the production smoke script. - `.github/workflows/release.yml` - Adds binary and container image release automation. - `scripts/check-swagger.sh` - Downloads modules before generating docs in a temporary workspace. +- `scripts/ci-smoke-deployment.sh` + - Starts StacyVM with the mock provider and runs deployment smoke probes in CI. - `cmd/stacyvm/cmd_serve.go` - Registers the mock provider when enabled in config. @@ -122,6 +125,7 @@ cd web && npm run build scripts/check-swagger.sh make release-build-all VERSION=phase-4-test scripts/smoke-deployment.sh http://127.0.0.1:7423 +scripts/ci-smoke-deployment.sh ``` GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swagger drift check stabilization. diff --git a/scripts/ci-smoke-deployment.sh b/scripts/ci-smoke-deployment.sh new file mode 100755 index 0000000..553f2d7 --- /dev/null +++ b/scripts/ci-smoke-deployment.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PORT="${STACYVM_SMOKE_PORT:-17423}" +API_KEY="${STACYVM_API_KEY:-ci-smoke-key}" +DB_PATH="${STACYVM_DATABASE_PATH:-${TMPDIR:-/tmp}/stacyvm-ci-smoke.db}" +LOG_PATH="${STACYVM_SMOKE_LOG:-${TMPDIR:-/tmp}/stacyvm-ci-smoke.log}" + +cd "$ROOT" + +rm -f "$DB_PATH" "$DB_PATH-shm" "$DB_PATH-wal" "$LOG_PATH" + +cleanup() { + if [[ -n "${server_pid:-}" ]]; then + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi +} +trap cleanup EXIT + +STACYVM_SERVER_PORT="$PORT" \ +STACYVM_PROVIDERS_DEFAULT=mock \ +STACYVM_PROVIDERS_MOCK_ENABLED=true \ +STACYVM_PROVIDERS_DOCKER_ENABLED=false \ +STACYVM_PROVIDERS_FIRECRACKER_ENABLED=false \ +STACYVM_AUTH_API_KEY="$API_KEY" \ +STACYVM_DATABASE_PATH="$DB_PATH" \ + ./stacyvm serve >"$LOG_PATH" 2>&1 & +server_pid="$!" + +for _ in $(seq 1 50); do + if curl --silent --fail --max-time 1 -H "X-API-Key: $API_KEY" "http://127.0.0.1:$PORT/api/v1/live" >/dev/null; then + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + echo "StacyVM server exited before becoming live. Logs:" >&2 + cat "$LOG_PATH" >&2 + exit 1 + fi + sleep 0.2 +done + +scripts/smoke-deployment.sh "http://127.0.0.1:$PORT" "$API_KEY" From df65377eaaffea117c09db168db43b8d7dbe1a49 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 10:26:01 +0530 Subject: [PATCH 037/147] ops: validate compose deployment smoke --- CHANGELOG.md | 2 ++ deploy/.env.example | 1 + deploy/docker-compose.yml | 2 +- docs/deployment.md | 14 ++++++++++++++ docs/releases/phase-4-production-deployment.md | 6 ++++++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eee8cc..26a7fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ This checkpoint adds the first production deployment and verification surface fo - Docker image builds now accept an explicit `VERSION` build argument and BuildKit target platform args. - Release artifacts now build into `dist/` with checksums instead of the repository root. - `stacyvm serve` now registers the mock provider when `providers.mock.enabled` is true. +- Production Compose now allows the Traefik host port to be overridden for smoke runs. +- Production Compose has been runtime-smoked with StacyVM, Traefik, Docker provider readiness, API probes, and live-preview routing. - README navigation now links to the production deployment guide. ### Verified diff --git a/deploy/.env.example b/deploy/.env.example index f9bc17b..804a86b 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -1,5 +1,6 @@ STACYVM_IMAGE=ghcr.io/stacyos/stacyvm:latest STACYVM_HOST_PORT=7423 +STACYVM_TRAEFIK_HOST_PORT=80 STACYVM_API_KEY=change-me-generate-at-least-32-bytes STACYVM_PREVIEW_DOMAIN=localhost STACYVM_LOG_LEVEL=info diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index e0aca62..3338fdf 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -32,7 +32,7 @@ services: - "--providers.docker.exposedbydefault=false" - "--entrypoints.web.address=:80" ports: - - "80:80" + - "${STACYVM_TRAEFIK_HOST_PORT:-80}:80" volumes: - ${STACYVM_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock:ro networks: diff --git a/docs/deployment.md b/docs/deployment.md index 5085698..b691833 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -55,6 +55,20 @@ docker build -t stacyvm:local .. STACYVM_IMAGE=stacyvm:local docker compose up -d ``` +For non-invasive smoke runs on a shared host, override the published ports: + +```bash +STACYVM_IMAGE=stacyvm:local STACYVM_HOST_PORT=17426 STACYVM_TRAEFIK_HOST_PORT=18080 docker compose up -d +``` + +Then validate the API surface: + +```bash +scripts/smoke-deployment.sh http://127.0.0.1:17426 "$STACYVM_API_KEY" +``` + +Live-preview routing can be checked by spawning a sandbox that serves port `3000` and requesting Traefik with `Host: 3000-.`. + ## systemd Use `deploy/stacyvm.service` when running the binary directly on a Linux host. diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md index 98f9e06..eac871a 100644 --- a/docs/releases/phase-4-production-deployment.md +++ b/docs/releases/phase-4-production-deployment.md @@ -62,6 +62,7 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - Added `scripts/smoke-deployment.sh` for liveness, health, readiness, and Prometheus deployment probes. - Added `docs/runtime-conformance.md` with host requirements and signoff checks for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. - Registered the mock provider in `stacyvm serve` when `providers.mock.enabled` is set, giving operators and CI a no-Docker smoke path. +- Validated the production Compose template with StacyVM, Traefik, Docker provider readiness, API smoke probes, and a port `3000` live-preview route. ## Code Changes By Area @@ -84,6 +85,7 @@ The goal of this phase is to make StacyVM easier to validate, release, and run o - `deploy/docker-compose.yml` - Adds a reusable production Compose template. + - Allows the Traefik host port to be overridden for smoke runs. - `deploy/stacyvm.production.yaml` - Adds a production baseline config. - `deploy/stacyvm.service` @@ -126,6 +128,10 @@ scripts/check-swagger.sh make release-build-all VERSION=phase-4-test scripts/smoke-deployment.sh http://127.0.0.1:7423 scripts/ci-smoke-deployment.sh +docker build --build-arg VERSION=phase-4-compose -t stacyvm:phase-4-compose . +docker compose -p stacyvm-phase4-smoke -f deploy/docker-compose.yml up -d +scripts/smoke-deployment.sh http://127.0.0.1:17426 phase4-compose-key +curl -H 'Host: 3000-.localhost' http://127.0.0.1:18080/ ``` GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swagger drift check stabilization. From 1277fbcb249ddbe680042725115548abd797d55b Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 10:33:46 +0530 Subject: [PATCH 038/147] docs: close out phase 4 release notes --- docs/releases/phase-4-production-deployment.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md index eac871a..e861a40 100644 --- a/docs/releases/phase-4-production-deployment.md +++ b/docs/releases/phase-4-production-deployment.md @@ -142,6 +142,13 @@ GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swa - Firecracker remains Linux/KVM-gated and should be rolled out only after host conformance checks pass. - PRoot remains gated on a real `proot` binary and a rootfs with the expected sandbox tooling. -## Next Phase 4 Direction +## Phase 4 Closeout -Remaining Phase 4 work should focus on release automation, container publishing, deployment smoke tests, and a clearer production conformance matrix for Docker, gVisor/Kata, Firecracker, and PRoot hosts. +Phase 4 implementation is complete. Release automation, container publishing workflow, deployment smoke testing, production deployment templates, CI coverage, and runtime conformance documentation are in place. + +The remaining work is external release/platform operation: + +- Trigger a real versioned release run when the project is ready to publish binaries and the GHCR image. +- Collect real-host runtime signoffs for optional host-gated runtimes such as gVisor, Kata, Firecracker, and PRoot. + +Phase 5 can proceed from this branch without more Phase 4 code work. From 6bc3fbfa37d78a3d378dacc34d07665fd00f1f27 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 10:45:19 +0530 Subject: [PATCH 039/147] feat: add admin API key and routes --- CHANGELOG.md | 20 ++++ README.md | 4 + cmd/stacyvm/cmd_serve.go | 7 +- deploy/.env.example | 1 + deploy/docker-compose.yml | 1 + deploy/stacyvm.env.example | 1 + deploy/stacyvm.production.yaml | 1 + docs/api.md | 37 ++++++- docs/deployment.md | 4 +- docs/releases/phase-5-admin-control-plane.md | 42 +++++++ internal/api/middleware/auth.go | 61 +++++++++- internal/api/middleware/auth_test.go | 87 +++++++++++++++ internal/api/server.go | 23 ++-- internal/api/server_test.go | 111 +++++++++++++++++++ internal/config/config.go | 6 +- internal/config/config_test.go | 5 + 16 files changed, 393 insertions(+), 18 deletions(-) create mode 100644 docs/releases/phase-5-admin-control-plane.md create mode 100644 internal/api/middleware/auth_test.go create mode 100644 internal/api/server_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 26a7fc5..61aec21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## Phase 5 Admin Control Plane - 2026-05-08 + +This checkpoint starts the Phase 5 operator control plane by separating admin access from regular API usage. + +### Added + +- Optional `auth.admin_api_key` / `STACYVM_AUTH_ADMIN_API_KEY` configuration. +- `X-Admin-API-Key` support for admin requests. +- `/api/v1/admin/*` route aliases for providers, quotas, diagnostics, JSON metrics, and Prometheus metrics. +- Admin key examples in deployment templates and docs. + +### Changed + +- Normal API and admin API keys can both authenticate regular API requests. +- Admin routes require the admin key when configured, with fallback to the regular API key only when no admin key is set. + +### Verified + +- `go test ./internal/api/middleware ./internal/config ./cmd/stacyvm` + ## Phase 4 Production Deployment - 2026-05-08 This checkpoint adds the first production deployment and verification surface for Phase 4: GitHub Actions CI, deployment templates, and an operator runbook. diff --git a/README.md b/README.md index e5b9f62..e91a445 100644 --- a/README.md +++ b/README.md @@ -431,6 +431,8 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | `GET` | `/metrics/prometheus` | Prometheus-compatible metrics | | `GET` | `/events` | Server-sent events stream | +Admin aliases for providers, quotas, diagnostics, and metrics are available under `/admin/*` and can be protected with `auth.admin_api_key`. + Full schemas, request/response examples, and error codes: **[docs/api.md](docs/api.md)**. OpenAPI spec: [docs/swagger.yaml](docs/swagger.yaml). @@ -521,6 +523,7 @@ defaults: auth: enabled: false api_key: "" + admin_api_key: "" # optional separate key for /api/v1/admin/* rate_limit: enabled: false @@ -554,6 +557,7 @@ pool: STACYVM_SERVER_PORT=8080 STACYVM_PROVIDERS_DEFAULT=firecracker STACYVM_AUTH_API_KEY=sk-xyz123 +STACYVM_AUTH_ADMIN_API_KEY=sk-admin-xyz123 STACYVM_RATE_LIMIT_ENABLED=true STACYVM_LOGGING_LEVEL=debug ``` diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index 5f3d1f7..1f5b60f 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -220,9 +220,10 @@ func runServe() error { rateLimitBucketTTL, _ := time.ParseDuration(cfg.RateLimit.BucketTTL) rateLimitCleanupInterval, _ := time.ParseDuration(cfg.RateLimit.CleanupInterval) srv := api.NewServer(api.ServerConfig{ - Addr: cfg.Server.Addr(), - APIKey: cfg.Auth.APIKey, - Version: version, + Addr: cfg.Server.Addr(), + APIKey: cfg.Auth.APIKey, + AdminAPIKey: cfg.Auth.AdminAPIKey, + Version: version, RateLimit: middleware.RateLimitConfig{ Enabled: cfg.RateLimit.Enabled, RequestsPerMinute: cfg.RateLimit.RequestsPerMinute, diff --git a/deploy/.env.example b/deploy/.env.example index 804a86b..b27e4a8 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -2,6 +2,7 @@ STACYVM_IMAGE=ghcr.io/stacyos/stacyvm:latest STACYVM_HOST_PORT=7423 STACYVM_TRAEFIK_HOST_PORT=80 STACYVM_API_KEY=change-me-generate-at-least-32-bytes +STACYVM_ADMIN_API_KEY=change-me-generate-a-separate-admin-key STACYVM_PREVIEW_DOMAIN=localhost STACYVM_LOG_LEVEL=info STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 3338fdf..464096b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -11,6 +11,7 @@ services: - ./stacyvm.production.yaml:/etc/stacyvm/stacyvm.yaml:ro environment: STACYVM_AUTH_API_KEY: ${STACYVM_API_KEY:?set STACYVM_API_KEY} + STACYVM_AUTH_ADMIN_API_KEY: ${STACYVM_ADMIN_API_KEY:-} STACYVM_DATABASE_PATH: ${STACYVM_DATABASE_PATH:-/var/lib/stacyvm/stacyvm.db} STACYVM_LOGGING_LEVEL: ${STACYVM_LOG_LEVEL:-info} STACYVM_SERVER_PREVIEW_DOMAIN: ${STACYVM_PREVIEW_DOMAIN:-localhost} diff --git a/deploy/stacyvm.env.example b/deploy/stacyvm.env.example index 80e386a..2c743c6 100644 --- a/deploy/stacyvm.env.example +++ b/deploy/stacyvm.env.example @@ -1,4 +1,5 @@ STACYVM_AUTH_API_KEY=change-me-generate-at-least-32-bytes +STACYVM_AUTH_ADMIN_API_KEY=change-me-generate-a-separate-admin-key STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db STACYVM_LOGGING_LEVEL=info STACYVM_LOGGING_FORMAT=json diff --git a/deploy/stacyvm.production.yaml b/deploy/stacyvm.production.yaml index 73fada3..05be030 100644 --- a/deploy/stacyvm.production.yaml +++ b/deploy/stacyvm.production.yaml @@ -73,6 +73,7 @@ defaults: auth: enabled: true api_key: "change-me-generate-at-least-32-bytes" + admin_api_key: "change-me-generate-a-separate-admin-key" rate_limit: enabled: true diff --git a/docs/api.md b/docs/api.md index 605d46f..aca0ebc 100644 --- a/docs/api.md +++ b/docs/api.md @@ -13,6 +13,7 @@ This document is the source of truth for the StacyVM HTTP API. The Python and Ty - [Authentication](#authentication) - [Conventions](#conventions) - [Errors](#errors) +- [Admin API](#admin-api) - [Sandboxes](#sandboxes) - [Files](#files) - [Templates](#templates) @@ -27,11 +28,12 @@ This document is the source of truth for the StacyVM HTTP API. The Python and Ty ## Authentication -Two optional headers, both off by default: +Optional headers: | Header | Purpose | Required when | |---|---|---| | `X-API-Key` | API key authentication | `auth.enabled: true` in `stacyvm.yaml` | +| `X-Admin-API-Key` | Admin API key authentication | `auth.admin_api_key` is configured and calling `/api/v1/admin/*` | | `X-User-ID` | Multi-tenant pool mode user identifier | `pool.enabled: true` | ```bash @@ -99,6 +101,39 @@ Errors return a JSON body with HTTP status reflecting the failure class: --- +## Admin API + +StacyVM supports an optional separate admin API key: + +```yaml +auth: + api_key: "sk-client" + admin_api_key: "sk-admin" +``` + +Use `X-Admin-API-Key` for admin requests. `X-API-Key` is still accepted when it matches the admin key. If `auth.admin_api_key` is empty, admin routes fall back to `auth.api_key` for backwards compatibility. + +Admin route aliases: + +| Method | Path | Purpose | +|---|---|---| +| `GET` | `/api/v1/admin/providers` | List providers with health details | +| `GET` | `/api/v1/admin/providers/{name}` | Provider detail | +| `POST` | `/api/v1/admin/providers/test` | Run provider health checks | +| `GET` | `/api/v1/admin/quotas` | List owner quota overrides | +| `GET` | `/api/v1/admin/quotas/summary` | Redacted quota coverage summary | +| `GET` | `/api/v1/admin/quotas/{ownerID}` | Get owner quota | +| `PUT` | `/api/v1/admin/quotas/{ownerID}` | Create or update owner quota | +| `GET` | `/api/v1/admin/quotas/{ownerID}/usage` | Owner usage against effective quota | +| `DELETE` | `/api/v1/admin/quotas/{ownerID}` | Delete owner quota | +| `GET` | `/api/v1/admin/diagnostics` | Redacted operational diagnostics | +| `GET` | `/api/v1/admin/metrics` | Structured JSON metrics | +| `GET` | `/api/v1/admin/metrics/prometheus` | Prometheus metrics | + +The existing non-admin paths remain available for compatibility in this phase. + +--- + ## Sandboxes ### Spawn a sandbox diff --git a/docs/deployment.md b/docs/deployment.md index b691833..770ed49 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -40,6 +40,8 @@ The files in `deploy/` provide a production-oriented Compose starting point: - `deploy/.env.example` lists the environment variables expected by the Compose file. - `deploy/stacyvm.env.example` is the systemd environment file template. +Use separate values for `STACYVM_API_KEY` and `STACYVM_ADMIN_API_KEY` in production. Admin routes live under `/api/v1/admin/*` and should be restricted to operator networks where possible. + ```bash cd deploy cp .env.example .env @@ -85,7 +87,7 @@ sudo install -m 0755 bin/stacyvm-agent /usr/local/bin/stacyvm-agent sudo install -m 0644 deploy/stacyvm.service /etc/systemd/system/stacyvm.service ``` -Edit `/etc/stacyvm/stacyvm.env` and set a real `STACYVM_AUTH_API_KEY`. Then enable the service: +Edit `/etc/stacyvm/stacyvm.env` and set real `STACYVM_AUTH_API_KEY` and `STACYVM_AUTH_ADMIN_API_KEY` values. Then enable the service: ```bash sudo systemctl daemon-reload diff --git a/docs/releases/phase-5-admin-control-plane.md b/docs/releases/phase-5-admin-control-plane.md new file mode 100644 index 0000000..306cc35 --- /dev/null +++ b/docs/releases/phase-5-admin-control-plane.md @@ -0,0 +1,42 @@ +# Phase 5 Admin Control Plane Release Notes + +Date: 2026-05-08 +Branch: `phase-5-admin-control-plane` + +## Summary + +Phase 5 starts the operator/admin control-plane work for StacyVM. This phase builds on Phase 3 quotas and Phase 4 production deployment by separating admin access from regular API usage and preparing the API surface for safer dashboard-driven operations. + +## What Changed + +### Admin Authentication + +- Added optional `auth.admin_api_key` config. +- Added `STACYVM_AUTH_ADMIN_API_KEY` environment variable support through the existing config loader. +- Added `X-Admin-API-Key` support for admin requests. +- Admin keys can authenticate normal API requests, but normal API keys cannot access admin routes when an admin key is configured. +- When no admin key is configured, admin routes fall back to `auth.api_key` for backwards compatibility. + +### Admin Route Namespace + +- Added `/api/v1/admin/*` operator route aliases for: + - providers + - quotas + - diagnostics + - JSON metrics + - Prometheus metrics +- Existing non-admin routes remain available for compatibility during this phase. + +### Deployment And Docs + +- Added admin key examples to production config, Compose env, systemd env, README, deployment docs, and API docs. + +## Verification + +```sh +go test ./internal/api/middleware ./internal/config ./cmd/stacyvm +``` + +## Next Phase 5 Direction + +The next slice should move dashboard quota/provider/diagnostics workflows onto the admin namespace and add persisted audit history for admin operations. diff --git a/internal/api/middleware/auth.go b/internal/api/middleware/auth.go index b3695cb..0c53c27 100644 --- a/internal/api/middleware/auth.go +++ b/internal/api/middleware/auth.go @@ -7,16 +7,18 @@ import ( ) func Auth(apiKey string) func(http.Handler) http.Handler { + return AuthAny(apiKey) +} + +func AuthAny(apiKeys ...string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if apiKey == "" { + if len(nonEmptyKeys(apiKeys...)) == 0 { next.ServeHTTP(w, r) return } - key := r.Header.Get("X-API-Key") - - if subtle.ConstantTimeCompare([]byte(key), []byte(apiKey)) != 1 { + if !matchesAnyKey(r.Header.Get("X-API-Key"), apiKeys...) && !matchesAnyKey(r.Header.Get("X-Admin-API-Key"), apiKeys...) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]string{ @@ -30,3 +32,54 @@ func Auth(apiKey string) func(http.Handler) http.Handler { }) } } + +func AdminAuth(adminAPIKey, fallbackAPIKey string) func(http.Handler) http.Handler { + if adminAPIKey == "" { + adminAPIKey = fallbackAPIKey + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if adminAPIKey == "" { + next.ServeHTTP(w, r) + return + } + + if !matchesAnyKey(r.Header.Get("X-Admin-API-Key"), adminAPIKey) && !matchesAnyKey(r.Header.Get("X-API-Key"), adminAPIKey) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + json.NewEncoder(w).Encode(map[string]string{ + "code": "FORBIDDEN", + "message": "admin API key required", + }) + return + } + + next.ServeHTTP(w, r) + }) + } +} + +func matchesAnyKey(candidate string, apiKeys ...string) bool { + if candidate == "" { + return false + } + for _, apiKey := range apiKeys { + if apiKey == "" { + continue + } + if subtle.ConstantTimeCompare([]byte(candidate), []byte(apiKey)) == 1 { + return true + } + } + return false +} + +func nonEmptyKeys(apiKeys ...string) []string { + keys := make([]string, 0, len(apiKeys)) + for _, key := range apiKeys { + if key != "" { + keys = append(keys, key) + } + } + return keys +} diff --git a/internal/api/middleware/auth_test.go b/internal/api/middleware/auth_test.go new file mode 100644 index 0000000..b53bda9 --- /dev/null +++ b/internal/api/middleware/auth_test.go @@ -0,0 +1,87 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestAuthAnyAcceptsPrimaryOrAdminKey(t *testing.T) { + handler := AuthAny("primary-key", "admin-key")(okHandler()) + + tests := []struct { + name string + header string + key string + want int + }{ + {name: "primary", header: "X-API-Key", key: "primary-key", want: http.StatusOK}, + {name: "admin via api header", header: "X-API-Key", key: "admin-key", want: http.StatusOK}, + {name: "admin via admin header", header: "X-Admin-API-Key", key: "admin-key", want: http.StatusOK}, + {name: "wrong", header: "X-API-Key", key: "wrong", want: http.StatusUnauthorized}, + {name: "missing", want: http.StatusUnauthorized}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + if tt.header != "" { + req.Header.Set(tt.header, tt.key) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.want, w.Body.String()) + } + }) + } +} + +func TestAdminAuthRequiresAdminKeyWhenConfigured(t *testing.T) { + handler := AdminAuth("admin-key", "primary-key")(okHandler()) + + tests := []struct { + name string + header string + key string + want int + }{ + {name: "admin via admin header", header: "X-Admin-API-Key", key: "admin-key", want: http.StatusOK}, + {name: "admin via api header", header: "X-API-Key", key: "admin-key", want: http.StatusOK}, + {name: "primary rejected", header: "X-API-Key", key: "primary-key", want: http.StatusForbidden}, + {name: "missing", want: http.StatusForbidden}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + if tt.header != "" { + req.Header.Set(tt.header, tt.key) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.want, w.Body.String()) + } + }) + } +} + +func TestAdminAuthFallsBackToPrimaryKey(t *testing.T) { + handler := AdminAuth("", "primary-key")(okHandler()) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("X-API-Key", "primary-key") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func okHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) +} diff --git a/internal/api/server.go b/internal/api/server.go index a7d9485..f0b4b11 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -19,10 +19,11 @@ import ( ) type ServerConfig struct { - Addr string - APIKey string - Version string - RateLimit middleware.RateLimitConfig + Addr string + APIKey string + AdminAPIKey string + Version string + RateLimit middleware.RateLimitConfig } type Server struct { @@ -57,8 +58,8 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr // API routes — with auth and CORS r.Group(func(r chi.Router) { - if cfg.APIKey != "" { - r.Use(middleware.Auth(cfg.APIKey)) + if cfg.APIKey != "" || cfg.AdminAPIKey != "" { + r.Use(middleware.AuthAny(cfg.APIKey, cfg.AdminAPIKey)) } // CORS @@ -66,7 +67,7 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key, X-Request-ID, X-User-ID") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key, X-Admin-API-Key, X-Request-ID, X-User-ID") if r.Method == "OPTIONS" { w.WriteHeader(http.StatusOK) return @@ -97,6 +98,14 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr r.Mount("/environments", environmentRoutes.Routes()) r.Mount("/quotas", quotaRoutes.Routes()) r.Get("/pool/status", sandboxRoutes.VMPoolStatus) + r.Route("/admin", func(r chi.Router) { + r.Use(middleware.AdminAuth(cfg.AdminAPIKey, cfg.APIKey)) + r.Mount("/providers", providerRoutes.Routes()) + r.Mount("/quotas", quotaRoutes.Routes()) + r.Get("/diagnostics", systemRoutes.Diagnostics) + r.Get("/metrics", systemRoutes.Metrics) + r.Get("/metrics/prometheus", systemRoutes.PrometheusMetrics) + }) r.Mount("/", systemRoutes.Routes()) }) }) diff --git a/internal/api/server_test.go b/internal/api/server_test.go new file mode 100644 index 0000000..fd36def --- /dev/null +++ b/internal/api/server_test.go @@ -0,0 +1,111 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/rs/zerolog" +) + +type noopBuildStarter struct{} + +func (noopBuildStarter) Enqueue(buildID string) error { return nil } + +func setupTestServer(t *testing.T, cfg ServerConfig) *Server { + t.Helper() + + st, err := store.NewSQLiteStore(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + + events := orchestrator.NewEventBus() + manager := orchestrator.NewManager(registry, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + templates := orchestrator.NewTemplateRegistry(st) + pool := orchestrator.NewPoolManager(manager, templates, zerolog.Nop()) + + return NewServer(cfg, registry, manager, events, templates, pool, st, noopBuildStarter{}, zerolog.Nop()) +} + +func TestAdminRoutesRequireAdminAPIKeyWhenConfigured(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + Version: "test", + }) + + tests := []struct { + name string + header string + key string + want int + }{ + {name: "client key forbidden", header: "X-API-Key", key: "client-key", want: http.StatusForbidden}, + {name: "admin api header ok", header: "X-API-Key", key: "admin-key", want: http.StatusOK}, + {name: "admin header ok", header: "X-Admin-API-Key", key: "admin-key", want: http.StatusOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set(tt.header, tt.key) + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.want, w.Body.String()) + } + }) + } +} + +func TestAdminRoutesFallbackToAPIKeyWhenAdminKeyUnset(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set("X-API-Key", "client-key") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestAdminAPIKeyCanAuthenticateRegularRoutes(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/live", nil) + req.Header.Set("X-Admin-API-Key", "admin-key") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 51a14d4..495126d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -135,8 +135,9 @@ type DefaultsConfig struct { } type AuthConfig struct { - Enabled bool `mapstructure:"enabled"` - APIKey string `mapstructure:"api_key"` + Enabled bool `mapstructure:"enabled"` + APIKey string `mapstructure:"api_key"` + AdminAPIKey string `mapstructure:"admin_api_key"` } type RateLimitConfig struct { @@ -225,6 +226,7 @@ func setDefaults(v *viper.Viper) { v.SetDefault("auth.enabled", false) v.SetDefault("auth.api_key", "") + v.SetDefault("auth.admin_api_key", "") v.SetDefault("rate_limit.enabled", false) v.SetDefault("rate_limit.requests_per_minute", 120) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 23d644c..c63ba0e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -78,6 +78,8 @@ rate_limit: key_by: "api_key" bucket_ttl: "30m" cleanup_interval: "2m" +auth: + admin_api_key: "admin-secret" pool: overflow: "queue" `), 0644); err != nil { @@ -94,4 +96,7 @@ pool: if !cfg.RateLimit.Enabled || cfg.RateLimit.KeyBy != "api_key" || cfg.RateLimit.BucketTTL != "30m" { t.Fatalf("unexpected rate limit config: %+v", cfg.RateLimit) } + if cfg.Auth.AdminAPIKey != "admin-secret" { + t.Fatalf("admin api key = %q, want admin-secret", cfg.Auth.AdminAPIKey) + } } From df21d4a2ed8b7944b102c499e49eebc24a2e28e5 Mon Sep 17 00:00:00 2001 From: ARPAN MONDAL Date: Fri, 8 May 2026 11:11:39 +0530 Subject: [PATCH 040/147] feat: wire dashboard to admin control plane --- CHANGELOG.md | 4 + docs/releases/phase-5-admin-control-plane.md | 10 +- web/src/api/client.ts | 113 +++++++++++++++++-- web/src/pages/Providers.tsx | 37 ++++-- web/src/pages/Settings.tsx | 47 +++++--- 5 files changed, 179 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61aec21..6afe6e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,15 +10,19 @@ This checkpoint starts the Phase 5 operator control plane by separating admin ac - `X-Admin-API-Key` support for admin requests. - `/api/v1/admin/*` route aliases for providers, quotas, diagnostics, JSON metrics, and Prometheus metrics. - Admin key examples in deployment templates and docs. +- Dashboard settings for separate regular and admin API keys. ### Changed - Normal API and admin API keys can both authenticate regular API requests. - Admin routes require the admin key when configured, with fallback to the regular API key only when no admin key is set. +- Dashboard provider and metrics calls now use the admin namespace. +- Provider health checks in the dashboard now call `/api/v1/admin/providers/test`. ### Verified - `go test ./internal/api/middleware ./internal/config ./cmd/stacyvm` +- `npm run build` ## Phase 4 Production Deployment - 2026-05-08 diff --git a/docs/releases/phase-5-admin-control-plane.md b/docs/releases/phase-5-admin-control-plane.md index 306cc35..7a40493 100644 --- a/docs/releases/phase-5-admin-control-plane.md +++ b/docs/releases/phase-5-admin-control-plane.md @@ -31,12 +31,20 @@ Phase 5 starts the operator/admin control-plane work for StacyVM. This phase bui - Added admin key examples to production config, Compose env, systemd env, README, deployment docs, and API docs. +### Dashboard Admin Workflows + +- Added dashboard settings for a regular API key and a separate admin API key. +- The shared web API client now sends `X-API-Key` and `X-Admin-API-Key` from browser settings. +- Provider list, provider detail, provider health tests, and JSON metrics now call `/api/v1/admin/*`. +- Provider cards now understand the backend `default`, latency, runtime count, capability, and error fields. + ## Verification ```sh go test ./internal/api/middleware ./internal/config ./cmd/stacyvm +npm run build ``` ## Next Phase 5 Direction -The next slice should move dashboard quota/provider/diagnostics workflows onto the admin namespace and add persisted audit history for admin operations. +The next slice should add dedicated dashboard views for quota management, diagnostics, and persisted admin audit history. diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 2aa8891..ec6a238 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -74,8 +74,14 @@ export interface CreateTemplateRequest { export interface Provider { name: string; + default?: boolean; is_default: boolean; healthy: boolean; + latency_ms?: number; + last_checked?: string; + error?: string; + capabilities?: string[]; + runtime_count?: number; } export interface HealthResponse { @@ -87,8 +93,22 @@ export interface HealthResponse { export interface MetricsResponse { goroutines: number; memory_alloc: number; + memory_sys?: number; + memory_heap_alloc?: number; + gc_cycles?: number; active_sandboxes: number; total_sandboxes: number; + sandboxes?: { + total: number; + active: number; + by_state: Record; + by_provider: Record; + }; + providers?: { + total: number; + healthy: number; + items: Provider[]; + }; } export interface SSEEvent { @@ -189,6 +209,12 @@ export interface EnvironmentSuggestionsResponse { suggestions: string[]; } +interface StoredAppSettings { + authEnabled?: boolean; + authToken?: string; + adminToken?: string; +} + // --------------------------------------------------------------------------- // API Error // --------------------------------------------------------------------------- @@ -210,21 +236,72 @@ export class ApiError extends Error { const BASE = '/api/v1'; +interface RequestOptions extends RequestInit { + admin?: boolean; +} + +function loadStoredSettings(): StoredAppSettings { + if (typeof window === 'undefined') return {}; + + try { + const stored = window.localStorage.getItem('stacyvm-settings'); + return stored ? (JSON.parse(stored) as StoredAppSettings) : {}; + } catch { + return {}; + } +} + +function normalizeHeaders(headers?: HeadersInit): Record { + if (!headers) return {}; + if (headers instanceof Headers) return Object.fromEntries(headers.entries()); + if (Array.isArray(headers)) return Object.fromEntries(headers); + return { ...headers }; +} + +function authHeaders(admin: boolean): Record { + const settings = loadStoredSettings(); + if (!settings.authEnabled) return {}; + + const headers: Record = {}; + const apiKey = settings.authToken?.trim(); + const adminKey = settings.adminToken?.trim(); + + if (apiKey) { + headers['X-API-Key'] = apiKey; + } + if (admin && adminKey) { + headers['X-Admin-API-Key'] = adminKey; + } + + return headers; +} + +function normalizeProvider(provider: Provider): Provider { + const isDefault = provider.is_default ?? provider.default ?? false; + return { + ...provider, + default: provider.default ?? isDefault, + is_default: isDefault, + }; +} + async function request( path: string, - options: RequestInit = {}, + options: RequestOptions = {}, ): Promise { const url = `${BASE}${path}`; + const { admin = false, headers: optionHeaders, ...fetchOptions } = options; const headers: Record = { - ...(options.headers as Record), + ...authHeaders(admin), + ...normalizeHeaders(optionHeaders), }; - if (options.body && typeof options.body === 'string') { + if (fetchOptions.body && typeof fetchOptions.body === 'string') { headers['Content-Type'] = 'application/json'; } const res = await fetch(url, { - ...options, + ...fetchOptions, headers, }); @@ -317,7 +394,7 @@ export async function execStreamNDJSON( const url = `${BASE}/sandboxes/${encodeURIComponent(sandboxId)}/exec`; const res = await fetch(url, { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { ...authHeaders(false), 'Content-Type': 'application/json' }, body: JSON.stringify({ command, stream: true }), signal, }); @@ -475,8 +552,10 @@ export async function spawnFromTemplate( // --------------------------------------------------------------------------- export async function listProviders(): Promise { - const result = await request('/providers'); - return result ?? []; + const result = await request('/admin/providers', { + admin: true, + }); + return (result ?? []).map(normalizeProvider); } export interface ProviderDetail { @@ -484,11 +563,22 @@ export interface ProviderDetail { healthy: boolean; default: boolean; sandbox_count: number; + health?: Provider; config: Record; } export async function getProviderDetail(name: string): Promise { - return request(`/providers/${encodeURIComponent(name)}`); + return request( + `/admin/providers/${encodeURIComponent(name)}`, + { admin: true }, + ); +} + +export async function testProviders(): Promise> { + return request>('/admin/providers/test', { + method: 'POST', + admin: true, + }); } // --------------------------------------------------------------------------- @@ -598,7 +688,12 @@ export async function getHealth(): Promise { } export async function getMetrics(): Promise { - return request('/metrics'); + const metrics = await request('/admin/metrics', { admin: true }); + return { + ...metrics, + active_sandboxes: metrics.active_sandboxes ?? metrics.sandboxes?.active ?? 0, + total_sandboxes: metrics.total_sandboxes ?? metrics.sandboxes?.total ?? 0, + }; } // --------------------------------------------------------------------------- diff --git a/web/src/pages/Providers.tsx b/web/src/pages/Providers.tsx index eb9af27..c725bb7 100644 --- a/web/src/pages/Providers.tsx +++ b/web/src/pages/Providers.tsx @@ -15,7 +15,13 @@ import { Settings, Hash, } from 'lucide-react'; -import { type Provider, type ProviderDetail, listProviders, getHealth, getProviderDetail } from '../api/client'; +import { + type Provider, + type ProviderDetail, + listProviders, + getProviderDetail, + testProviders, +} from '../api/client'; import { ProviderCardSkeleton } from '../components/Skeleton'; import { useToast } from '../hooks/useToast'; @@ -83,16 +89,15 @@ export default function Providers() { const handleTestConnection = async (providerName: string) => { setTestingProvider(providerName); try { - // Test via the health endpoint (which validates backend connectivity) - const health = await getHealth(); - const ok = health.status === 'ok'; + const results = await testProviders(); + const ok = results[providerName] ?? false; const message = ok - ? `Connected successfully (uptime: ${health.uptime})` - : `Health check returned: ${health.status}`; + ? 'Provider health check passed' + : 'Provider health check failed'; setTestResults((prev) => ({ ...prev, [providerName]: { ok, message } })); addToast({ type: ok ? 'success' : 'warning', - title: `${providerName}: ${ok ? 'Connected' : 'Unhealthy'}`, + title: `${providerName}: ${ok ? 'Healthy' : 'Unhealthy'}`, message, }); } catch (err) { @@ -277,6 +282,17 @@ function ProviderCard({

{info.description}

+
+ {provider.latency_ms !== undefined && ( + {provider.latency_ms}ms health + )} + {provider.runtime_count !== undefined && ( + {provider.runtime_count} runtime{provider.runtime_count !== 1 ? 's' : ''} + )} + {provider.error && ( + {provider.error} + )} +
{/* Test button + expand */} @@ -336,6 +352,13 @@ function ProviderCard({ {detail.sandbox_count} active sandbox{detail.sandbox_count !== 1 ? 'es' : ''} + {detail.health?.latency_ms !== undefined && ( +
+ + {detail.health.latency_ms}ms + health latency +
+ )} {/* Config table */} diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 16eb393..bed45ef 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -20,6 +20,7 @@ interface AppSettings { poolSize: number; authEnabled: boolean; authToken: string; + adminToken: string; serverPort: number; serverHost: string; theme: 'dark' | 'light' | 'system'; @@ -32,6 +33,7 @@ const DEFAULT_SETTINGS: AppSettings = { poolSize: 5, authEnabled: false, authToken: '', + adminToken: '', serverPort: 7423, serverHost: 'localhost', theme: 'dark', @@ -293,11 +295,11 @@ export default function Settings() { {activeSection === 'auth' && (