diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a6e4f69 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +.git +.github +.codex +dist +bin +stacyvm +stacyvm-agent +checksums.txt + +web/node_modules +web/dist +sdk/js/node_modules + +images/evm/contracts/lib +images/evm/frontend/node_modules +images/evm/frontend/.next + +*.log +*.db +*.db-shm +*.db-wal +.env diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..8c1c3a4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,71 @@ +name: Bug report +description: Report a reproducible StacyVM bug with the evidence needed for maintainers to triage it. +title: "[Bug]: " +labels: ["bug", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report this. For public self-serve installs, attach the support evidence below whenever it applies. Redact anything you cannot share publicly. + - type: textarea + id: summary + attributes: + label: What happened? + description: Describe the observed behavior and the expected behavior. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Reproduction steps + description: Include commands, API calls, SDK snippets, or config changes. + placeholder: | + 1. Run ... + 2. Call ... + 3. Observe ... + validations: + required: true + - type: dropdown + id: install_mode + attributes: + label: Install mode + options: + - Local/mock provider + - Single-node Docker/runc + - Docker with gVisor + - Docker with Kata + - Firecracker + - PRoot + - E2B/custom provider + - Multi-worker/enterprise prototype + validations: + required: true + - type: textarea + id: support_evidence + attributes: + label: Support evidence + description: Attach or paste the relevant command output. Use fenced code blocks for logs. + value: | + - `stacyvm support bundle --output support.json`: + - `stacyvm config lint --production --file `: + - `stacyvm upgrade rehearse --config --database `: + - `stacyvm doctor --production`: + - `scripts/certify-runtime.sh --format markdown --output -certification.md`: + - `scripts/verify-release.sh ` or installer verification output: + validations: + required: false + - type: textarea + id: environment + attributes: + label: Environment + description: Include OS, architecture, StacyVM version/commit, provider runtime version, Docker/Firecracker/gVisor/Kata/PRoot version, and whether this is CI or a host install. + validations: + required: true + - type: textarea + id: logs + attributes: + label: Logs + description: Paste redacted server logs, SDK stack traces, or diagnostics JSON. + render: shell + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d79a77e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Public support matrix + url: https://github.com/StacyOS/stacyvm/blob/main/docs/public-support-matrix.md + about: Check supported install modes and required support evidence before opening an issue. + - name: Production readiness checklist + url: https://github.com/StacyOS/stacyvm/blob/main/docs/production-readiness.md + about: Review the current production-readiness gates and known platform requirements. diff --git a/.github/ISSUE_TEMPLATE/support_request.yml b/.github/ISSUE_TEMPLATE/support_request.yml new file mode 100644 index 0000000..5dee663 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support_request.yml @@ -0,0 +1,63 @@ +name: Production support request +description: Ask for help with a public self-serve or production-style StacyVM install. +title: "[Support]: " +labels: ["support", "needs-triage"] +body: + - type: markdown + attributes: + value: | + Use this form for install, upgrade, runtime certification, and production-readiness help. The public support matrix explains which modes are supported, host-certified, preview, experimental, or planned. + - type: dropdown + id: support_level + attributes: + label: Runtime or deployment mode + options: + - Single-node Docker/runc + - Docker with gVisor + - Docker with Kata + - Firecracker + - PRoot + - E2B/custom provider + - Multi-worker/enterprise prototype + - Other + validations: + required: true + - type: textarea + id: goal + attributes: + label: What are you trying to do? + description: Describe the target setup, workload, and what production-ready means for your environment. + validations: + required: true + - type: checkboxes + id: evidence_checklist + attributes: + label: Evidence checklist + options: + - label: I ran release or installer verification. + - label: I ran production config lint with the same environment the service uses. + - label: I ran upgrade rehearsal before changing binaries or images. + - label: I ran `stacyvm doctor --production` on the target host. + - label: I generated a redacted support bundle. + - label: I generated runtime certification output for host-certified runtimes. + - type: textarea + id: evidence + attributes: + label: Evidence output + description: Paste or attach redacted output for every checked item above. + value: | + - Release/install verification: + - Config lint: + - Upgrade rehearsal: + - Doctor: + - Support bundle: + - Runtime certification: + validations: + required: true + - type: textarea + id: blocker + attributes: + label: Current blocker + description: What is preventing you from proceeding? + validations: + required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b6e4804 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,289 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + - 'phase-*' + - 'feat/*' + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + go: + name: Go tests and build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run Go tests + run: make test + + - name: Build CLI + run: make build + + swagger: + name: Swagger drift + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Check generated Swagger docs + run: scripts/check-swagger.sh + + deployment-smoke: + name: Deployment smoke + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build CLI + run: make build + + - name: Run mock-provider deployment smoke + run: scripts/ci-smoke-deployment.sh + + upgrade-migration: + name: Upgrade and migration checks + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run upgrade and migration checks + run: scripts/ci-upgrade-migration.sh + + cluster-conformance: + name: Cluster conformance checks + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: stacyvm + POSTGRES_PASSWORD: stacyvm + POSTGRES_DB: stacyvm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U stacyvm -d stacyvm" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run cluster conformance checks + env: + STACYVM_POSTGRES_TEST_DSN: postgres://stacyvm:stacyvm@127.0.0.1:5432/stacyvm?sslmode=disable + run: scripts/ci-cluster-conformance.sh + + remote-worker-postgres-smoke: + name: Remote worker Postgres smoke + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: stacyvm + POSTGRES_PASSWORD: stacyvm + POSTGRES_DB: stacyvm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U stacyvm -d stacyvm" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build CLI + run: make build + + - name: Run remote worker smoke with Postgres store + env: + STACYVM_REMOTE_SMOKE_DATABASE_DRIVER: postgres + STACYVM_REMOTE_SMOKE_DATABASE_DSN: postgres://stacyvm:stacyvm@127.0.0.1:5432/stacyvm?sslmode=disable + run: scripts/smoke-remote-worker.sh ./stacyvm + + remote-worker-mtls-smoke: + name: Remote worker mTLS smoke + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build CLI + run: make build + + - name: Run remote worker mTLS smoke with ephemeral certs + run: scripts/smoke-remote-worker.sh ./stacyvm --mtls + + runtime-certification: + name: Runtime certification (Docker) + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build CLI + run: make build + + - name: Run Docker runtime certification with StacyVM integration smoke + run: | + scripts/certify-runtime.sh docker \ + --stacyvm-bin ./stacyvm \ + --format markdown \ + --output /tmp/docker-certification.md + cat /tmp/docker-certification.md + + - name: Upload certification report + if: always() + uses: actions/upload-artifact@v4 + with: + name: docker-runtime-certification + path: /tmp/docker-certification.md + + public-release-sanity: + name: Public release sanity + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run public release sanity checks + run: scripts/ci-public-release-sanity.sh + + web: + name: Web build + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: web/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build web app + run: npm run build + + sdk-js: + name: TypeScript SDK build + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/js + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Build TypeScript SDK + run: bun run build + + - name: Run TypeScript SDK parity tests + run: bun test + + sdk-python: + name: Python SDK tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Python SDK + run: python -m pip install -e sdk/python + + - name: Compile Python SDK + run: python -m compileall sdk/python/stacyvm + + - name: Import Python SDK + run: python -c "import stacyvm; print(stacyvm.__version__)" + + - name: Run Python SDK parity tests + run: python -m unittest sdk/python/tests/test_client_parity.py diff --git a/.github/workflows/public-readiness-certification.yml b/.github/workflows/public-readiness-certification.yml new file mode 100644 index 0000000..ebdf987 --- /dev/null +++ b/.github/workflows/public-readiness-certification.yml @@ -0,0 +1,86 @@ +name: Public Readiness Certification + +on: + workflow_dispatch: + inputs: + version: + description: 'Published release tag to certify, for example v0.14.3' + required: true + type: string + runtime: + description: 'Runtime to certify on the GitHub-hosted Linux runner' + required: true + default: docker + type: choice + options: + - docker + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +permissions: + contents: read + +jobs: + public-readiness-certification: + name: Certify published release + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Validate published release assets and Linux installer + env: + GH_TOKEN: ${{ github.token }} + STACYVM_VALIDATE_INSTALLER: "true" + run: scripts/post-release-validate.sh "${{ inputs.version }}" + + - name: Download published StacyVM binary + env: + VERSION: ${{ inputs.version }} + run: | + curl -fsSL -o stacyvm-linux-amd64 \ + "https://github.com/StacyOS/stacyvm/releases/download/${VERSION}/stacyvm-linux-amd64" + chmod +x stacyvm-linux-amd64 + + - name: Certify runtime with StacyVM integration smoke + env: + STACYVM_RUNTIME: ${{ inputs.runtime }} + run: | + mkdir -p public-readiness-evidence + scripts/certify-runtime.sh "$STACYVM_RUNTIME" \ + --format markdown \ + --output "public-readiness-evidence/${STACYVM_RUNTIME}-certification.md" \ + --stacyvm-bin "./stacyvm-linux-amd64" + + - name: Write certification summary + env: + VERSION: ${{ inputs.version }} + STACYVM_RUNTIME: ${{ inputs.runtime }} + run: | + { + echo "# Public Readiness Certification" + echo + echo "- Version: \`${VERSION}\`" + echo "- Runtime: \`${STACYVM_RUNTIME}\`" + echo "- Release validation: \`PASS\`" + echo "- Installer verify-only: \`PASS\`" + echo "- Runtime certification: \`PASS\`" + echo "- Evidence generated at: \`$(date -u '+%Y-%m-%dT%H:%M:%SZ')\`" + echo + echo "## Runtime Report" + echo + cat "public-readiness-evidence/${STACYVM_RUNTIME}-certification.md" + } > public-readiness-evidence/public-readiness-certification.md + + cat public-readiness-evidence/public-readiness-certification.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload public readiness evidence + uses: actions/upload-artifact@v4 + with: + name: public-readiness-evidence-${{ inputs.version }}-${{ inputs.runtime }} + path: public-readiness-evidence/* + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5ae1b5b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,215 @@ +name: Release + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Release version or image tag, for example v0.4.0' + required: true + type: string + publish_image: + description: 'Publish the container image to GHCR' + required: true + default: true + type: boolean + create_release: + description: 'Create a GitHub release with binary artifacts' + required: true + default: false + type: boolean + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + IMAGE_NAME: ghcr.io/stacyos/stacyvm + +permissions: + actions: read + contents: write + packages: write + id-token: write + +jobs: + release-artifacts: + name: Build release artifacts + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Resolve release version + id: version + shell: bash + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT" + else + echo "value=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + fi + + - name: Build binaries + run: make release-build-all VERSION="${{ steps.version.outputs.value }}" + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Sign release artifacts + shell: bash + run: | + for artifact in dist/*; do + cosign sign-blob --yes \ + --output-signature "${artifact}.sig" \ + --output-certificate "${artifact}.pem" \ + "${artifact}" + done + + - name: Upload release artifacts + uses: actions/upload-artifact@v4 + with: + name: stacyvm-release-${{ steps.version.outputs.value }} + path: dist/* + if-no-files-found: error + + - name: Create GitHub release + if: startsWith(github.ref, 'refs/tags/') || inputs.create_release == true + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ steps.version.outputs.value }}" dist/* \ + --title "${{ steps.version.outputs.value }}" \ + --generate-notes + + container-image: + name: Publish container image + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') || inputs.publish_image == true + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve image version + id: version + shell: bash + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "value=${{ inputs.version }}" >> "$GITHUB_OUTPUT" + else + echo "value=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ steps.version.outputs.value }} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + + - name: Build and publish image + id: build + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + VERSION=${{ steps.version.outputs.value }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Sign published image digest + run: cosign sign --yes "${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }}" + + public-readiness-certification: + name: Public readiness certification + needs: + - release-artifacts + - container-image + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Resolve release version + id: version + shell: bash + run: echo "value=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + + - name: Validate published release assets and Linux installer + env: + GH_TOKEN: ${{ github.token }} + STACYVM_VALIDATE_INSTALLER: "true" + run: scripts/post-release-validate.sh "${{ steps.version.outputs.value }}" + + - name: Download published StacyVM binary + run: | + curl -fsSL -o stacyvm-linux-amd64 \ + "https://github.com/StacyOS/stacyvm/releases/download/${{ steps.version.outputs.value }}/stacyvm-linux-amd64" + chmod +x stacyvm-linux-amd64 + + - name: Certify Docker runtime with StacyVM integration smoke + run: | + mkdir -p public-readiness-evidence + scripts/certify-runtime.sh docker \ + --format markdown \ + --output public-readiness-evidence/docker-certification.md \ + --stacyvm-bin ./stacyvm-linux-amd64 + + - name: Write certification summary + run: | + { + echo "# Public Readiness Certification" + echo + echo "- Version: \`${{ steps.version.outputs.value }}\`" + echo "- Runtime: \`docker\`" + echo "- Release validation: \`PASS\`" + echo "- Installer verify-only: \`PASS\`" + echo "- Runtime certification: \`PASS\`" + echo "- Evidence generated at: \`$(date -u '+%Y-%m-%dT%H:%M:%SZ')\`" + echo + echo "## Runtime Report" + echo + cat public-readiness-evidence/docker-certification.md + } > public-readiness-evidence/public-readiness-certification.md + + cat public-readiness-evidence/public-readiness-certification.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload public readiness evidence + uses: actions/upload-artifact@v4 + with: + name: public-readiness-evidence-${{ steps.version.outputs.value }}-docker + path: public-readiness-evidence/* + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 72bbb2d..f731b52 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,14 @@ web/dist/ .env .env.local +# TLS certificates and private keys — never commit real certs or keys to the repo +*.pem +*.key +*.crt +*.p12 +*.pfx +*.csr + # Coverage coverage/ *.cover diff --git a/.mintignore b/.mintignore new file mode 100644 index 0000000..5a8cbbc --- /dev/null +++ b/.mintignore @@ -0,0 +1,20 @@ +node_modules +.git +.github +.codex +cmd +deploy +dist +tmp +coverage +examples +images +internal +scripts +sdk +tests +tui +web +*.db +*.log +docs/docs.go diff --git a/.mintlify/Assistant.md b/.mintlify/Assistant.md new file mode 100644 index 0000000..9e7b73c --- /dev/null +++ b/.mintlify/Assistant.md @@ -0,0 +1,25 @@ +You are the StacyVM documentation assistant. + +## Tone + +- Be concise, technical, and direct. +- Assume readers are developers or infrastructure operators. +- Prefer runnable commands and links to relevant docs pages. + +## Product Context + +StacyVM is self-hosted execution infrastructure for autonomous software systems. +It provides isolated sandbox runtimes through Docker, Firecracker, PRoot, and remote workers. + +## Support Boundaries + +- Treat Docker public self-serve support as certified only for the published release path that has public readiness certification evidence. +- Do not claim Firecracker, gVisor, Kata, or PRoot are production-certified unless the user has host-level runtime certification output for that runtime. +- For production questions, point users to the production readiness, deployment, runtime certification, and public support matrix pages. + +## Terminology + +- Use "sandbox" for an isolated execution environment. +- Use "provider" for Docker, Firecracker, PRoot, mock, or remote worker runtime implementations. +- Use "runtime certification" for host-level proof generated by `scripts/certify-runtime.sh`. +- Use "public readiness certification" for GitHub release validation plus Docker runtime smoke evidence. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bc3cfd7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,509 @@ +# Changelog + +## Phase 14 Public-Readiness Repair - 2026-05-10 + +Security fixes required before public launch. No new features. + +### Fixed (security) + +- **RS256 JWT verification used `hash=0` instead of `crypto.SHA256`** — tokens from real OIDC providers (Google Workspace, Okta, Azure AD, Cloudflare Access) were rejected because they sign with SHA256, but verification was using `rsa.VerifyPKCS1v15(key, 0, ...)`. Fixed to `rsa.VerifyPKCS1v15(key, crypto.SHA256, ...)`. Tests updated to sign with `crypto.SHA256` and a new regression test `TestVerifyJWT_RS256UsesSHA256` proves hash=0 signatures are rejected. +- **Admin routes unprotected in OIDC-only mode** — when no `auth.admin_api_key` was configured and only OIDC was set, `AdminAuth` passed anonymous requests through silently (the `adminAPIKey == ""` branch fell through) and `RequireScope(ScopeAdmin)` was not applied. Fixed: `RequireScope(ScopeAdmin)` now fires whenever any auth is configured (`authConfigured`), not just when API keys are set. `AdminAuth` now correctly passes through only OIDC-established identities (Bearer token header) so API-key role promotion still works. +- **`AuthAny` rejected valid OIDC tokens in mixed mode** — when both OIDC and API keys were configured, a valid Bearer token was accepted by `OIDCAuth` but then rejected by `AuthAny` (which always looked for an API key header). Fixed: `AuthAny` now skips its check when OIDC already set an identity via `Authorization` header. +- **`PolicyEnforcer` was never mounted** — policy CRUD routes existed and policies could be created, but they were never evaluated on sandbox creation. Fixed: `PolicyEnforcer(st)` is now applied around `POST /api/v1/sandboxes` when a policy store is wired in. +- **Worker token issuer accepted non-worker scopes** — a caller could request `admin:*` in a worker token. The scope was silently filtered at verification, but the issuer should reject it outright. Fixed: `POST /api/v1/admin/worker-tokens` now explicitly validates that all requested scopes begin with `worker:` and returns 400 if any non-worker scope is present. + +### Added + +- **Auth matrix regression tests** (`internal/api/auth_matrix_test.go`) — 7 tests covering API-key only, OIDC only (normal route, viewer-cannot-spawn, admin route blocked for anonymous/non-admin/admin), mixed OIDC+API-key, worker token not treated as Bearer, and token issuer scope rejection. +- `TestVerifyJWT_RS256UsesSHA256` — explicitly proves the verifier uses `crypto.SHA256` and rejects hash=0 signatures. + +--- + +## Phase 14 Enterprise Governance - 2026-05-10 + +This release closes the enterprise production readiness gates: OIDC/SSO, RBAC, multi-tenancy, policy controls, and HA event durability. + +### Added + +- **OIDC/JWT authentication** — RS256 and ES256/ES384/ES512 Bearer token validation with configurable JWKS URL, static PEM public key (RSA or EC), issuer, audience, and clock-skew tolerance. Config keys: `auth.oidc_enabled`, `auth.oidc_issuer`, `auth.oidc_jwks_url`, `auth.oidc_public_key_file`. +- **RBAC roles** — `viewer` (read-only), `operator` (spawn/exec/files), `tenant_admin` (per-tenant admin) beyond existing `api` and `admin`. OIDC group-to-role mapping via `auth.oidc_admin_groups`, `auth.oidc_operator_groups`, `auth.oidc_viewer_groups`. +- **RBAC scope enforcement on sandbox routes** — read operations require `read:*`, mutating operations require `api:*`. Viewer-role users cannot spawn, exec, or destroy sandboxes. +- **Tenant/project model** — `tenants`, `tenant_members`, `policies` tables (migration 11). `tenant_id` on sandboxes, admin audit logs, and operation audit logs. Tenant CRUD, per-tenant member RBAC, per-tenant audit export, and per-tenant policy management via `/api/v1/admin/tenants`. +- **Policy controls** — per-tenant allow/deny rules for `image`, `provider`, and `network` resources with glob pattern matching and priority ordering. Policy enforcement runs at spawn time without consuming the request body. +- **Sandbox tenant scoping** — List, Get, Destroy, Exec, file, and log operations enforce tenant boundaries for OIDC-authenticated callers. +- **Centralized worker token issuer** — `POST /api/v1/admin/worker-tokens` mints short-lived signed worker tokens so workers do not need direct access to `auth.worker_signing_key`. Workers can call the issuer via `--bootstrap-admin-key`. +- **Durable EventBus (HA)** — When `database.driver` is `postgres`, a Postgres LISTEN/NOTIFY bridge propagates lifecycle events across control-plane replicas. Each bridge stamps events with an instance UUID to prevent double-delivery on the originating replica. +- **Postgres backup and rehearsal** — `stacyvm db pg-backup ` wraps `pg_dump`. `stacyvm db pg-rehearse` verifies schema state and checks all expected tables exist before upgrades. +- **Admin UI tenant management** — Tenants page in the web dashboard with tenant lifecycle, member RBAC, policy management, and one-click per-tenant audit export. +- **Config lint for OIDC** — `stacyvm config lint --production` validates OIDC issuer, JWKS URL, audience, and group-to-role mappings. +- **Worker RPC mTLS smoke** — `scripts/smoke-remote-worker.sh --mtls` generates an ephemeral CA + TLS certs and runs the full remote-worker smoke over HTTPS with mutual TLS auth. +- **Runtime certification integration smoke** — `scripts/certify-runtime.sh --stacyvm-bin` auto-starts a local StacyVM server, spawns a sandbox with the target runtime, execs a command, and destroys it to prove end-to-end functionality. + +### Fixed + +- Worker heartbeat now advertises `https://` RPC URL when `worker.rpc_tls.enabled = true`. +- Policy enforcement middleware buffers request body before inspection so downstream handlers can still decode it. +- Durable bridge no longer delivers events twice to local subscribers on the publishing instance. +- `stacyvm config lint` markdown output `printf` usage fixed for portability. + +### Security + +- RBAC scopes enforced: viewer tokens cannot reach any mutating sandbox route. +- Tenant isolation enforced across all sandbox operations: List, Get, Destroy, Exec, file reads/writes/deletes, and console logs. +- ES256 JWT support added — required for Google Workspace, Cloudflare Access, and Azure AD OIDC integrations. + +--- + +## Phase 14 Worker Identity Hardening - 2026-05-09 + +This checkpoint starts hardening remote worker identity for public and enterprise multi-worker deployments. + +### Added + +- HMAC-SHA256 signed worker token format with `stacyvm-worker-v1` prefix. +- Signed worker token claims for `worker_id`, `jti`, `aud`, optional worker scopes, `iat`, `nbf`, and `exp`. +- Worker auth verification for signed tokens through `auth.worker_signing_key`. +- Additional signed-token verification keys through `auth.worker_signing_keys` for no-downtime key rotation. +- Emergency signed-token revocation through `auth.worker_revoked_token_ids`. +- Worker token issuer JSON output, explicit token IDs, and delayed-validity `nbf` support. +- `stacyvm worker token inspect ` for unverified signed-token metadata and `jti` recovery during incident response. +- `stacyvm worker token verify ` for signature, rotation-key, audience, worker ID, and revocation validation. +- `stacyvm worker token rotation-plan` for no-secret signing-key rotation checklists and validation commands. +- Worker secret file flags for runtime worker tokens, runtime signing keys, token issuance signing keys, and verification rotation keys. +- Signed-token worker ID matching and expiry enforcement. +- Signed-token not-before, future issued-at, clock-skew, and max-lifetime enforcement. +- Signed-token audience separation for worker-to-control-plane and control-plane-to-worker RPC calls. +- Worker scope filtering so signed tokens cannot grant non-worker scopes. +- Worker runtime token derivation for heartbeat and lease-renewal calls when `auth.worker_signing_key` is configured and no static worker token is provided. +- Worker token-file reload support so sidecar-issued short-lived worker tokens can rotate without restarting workers. +- Signed control-plane-to-worker RPC tokens for remote worker calls when no shared `auth.worker_token` is configured. +- `stacyvm worker token ` for issuing signed worker tokens. +- Config loading and defaults for `auth.worker_signing_key` and `auth.worker_signing_keys`. +- Config lint awareness for signed worker credentials. +- Config lint warnings for shared worker tokens left enabled beside signed worker tokens and invalid signing-key rotation state. +- Worker RPC mTLS config under `worker.rpc_tls`. +- TLS server support for inbound worker RPC. +- TLS client support for control-plane calls to remote worker RPC. +- Config-level worker secret file support through `auth.worker_token_file` and `auth.worker_signing_key_file`. +- Worker RPC mTLS conformance test with generated CA, server, and client certificates. +- Config lint checks for worker RPC TLS certificate and CA settings. +- Config lint source checks for file-backed versus inline worker token and signing-key secrets. +- Cluster conformance coverage for signed-token migration lint warnings. +- `scripts/certify-worker-identity.sh` for signed-token lifecycle certification reports on target hosts. +- Cluster conformance coverage for worker identity certification report generation. +- Phase 14 release notes under `docs/releases/phase-14-worker-identity-hardening.md`. + +### Changed + +- Worker auth now accepts signed tokens, per-worker static tokens, or the shared staging token while preserving existing compatibility. +- Config lint now warns when revoked signed-token IDs are configured without signed worker-token verification. +- Worker RPC docs now include an issue, inspect, verify, and revoke runbook for signed worker tokens. +- Worker RPC docs now include a generated no-secret rotation-plan workflow for signed worker token keys. +- Worker token runbooks now prefer secret-mounted files over shell history or environment variables for long-lived worker secrets. +- Production readiness notes now reflect signed worker identity certification and the remaining target-network/runtime signoff gates. +- Public support matrix now describes multi-worker as preview with signed identity, worker RPC routing, mTLS wiring, and explicit certification evidence. +- Cluster conformance documentation now treats signed worker tokens as the production-aligned worker identity path. +- API and worker RPC docs now describe signed worker token behavior, signing-key rotation, and worker RPC mTLS. +- Threat model and remote-worker staging docs now treat signed worker tokens as implemented Phase 14 identity controls instead of future work. +- Runtime certification and public support guidance now document reloadable worker token files for external issuer handoff. +- Config examples now prefer secret-mounted worker credential files for production services. +- Worker identity config lint now reports file-backed worker credential sources so inline secrets are visible during production review. + +## Phase 13 Cluster Store And Worker Identity - 2026-05-09 + +This checkpoint starts the enterprise multi-worker production track after Phase 12 completed remote sandbox I/O routing. + +### Added + +- Driver-based store factory through `store.Open`. +- `database.driver` config with SQLite as the default. +- `database.dsn` config for Postgres-backed cluster storage. +- Linked Postgres store implementation through the pgx stdlib driver. +- Config validation for database driver selection and required Postgres DSN. +- Reusable store contract test harness wired to SQLite. +- Cross-store contract coverage for sandbox lifecycle, workers, leases, audits, exec logs, quotas, provider configs, templates, environment builds, artifacts, and registry connections. +- Per-worker token map support through `auth.worker_tokens`. +- Worker auth scopes for heartbeat, spawn, destroy, status, exec, files, logs, and leases. +- Config linting guidance for staging shared worker tokens versus production per-worker credentials. +- Cluster conformance CI script and workflow job. +- Cluster conformance matrix documenting store, worker identity, runtime, and promotion gates. +- Postgres-native migration definitions matching the current SQLite schema version cadence. +- Store migration tests that keep Postgres schema coverage aligned with SQLite migrations. +- Live Postgres store contract path through `STACYVM_POSTGRES_TEST_DSN`. +- Live Postgres lease race/concurrency coverage. +- Live Postgres migration rehearsal coverage. +- Remote worker smoke support for Postgres-backed control planes. +- Phase 13 release notes under `docs/releases/phase-13-cluster-store-and-worker-identity.md`. + +### Changed + +- `stacyvm serve` now opens persistence through the driver-based store factory. +- `stacyvm config lint` now passes Postgres configs with a valid DSN. +- Worker lease renewal now checks the dedicated `worker:lease` scope instead of reusing heartbeat-only authorization. + +## Phase 12 Remote Sandbox I/O Routing - 2026-05-09 + +This checkpoint starts extending remote worker routing beyond sandbox lifecycle operations. + +### Added + +- `worker.exec` in the worker RPC contract. +- Worker protocol exec request and result payloads. +- `worker:exec` scope constant for future worker token scoping. +- Worker-side non-streaming exec handling through the provider registry. +- Worker-side live NDJSON streaming exec handling through the provider registry. +- Typed worker RPC client support for exec calls. +- Typed worker RPC client support for live exec-stream calls. +- Worker RPC contract and client support for file write, read, list, delete, move, chmod, stat, and glob. +- Worker RPC contract and client support for remote console logs. +- Worker preview domain advertisement through heartbeat capacity. +- Remote worker drain/offline reconciliation policy for owned sandboxes. +- Control-plane routing for non-streaming exec on remote-owned sandboxes. +- Control-plane routing for exec-stream calls on remote-owned sandboxes. +- Control-plane routing for remote-owned sandbox file APIs. +- Control-plane routing for remote-owned sandbox console logs. +- Control-plane preview-domain routing for remote-owned sandboxes. +- Phase 12 release notes under `docs/releases/phase-12-remote-sandbox-io-routing.md`. + +### Changed + +- Remote-owned sandbox exec now uses persisted worker ownership and provider runtime ID instead of the local provider. +- Remote exec preserves existing event, audit, metric, timeout, and exec-log behavior. +- Remote exec-stream keeps the manager streaming API shape while forwarding live NDJSON chunks over worker RPC. +- Remote file APIs now use persisted worker ownership and provider runtime ID instead of local provider state. +- Remote console logs now use persisted worker ownership and provider runtime ID instead of local provider state. +- Remote-owned sandbox responses now use the owning worker's preview domain when the worker advertises one. +- Remote-owned sandboxes on stale/offline workers are marked `unhealthy`; expired remote-owned sandboxes are marked `expired` and release their lease. +- Draining workers keep existing ownership but stay out of new placement. +- Pool-mode default workdir is no longer applied to remote sandboxes just because their provider runtime ID is stored in `VMID`. + +## Phase 11 Remote Worker Runtime - 2026-05-09 + +This checkpoint starts the remote worker runtime track on top of the Phase 10 worker registry, lease, and RPC contract foundation. + +### Added + +- `stacyvm worker` command for running a remote worker process. +- Worker command flags for ID, control-plane URL, worker token, heartbeat interval, and one-shot heartbeat smoke tests. +- Worker RPC listen flag for inbound control-plane-to-worker calls. +- `worker.id`, `worker.control_plane_url`, `worker.listen_addr`, `worker.heartbeat_interval`, and `worker.shutdown_timeout` config. +- `auth.worker_token` for worker-to-control-plane authentication. +- Dedicated worker auth role with `worker:heartbeat` scope. +- Worker-only heartbeat endpoint at `POST /api/v1/worker/{workerID}/heartbeat`. +- `internal/worker` heartbeat client and runtime loop. +- Worker-side `/rpc` handler for `workerproto.Request` envelopes. +- Worker RPC status handling through `worker.status`. +- Worker-authenticated lease renewal endpoint for durable control-plane leases. +- Worker RPC lease renewal handling through `worker.renew_lease`. +- Worker-side spawn RPC handling through `worker.spawn`. +- Typed worker RPC client methods for spawn and status calls. +- Control-plane remote spawn assignment for scheduler-selected workers that advertise an RPC URL. +- Remote-owned sandbox status refresh through `worker.status`. +- Worker-side destroy RPC handling through `worker.destroy`. +- Control-plane remote destroy routing for remote-owned sandboxes. +- Two-process remote worker staging guide and mock smoke script. +- Worker shutdown drain state that rejects new spawn assignments and reports `draining` heartbeats. +- Phase 11 release notes under `docs/releases/phase-11-remote-worker-runtime.md`. + +### Changed + +- Worker heartbeats can now be submitted without reusing API/admin keys. +- Worker heartbeat requests are rejected when the authenticated worker ID does not match the requested worker path. +- Worker lease renewal validates resource, holder, and expiry before renewing durable control-plane leases. +- Worker spawn validates lease ownership before creating a provider runtime and returns both control-plane sandbox ID and provider runtime ID. +- Remote spawn persists selected `worker_id` and provider runtime ID for later status/destroy routing. +- Sandbox reads refresh and persist state changes reported by the owning remote worker. +- Remote destroy validates worker lease ownership, tears down the provider runtime, marks the sandbox destroyed, and releases the durable lease. +- Production readiness documentation now reflects the Phase 11 remote spawn/status/destroy transport state. +- Scheduler placement avoids draining workers because they no longer report `online`. +- Config validation now covers worker heartbeat and shutdown durations. + +## Phase 10 Multi-Worker Foundation - 2026-05-09 + +This checkpoint starts the enterprise and multi-worker readiness track. + +### Added + +- SQLite-backed worker registry storage with heartbeat, provider, capability, and capacity fields. +- Store-level worker CRUD methods for future scheduler and worker ownership work. +- Local worker registration on API server startup so single-node deployments report as a worker. +- Worker registry API: + - `GET /api/v1/workers` + - `GET /api/v1/workers/{workerID}` + - `POST /api/v1/admin/workers/{workerID}/heartbeat` + - `DELETE /api/v1/admin/workers/{workerID}` +- Worker ownership on sandbox records through persisted `worker_id`. +- Worker-aware spawn admission that evaluates eligible workers by status, heartbeat freshness, provider support, capacity, and active sandbox count. +- Durable lease storage with acquire, renew, release, get, and list semantics for future distributed sandbox ownership. +- Lease enforcement around local spawn/adopt/destroy lifecycle paths. +- Periodic local worker heartbeat refresh while the API server is running. +- Worker RPC contract types under `internal/workerproto`. +- Worker authentication scopes and cluster store semantics documentation. +- Diagnostics worker summary with online, stale, unhealthy, and total counts. +- Diagnostics lease summary with active, expired, total, and per-holder counts. +- Diagnostics sandbox summaries grouped by worker ID. +- Prometheus worker count metrics by status. +- Prometheus sandbox ownership metrics by worker ID. +- Prometheus lease count metrics by status. +- Phase 10 release notes under `docs/releases/phase-10-multi-worker-foundation.md`. + +### Changed + +- Diagnostics now include worker registry state alongside provider, sandbox, scheduler, quota, rate-limit, and operation data. +- The public API exposes read-only worker discovery while heartbeat and delete operations live under the admin namespace. +- Scheduler status now reports the active local worker ID. +- Scheduler status now reports the selected worker and eligible worker count while remote execution remains gated on worker RPC. +- Lease acquisition is holder-checked and expiry-aware so later workers can safely fence lifecycle ownership. +- Destroy now requires the local worker to acquire or hold the sandbox lease before mutating provider or store state. +- Worker placement now treats stale local worker records the same as stale remote workers; the heartbeat loop keeps the local worker fresh in real server runs. +- Remote worker execution remains gated until a network transport enforces the worker RPC contract. + +## Phase 9 Public Self-Serve Release Trust - 2026-05-08 + +This checkpoint starts the Phase 9 public self-serve production readiness track. + +### Added + +- Sigstore keyless signing for release binaries and `checksums.txt`. +- Sigstore signing for published GHCR image digests. +- `scripts/verify-release.sh` for public release signature and checksum verification. +- Installer Sigstore verification when `cosign` is available. +- `STACYVM_REQUIRE_SIGNATURES=true` installer mode for fail-closed public installs. +- Phase 9 acceptance criteria in the production readiness checklist. +- Upgrade and SQLite migration checks in CI through `scripts/ci-upgrade-migration.sh`. +- Diagnostics remediation links for production readiness, runtime certification, release verification, support bundles, and upgrade rollback. +- Public self-serve limitations and support matrix under `docs/public-support-matrix.md`. +- Public release sanity checks in CI for installer/verifier shell syntax, release builds, and checksum validation. +- `scripts/post-release-validate.sh` for post-tag release asset, signature, checksum, and installer verify-only validation. +- Mock-based TypeScript and Python SDK parity smoke tests in CI. +- GitHub bug and production support issue templates that request support bundle, config lint, upgrade rehearsal, runtime certification, and release verification evidence. + +### Changed + +- Release documentation now explains binary, checksum, and container signature verification. +- Production readiness documentation now marks Phase 9 upgrade/migration CI and public limitation docs as complete. +- Docker integration tests are opt-in with `STACYVM_DOCKER_INTEGRATION=1` so default CI remains independent of Docker Hub and host daemon state. +- TypeScript SDK spawn options now include `template`, matching Python spawn behavior. +- Python SDK now exposes `templates` and `providers()` helpers for closer TypeScript parity. +- `scripts/install.sh` supports `STACYVM_VERIFY_ONLY=true` for release validation without installing binaries or touching host setup. + +## Phase 8 Single-Node Production - 2026-05-08 + +This checkpoint starts the Phase 8 single-node production readiness track. + +### Added + +- `stacyvm db backup` for consistent SQLite backups with integrity validation. +- `stacyvm db restore` with explicit confirmation, backup validation, pre-restore safety copy, and stale WAL/SHM cleanup. +- `stacyvm config lint --production` for deterministic single-node production config validation. +- `stacyvm upgrade rehearse` for pre-upgrade config, database, backup path, live-check, and rollback guidance. +- `stacyvm support bundle` for redacted operator diagnostics. +- Runtime certification reports through `scripts/certify-runtime.sh --format json|markdown --output `. +- Phase 8 release notes under `docs/releases/phase-8-single-node-production.md`. + +### Changed + +- Deployment and release docs now include production config linting before upgrades and release tags. +- Deployment docs now cover upgrade rehearsal, rollback, and support bundle generation. +- Runtime certification docs now require host-generated certification artifacts for runtime signoff. + +## Phase 7 Release Candidate Hardening - 2026-05-08 + +This checkpoint starts the Phase 7 release-candidate hardening track. + +### Added + +- Initial `stacyvm doctor` command with local and production diagnostic modes. +- Production readiness checklist under `docs/production-readiness.md`. +- Threat model under `docs/threat-model.md`. +- Phase 7 release notes under `docs/releases/phase-7-release-candidate-hardening.md`. + +### Changed + +- Made exec command mode explicit with backwards-compatible `shell` mode and safer `argv` mode. +- Added operation audit persistence for sandbox lifecycle, exec, and file operations. +- Tightened pooled file path traversal handling and expanded traversal tests across file operations. +- Added runtime host certification script and documentation. +- Added remediation guidance to `stacyvm doctor` output. + +## Phase 6 Security Governance - 2026-05-08 + +This checkpoint starts the Phase 6 security and governance work on top of the Phase 5 admin control plane. + +### Added + +- Request-scoped authentication identities with `api` and `admin` roles. +- Initial scope metadata for authenticated requests: `api:*` and `admin:*`. +- Route-level scope enforcement for authenticated admin routes. +- Admin audit fallback attribution now includes the authenticated role and key header when no explicit actor header is supplied. +- Configurable admin fallback through `auth.admin_fallback_enabled`. +- Production security governance guide with admin hardening, key handling, audit retention, and OIDC/SSO design groundwork. +- Phase 6 release notes under `docs/releases/phase-6-security-governance.md`. + +## Phase 5 Admin Control Plane - 2026-05-08 + +This checkpoint starts the Phase 5 operator control plane by separating admin access from regular API usage. + +### Added + +- Optional `auth.admin_api_key` / `STACYVM_AUTH_ADMIN_API_KEY` configuration. +- `X-Admin-API-Key` support for admin requests. +- `/api/v1/admin/*` route aliases for providers, quotas, diagnostics, JSON metrics, and Prometheus metrics. +- Admin key examples in deployment templates and docs. +- Dashboard settings for separate regular and admin API keys. +- Operations dashboard page for admin quota controls and diagnostics. +- Persisted admin audit log storage and `/api/v1/admin/audit`. +- Admin control-plane operator guide under `docs/admin-control-plane.md`. +- Config-driven admin audit retention through `auth.admin_audit_retention`. + +### Changed + +- Normal API and admin API keys can both authenticate regular API requests. +- Admin routes require the admin key when configured, with fallback to the regular API key only when no admin key is set. +- Dashboard provider and metrics calls now use the admin namespace. +- Provider health checks in the dashboard now call `/api/v1/admin/providers/test`. +- Owner quota list, save, delete, summary, usage, and diagnostics workflows are available from the dashboard. +- Admin route access is recorded with redacted request metadata and shown in the Operations dashboard. +- Admin audit history can be filtered by actor, method, status, and path, then exported as CSV. +- Admin audit pruning removes records older than the configured retention window after successful admin audit writes. + +### Verified + +- `go test ./internal/api/middleware ./internal/config ./cmd/stacyvm` +- `npm run build` + +## Phase 4 Production Deployment - 2026-05-08 + +This checkpoint adds the first production deployment and verification surface for Phase 4: GitHub Actions CI, deployment templates, and an operator runbook. + +### Added + +- GitHub Actions workflow for Go tests/build, Swagger drift, web build, TypeScript SDK build, and Python SDK import checks. +- Production Docker Compose template with StacyVM and Traefik for live previews. +- Production baseline config with auth, rate limiting, sandbox caps, queueing, JSON logs, and persistent SQLite state. +- systemd unit and environment template for binary-based Linux installs. +- Deployment guide covering host requirements, health probes, Prometheus metrics, reverse proxy setup, backups, upgrades, and provider notes. +- Release workflow for GitHub releases and GHCR container image publishing. +- Release runbook documenting tags, manual dispatch, binary artifacts, image tags, and preflight checks. +- `.dockerignore` for smaller and safer Docker build contexts. +- Deployment smoke script for live, health, readiness, and Prometheus probes. +- CI deployment smoke job using the mock provider. +- Runtime conformance matrix for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. +- Phase 4 release notes under `docs/releases/phase-4-production-deployment.md`. + +### Changed + +- Swagger drift checks now download Go modules before invoking `swag`, which makes cold CI runners reliable. +- CI opts into Node 24-based JavaScript actions to address the GitHub Actions Node 20 deprecation warning. +- Docker image builds now accept an explicit `VERSION` build argument and BuildKit target platform args. +- Release artifacts now build into `dist/` with checksums instead of the repository root. +- `stacyvm serve` now registers the mock provider when `providers.mock.enabled` is true. +- Production Compose now allows the Traefik host port to be overridden for smoke runs. +- Production Compose has been runtime-smoked with StacyVM, Traefik, Docker provider readiness, API probes, and live-preview routing. +- README navigation now links to the production deployment guide. + +### Verified + +- `docker compose --env-file deploy/.env.example -f deploy/docker-compose.yml config` +- YAML parsing for deployment templates +- `git diff --check` +- `go test ./...` +- `cd web && npm run build` +- `scripts/check-swagger.sh` +- `make release-build-all VERSION=phase-4-test` + +## Phase 3 Quotas And Scheduling - 2026-05-08 + +This checkpoint adds the first production multi-tenant control plane: persisted owner quotas, API rate limiting, spawn backpressure, scheduler visibility, admission preflight, and SDK helpers. + +### Added + +- Persisted owner quota policies for max sandboxes, max TTL, and max exec timeout. +- Owner quota APIs, including usage and redacted summary endpoints. +- Spawn admission decisions and `POST /api/v1/sandboxes/admission`. +- Configurable spawn overflow queue with queue timeout and maximum queue depth. +- Optional API rate limiting by owner, API key, or IP address. +- Scheduler, quota, and rate-limit metrics in JSON diagnostics/metrics and Prometheus output. +- TypeScript and Python SDK helpers for admission preflight and quota summary. + +### Changed + +- Spawn admission is serialized to avoid concurrent over-admission. +- Queued spawns wake when capacity opens or owner quotas change. +- Rate-limit bucket keys are hashed before storage. +- Streaming exec cancellation is no longer reported as a timeout. +- Streaming exec preflight errors now use the same API error mapping as non-streaming exec. +- OpenAPI docs were regenerated for the Phase 3 API surface. + +### Verified + +- `go test ./internal/api/routes ./internal/orchestrator` +- `make build` +- `cd web && npm run build` +- `make test` + +## Phase 2 Observability And Ops - 2026-05-08 + +This checkpoint adds production operations surfaces for health checks, diagnostics, metrics, audit events, and runtime limits. + +### Added + +- Liveness endpoint at `/api/v1/live`. +- Readiness endpoint at `/api/v1/ready` with detailed provider health. +- Redacted diagnostics endpoint at `/api/v1/diagnostics`. +- Structured JSON operation metrics on `/api/v1/metrics`. +- Prometheus-compatible metrics endpoint at `/api/v1/metrics/prometheus`. +- Provider health detail with latency, last checked time, capabilities, error reason, and runtime inventory count when supported. +- Operational audit events for exec failures, exec timeouts, provider failures, resource limits, and reconciliation actions. +- Configurable operational limits for max TTL, default/max exec timeout, max sandboxes, and max sandboxes per owner. + +### Changed + +- `/api/v1/metrics` now includes sandbox state/provider breakdown, provider health, event bus stats, and operation metrics. +- `/api/v1/providers` and `/api/v1/providers/{name}` now expose richer provider health details. +- Diagnostics include store health, build/runtime data, sandbox counts, provider health, event stats, operation metrics, and explicit redaction categories. +- Manager-level spawn and exec flows now enforce configured operational limits centrally. + +### Verified + +- `make test` +- `make build` +- `cd web && npm run build` + +## Phase 1 Foundation Hardening - 2026-05-08 + +This checkpoint closes the Phase 1 reliability and production-readiness foundation. + +### Added + +- Provider contract documentation in `docs/provider-contract.md`. +- Typed provider errors for sandbox lifecycle, provider availability, exec timeout, and resource-limit failures. +- Typed store errors for not-found and conflict cases. +- Shared API route error mapping with explicit `404`, `408`, `429`, and `503` responses. +- Provider conformance harness covering lifecycle, exec, streaming exec, and file operations. +- Mock, Docker, Custom, PRoot, and Firecracker conformance coverage, with PRoot and Firecracker gated on platform dependencies. +- Startup reconciliation that refreshes persisted sandbox state from provider runtime state. +- Docker runtime inventory and adoption for StacyVM containers missing from SQLite after process restart. +- Streaming exec timeout handling that emits an explicit stderr timeout chunk. +- Non-Linux `stacyvm-agent` stub so repository builds work on macOS while the real agent remains Linux-only. + +### Changed + +- Sandbox, template, environment, and provider routes now use typed errors instead of string matching. +- Docker sandboxes now include richer `stacyvm.*` labels for reconciliation and metadata recovery. +- Docker missing-container paths now map to `ErrSandboxNotFound`. +- Manager `Exec` and `ExecStream` now consistently honor caller-supplied timeouts. +- Provider comments now point implementers to the documented contract and conformance tests. + +### Verified + +- `make test` +- `make build` +- `cd web && npm run build` +- Docker provider conformance and runtime inventory tests with Docker daemon access + +### Platform Notes + +- Firecracker conformance is available on Linux hosts with `/dev/kvm`, Firecracker, kernel, rootfs, and agent paths configured. +- PRoot conformance is available when `proot` and a usable rootfs are installed. +- Local sandboxed test runs still need permission to bind `httptest` sockets for the full integration suite. diff --git a/Dockerfile b/Dockerfile index 306fcdc..cda445c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,15 @@ -FROM golang:1.25-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=$(git describe --tags --always 2>/dev/null || echo dev)" -o /stacyvm ./cmd/stacyvm -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o /stacyvm-agent ./cmd/stacyvm-agent +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -ldflags="-s -w -X main.version=$VERSION" -o /stacyvm ./cmd/stacyvm +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -ldflags="-s -w" -o /stacyvm-agent ./cmd/stacyvm-agent FROM alpine:3.20 diff --git a/Makefile b/Makefile index 4a4eac6..2ed6fa1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: build build-agent build-android build-agent-arm64 test lint clean serve dev release-build release-build-all VERSION ?= $(shell git describe --tags --always 2>/dev/null || echo dev) +DIST_DIR ?= dist # Build the server/CLI binary build: @@ -37,10 +38,14 @@ web: serve: build ./stacyvm serve +# Check local prerequisites, build, and start a development server +dev: + ./scripts/dev.sh + # Clean build artifacts clean: rm -f stacyvm stacyvm-agent - rm -rf bin/ web/dist/ + rm -rf bin/ web/dist/ $(DIST_DIR)/ # Run go vet lint: @@ -48,12 +53,13 @@ lint: # Build static release binaries + checksums (amd64 only) release-build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o stacyvm-linux-amd64 ./cmd/stacyvm - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o stacyvm-agent-linux-amd64 ./cmd/stacyvm-agent - sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 > checksums.txt + mkdir -p $(DIST_DIR) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(DIST_DIR)/stacyvm-linux-amd64 ./cmd/stacyvm + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(DIST_DIR)/stacyvm-agent-linux-amd64 ./cmd/stacyvm-agent + cd $(DIST_DIR) && sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 > checksums.txt # Build release binaries for all architectures (amd64 + arm64) release-build-all: release-build - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o stacyvm-linux-arm64 ./cmd/stacyvm - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o stacyvm-agent-linux-arm64 ./cmd/stacyvm-agent - sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 stacyvm-linux-arm64 stacyvm-agent-linux-arm64 > checksums.txt + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(DIST_DIR)/stacyvm-linux-arm64 ./cmd/stacyvm + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o $(DIST_DIR)/stacyvm-agent-linux-arm64 ./cmd/stacyvm-agent + cd $(DIST_DIR) && sha256sum stacyvm-linux-amd64 stacyvm-agent-linux-amd64 stacyvm-linux-arm64 stacyvm-agent-linux-arm64 > checksums.txt diff --git a/README.md b/README.md index 68d7e4a..8f25197 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ On bare metal? Firecracker microVMs in ~28ms.
On Kubernetes? gVisor or Kata containers.
Need 100 sandboxes but only have 20 VMs? Pool mode.
Need to expose localhost:3000 from inside the sandbox? Live preview, one method call.

-Self-hosted. Single binary. Python & TypeScript SDKs. MIT licensed. No cloud required. +Self-hosted. Single binary. Python & TypeScript SDKs. Apache 2.0 licensed. No cloud required.

@@ -23,7 +23,7 @@ Self-hosted. Single binary. Python & TypeScript SDKs. MIT licensed. No cloud Forks Issues Go - MIT + Apache 2.0 Platform

@@ -33,7 +33,8 @@ Self-hosted. Single binary. Python & TypeScript SDKs. MIT licensed. No cloud Providers • Live Preview • Pool Mode • - API Reference • + Deployment • + API Reference • Contributing

@@ -56,6 +57,11 @@ Self-hosted. Single binary. Python & TypeScript SDKs. MIT licensed. No cloud - [Providers, pool, system](#providers-pool-system) - [CLI](#cli) - [Configuration](#configuration) +- [Enterprise / Multi-worker](#enterprise--multi-worker) + - [OIDC & RBAC](#oidc--rbac) + - [Multi-tenancy & policy controls](#multi-tenancy--policy-controls) + - [Remote workers & mTLS](#remote-workers--mtls) +- [Production deployment](#production-deployment) - [Templates](#templates-1) - [Security defaults](#security-defaults) - [Architecture](#architecture) @@ -130,7 +136,7 @@ You're building an AI agent. It generates code. That code needs to run somewhere | File API (read/write/glob) | ✅ 9 methods | ✅ | ❌ | ❌ | ❌ | ❌ | | Python + TS SDKs | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | | Your data stays local | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| License | MIT | Partial | Apache 2.0 | Apache 2.0 | Proprietary | N/A | +| License | Apache 2.0 | Partial | Apache 2.0 | Apache 2.0 | Proprietary | N/A | > **On speed:** Zeroboot's 0.8ms is real — they bypass Firecracker's VMM entirely and `mmap(MAP_PRIVATE)` the snapshot memory as copy-on-write. But there's no disk, no network, and I/O is serial UART only. StacyVM's 28ms gives you a full sandbox with networking, file system, virtio, and multi-vCPU. Different tools for different jobs. @@ -260,6 +266,8 @@ client = Client("http://localhost:7423", user_id="alice@example.com") const client = new Client({ baseUrl: "http://localhost:7423", userId: "alice@example.com" }); ``` +User IDs are trimmed by the server. They must be 128 characters or fewer and cannot contain whitespace, control characters, or path separators. + Hardening knobs (Docker provider): ```yaml @@ -373,6 +381,7 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | Method | Endpoint | Description | |---|---|---| | `POST` | `/sandboxes` | Spawn a sandbox | +| `POST` | `/sandboxes/admission` | Preflight quota and scheduler admission | | `GET` | `/sandboxes` | List active sandboxes | | `DELETE` | `/sandboxes` | Prune expired sandboxes | | `GET` | `/sandboxes/{id}` | Get sandbox details | @@ -382,6 +391,8 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | `GET` | `/sandboxes/{id}/exec/ws` | Execute over WebSocket | | `GET` | `/sandboxes/{id}/logs` | Console logs | +Exec requests default to backwards-compatible shell mode. Set `mode: "argv"` with `args` to run direct process arguments without shell interpolation. + ### Files (per sandbox) | Method | Endpoint | Description | @@ -413,13 +424,27 @@ Auth: pass `X-API-Key: ` if `auth.enabled: true`. For pool mode, also | `GET` | `/providers` | List configured providers | | `GET` | `/providers/{name}` | Provider details + sandbox count | | `POST` | `/providers/test` | Health-check all providers | +| `GET` | `/quotas` | List owner quota overrides | +| `GET` | `/quotas/summary` | Redacted owner quota policy counts | +| `PUT` | `/quotas/{ownerID}` | Create or update owner quota | +| `GET` | `/quotas/{ownerID}/usage` | Owner usage against effective quota | +| `GET` | `/workers` | List registered workers and heartbeat state | +| `GET` | `/workers/{workerID}` | Get one worker registry record | +| `POST` | `/worker/{workerID}/heartbeat` | Remote worker heartbeat with worker token | +| `POST` | `/worker/{workerID}/leases/{resourceID}/renew` | Remote worker lease renewal | | `GET` | `/pool/status` | Pool VM and user counts | | `GET` | `/snapshots` | Available VM snapshots | | `GET` | `/health` | Health check | +| `GET` | `/ready` | Readiness check | +| `GET` | `/diagnostics` | Redacted operational diagnostics | | `GET` | `/metrics` | Runtime metrics (goroutines, alloc, sandbox counts) | +| `GET` | `/metrics/prometheus` | Prometheus-compatible metrics | | `GET` | `/events` | Server-sent events stream | -Full schemas, request/response examples, and error codes: **[docs/api.md](docs/api.md)**. +Admin aliases for providers, quotas, workers, diagnostics, and metrics are available under `/admin/*` and can be protected with `auth.admin_api_key`. Worker registry deletion remains admin-only under `/admin/workers/*`; remote worker heartbeat and lease renewal use worker-only `/worker/*` endpoints. +For the operator dashboard, quota workflows, diagnostics, and persisted admin audit history, see [docs/admin-control-plane.md](docs/admin-control-plane.md). + +Full schemas, request/response examples, and error codes: **[docs/rest-api.md](docs/rest-api.md)**. OpenAPI spec: [docs/swagger.yaml](docs/swagger.yaml). --- @@ -428,11 +453,18 @@ OpenAPI spec: [docs/swagger.yaml](docs/swagger.yaml). ```bash stacyvm serve # start the API server +stacyvm worker --once # send one remote-worker heartbeat +stacyvm worker --listen 127.0.0.1:7430 # heartbeat + worker RPC server stacyvm spawn --image python:3.12 --ttl 1h # spawn -stacyvm exec sb-a1b2c3d4 -- python3 app.py # run a command in a sandbox +stacyvm exec sb-a1b2c3d4 -- python3 app.py # run argv mode in a sandbox +stacyvm exec sb-a1b2c3d4 --shell -- "echo $HOME && pwd" stacyvm list # list active sandboxes stacyvm kill sb-a1b2c3d4 # destroy stacyvm build-image python:3.12 # pre-build rootfs (Firecracker) +stacyvm config lint --production # production config lint gate +stacyvm db backup /backup/stacyvm.db # consistent SQLite backup +stacyvm upgrade rehearse --database stacyvm.db # rehearse single-node upgrade +stacyvm support bundle support.json # redacted support diagnostics stacyvm tui # interactive dashboard stacyvm version # version info ``` @@ -451,6 +483,24 @@ server: host: "0.0.0.0" port: 7423 preview_domain: "localhost" # used to build live-preview URLs + cors_allowed_origins: ["*"] # set explicit https:// origins in production + +worker: + id: "" # defaults to hostname + control_plane_url: "http://localhost:7423" + listen_addr: "" # set to enable inbound worker RPC + heartbeat_interval: "30s" + shutdown_timeout: "10s" + rpc_tls: + enabled: false + server_cert_file: "" + server_key_file: "" + client_ca_file: "" + ca_file: "" + client_cert_file: "" + client_key_file: "" + server_name: "" + insecure_skip_verify: false providers: default: "docker" @@ -497,13 +547,53 @@ defaults: image: "alpine:latest" memory_mb: 1024 vcpus: 1 + max_ttl: "24h" + default_exec_timeout: "0s" # disabled unless set + max_exec_timeout: "10m" + max_sandboxes: 0 # 0 = unlimited + max_sandboxes_per_owner: 0 # 0 = unlimited + spawn_overflow: "reject" # reject or queue when sandbox capacity is full + spawn_queue_timeout: "30s" + max_spawn_queue: 100 auth: enabled: false api_key: "" + admin_api_key: "" # optional separate key for /api/v1/admin/* + worker_token: "" # shared staging worker token + worker_token_file: "" # file containing shared worker token + worker_tokens: {} # production map of worker_id: token + worker_signing_key: "" # production signed worker token verification key + worker_signing_key_file: "" # file containing active worker signing key + worker_signing_keys: [] # old verification keys accepted during rotation + worker_revoked_token_ids: [] # signed worker token jti values rejected during incidents + admin_fallback_enabled: true # false requires admin_api_key for admin routes + admin_audit_retention: "0s" # 0s disables native audit pruning + + # OIDC/SSO — enterprise single sign-on (RS256 and ES256/ES384/ES512) + oidc_enabled: false + oidc_issuer: "" # e.g. https://accounts.google.com + oidc_audience: "" # your application's client ID / audience + oidc_jwks_url: "" # IdP's JWKS endpoint for key verification + oidc_public_key_file: "" # alternative: static PEM public key file + oidc_groups_claim: "groups" # JWT claim containing group memberships + oidc_tenant_claim: "tenant_id" # JWT claim containing tenant identifier + oidc_admin_groups: [] # groups that receive the admin role + oidc_operator_groups: [] # groups that receive the operator role + oidc_viewer_groups: [] # groups that receive the read-only viewer role + +rate_limit: + enabled: false + requests_per_minute: 120 + burst: 60 + key_by: "owner" # owner, api_key, or ip + bucket_ttl: "15m" + cleanup_interval: "1m" database: + driver: "sqlite" # sqlite; postgres config is reserved for cluster builds path: "stacyvm.db" + dsn: "" # required for future postgres-backed cluster mode logging: level: "info" # debug | info | warn | error @@ -526,11 +616,116 @@ pool: STACYVM_SERVER_PORT=8080 STACYVM_PROVIDERS_DEFAULT=firecracker STACYVM_AUTH_API_KEY=sk-xyz123 +STACYVM_AUTH_ADMIN_API_KEY=sk-admin-xyz123 +STACYVM_AUTH_ADMIN_FALLBACK_ENABLED=false +STACYVM_RATE_LIMIT_ENABLED=true STACYVM_LOGGING_LEVEL=debug ``` --- +## Enterprise / Multi-worker + +StacyVM ships everything you need to run as multi-tenant infrastructure. All features below are stable and covered by CI. + +### OIDC & RBAC + +Enable enterprise SSO by pointing StacyVM at any RFC 7517-compliant identity provider: + +```yaml +auth: + enabled: true + oidc_enabled: true + oidc_issuer: "https://accounts.google.com" # or Okta, Cloudflare, Azure AD + oidc_audience: "my-stacyvm" + oidc_jwks_url: "https://www.googleapis.com/oauth2/v3/certs" + oidc_admin_groups: ["stacyvm-admins"] + oidc_operator_groups: ["stacyvm-operators"] + oidc_viewer_groups: ["stacyvm-viewers"] +``` + +Callers send a standard Bearer token. StacyVM validates it (RS256 and ES256/ES384/ES512 are both supported) and maps IdP group membership to one of four roles: + +| Role | Can do | +|---|---| +| `viewer` | List and inspect sandboxes | +| `api` / `operator` | Spawn, exec, read/write files, destroy | +| `admin` | Everything + quotas, workers, provider config, tenants | +| `tenant_admin` | Admin within their own tenant | + +Scope enforcement is applied on every route when auth is configured. A viewer token cannot spawn or exec — it gets 403. + +Run `stacyvm config lint --production` to validate your OIDC config before exposing it. + +### Multi-tenancy & policy controls + +Create isolated tenants and restrict what each one can use: + +```bash +# Create a tenant +curl -X POST /api/v1/admin/tenants \ + -d '{"id":"acme","name":"Acme Corp","owner_id":"user-alice"}' + +# Add a member with operator role +curl -X PUT /api/v1/admin/tenants/acme/members/user-bob \ + -d '{"role":"operator"}' + +# Allow only trusted images +curl -X POST /api/v1/admin/tenants/acme/policies \ + -d '{"resource_type":"image","effect":"allow","pattern":"alpine:*","priority":10}' + +# Block untrusted networks +curl -X POST /api/v1/admin/tenants/acme/policies \ + -d '{"resource_type":"network","effect":"deny","pattern":"host","priority":1}' + +# Export per-tenant audit log +curl /api/v1/admin/tenants/acme/audit +``` + +Each tenant's sandboxes, audit logs, and policies are fully isolated. OIDC callers are automatically scoped to their tenant via the configurable `oidc_tenant_claim`. + +### Remote workers & mTLS + +Run a distributed cluster with signed worker tokens and mutual TLS on the RPC channel: + +```yaml +# Control plane +auth: + worker_signing_key: "<32-byte secret>" +worker: + rpc_tls: + enabled: true + ca_file: /etc/stacyvm/ca.crt + client_cert_file: /etc/stacyvm/cp-client.crt + client_key_file: /etc/stacyvm/cp-client.key +``` + +```bash +# Worker — receives short-lived signed tokens from the control plane issuer +# (no direct access to the signing key needed) +stacyvm worker \ + --control-plane https://cp.internal:7423 \ + --bootstrap-admin-key "$STACYVM_ADMIN_KEY" \ + --bootstrap-token-ttl 5m \ + --listen 0.0.0.0:7430 +``` + +Key rotation, token revocation, mTLS cert management, and the full enterprise signoff checklist are documented in [docs/enterprise-signoff-runbook.md](docs/enterprise-signoff-runbook.md). + +--- + +## Production deployment + +Use [docs/deployment.md](docs/deployment.md) for production setup guidance, including Docker Compose and systemd templates, auth, explicit CORS origins, rate-limit defaults, health/readiness probes, Prometheus scraping, backup steps, and provider-specific rollout notes. Remote worker staging guidance lives in [docs/remote-worker-staging.md](docs/remote-worker-staging.md). Runtime signoff expectations live in [docs/runtime-conformance.md](docs/runtime-conformance.md), public self-serve support expectations live in [docs/public-support-matrix.md](docs/public-support-matrix.md), public announcement evidence lives in [docs/public-readiness-evidence.md](docs/public-readiness-evidence.md), and release-candidate gates live in [docs/production-readiness.md](docs/production-readiness.md). The reusable templates live under [`deploy/`](deploy/). + +For enterprise multi-worker deployments, follow [docs/enterprise-signoff-runbook.md](docs/enterprise-signoff-runbook.md) — it covers mTLS smoke with deployment-issued certificates, per-host runtime certification, Postgres migration rehearsal, OIDC token validation, and the full pre-go-live checklist. + +Run `stacyvm doctor --production` on a target host before treating it as production-ready. Runtime host certification checks live in [docs/runtime-certification.md](docs/runtime-certification.md). + +Release automation and GHCR publishing are documented in [docs/releasing.md](docs/releasing.md). + +--- + ## Templates Templates are pre-baked sandbox specs stored server-side. Define once, spawn many times. @@ -577,7 +772,9 @@ Every sandbox ships locked down. You opt *in* to less restriction, not out. With the Firecracker provider you also get: dedicated kernel per sandbox, vsock-only host-guest communication (no TCP between host and guest), and ephemeral rootfs destroyed on teardown. -Full security model and reporting policy: [SECURITY.md](SECURITY.md). +For enterprise deployments, OIDC/JWT authentication (RS256 + ES256) and RBAC role enforcement replace static API keys. Scope checks are applied on every sandbox route — a viewer-role token cannot spawn or exec. See the [Enterprise / Multi-worker](#enterprise--multi-worker) section above. + +Full security model and reporting policy: [SECURITY.md](SECURITY.md). Production admin hardening and identity-provider planning: [docs/security-governance.md](docs/security-governance.md). Release-candidate threat model: [docs/threat-model.md](docs/threat-model.md). Worker RPC and multi-worker trust boundary: [docs/worker-rpc-contract.md](docs/worker-rpc-contract.md). --- @@ -641,6 +838,14 @@ curl -fsSL https://github.com/StacyOs/stacyvm/releases/latest/download/stacyvm-l chmod +x stacyvm && sudo mv stacyvm /usr/local/bin/ ``` +For public installs, verify the release first: + +```bash +scripts/verify-release.sh v0.4.0 amd64 +``` + +The installer verifies Sigstore signatures automatically when `cosign` is installed. Set `STACYVM_REQUIRE_SIGNATURES=true` to fail closed when signature verification is unavailable. + --- ## Project layout @@ -670,6 +875,7 @@ stacyvm/ ## Roadmap +**Single-node & public self-serve** - [x] Firecracker provider (KVM microVMs, ~28ms snapshot restore) - [x] Docker provider (OCI containers, seccomp, no KVM needed) - [x] gVisor support (user-space kernel via runsc runtime) @@ -680,11 +886,29 @@ stacyvm/ - [x] Template system + warm pools - [x] PRoot provider (root-less, KVM-less) - [x] E2B + custom HTTP provider +- [x] Signed release binaries + Sigstore verification +- [x] `stacyvm doctor`, config lint, upgrade rehearsal, support bundle + +**Enterprise / multi-worker** +- [x] Postgres store with durable leases and migration rehearsal +- [x] Remote worker registry, placement, RPC routing +- [x] Signed worker tokens (HMAC-SHA256, rotation, revocation) +- [x] Worker RPC mutual TLS (mTLS) +- [x] Centralized worker token issuance (no signing key on workers) +- [x] Durable event bus (Postgres LISTEN/NOTIFY for HA replicas) +- [x] OIDC/SSO — RS256 + ES256/ES384/ES512, JWKS, Google/Okta/Cloudflare/Azure +- [x] RBAC roles — viewer, operator, admin, tenant_admin with scope enforcement +- [x] Multi-tenancy — tenant model, member RBAC, per-tenant audit export +- [x] Policy controls — image/provider/network allow-deny per tenant +- [x] Enterprise signoff runbook + runtime certification script + +**Planned** - [ ] Live Preview for Firecracker - [ ] Kata Containers provider (K8s-native) - [ ] Persistent volumes across sandboxes - [ ] MCP server mode - [ ] GPU passthrough +- [ ] Centralized token issuer as a standalone sidecar service --- @@ -696,7 +920,7 @@ If you find a security issue, do **not** open a public issue — follow [SECURIT ## License -[MIT](LICENSE) — use it however you want. +[Apache 2.0](LICENSE) — use it however you want. --- diff --git a/SECURITY.md b/SECURITY.md index cfcd7c0..ed2aa01 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,26 +12,51 @@ We will acknowledge receipt within 48 hours and aim to release a fix within 7 da ## Security Model -StacyVM provides hardware-level isolation via Firecracker microVMs: +### Sandbox isolation -- Each sandbox runs in its own KVM virtual machine with a dedicated kernel +- Each Firecracker sandbox runs in its own KVM virtual machine with a dedicated kernel - No shared kernel between sandboxes or between sandbox and host - Per-VM rootfs — destroyed on teardown - Host-guest communication via virtio-vsock (no network exposure) -- Optional API key authentication on all endpoints +- Docker sandboxes use dropped capabilities (`CAP_ALL`), seccomp, pids limit, and memory/CPU limits - TTL-based auto-expiry prevents resource leaks +### Authentication and authorisation + +- **API key auth** — static API and admin keys; `stacyvm config lint --production` enforces minimum entropy and key separation +- **OIDC/JWT auth** — RS256 and ES256/ES384/ES512 Bearer token verification with configurable JWKS endpoint, audience, and issuer; supports Google Workspace, Okta, Cloudflare Access, Azure AD, and any RFC 7517-compliant IdP +- **RBAC** — `viewer` (read-only), `api`, `operator`, `admin`, `tenant_admin` roles with scope enforcement on every API route; OIDC group-to-role mapping is configurable +- **Multi-tenancy** — sandboxes, audit logs, and policies are scoped to tenants; cross-tenant access returns 404 + +### Worker security + +- Remote workers authenticate with HMAC-SHA256 signed tokens (`stacyvm-worker-v1` format) or individually rotatable static tokens +- Worker RPC supports mutual TLS (mTLS) for network-level transport identity +- Signed tokens carry `worker_id`, `jti`, `aud`, `exp`, `iat`; emergency revocation is supported via `auth.worker_revoked_token_ids` +- Workers can receive short-lived signed tokens from the centralized issuer (`POST /api/v1/admin/worker-tokens`) without needing direct access to the signing key + +### Audit + +- All admin operations are persisted in `admin_audit_logs` with actor, method, path, status, and tenant +- All sandbox lifecycle, exec, and file operations are persisted in `operation_audit_logs` +- Per-tenant audit export is available at `GET /api/v1/admin/tenants/{id}/audit` +- Audit records redacted automatically in support bundles + ## Scope The following are in scope for security reports: -- VM escape vulnerabilities -- API authentication bypass +- VM or container escape vulnerabilities +- API authentication or authorisation bypass +- OIDC JWT validation bypass (algorithm confusion, `alg: none`, signature skip) +- Cross-tenant data access - Unauthorized access to host filesystem or network - Denial of service via resource exhaustion -- Command injection through the API +- Command injection through the API or exec interface +- Worker impersonation or signed-token forgery ## Out of Scope -- The mock provider intentionally runs commands on the host (development only) -- Self-hosted deployments without API key auth enabled +- The mock provider intentionally runs commands on the host and is for development only +- Self-hosted deployments where the operator has explicitly disabled auth (`auth.enabled: false`) — this is flagged as a production failure by `stacyvm doctor --production` and `stacyvm config lint --production` +- Vulnerabilities in third-party runtimes (Docker, Firecracker, gVisor, Kata) that are outside StacyVM's control diff --git a/assets/stacy-logo-dark.png b/assets/stacy-logo-dark.png new file mode 100644 index 0000000..05226d8 Binary files /dev/null and b/assets/stacy-logo-dark.png differ diff --git a/assets/stacy-logo-light.png b/assets/stacy-logo-light.png new file mode 100644 index 0000000..34064f0 Binary files /dev/null and b/assets/stacy-logo-light.png differ diff --git a/assets/stacy-mark-dark.png b/assets/stacy-mark-dark.png new file mode 100644 index 0000000..b188bab Binary files /dev/null and b/assets/stacy-mark-dark.png differ diff --git a/assets/stacy-mark-orange.png b/assets/stacy-mark-orange.png new file mode 100644 index 0000000..6a95f95 Binary files /dev/null and b/assets/stacy-mark-orange.png differ diff --git a/assets/stacyvm-favicon.svg b/assets/stacyvm-favicon.svg new file mode 100644 index 0000000..d2d3f8b --- /dev/null +++ b/assets/stacyvm-favicon.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/assets/stacyvm-logo-dark.svg b/assets/stacyvm-logo-dark.svg new file mode 100644 index 0000000..5852240 --- /dev/null +++ b/assets/stacyvm-logo-dark.svg @@ -0,0 +1,6 @@ + + + + + STACYVM + diff --git a/assets/stacyvm-logo-light.svg b/assets/stacyvm-logo-light.svg new file mode 100644 index 0000000..591b8d8 --- /dev/null +++ b/assets/stacyvm-logo-light.svg @@ -0,0 +1,6 @@ + + + + + STACYVM + diff --git a/cmd/stacyvm-agent/main.go b/cmd/stacyvm-agent/main.go index 1a105f7..2d2f095 100644 --- a/cmd/stacyvm-agent/main.go +++ b/cmd/stacyvm-agent/main.go @@ -1,3 +1,5 @@ +//go:build linux + // stacyvm-agent runs inside a Firecracker VM and serves exec/file requests // over vsock. Build with: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/stacyvm-agent ./cmd/stacyvm-agent package main @@ -173,6 +175,24 @@ func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } +func buildExecCommand(params agentproto.ExecParams) ([]string, error) { + switch strings.TrimSpace(params.Mode) { + case "", "shell": + shellCmd := params.Command + for _, arg := range params.Args { + shellCmd += " " + shellQuote(arg) + } + return []string{"/bin/sh", "-c", shellCmd}, nil + case "argv": + if strings.TrimSpace(params.Command) == "" { + return nil, fmt.Errorf("argv exec mode requires command") + } + return append([]string{params.Command}, params.Args...), nil + default: + return nil, fmt.Errorf("unsupported exec mode %q", params.Mode) + } +} + func handleExec(w io.Writer, req *agentproto.Request) { var params agentproto.ExecParams if err := agentproto.UnmarshalParams(req.Params, ¶ms); err != nil { @@ -183,15 +203,13 @@ func handleExec(w io.Writer, req *agentproto.Request) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - // Build full shell command: if args provided, shell-quote and append them - shellCmd := params.Command - if len(params.Args) > 0 { - for _, a := range params.Args { - shellCmd += " " + shellQuote(a) - } + args, err := buildExecCommand(params) + if err != nil { + sendError(w, req.ID, err.Error()) + return } - cmd := exec.CommandContext(ctx, "/bin/sh", "-c", shellCmd) + cmd := exec.CommandContext(ctx, args[0], args[1:]...) if params.WorkDir != "" { cmd.Dir = params.WorkDir } @@ -206,7 +224,7 @@ func handleExec(w io.Writer, req *agentproto.Request) { cmd.Stdout = &stdoutBuf cmd.Stderr = &stderrBuf - err := cmd.Run() + err = cmd.Run() exitCode := 0 if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { @@ -238,15 +256,15 @@ func handleExecStream(w io.Writer, req *agentproto.Request) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - // Build full shell command: if args provided, shell-quote and append them - shellCmd := params.Command - if len(params.Args) > 0 { - for _, a := range params.Args { - shellCmd += " " + shellQuote(a) - } + args, err := buildExecCommand(params) + if err != nil { + agentproto.SendStreamResponse(w, &agentproto.StreamResponse{ + ID: req.ID, Error: err.Error(), Done: true, + }) + return } - cmd := exec.CommandContext(ctx, "/bin/sh", "-c", shellCmd) + cmd := exec.CommandContext(ctx, args[0], args[1:]...) if params.WorkDir != "" { cmd.Dir = params.WorkDir } diff --git a/cmd/stacyvm-agent/main_unsupported.go b/cmd/stacyvm-agent/main_unsupported.go new file mode 100644 index 0000000..bd90e76 --- /dev/null +++ b/cmd/stacyvm-agent/main_unsupported.go @@ -0,0 +1,9 @@ +//go:build !linux + +package main + +import "fmt" + +func main() { + fmt.Println("stacyvm-agent only runs on Linux") +} diff --git a/cmd/stacyvm/cmd_config.go b/cmd/stacyvm/cmd_config.go new file mode 100644 index 0000000..f268f47 --- /dev/null +++ b/cmd/stacyvm/cmd_config.go @@ -0,0 +1,452 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/config" + "github.com/spf13/cobra" +) + +func newConfigCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Inspect and validate StacyVM configuration", + } + cmd.AddCommand(newConfigLintCmd()) + return cmd +} + +func newConfigLintCmd() *cobra.Command { + var file string + var production bool + cmd := &cobra.Command{ + Use: "lint", + Short: "Lint configuration for operational safety", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadLintConfig(file) + if err != nil { + return err + } + checks := lintConfig(cfg, production) + failed := printDoctorChecks(checks) + if failed > 0 { + return fmt.Errorf("config lint found %d failing check(s)", failed) + } + return nil + }, + } + cmd.Flags().StringVar(&file, "file", "", "config file to lint; defaults to normal StacyVM config lookup") + cmd.Flags().BoolVar(&production, "production", false, "treat production hardening warnings as failures") + return cmd +} + +func loadLintConfig(file string) (*config.Config, error) { + if strings.TrimSpace(file) == "" { + return config.Load() + } + return config.LoadFile(file) +} + +func lintConfig(cfg *config.Config, production bool) []doctorCheck { + checks := []doctorCheck{ + {Name: "config", Status: doctorPass, Message: "syntax and schema validation passed"}, + } + checks = append(checks, lintAuthConfig(cfg, production)...) + checks = append(checks, lintOIDCConfig(cfg, production)...) + checks = append(checks, lintServerConfig(cfg, production)...) + checks = append(checks, lintRateLimitConfig(cfg, production)...) + checks = append(checks, lintDatabaseConfig(cfg, production)...) + checks = append(checks, lintRuntimeLimits(cfg, production)...) + checks = append(checks, lintLoggingConfig(cfg, production)...) + checks = append(checks, lintWorkerRPCConfig(cfg, production)...) + checks = append(checks, lintProviderConfig(cfg, production)...) + return checks +} + +func lintServerConfig(cfg *config.Config, production bool) []doctorCheck { + origins := cleanCORSOrigins(cfg.Server.CORSAllowedOrigins) + if len(origins) == 0 { + return []doctorCheck{{Name: "server.cors_allowed_origins", Status: severityForProduction(production), Message: "no CORS origins configured", Remediation: "Set server.cors_allowed_origins to the exact web console or API client origins that may call StacyVM."}} + } + if containsCORSWildcard(origins) { + return []doctorCheck{{Name: "server.cors_allowed_origins", Status: severityForProduction(production), Message: "CORS allows every origin", Remediation: "Set server.cors_allowed_origins to explicit https:// origins before exposing StacyVM publicly."}} + } + return []doctorCheck{{Name: "server.cors_allowed_origins", Status: doctorPass, Message: fmt.Sprintf("%d explicit origin(s)", len(origins))}} +} + +func cleanCORSOrigins(origins []string) []string { + cleaned := make([]string, 0, len(origins)) + for _, origin := range origins { + origin = strings.TrimSpace(origin) + if origin != "" { + cleaned = append(cleaned, origin) + } + } + return cleaned +} + +func containsCORSWildcard(origins []string) bool { + for _, origin := range origins { + if strings.TrimSpace(origin) == "*" { + return true + } + } + return false +} + +func lintAuthConfig(cfg *config.Config, production bool) []doctorCheck { + var checks []doctorCheck + if !cfg.Auth.Enabled { + checks = append(checks, doctorCheck{Name: "auth.enabled", Status: severityForProduction(production), Message: "authentication is disabled", Remediation: "Set auth.enabled=true before exposing StacyVM beyond a trusted local network."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.enabled", Status: doctorPass, Message: "enabled"}) + } + + checks = append(checks, lintSecret("auth.api_key", cfg.Auth.APIKey, production, "Set STACYVM_AUTH_API_KEY or auth.api_key to a random 32+ byte secret.")) + checks = append(checks, lintSecret("auth.admin_api_key", cfg.Auth.AdminAPIKey, production, "Set STACYVM_AUTH_ADMIN_API_KEY or auth.admin_api_key to a separate random 32+ byte secret.")) + if cfg.Auth.WorkerSigningKey != "" { + checks = append(checks, lintSecret("auth.worker_signing_key", cfg.Auth.WorkerSigningKey, production, "Set STACYVM_AUTH_WORKER_SIGNING_KEY or auth.worker_signing_key to a random 32+ byte secret used to verify signed worker tokens.")) + checks = append(checks, lintWorkerSecretSource("auth.worker_signing_key_file", cfg.Auth.WorkerSigningKeyFile, "auth.worker_signing_key", "Mount the worker signing key through auth.worker_signing_key_file for production services.")...) + checks = append(checks, lintWorkerSigningKeyRotation(cfg.Auth.WorkerSigningKey, cfg.Auth.WorkerSigningKeys)...) + if len(cfg.Auth.WorkerRevokedTokenIDs) > 0 { + checks = append(checks, doctorCheck{Name: "auth.worker_revoked_token_ids", Status: doctorPass, Message: fmt.Sprintf("%d revoked worker token id(s) configured", len(cfg.Auth.WorkerRevokedTokenIDs))}) + } + if cfg.Auth.WorkerToken != "" { + checks = append(checks, doctorCheck{Name: "auth.worker_token", Status: doctorWarn, Message: "shared worker token still configured with signed worker tokens", Remediation: "Remove auth.worker_token after workers and worker RPC clients use short-lived signed worker tokens."}) + checks = append(checks, lintWorkerSecretSource("auth.worker_token_file", cfg.Auth.WorkerTokenFile, "auth.worker_token", "Mount shared staging worker tokens through auth.worker_token_file when they cannot be removed yet.")...) + } + if len(cfg.Auth.WorkerTokens) > 0 { + checks = append(checks, doctorCheck{Name: "auth.worker_tokens", Status: doctorWarn, Message: fmt.Sprintf("%d static per-worker token(s) still configured", len(cfg.Auth.WorkerTokens)), Remediation: "Prefer short-lived signed worker tokens for production workers; keep static worker tokens only during migration."}) + } + } else if len(cfg.Auth.WorkerSigningKeys) > 0 { + checks = append(checks, doctorCheck{Name: "auth.worker_signing_key", Status: severityForProduction(production), Message: "additional verification keys configured without a primary signing key", Remediation: "Set auth.worker_signing_key to the active signing key and keep old keys in auth.worker_signing_keys only during rotation."}) + checks = append(checks, lintWorkerRevokedTokenIDsWithoutSigningKey(cfg.Auth.WorkerRevokedTokenIDs)...) + } else if len(cfg.Auth.WorkerTokens) > 0 { + checks = append(checks, lintWorkerRevokedTokenIDsWithoutSigningKey(cfg.Auth.WorkerRevokedTokenIDs)...) + checks = append(checks, doctorCheck{Name: "auth.worker_tokens", Status: doctorPass, Message: fmt.Sprintf("%d per-worker token(s) configured", len(cfg.Auth.WorkerTokens))}) + } else if cfg.Auth.WorkerToken != "" { + checks = append(checks, lintWorkerRevokedTokenIDsWithoutSigningKey(cfg.Auth.WorkerRevokedTokenIDs)...) + checks = append(checks, doctorCheck{Name: "auth.worker_tokens", Status: doctorWarn, Message: "using shared worker token", Remediation: "Configure auth.worker_tokens for production workers so each worker has an individually rotatable credential."}) + checks = append(checks, lintWorkerSecretSource("auth.worker_token_file", cfg.Auth.WorkerTokenFile, "auth.worker_token", "Mount shared staging worker tokens through auth.worker_token_file when they cannot be replaced yet.")...) + } else { + checks = append(checks, lintWorkerRevokedTokenIDsWithoutSigningKey(cfg.Auth.WorkerRevokedTokenIDs)...) + checks = append(checks, doctorCheck{Name: "auth.worker_tokens", Status: doctorWarn, Message: "no worker credentials configured", Remediation: "Set auth.worker_token for staging, auth.worker_tokens for migration, or auth.worker_signing_key for production signed worker identity."}) + } + if cfg.Auth.APIKey != "" && cfg.Auth.APIKey == cfg.Auth.AdminAPIKey { + checks = append(checks, doctorCheck{Name: "auth.key_separation", Status: severityForProduction(production), Message: "regular and admin API keys match", Remediation: "Use separate keys so admin endpoints can be rotated and restricted independently."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.key_separation", Status: doctorPass, Message: "regular and admin keys are separate"}) + } + if cfg.Auth.AdminFallbackEnabled { + checks = append(checks, doctorCheck{Name: "auth.admin_fallback_enabled", Status: severityForProduction(production), Message: "admin fallback is enabled", Remediation: "Set auth.admin_fallback_enabled=false in production."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.admin_fallback_enabled", Status: doctorPass, Message: "disabled"}) + } + retention, _ := time.ParseDuration(cfg.Auth.AdminAuditRetention) + if retention <= 0 { + checks = append(checks, doctorCheck{Name: "auth.admin_audit_retention", Status: severityForProduction(production), Message: "admin audit retention is disabled", Remediation: "Set auth.admin_audit_retention to a positive retention window such as 2160h."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.admin_audit_retention", Status: doctorPass, Message: retention.String()}) + } + return checks +} + +func lintWorkerSecretSource(fileName, filePath, inlineName, remediation string) []doctorCheck { + if strings.TrimSpace(filePath) != "" { + return []doctorCheck{{Name: fileName, Status: doctorPass, Message: "configured"}} + } + return []doctorCheck{{Name: fileName, Status: doctorWarn, Message: fmt.Sprintf("%s is configured inline", inlineName), Remediation: remediation}} +} + +func lintWorkerRevokedTokenIDsWithoutSigningKey(tokenIDs []string) []doctorCheck { + if len(tokenIDs) == 0 { + return nil + } + return []doctorCheck{{ + Name: "auth.worker_revoked_token_ids", + Status: doctorWarn, + Message: fmt.Sprintf("%d revoked worker token id(s) configured without signed worker tokens", len(tokenIDs)), + Remediation: "Keep auth.worker_revoked_token_ids only alongside auth.worker_signing_key during an incident-response window.", + }} +} + +func lintWorkerSigningKeyRotation(primary string, keys []string) []doctorCheck { + if len(keys) == 0 { + return nil + } + primary = strings.TrimSpace(primary) + seen := map[string]struct{}{} + for _, key := range keys { + key = strings.TrimSpace(key) + if key == "" { + continue + } + if key == primary { + return []doctorCheck{{Name: "auth.worker_signing_keys", Status: doctorWarn, Message: "rotation keys include the active worker signing key", Remediation: "Keep only previous verification keys in auth.worker_signing_keys; auth.worker_signing_key is already accepted."}} + } + if _, ok := seen[key]; ok { + return []doctorCheck{{Name: "auth.worker_signing_keys", Status: doctorWarn, Message: "duplicate worker signing rotation key configured", Remediation: "Remove duplicate entries from auth.worker_signing_keys so rotation state is unambiguous."}} + } + seen[key] = struct{}{} + } + return []doctorCheck{{Name: "auth.worker_signing_keys", Status: doctorPass, Message: fmt.Sprintf("%d additional verification key(s) configured", len(seen))}} +} + +func lintSecret(name, value string, production bool, remediation string) doctorCheck { + if strings.TrimSpace(value) == "" { + return doctorCheck{Name: name, Status: severityForProduction(production), Message: "missing secret", Remediation: remediation} + } + lower := strings.ToLower(value) + if strings.Contains(lower, "change-me") || strings.Contains(lower, "changeme") || strings.Contains(lower, "replace-me") { + return doctorCheck{Name: name, Status: severityForProduction(production), Message: "placeholder secret is still configured", Remediation: remediation} + } + if len(value) < 32 { + return doctorCheck{Name: name, Status: severityForProduction(production), Message: "secret is shorter than 32 bytes", Remediation: remediation} + } + return doctorCheck{Name: name, Status: doctorPass, Message: "configured"} +} + +func lintOIDCConfig(cfg *config.Config, production bool) []doctorCheck { + if !cfg.Auth.OIDCEnabled { + return []doctorCheck{{Name: "auth.oidc_enabled", Status: doctorPass, Message: "disabled (API key auth only)"}} + } + var checks []doctorCheck + checks = append(checks, doctorCheck{Name: "auth.oidc_enabled", Status: doctorPass, Message: "enabled"}) + + // Issuer is required for audience and signature validation. + if strings.TrimSpace(cfg.Auth.OIDCIssuer) == "" { + checks = append(checks, doctorCheck{ + Name: "auth.oidc_issuer", + Status: severityForProduction(production), + Message: "OIDC issuer is not set", + Remediation: "Set auth.oidc_issuer to your identity provider's issuer URL (e.g. https://accounts.google.com).", + }) + } else { + checks = append(checks, doctorCheck{Name: "auth.oidc_issuer", Status: doctorPass, Message: cfg.Auth.OIDCIssuer}) + } + + // At least one of JWKS URL or static public key must be set. + hasKey := strings.TrimSpace(cfg.Auth.OIDCJWKSUrl) != "" || strings.TrimSpace(cfg.Auth.OIDCPublicKey) != "" + if !hasKey { + checks = append(checks, doctorCheck{ + Name: "auth.oidc_jwks_url", + Status: severityForProduction(production), + Message: "no OIDC verification key configured", + Remediation: "Set auth.oidc_jwks_url to your identity provider's JWKS endpoint, or set auth.oidc_public_key_file to a PEM public key.", + }) + } else { + if strings.TrimSpace(cfg.Auth.OIDCJWKSUrl) != "" { + checks = append(checks, doctorCheck{Name: "auth.oidc_jwks_url", Status: doctorPass, Message: cfg.Auth.OIDCJWKSUrl}) + } + if strings.TrimSpace(cfg.Auth.OIDCPublicKey) != "" { + src := "inline" + if strings.TrimSpace(cfg.Auth.OIDCPublicKeyFile) != "" { + src = "file-backed" + } + checks = append(checks, doctorCheck{Name: "auth.oidc_public_key", Status: doctorPass, Message: src}) + if src == "inline" { + checks = append(checks, doctorCheck{ + Name: "auth.oidc_public_key_file", + Status: doctorWarn, + Message: "OIDC public key is configured inline", + Remediation: "Mount the OIDC public key via auth.oidc_public_key_file in production deployments.", + }) + } + } + } + + // Audience is strongly recommended for production. + if strings.TrimSpace(cfg.Auth.OIDCAudience) == "" { + checks = append(checks, doctorCheck{ + Name: "auth.oidc_audience", + Status: doctorWarn, + Message: "OIDC audience not set; any token issued by this provider will be accepted", + Remediation: "Set auth.oidc_audience to restrict token acceptance to your specific application.", + }) + } else { + checks = append(checks, doctorCheck{Name: "auth.oidc_audience", Status: doctorPass, Message: cfg.Auth.OIDCAudience}) + } + + // At least one group mapping should exist, otherwise all OIDC users get the default API role. + hasGroupMapping := len(cfg.Auth.OIDCAdminGroups) > 0 || len(cfg.Auth.OIDCOperatorGroups) > 0 || len(cfg.Auth.OIDCViewerGroups) > 0 + if !hasGroupMapping { + checks = append(checks, doctorCheck{ + Name: "auth.oidc_groups", + Status: doctorWarn, + Message: "no OIDC group-to-role mappings configured; all authenticated users get the default api role", + Remediation: "Set auth.oidc_admin_groups, auth.oidc_operator_groups, or auth.oidc_viewer_groups to restrict access by group membership.", + }) + } else { + checks = append(checks, doctorCheck{ + Name: "auth.oidc_groups", + Status: doctorPass, + Message: fmt.Sprintf("admin=%d operator=%d viewer=%d groups configured", len(cfg.Auth.OIDCAdminGroups), len(cfg.Auth.OIDCOperatorGroups), len(cfg.Auth.OIDCViewerGroups)), + }) + } + + return checks +} + +func lintRateLimitConfig(cfg *config.Config, production bool) []doctorCheck { + var checks []doctorCheck + if !cfg.RateLimit.Enabled { + checks = append(checks, doctorCheck{Name: "rate_limit.enabled", Status: severityForProduction(production), Message: "rate limiting is disabled", Remediation: "Set rate_limit.enabled=true and choose bounded request limits."}) + } else { + checks = append(checks, doctorCheck{Name: "rate_limit.enabled", Status: doctorPass, Message: "enabled"}) + } + if cfg.RateLimit.RequestsPerMinute <= 0 || cfg.RateLimit.Burst <= 0 { + checks = append(checks, doctorCheck{Name: "rate_limit.capacity", Status: severityForProduction(production), Message: "requests_per_minute and burst must be positive", Remediation: "Set positive rate_limit.requests_per_minute and rate_limit.burst values."}) + } else { + checks = append(checks, doctorCheck{Name: "rate_limit.capacity", Status: doctorPass, Message: fmt.Sprintf("%d rpm, burst %d", cfg.RateLimit.RequestsPerMinute, cfg.RateLimit.Burst)}) + } + if cfg.RateLimit.KeyBy == "ip" { + checks = append(checks, doctorCheck{Name: "rate_limit.key_by", Status: doctorWarn, Message: "IP-based limiting can collapse unrelated users behind NAT", Remediation: "Prefer rate_limit.key_by=api_key or owner for authenticated production deployments."}) + } else { + checks = append(checks, doctorCheck{Name: "rate_limit.key_by", Status: doctorPass, Message: cfg.RateLimit.KeyBy}) + } + return checks +} + +func lintDatabaseConfig(cfg *config.Config, production bool) []doctorCheck { + driver := strings.ToLower(strings.TrimSpace(cfg.Database.Driver)) + if driver == "" { + driver = "sqlite" + } + if driver == "postgres" || driver == "postgresql" { + if cfg.Database.DSN == "" { + return []doctorCheck{{Name: "database.dsn", Status: doctorFail, Message: "postgres DSN is required"}} + } + return []doctorCheck{{Name: "database.driver", Status: doctorPass, Message: "postgres"}} + } + if filepath.IsAbs(cfg.Database.Path) { + return []doctorCheck{{Name: "database.path", Status: doctorPass, Message: cfg.Database.Path}} + } + return []doctorCheck{{Name: "database.path", Status: severityForProduction(production), Message: "database path is relative", Remediation: "Use an absolute path on durable storage, for example /var/lib/stacyvm/stacyvm.db."}} +} + +func lintRuntimeLimits(cfg *config.Config, production bool) []doctorCheck { + var checks []doctorCheck + if cfg.Defaults.MaxSandboxes <= 0 || cfg.Defaults.MaxSandboxesPerOwner <= 0 { + checks = append(checks, doctorCheck{Name: "defaults.sandbox_caps", Status: severityForProduction(production), Message: "global and per-owner sandbox caps must be positive", Remediation: "Set defaults.max_sandboxes and defaults.max_sandboxes_per_owner to bounded production values."}) + } else { + checks = append(checks, doctorCheck{Name: "defaults.sandbox_caps", Status: doctorPass, Message: fmt.Sprintf("global %d, per-owner %d", cfg.Defaults.MaxSandboxes, cfg.Defaults.MaxSandboxesPerOwner)}) + } + if cfg.Defaults.MaxSpawnQueue <= 0 { + checks = append(checks, doctorCheck{Name: "defaults.max_spawn_queue", Status: severityForProduction(production), Message: "spawn queue is unbounded or disabled by capacity", Remediation: "Set defaults.max_spawn_queue to a positive bounded value."}) + } else { + checks = append(checks, doctorCheck{Name: "defaults.max_spawn_queue", Status: doctorPass, Message: fmt.Sprintf("%d", cfg.Defaults.MaxSpawnQueue)}) + } + defaultExec, _ := time.ParseDuration(cfg.Defaults.DefaultExecTimeout) + maxExec, _ := time.ParseDuration(cfg.Defaults.MaxExecTimeout) + if defaultExec <= 0 || maxExec <= 0 { + checks = append(checks, doctorCheck{Name: "defaults.exec_timeouts", Status: severityForProduction(production), Message: "exec timeouts should be positive", Remediation: "Set defaults.default_exec_timeout and defaults.max_exec_timeout to positive durations."}) + } else { + checks = append(checks, doctorCheck{Name: "defaults.exec_timeouts", Status: doctorPass, Message: fmt.Sprintf("default %s, max %s", defaultExec, maxExec)}) + } + maxTTL, _ := time.ParseDuration(cfg.Defaults.MaxTTL) + if maxTTL <= 0 { + checks = append(checks, doctorCheck{Name: "defaults.max_ttl", Status: severityForProduction(production), Message: "max TTL should be positive", Remediation: "Set defaults.max_ttl to a positive duration so sandboxes cannot run forever."}) + } else { + checks = append(checks, doctorCheck{Name: "defaults.max_ttl", Status: doctorPass, Message: maxTTL.String()}) + } + return checks +} + +func lintLoggingConfig(cfg *config.Config, production bool) []doctorCheck { + if cfg.Logging.Format != "json" { + return []doctorCheck{{Name: "logging.format", Status: severityForProduction(production), Message: "logs are not JSON formatted", Remediation: "Set logging.format=json so production log collectors can parse records reliably."}} + } + return []doctorCheck{{Name: "logging.format", Status: doctorPass, Message: "json"}} +} + +func lintWorkerRPCConfig(cfg *config.Config, production bool) []doctorCheck { + rpcTLS := cfg.Worker.RPCTLS + if !rpcTLS.Enabled { + return []doctorCheck{{Name: "worker.rpc_tls.enabled", Status: doctorWarn, Message: "worker RPC TLS is disabled", Remediation: "Enable worker.rpc_tls.enabled with server, client, and CA certificates before exposing worker RPC across a network."}} + } + var checks []doctorCheck + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.enabled", Status: doctorPass, Message: "enabled"}) + if rpcTLS.ServerCertFile == "" || rpcTLS.ServerKeyFile == "" { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.server_cert", Status: severityForProduction(production), Message: "server cert and key are required", Remediation: "Set worker.rpc_tls.server_cert_file and worker.rpc_tls.server_key_file on worker nodes."}) + } else { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.server_cert", Status: doctorPass, Message: "configured"}) + } + if rpcTLS.ClientCAFile == "" { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.client_ca", Status: severityForProduction(production), Message: "client CA is not configured", Remediation: "Set worker.rpc_tls.client_ca_file so worker RPC servers require trusted control-plane client certificates."}) + } else { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.client_ca", Status: doctorPass, Message: "configured"}) + } + if rpcTLS.CAFile == "" { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.ca", Status: severityForProduction(production), Message: "server CA is not configured", Remediation: "Set worker.rpc_tls.ca_file so control planes verify worker RPC server certificates."}) + } else { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.ca", Status: doctorPass, Message: "configured"}) + } + if rpcTLS.ClientCertFile == "" || rpcTLS.ClientKeyFile == "" { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.client_cert", Status: severityForProduction(production), Message: "client cert and key are required", Remediation: "Set worker.rpc_tls.client_cert_file and worker.rpc_tls.client_key_file on control-plane nodes."}) + } else { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.client_cert", Status: doctorPass, Message: "configured"}) + } + if rpcTLS.InsecureSkipVerify { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.insecure_skip_verify", Status: severityForProduction(production), Message: "TLS certificate verification is disabled", Remediation: "Set worker.rpc_tls.insecure_skip_verify=false outside throwaway local tests."}) + } else { + checks = append(checks, doctorCheck{Name: "worker.rpc_tls.insecure_skip_verify", Status: doctorPass, Message: "disabled"}) + } + return checks +} + +func lintProviderConfig(cfg *config.Config, production bool) []doctorCheck { + if cfg.Providers.Default != "docker" || !cfg.Providers.Docker.Enabled { + return []doctorCheck{{Name: "providers.default", Status: doctorWarn, Message: fmt.Sprintf("default provider is %q", cfg.Providers.Default), Remediation: "For single-node broad compatibility, validate non-Docker providers with runtime certification before production."}} + } + + var checks []doctorCheck + docker := cfg.Providers.Docker + if docker.Runtime == "" { + checks = append(checks, doctorCheck{Name: "docker.runtime", Status: severityForProduction(production), Message: "runtime is empty", Remediation: "Set providers.docker.runtime explicitly, such as runc, runsc, or kata."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.runtime", Status: doctorPass, Message: docker.Runtime}) + } + if docker.NetworkMode == "" { + checks = append(checks, doctorCheck{Name: "docker.network_mode", Status: severityForProduction(production), Message: "network mode is empty", Remediation: "Set providers.docker.network_mode explicitly."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.network_mode", Status: doctorPass, Message: docker.NetworkMode}) + } + if docker.SeccompProfile == "" || docker.SeccompProfile == "unconfined" { + checks = append(checks, doctorCheck{Name: "docker.seccomp_profile", Status: severityForProduction(production), Message: "seccomp is not enforced", Remediation: "Use Docker's default seccomp profile or a custom restrictive profile."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.seccomp_profile", Status: doctorPass, Message: docker.SeccompProfile}) + } + if len(docker.AddedCaps) > 0 { + checks = append(checks, doctorCheck{Name: "docker.added_caps", Status: severityForProduction(production), Message: "extra Linux capabilities are configured", Remediation: "Remove providers.docker.added_caps unless a certified workload requires them."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.added_caps", Status: doctorPass, Message: "none"}) + } + if !containsString(docker.DroppedCaps, "ALL") { + checks = append(checks, doctorCheck{Name: "docker.dropped_caps", Status: severityForProduction(production), Message: "ALL capabilities are not dropped by default", Remediation: "Set providers.docker.dropped_caps=[\"ALL\"] and add back only certified capabilities."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.dropped_caps", Status: doctorPass, Message: strings.Join(docker.DroppedCaps, ",")}) + } + if docker.Memory == "" || docker.CPUs == "" || docker.PidsLimit <= 0 { + checks = append(checks, doctorCheck{Name: "docker.resource_limits", Status: severityForProduction(production), Message: "memory, CPU, and pids limits must be configured", Remediation: "Set providers.docker.memory, providers.docker.cpus, and providers.docker.pids_limit."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.resource_limits", Status: doctorPass, Message: fmt.Sprintf("memory %s, cpus %s, pids %d", docker.Memory, docker.CPUs, docker.PidsLimit)}) + } + if docker.User == "" { + checks = append(checks, doctorCheck{Name: "docker.user", Status: severityForProduction(production), Message: "containers may run as image default user", Remediation: "Set providers.docker.user to a non-root UID/GID such as 1000:1000 when images support it."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.user", Status: doctorPass, Message: docker.User}) + } + return checks +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/cmd/stacyvm/cmd_config_test.go b/cmd/stacyvm/cmd_config_test.go new file mode 100644 index 0000000..6774678 --- /dev/null +++ b/cmd/stacyvm/cmd_config_test.go @@ -0,0 +1,313 @@ +package main + +import ( + "testing" + + "github.com/StacyOs/stacyvm/internal/config" +) + +func TestLintConfigProductionBaselinePasses(t *testing.T) { + cfg := validProductionConfig() + + checks := lintConfig(cfg, true) + for _, check := range checks { + if check.Status == doctorFail { + t.Fatalf("%s failed: %s", check.Name, check.Message) + } + } +} + +func TestLintConfigProductionCatchesUnsafeSettings(t *testing.T) { + cfg := validProductionConfig() + cfg.Auth.Enabled = false + cfg.Auth.APIKey = "change-me-generate-at-least-32-bytes" + cfg.Auth.AdminAPIKey = cfg.Auth.APIKey + cfg.Auth.AdminFallbackEnabled = true + cfg.Auth.AdminAuditRetention = "0s" + cfg.Server.CORSAllowedOrigins = []string{"*"} + cfg.RateLimit.Enabled = false + cfg.Database.Path = "stacyvm.db" + cfg.Defaults.MaxSandboxes = 0 + cfg.Defaults.DefaultExecTimeout = "0s" + cfg.Logging.Format = "console" + cfg.Providers.Docker.SeccompProfile = "unconfined" + cfg.Providers.Docker.AddedCaps = []string{"SYS_ADMIN"} + cfg.Providers.Docker.DroppedCaps = nil + cfg.Providers.Docker.PidsLimit = 0 + cfg.Providers.Docker.User = "" + + checks := lintConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + for _, name := range []string{ + "auth.enabled", + "auth.api_key", + "auth.key_separation", + "auth.admin_fallback_enabled", + "auth.admin_audit_retention", + "server.cors_allowed_origins", + "rate_limit.enabled", + "database.path", + "defaults.sandbox_caps", + "defaults.exec_timeouts", + "logging.format", + "docker.seccomp_profile", + "docker.added_caps", + "docker.dropped_caps", + "docker.resource_limits", + "docker.user", + } { + if statuses[name] != doctorFail { + t.Fatalf("%s status = %s, want %s", name, statuses[name], doctorFail) + } + } +} + +func TestLintDatabaseConfigPassesForPostgresWithDSN(t *testing.T) { + cfg := validProductionConfig() + cfg.Database = config.DatabaseConfig{ + Driver: "postgres", + DSN: "postgres://stacyvm@example/stacyvm", + } + + checks := lintDatabaseConfig(cfg, true) + if len(checks) != 1 { + t.Fatalf("checks = %+v, want one check", checks) + } + if checks[0].Name != "database.driver" || checks[0].Status != doctorPass { + t.Fatalf("unexpected check: %+v", checks[0]) + } +} + +func TestLintServerConfigRequiresExplicitCORSInProduction(t *testing.T) { + cfg := validProductionConfig() + cfg.Server.CORSAllowedOrigins = []string{"*"} + + checks := lintServerConfig(cfg, true) + if len(checks) != 1 { + t.Fatalf("checks = %+v, want one check", checks) + } + if checks[0].Name != "server.cors_allowed_origins" || checks[0].Status != doctorFail { + t.Fatalf("unexpected check: %+v", checks[0]) + } +} + +func TestLintServerConfigAcceptsExplicitCORSOrigins(t *testing.T) { + cfg := validProductionConfig() + + checks := lintServerConfig(cfg, true) + if len(checks) != 1 { + t.Fatalf("checks = %+v, want one check", checks) + } + if checks[0].Name != "server.cors_allowed_origins" || checks[0].Status != doctorPass { + t.Fatalf("unexpected check: %+v", checks[0]) + } +} + +func TestLintAuthConfigAcceptsWorkerSigningKey(t *testing.T) { + cfg := validProductionConfig() + cfg.Auth.WorkerSigningKey = "worker-signing-key-with-at-least-32-bytes" + cfg.Auth.WorkerTokens = map[string]string{"worker-a": "legacy-token"} + + checks := lintAuthConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + if statuses["auth.worker_signing_key"] != doctorPass { + t.Fatalf("auth.worker_signing_key status = %s, want %s", statuses["auth.worker_signing_key"], doctorPass) + } + if statuses["auth.worker_tokens"] != doctorWarn { + t.Fatalf("auth.worker_tokens status = %s, want migration warning", statuses["auth.worker_tokens"]) + } +} + +func TestLintAuthConfigReportsWorkerSecretFileSources(t *testing.T) { + cfg := validProductionConfig() + cfg.Auth.WorkerSigningKey = "worker-signing-key-with-at-least-32-bytes" + cfg.Auth.WorkerSigningKeyFile = "/run/secrets/stacyvm-worker-signing-key" + cfg.Auth.WorkerToken = "shared-worker-token-with-at-least-32-bytes" + cfg.Auth.WorkerTokenFile = "/run/secrets/stacyvm-worker-token" + + checks := lintAuthConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + if statuses["auth.worker_signing_key_file"] != doctorPass { + t.Fatalf("auth.worker_signing_key_file status = %s, want %s", statuses["auth.worker_signing_key_file"], doctorPass) + } + if statuses["auth.worker_token_file"] != doctorPass { + t.Fatalf("auth.worker_token_file status = %s, want %s", statuses["auth.worker_token_file"], doctorPass) + } + + cfg.Auth.WorkerSigningKeyFile = "" + cfg.Auth.WorkerTokenFile = "" + checks = lintAuthConfig(cfg, true) + statuses = map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + if statuses["auth.worker_signing_key_file"] != doctorWarn { + t.Fatalf("auth.worker_signing_key_file inline status = %s, want %s", statuses["auth.worker_signing_key_file"], doctorWarn) + } + if statuses["auth.worker_token_file"] != doctorWarn { + t.Fatalf("auth.worker_token_file inline status = %s, want %s", statuses["auth.worker_token_file"], doctorWarn) + } +} + +func TestLintAuthConfigWarnsRevokedWorkerTokenIDsWithoutSigningKey(t *testing.T) { + cfg := validProductionConfig() + cfg.Auth.WorkerRevokedTokenIDs = []string{"revoked-token-id"} + + checks := lintAuthConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + if statuses["auth.worker_revoked_token_ids"] != doctorWarn { + t.Fatalf("auth.worker_revoked_token_ids status = %s, want %s", statuses["auth.worker_revoked_token_ids"], doctorWarn) + } +} + +func TestLintAuthConfigWarnsSharedWorkerTokenWithSigningKey(t *testing.T) { + cfg := validProductionConfig() + cfg.Auth.WorkerSigningKey = "worker-signing-key-with-at-least-32-bytes" + cfg.Auth.WorkerToken = "shared-worker-token-with-at-least-32-bytes" + + checks := lintAuthConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + if statuses["auth.worker_token"] != doctorWarn { + t.Fatalf("auth.worker_token status = %s, want %s", statuses["auth.worker_token"], doctorWarn) + } +} + +func TestLintAuthConfigWarnsInvalidWorkerSigningKeyRotation(t *testing.T) { + cfg := validProductionConfig() + cfg.Auth.WorkerSigningKey = "worker-signing-key-with-at-least-32-bytes" + cfg.Auth.WorkerSigningKeys = []string{"worker-signing-key-with-at-least-32-bytes"} + + checks := lintAuthConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + if statuses["auth.worker_signing_keys"] != doctorWarn { + t.Fatalf("auth.worker_signing_keys status = %s, want %s", statuses["auth.worker_signing_keys"], doctorWarn) + } + + cfg.Auth.WorkerSigningKeys = []string{ + "old-worker-signing-key-with-at-least-32-bytes", + "old-worker-signing-key-with-at-least-32-bytes", + } + checks = lintAuthConfig(cfg, true) + statuses = map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + if statuses["auth.worker_signing_keys"] != doctorWarn { + t.Fatalf("auth.worker_signing_keys duplicate status = %s, want %s", statuses["auth.worker_signing_keys"], doctorWarn) + } +} + +func TestLintWorkerRPCConfigChecksMTLSInputs(t *testing.T) { + cfg := validProductionConfig() + cfg.Worker.RPCTLS.Enabled = true + + checks := lintWorkerRPCConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + for _, name := range []string{ + "worker.rpc_tls.server_cert", + "worker.rpc_tls.client_ca", + "worker.rpc_tls.ca", + "worker.rpc_tls.client_cert", + } { + if statuses[name] != doctorFail { + t.Fatalf("%s status = %s, want %s", name, statuses[name], doctorFail) + } + } +} + +func TestLintWorkerRPCConfigPassesCompleteMTLSInputs(t *testing.T) { + cfg := validProductionConfig() + cfg.Worker.RPCTLS = config.WorkerRPCTLSConfig{ + Enabled: true, + ServerCertFile: "/etc/stacyvm/tls/worker.crt", + ServerKeyFile: "/etc/stacyvm/tls/worker.key", + ClientCAFile: "/etc/stacyvm/tls/control-plane-ca.crt", + CAFile: "/etc/stacyvm/tls/worker-ca.crt", + ClientCertFile: "/etc/stacyvm/tls/control-plane.crt", + ClientKeyFile: "/etc/stacyvm/tls/control-plane.key", + } + + checks := lintWorkerRPCConfig(cfg, true) + for _, check := range checks { + if check.Status == doctorFail { + t.Fatalf("%s failed: %s", check.Name, check.Message) + } + } +} + +func validProductionConfig() *config.Config { + return &config.Config{ + Server: config.ServerConfig{ + CORSAllowedOrigins: []string{"https://console.example.com"}, + }, + Auth: config.AuthConfig{ + Enabled: true, + APIKey: "regular-api-key-with-at-least-32-bytes", + AdminAPIKey: "admin-api-key-with-at-least-32-bytesxx", + AdminFallbackEnabled: false, + AdminAuditRetention: "2160h", + }, + RateLimit: config.RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 120, + Burst: 60, + KeyBy: "api_key", + }, + Database: config.DatabaseConfig{ + Path: "/var/lib/stacyvm/stacyvm.db", + }, + Defaults: config.DefaultsConfig{ + MaxSandboxes: 100, + MaxSandboxesPerOwner: 10, + MaxSpawnQueue: 100, + DefaultExecTimeout: "30s", + MaxExecTimeout: "10m", + MaxTTL: "24h", + }, + Logging: config.LoggingConfig{ + Format: "json", + }, + Providers: config.ProvidersConfig{ + Default: "docker", + Docker: config.DockerConfig{ + Enabled: true, + Runtime: "runc", + NetworkMode: "stacyvm-network", + SeccompProfile: "default", + Memory: "512m", + CPUs: "1", + PidsLimit: 256, + User: "1000:1000", + DroppedCaps: []string{"ALL"}, + AddedCaps: []string{}, + }, + }, + } +} diff --git a/cmd/stacyvm/cmd_db.go b/cmd/stacyvm/cmd_db.go new file mode 100644 index 0000000..792d248 --- /dev/null +++ b/cmd/stacyvm/cmd_db.go @@ -0,0 +1,375 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/config" + "github.com/spf13/cobra" + _ "modernc.org/sqlite" +) + +func newDBCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "db", + Short: "Manage the StacyVM database (SQLite or Postgres)", + } + cmd.AddCommand(newDBBackupCmd(), newDBRestoreCmd(), newDBPgBackupCmd(), newDBPgRehearseCmd()) + return cmd +} + +func newDBBackupCmd() *cobra.Command { + var dbPath string + var force bool + cmd := &cobra.Command{ + Use: "backup ", + Short: "Create a consistent SQLite backup", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + source, err := resolveDatabasePath(dbPath) + if err != nil { + return err + } + return backupSQLite(cmd.Context(), source, args[0], force) + }, + } + cmd.Flags().StringVar(&dbPath, "database", "", "database path; defaults to configured database.path") + cmd.Flags().BoolVar(&force, "force", false, "overwrite an existing backup output file") + return cmd +} + +func newDBRestoreCmd() *cobra.Command { + var dbPath string + var yes bool + cmd := &cobra.Command{ + Use: "restore ", + Short: "Restore SQLite database from a backup", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if !yes { + return fmt.Errorf("restore replaces the active database; rerun with --yes after stopping stacyvm") + } + target, err := resolveDatabasePath(dbPath) + if err != nil { + return err + } + return restoreSQLite(cmd.Context(), args[0], target) + }, + } + cmd.Flags().StringVar(&dbPath, "database", "", "database path; defaults to configured database.path") + cmd.Flags().BoolVar(&yes, "yes", false, "confirm the service is stopped and restore should proceed") + return cmd +} + +func resolveDatabasePath(override string) (string, error) { + if override != "" { + return filepath.Abs(override) + } + cfg, err := config.Load() + if err != nil { + return "", err + } + return filepath.Abs(cfg.Database.Path) +} + +func backupSQLite(ctx context.Context, source, output string, force bool) error { + sourceAbs, err := filepath.Abs(source) + if err != nil { + return err + } + outputAbs, err := filepath.Abs(output) + if err != nil { + return err + } + if _, err := os.Stat(sourceAbs); err != nil { + return fmt.Errorf("source database unavailable: %w", err) + } + if _, err := os.Stat(outputAbs); err == nil && !force { + return fmt.Errorf("backup output already exists: %s; use --force to overwrite", outputAbs) + } else if err != nil && !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(filepath.Dir(outputAbs), 0750); err != nil { + return err + } + if force { + if err := os.Remove(outputAbs); err != nil && !os.IsNotExist(err) { + return err + } + } + + db, err := sql.Open("sqlite", sourceAbs+"?mode=ro&_busy_timeout=5000") + if err != nil { + return err + } + defer db.Close() + + if _, err := db.ExecContext(ctx, "VACUUM INTO ?", outputAbs); err != nil { + return fmt.Errorf("creating sqlite backup: %w", err) + } + if err := checkSQLiteIntegrity(ctx, outputAbs); err != nil { + return fmt.Errorf("backup integrity check failed: %w", err) + } + fmt.Printf("backup written: %s\n", outputAbs) + return nil +} + +func restoreSQLite(ctx context.Context, backup, target string) error { + backupAbs, err := filepath.Abs(backup) + if err != nil { + return err + } + targetAbs, err := filepath.Abs(target) + if err != nil { + return err + } + if err := checkSQLiteIntegrity(ctx, backupAbs); err != nil { + return fmt.Errorf("backup integrity check failed: %w", err) + } + if err := os.MkdirAll(filepath.Dir(targetAbs), 0750); err != nil { + return err + } + if _, err := os.Stat(targetAbs); err == nil { + safety := targetAbs + ".pre-restore-" + time.Now().UTC().Format("20060102T150405Z") + if err := copyFile(targetAbs, safety, 0600); err != nil { + return fmt.Errorf("creating pre-restore safety copy: %w", err) + } + fmt.Printf("existing database safety copy: %s\n", safety) + } else if err != nil && !os.IsNotExist(err) { + return err + } + for _, suffix := range []string{"-wal", "-shm"} { + if err := os.Remove(targetAbs + suffix); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("removing stale sqlite sidecar %s: %w", targetAbs+suffix, err) + } + } + if err := copyFile(backupAbs, targetAbs, 0600); err != nil { + return fmt.Errorf("restoring database: %w", err) + } + fmt.Printf("database restored: %s\n", targetAbs) + return nil +} + +func checkSQLiteIntegrity(ctx context.Context, path string) error { + if _, err := os.Stat(path); err != nil { + return err + } + db, err := sql.Open("sqlite", path+"?mode=ro&_busy_timeout=5000") + if err != nil { + return err + } + defer db.Close() + + var result string + if err := db.QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&result); err != nil { + return err + } + if result != "ok" { + return fmt.Errorf("integrity_check returned %q", result) + } + return nil +} + +// newDBPgBackupCmd creates a Postgres backup using pg_dump. +func newDBPgBackupCmd() *cobra.Command { + var dsn string + var format string + var cfgFile string + cmd := &cobra.Command{ + Use: "pg-backup ", + Short: "Backup a Postgres database using pg_dump", + Long: `Creates a Postgres backup using pg_dump. + +Requires pg_dump to be installed. The DSN can be provided via --dsn, +STACYVM_DATABASE_DSN environment variable, or from the config file. + +Example: + stacyvm db pg-backup backup-$(date +%Y%m%d).sql + stacyvm db pg-backup --format custom backup.dump`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + output := args[0] + if dsn == "" { + var cfg *config.Config + var err error + if cfgFile != "" { + cfg, err = config.LoadFile(cfgFile) + } else { + cfg, err = config.Load() + } + if err == nil { + dsn = cfg.Database.DSN + } + } + if dsn == "" { + dsn = os.Getenv("STACYVM_DATABASE_DSN") + } + if dsn == "" { + return fmt.Errorf("no Postgres DSN configured; use --dsn or set database.dsn / STACYVM_DATABASE_DSN") + } + return pgBackup(cmd.Context(), dsn, output, format) + }, + } + cmd.Flags().StringVar(&dsn, "dsn", "", "Postgres DSN (overrides config)") + cmd.Flags().StringVar(&format, "format", "plain", "pg_dump format: plain, custom, directory, tar") + cmd.Flags().StringVar(&cfgFile, "config", "", "config file path") + return cmd +} + +// newDBPgRehearseCmd runs a Postgres migration rehearsal (dry-run apply + rollback check). +func newDBPgRehearseCmd() *cobra.Command { + var dsn string + var cfgFile string + cmd := &cobra.Command{ + Use: "pg-rehearse", + Short: "Rehearse Postgres migration safety: verify all migrations apply cleanly", + Long: `Connects to the Postgres database and verifies that all pending migrations +can be applied. This is a read-only rehearsal check — it reports the current +schema version and the next migration versions that would be applied. + +Run this before every enterprise production upgrade: + + stacyvm db pg-rehearse --dsn + stacyvm db pg-rehearse --config stacyvm.yaml`, + RunE: func(cmd *cobra.Command, args []string) error { + if dsn == "" { + var cfg *config.Config + var err error + if cfgFile != "" { + cfg, err = config.LoadFile(cfgFile) + } else { + cfg, err = config.Load() + } + if err == nil { + dsn = cfg.Database.DSN + } + } + if dsn == "" { + dsn = os.Getenv("STACYVM_DATABASE_DSN") + } + if dsn == "" { + return fmt.Errorf("no Postgres DSN configured; use --dsn or set database.dsn / STACYVM_DATABASE_DSN") + } + return pgRehearseCheck(cmd.Context(), dsn) + }, + } + cmd.Flags().StringVar(&dsn, "dsn", "", "Postgres DSN (overrides config)") + cmd.Flags().StringVar(&cfgFile, "config", "", "config file path") + return cmd +} + +func pgBackup(ctx context.Context, dsn, output, format string) error { + pgDump, err := exec.LookPath("pg_dump") + if err != nil { + return fmt.Errorf("pg_dump not found in PATH; install postgresql-client to use Postgres backups") + } + + formatFlag := "--format=" + format + args := []string{formatFlag, "--file=" + output, dsn} + + cmd := exec.CommandContext(ctx, pgDump, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + fmt.Printf("running: pg_dump %s\n", strings.Join(args, " ")) + if err := cmd.Run(); err != nil { + return fmt.Errorf("pg_dump failed: %w", err) + } + fmt.Printf("backup written: %s\n", output) + return nil +} + +// expectedTables are the production-aligned tables that must exist after migration. +var expectedPostgresTables = []string{ + "sandboxes", "exec_logs", "provider_configs", "templates", + "environment_specs", "environment_builds", "environment_artifacts", + "registry_connections", "owner_quotas", "admin_audit_logs", + "operation_audit_logs", "workers", "leases", + "tenants", "tenant_members", "policies", +} + +func pgRehearseCheck(ctx context.Context, dsn string) error { + db, err := sql.Open("pgx", dsn) + if err != nil { + return fmt.Errorf("connecting to Postgres: %w", err) + } + defer db.Close() + + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("Postgres connection failed: %w", err) + } + fmt.Println("connection: OK") + + // Check applied migrations. + rows, err := db.QueryContext(ctx, "SELECT version FROM schema_migrations ORDER BY version ASC") + if err != nil { + fmt.Println("schema_migrations: not found — database is uninitialized (will be created on first startup)") + fmt.Println("pg-rehearse: PASS (fresh database)") + return nil + } + defer rows.Close() + + var applied []int + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + return err + } + applied = append(applied, v) + } + if err := rows.Err(); err != nil { + return err + } + fmt.Printf("schema_migrations: %d applied — versions %v\n", len(applied), applied) + + // Verify that all expected tables exist. + missing := []string{} + for _, table := range expectedPostgresTables { + var exists bool + err := db.QueryRowContext(ctx, + `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name=$1)`, + table, + ).Scan(&exists) + if err != nil { + return fmt.Errorf("checking table %q: %w", table, err) + } + if !exists { + missing = append(missing, table) + } + } + + if len(missing) > 0 { + return fmt.Errorf("pg-rehearse: FAIL — missing tables: %s\nRun the server once to apply migrations, or check migration history", + strings.Join(missing, ", ")) + } + + fmt.Printf("tables: all %d expected tables present\n", len(expectedPostgresTables)) + fmt.Println("pg-rehearse: PASS — schema is production-aligned") + return nil +} + +func copyFile(src, dst string, perm os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) + if err != nil { + return err + } + defer out.Close() + + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Sync() +} diff --git a/cmd/stacyvm/cmd_db_test.go b/cmd/stacyvm/cmd_db_test.go new file mode 100644 index 0000000..32f0f94 --- /dev/null +++ b/cmd/stacyvm/cmd_db_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" +) + +func TestBackupAndRestoreSQLite(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + source := filepath.Join(dir, "stacyvm.db") + backup := filepath.Join(dir, "backup", "stacyvm.db") + target := filepath.Join(dir, "restored.db") + + writeTestSQLite(t, source, "phase8") + if err := backupSQLite(ctx, source, backup, false); err != nil { + t.Fatalf("backup sqlite: %v", err) + } + if err := checkSQLiteIntegrity(ctx, backup); err != nil { + t.Fatalf("backup integrity: %v", err) + } + + writeTestSQLite(t, target, "old") + if err := os.WriteFile(target+"-wal", []byte("stale wal"), 0600); err != nil { + t.Fatalf("write stale wal: %v", err) + } + if err := os.WriteFile(target+"-shm", []byte("stale shm"), 0600); err != nil { + t.Fatalf("write stale shm: %v", err) + } + if err := restoreSQLite(ctx, backup, target); err != nil { + t.Fatalf("restore sqlite: %v", err) + } + got := readTestSQLiteValue(t, target) + if got != "phase8" { + t.Fatalf("restored value = %q, want phase8", got) + } + matches, err := filepath.Glob(target + ".pre-restore-*") + if err != nil { + t.Fatalf("glob safety copy: %v", err) + } + if len(matches) != 1 { + t.Fatalf("expected one safety copy, got %d", len(matches)) + } + for _, sidecar := range []string{target + "-wal", target + "-shm"} { + if _, err := os.Stat(sidecar); !os.IsNotExist(err) { + t.Fatalf("expected stale sidecar %s to be removed, got %v", sidecar, err) + } + } +} + +func TestBackupSQLiteRefusesOverwriteWithoutForce(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + source := filepath.Join(dir, "stacyvm.db") + backup := filepath.Join(dir, "backup.db") + writeTestSQLite(t, source, "phase8") + if err := os.WriteFile(backup, []byte("exists"), 0600); err != nil { + t.Fatalf("write existing backup: %v", err) + } + + err := backupSQLite(ctx, source, backup, false) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("expected overwrite refusal, got %v", err) + } + if err := backupSQLite(ctx, source, backup, true); err != nil { + t.Fatalf("backup with force: %v", err) + } +} + +func writeTestSQLite(t *testing.T, path, value string) { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS sanity (value TEXT NOT NULL)`); err != nil { + t.Fatalf("create table: %v", err) + } + if _, err := db.Exec(`DELETE FROM sanity`); err != nil { + t.Fatalf("delete table: %v", err) + } + if _, err := db.Exec(`INSERT INTO sanity (value) VALUES (?)`, value); err != nil { + t.Fatalf("insert value: %v", err) + } +} + +func readTestSQLiteValue(t *testing.T, path string) string { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + var value string + if err := db.QueryRow(`SELECT value FROM sanity LIMIT 1`).Scan(&value); err != nil { + t.Fatalf("select value: %v", err) + } + return value +} diff --git a/cmd/stacyvm/cmd_doctor.go b/cmd/stacyvm/cmd_doctor.go new file mode 100644 index 0000000..f5b35c1 --- /dev/null +++ b/cmd/stacyvm/cmd_doctor.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/config" + "github.com/spf13/cobra" +) + +type doctorStatus string + +const ( + doctorPass doctorStatus = "PASS" + doctorWarn doctorStatus = "WARN" + doctorFail doctorStatus = "FAIL" +) + +type doctorCheck struct { + Name string + Status doctorStatus + Message string + Remediation string +} + +func newDoctorCmd() *cobra.Command { + var production bool + cmd := &cobra.Command{ + Use: "doctor", + Short: "Run local production-readiness diagnostics", + RunE: func(cmd *cobra.Command, args []string) error { + checks := runDoctor(cmd.Context(), production) + failed := printDoctorChecks(checks) + if failed > 0 { + return fmt.Errorf("doctor found %d failing check(s)", failed) + } + return nil + }, + } + cmd.Flags().BoolVar(&production, "production", false, "treat production hardening warnings as failures") + return cmd +} + +func runDoctor(ctx context.Context, production bool) []doctorCheck { + var checks []doctorCheck + + cfg, err := config.Load() + if err != nil { + return []doctorCheck{{ + Name: "config", + Status: doctorFail, + Message: err.Error(), + Remediation: "Check config file/env values; see docs/configuration.md.", + }} + } + + checks = append(checks, + checkConfig(cfg, production)..., + ) + checks = append(checks, + checkDocker(ctx, cfg)..., + ) + checks = append(checks, + checkFirecracker(cfg)..., + ) + checks = append(checks, + checkPRoot(cfg)..., + ) + return checks +} + +func checkConfig(cfg *config.Config, production bool) []doctorCheck { + checks := []doctorCheck{ + {Name: "config", Status: doctorPass, Message: "loaded successfully"}, + } + + if cfg.Auth.APIKey == "" { + checks = append(checks, doctorCheck{Name: "auth.api_key", Status: severityForProduction(production), Message: "missing API key; endpoints are unauthenticated", Remediation: "Set STACYVM_AUTH_API_KEY or auth.api_key to a random 32+ byte secret."}) + } else if len(cfg.Auth.APIKey) < 32 { + checks = append(checks, doctorCheck{Name: "auth.api_key", Status: severityForProduction(production), Message: "API key is shorter than the recommended 32 bytes", Remediation: "Rotate auth.api_key to a random 32+ byte value before production."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.api_key", Status: doctorPass, Message: "configured"}) + } + + if cfg.Auth.AdminAPIKey == "" { + checks = append(checks, doctorCheck{Name: "auth.admin_api_key", Status: severityForProduction(production), Message: "missing dedicated admin API key", Remediation: "Set STACYVM_AUTH_ADMIN_API_KEY to a separate random 32+ byte secret."}) + } else if cfg.Auth.AdminAPIKey == cfg.Auth.APIKey { + checks = append(checks, doctorCheck{Name: "auth.admin_api_key", Status: severityForProduction(production), Message: "admin API key matches regular API key", Remediation: "Use separate regular and admin keys so admin routes are isolated."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.admin_api_key", Status: doctorPass, Message: "configured separately"}) + } + + if cfg.Auth.AdminFallbackEnabled { + checks = append(checks, doctorCheck{Name: "auth.admin_fallback_enabled", Status: severityForProduction(production), Message: "admin fallback is enabled; production should require a dedicated admin key", Remediation: "Set auth.admin_fallback_enabled=false in production."}) + } else { + checks = append(checks, doctorCheck{Name: "auth.admin_fallback_enabled", Status: doctorPass, Message: "disabled"}) + } + + driver := strings.ToLower(strings.TrimSpace(cfg.Database.Driver)) + if driver == "" { + driver = "sqlite" + } + if driver == "postgres" || driver == "postgresql" { + checks = append(checks, doctorCheck{Name: "database.driver", Status: doctorWarn, Message: "postgres configured, but this build does not link a postgres store driver yet", Remediation: "Use sqlite for this build or deploy a build that includes the postgres store driver."}) + } else { + dbDir := filepath.Dir(cfg.Database.Path) + if dbDir == "." || dbDir == "" { + checks = append(checks, doctorCheck{Name: "database.path", Status: doctorWarn, Message: "database path is relative; production should use persistent storage", Remediation: "Use an absolute database.path on durable disk and test backup/restore."}) + } else if info, err := os.Stat(dbDir); err != nil { + checks = append(checks, doctorCheck{Name: "database.path", Status: severityForProduction(production), Message: fmt.Sprintf("database directory unavailable: %v", err), Remediation: "Create the database parent directory and ensure the StacyVM process can write to it."}) + } else if !info.IsDir() { + checks = append(checks, doctorCheck{Name: "database.path", Status: doctorFail, Message: "database parent path is not a directory", Remediation: "Point database.path at a file whose parent is a real directory."}) + } else { + checks = append(checks, doctorCheck{Name: "database.path", Status: doctorPass, Message: dbDir}) + } + } + + return checks +} + +func checkDocker(ctx context.Context, cfg *config.Config) []doctorCheck { + if !cfg.Providers.Docker.Enabled && cfg.Providers.Default != "docker" { + return []doctorCheck{{Name: "docker", Status: doctorWarn, Message: "Docker provider is not enabled/default", Remediation: "Enable Docker only on hosts intended to run Docker sandboxes."}} + } + + var checks []doctorCheck + if _, err := exec.LookPath("docker"); err != nil { + return []doctorCheck{{Name: "docker.cli", Status: doctorFail, Message: "docker CLI not found in PATH", Remediation: "Install Docker CLI or disable the Docker provider."}} + } + checks = append(checks, doctorCheck{Name: "docker.cli", Status: doctorPass, Message: "found"}) + + if out, err := runDoctorCommand(ctx, 3*time.Second, "docker", "info", "--format", "{{.ServerVersion}}"); err != nil { + checks = append(checks, doctorCheck{Name: "docker.daemon", Status: doctorFail, Message: strings.TrimSpace(err.Error() + " " + out), Remediation: "Start Docker and ensure the StacyVM user can reach the Docker daemon."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.daemon", Status: doctorPass, Message: "server " + strings.TrimSpace(out)}) + } + + if cfg.Providers.Docker.NetworkMode == "" { + checks = append(checks, doctorCheck{Name: "docker.network_mode", Status: doctorWarn, Message: "empty network mode; explicit mode is recommended", Remediation: "Set providers.docker.network_mode explicitly; prefer none/allowlisted networking for untrusted workloads."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.network_mode", Status: doctorPass, Message: cfg.Providers.Docker.NetworkMode}) + } + if len(cfg.Providers.Docker.DroppedCaps) == 0 { + checks = append(checks, doctorCheck{Name: "docker.dropped_caps", Status: doctorWarn, Message: "no dropped capabilities configured", Remediation: "Configure dropped capabilities, seccomp, pids, memory, and CPU limits before production."}) + } else { + checks = append(checks, doctorCheck{Name: "docker.dropped_caps", Status: doctorPass, Message: strings.Join(cfg.Providers.Docker.DroppedCaps, ",")}) + } + return checks +} + +func checkFirecracker(cfg *config.Config) []doctorCheck { + if !cfg.Providers.Firecracker.Enabled && cfg.Providers.Default != "firecracker" { + return []doctorCheck{{Name: "firecracker", Status: doctorWarn, Message: "Firecracker provider is not enabled/default", Remediation: "Enable Firecracker only on Linux/KVM hosts prepared for VM workloads."}} + } + + checks := []doctorCheck{} + if _, err := exec.LookPath(filepath.Base(cfg.Providers.Firecracker.FirecrackerPath)); err != nil { + if _, statErr := os.Stat(cfg.Providers.Firecracker.FirecrackerPath); statErr != nil { + checks = append(checks, doctorCheck{Name: "firecracker.binary", Status: doctorFail, Message: "Firecracker binary unavailable", Remediation: "Install Firecracker or set providers.firecracker.firecracker_path."}) + } else { + checks = append(checks, doctorCheck{Name: "firecracker.binary", Status: doctorPass, Message: cfg.Providers.Firecracker.FirecrackerPath}) + } + } else { + checks = append(checks, doctorCheck{Name: "firecracker.binary", Status: doctorPass, Message: "found in PATH"}) + } + + checks = append(checks, fileCheck("firecracker.kvm", "/dev/kvm", false)) + checks = append(checks, fileCheck("firecracker.kernel", cfg.Providers.Firecracker.KernelPath, false)) + checks = append(checks, fileCheck("firecracker.agent", cfg.Providers.Firecracker.AgentPath, false)) + return checks +} + +func checkPRoot(cfg *config.Config) []doctorCheck { + if !cfg.Providers.PRoot.Enabled && cfg.Providers.Default != "proot" { + return []doctorCheck{{Name: "proot", Status: doctorWarn, Message: "PRoot provider is not enabled/default", Remediation: "Enable PRoot only on hosts with proot, rootfs, and workspace base configured."}} + } + + var checks []doctorCheck + if _, err := exec.LookPath(cfg.Providers.PRoot.PRootBinary); err != nil { + checks = append(checks, doctorCheck{Name: "proot.binary", Status: doctorFail, Message: "proot binary unavailable", Remediation: "Install proot or set providers.proot.proot_binary."}) + } else { + checks = append(checks, doctorCheck{Name: "proot.binary", Status: doctorPass, Message: cfg.Providers.PRoot.PRootBinary}) + } + checks = append(checks, fileCheck("proot.rootfs", cfg.Providers.PRoot.RootfsPath, true)) + checks = append(checks, fileCheck("proot.workspace_base", cfg.Providers.PRoot.WorkspaceBase, true)) + return checks +} + +func fileCheck(name, path string, wantDir bool) doctorCheck { + if strings.TrimSpace(path) == "" { + return doctorCheck{Name: name, Status: doctorFail, Message: "path is empty", Remediation: "Configure this path before enabling the provider."} + } + info, err := os.Stat(path) + if err != nil { + return doctorCheck{Name: name, Status: doctorFail, Message: err.Error(), Remediation: "Create the path or update provider configuration to the correct location."} + } + if wantDir && !info.IsDir() { + return doctorCheck{Name: name, Status: doctorFail, Message: "path is not a directory", Remediation: "Configure a directory path for this check."} + } + return doctorCheck{Name: name, Status: doctorPass, Message: path} +} + +func severityForProduction(production bool) doctorStatus { + if production { + return doctorFail + } + return doctorWarn +} + +func runDoctorCommand(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + cmdCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + out, err := exec.CommandContext(cmdCtx, name, args...).CombinedOutput() + return string(out), err +} + +func printDoctorChecks(checks []doctorCheck) int { + failed := 0 + for _, check := range checks { + if check.Status == doctorFail { + failed++ + } + fmt.Printf("[%s] %s: %s\n", check.Status, check.Name, check.Message) + if check.Status != doctorPass && check.Remediation != "" { + fmt.Printf(" fix: %s\n", check.Remediation) + } + } + return failed +} diff --git a/cmd/stacyvm/cmd_doctor_test.go b/cmd/stacyvm/cmd_doctor_test.go new file mode 100644 index 0000000..9cecf0c --- /dev/null +++ b/cmd/stacyvm/cmd_doctor_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "testing" + + "github.com/StacyOs/stacyvm/internal/config" +) + +func TestSeverityForProduction(t *testing.T) { + if got := severityForProduction(false); got != doctorWarn { + t.Fatalf("non-production severity = %s, want %s", got, doctorWarn) + } + if got := severityForProduction(true); got != doctorFail { + t.Fatalf("production severity = %s, want %s", got, doctorFail) + } +} + +func TestCheckConfigProductionAuthPosture(t *testing.T) { + cfg := &config.Config{} + cfg.Auth.APIKey = "short" + cfg.Auth.AdminAPIKey = "short" + cfg.Auth.AdminFallbackEnabled = true + cfg.Database.Path = "stacyvm.db" + + checks := checkConfig(cfg, true) + statuses := map[string]doctorStatus{} + for _, check := range checks { + statuses[check.Name] = check.Status + } + + for _, name := range []string{"auth.api_key", "auth.admin_api_key", "auth.admin_fallback_enabled"} { + if statuses[name] != doctorFail { + t.Fatalf("%s status = %s, want %s", name, statuses[name], doctorFail) + } + } +} diff --git a/cmd/stacyvm/cmd_exec.go b/cmd/stacyvm/cmd_exec.go index 9ca97f5..48af42a 100644 --- a/cmd/stacyvm/cmd_exec.go +++ b/cmd/stacyvm/cmd_exec.go @@ -9,30 +9,43 @@ import ( ) func newExecCmd() *cobra.Command { + var shell bool cmd := &cobra.Command{ Use: "exec -- ", Short: "Execute a command in a sandbox", Example: ` stacyvm exec sb-a1b2c3d4 -- echo hello - stacyvm exec sb-a1b2c3d4 -- ls -la`, + stacyvm exec sb-a1b2c3d4 -- ls -la + stacyvm exec sb-a1b2c3d4 --shell -- "echo $HOME && pwd"`, Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { sandboxID := args[0] - // Find everything after "--" - var command string + // Find everything after "--". + var commandArgs []string dashIdx := cmd.ArgsLenAtDash() if dashIdx >= 0 && dashIdx < len(args) { - command = strings.Join(args[dashIdx:], " ") + commandArgs = args[dashIdx:] } else if len(args) > 1 { - command = strings.Join(args[1:], " ") + commandArgs = args[1:] } else { return fmt.Errorf("no command specified; use: stacyvm exec -- ") } + if len(commandArgs) == 0 { + return fmt.Errorf("no command specified; use: stacyvm exec -- ") + } + + req := orchestrator.ExecRequest{Mode: "argv", Command: commandArgs[0]} + if len(commandArgs) > 1 { + req.Args = commandArgs[1:] + } + if shell { + req.Mode = "shell" + req.Command = strings.Join(commandArgs, " ") + req.Args = nil + } c := getClient() - resp, err := c.do("POST", "/api/v1/sandboxes/"+sandboxID+"/exec", orchestrator.ExecRequest{ - Command: command, - }) + resp, err := c.do("POST", "/api/v1/sandboxes/"+sandboxID+"/exec", req) if err != nil { return err } @@ -54,5 +67,6 @@ func newExecCmd() *cobra.Command { return nil }, } + cmd.Flags().BoolVar(&shell, "shell", false, "run the command through /bin/sh -c instead of argv mode") return cmd } diff --git a/cmd/stacyvm/cmd_serve.go b/cmd/stacyvm/cmd_serve.go index 52c0763..799f665 100644 --- a/cmd/stacyvm/cmd_serve.go +++ b/cmd/stacyvm/cmd_serve.go @@ -4,10 +4,12 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "time" "github.com/StacyOs/stacyvm/internal/api" + "github.com/StacyOs/stacyvm/internal/api/middleware" "github.com/StacyOs/stacyvm/internal/config" "github.com/StacyOs/stacyvm/internal/environments" "github.com/StacyOs/stacyvm/internal/orchestrator" @@ -46,17 +48,37 @@ func runServe() error { } // Store - st, err := store.NewSQLiteStore(cfg.Database.Path) + st, err := store.Open(store.Config{ + Driver: cfg.Database.Driver, + Path: cfg.Database.Path, + DSN: cfg.Database.DSN, + }) if err != nil { return err } defer st.Close() - // Event bus + // Event bus — attach a durable Postgres LISTEN/NOTIFY bridge when running + // in Postgres mode so events reach all control-plane replicas in HA setups. events := orchestrator.NewEventBus() + if strings.EqualFold(strings.TrimSpace(cfg.Database.Driver), "postgres") || + strings.EqualFold(strings.TrimSpace(cfg.Database.Driver), "postgresql") { + bridge, bridgeErr := orchestrator.NewDurableBridge(context.Background(), cfg.Database.DSN, events, logger) + if bridgeErr != nil { + logger.Warn().Err(bridgeErr).Msg("durable event bridge unavailable; events will not cross control-plane replicas") + } else { + defer bridge.Stop() + logger.Info().Msg("durable event bridge attached (Postgres LISTEN/NOTIFY)") + } + } // Provider registry registry := providers.NewRegistry() + if cfg.Providers.Mock.Enabled { + mock := providers.NewMockProvider() + registry.Register(mock) + logger.Info().Msg("mock provider registered") + } if cfg.Providers.Firecracker.Enabled { fc := providers.NewFirecrackerProvider(providers.FirecrackerProviderConfig{ FirecrackerPath: cfg.Providers.Firecracker.FirecrackerPath, @@ -163,6 +185,10 @@ func runServe() error { // Manager ttl, _ := time.ParseDuration(cfg.Defaults.TTL) + maxTTL, _ := time.ParseDuration(cfg.Defaults.MaxTTL) + defaultExecTimeout, _ := time.ParseDuration(cfg.Defaults.DefaultExecTimeout) + maxExecTimeout, _ := time.ParseDuration(cfg.Defaults.MaxExecTimeout) + spawnQueueTimeout, _ := time.ParseDuration(cfg.Defaults.SpawnQueueTimeout) mgr := orchestrator.NewManager(registry, st, events, logger, orchestrator.ManagerConfig{ DefaultTTL: ttl, DefaultImage: cfg.Defaults.Image, @@ -170,7 +196,24 @@ func runServe() error { DefaultVCPUs: cfg.Defaults.VCPUs, Pool: cfg.Pool, PreviewDomain: cfg.Server.PreviewDomain, + Limits: orchestrator.OperationalLimits{ + MaxSandboxes: cfg.Defaults.MaxSandboxes, + MaxSandboxesPerOwner: cfg.Defaults.MaxSandboxesPerOwner, + DefaultExecTimeout: defaultExecTimeout, + MaxExecTimeout: maxExecTimeout, + MaxTTL: maxTTL, + SpawnOverflow: cfg.Defaults.SpawnOverflow, + SpawnQueueTimeout: spawnQueueTimeout, + MaxSpawnQueue: cfg.Defaults.MaxSpawnQueue, + }, + WorkerToken: cfg.Auth.WorkerToken, + WorkerSigningKey: cfg.Auth.WorkerSigningKey, + WorkerRevokedTokenIDs: cfg.Auth.WorkerRevokedTokenIDs, + WorkerRPCTLS: workerTLSConfig(cfg.Worker.RPCTLS), }) + if err := mgr.Reconcile(context.Background()); err != nil { + return err + } mgr.Start() mgr.InitVMPool() defer mgr.Stop() @@ -194,10 +237,41 @@ func runServe() error { } // Server + rateLimitBucketTTL, _ := time.ParseDuration(cfg.RateLimit.BucketTTL) + rateLimitCleanupInterval, _ := time.ParseDuration(cfg.RateLimit.CleanupInterval) + adminAuditRetention, _ := time.ParseDuration(cfg.Auth.AdminAuditRetention) srv := api.NewServer(api.ServerConfig{ - Addr: cfg.Server.Addr(), - APIKey: cfg.Auth.APIKey, - Version: version, + Addr: cfg.Server.Addr(), + APIKey: cfg.Auth.APIKey, + AdminAPIKey: cfg.Auth.AdminAPIKey, + AdminFallbackDisabled: !cfg.Auth.AdminFallbackEnabled, + AdminAuditRetention: adminAuditRetention, + CORSAllowedOrigins: cfg.Server.CORSAllowedOrigins, + WorkerToken: cfg.Auth.WorkerToken, + WorkerTokens: cfg.Auth.WorkerTokens, + WorkerSigningKey: cfg.Auth.WorkerSigningKey, + WorkerSigningKeys: cfg.Auth.WorkerSigningKeys, + WorkerRevokedTokenIDs: cfg.Auth.WorkerRevokedTokenIDs, + Version: version, + RateLimit: middleware.RateLimitConfig{ + Enabled: cfg.RateLimit.Enabled, + RequestsPerMinute: cfg.RateLimit.RequestsPerMinute, + Burst: cfg.RateLimit.Burst, + KeyBy: cfg.RateLimit.KeyBy, + BucketTTL: rateLimitBucketTTL, + CleanupInterval: rateLimitCleanupInterval, + }, + OIDC: middleware.OIDCConfig{ + Issuer: cfg.Auth.OIDCIssuer, + Audience: cfg.Auth.OIDCAudience, + JWKSUrl: cfg.Auth.OIDCJWKSUrl, + PublicKeyPEM: cfg.Auth.OIDCPublicKey, + GroupsClaim: cfg.Auth.OIDCGroupsClaim, + TenantClaim: cfg.Auth.OIDCTenantClaim, + AdminGroups: cfg.Auth.OIDCAdminGroups, + OperatorGroups: cfg.Auth.OIDCOperatorGroups, + ViewerGroups: cfg.Auth.OIDCViewerGroups, + }, }, registry, mgr, events, templates, pool, st, envBuilds, logger) // Graceful shutdown diff --git a/cmd/stacyvm/cmd_support.go b/cmd/stacyvm/cmd_support.go new file mode 100644 index 0000000..117a202 --- /dev/null +++ b/cmd/stacyvm/cmd_support.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "runtime" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/config" + "github.com/spf13/cobra" +) + +type supportBundle struct { + GeneratedAt string `json:"generated_at"` + Version string `json:"version"` + Runtime map[string]string `json:"runtime"` + Config map[string]interface{} `json:"config,omitempty"` + ConfigLint []doctorCheck `json:"config_lint,omitempty"` + Doctor []doctorCheck `json:"doctor,omitempty"` + ServerDiagnostics map[string]interface{} `json:"server_diagnostics,omitempty"` + CollectionErrors []string `json:"collection_errors,omitempty"` + Redactions []string `json:"redactions"` +} + +func newSupportCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "support", + Short: "Collect redacted support diagnostics", + } + cmd.AddCommand(newSupportBundleCmd()) + return cmd +} + +func newSupportBundleCmd() *cobra.Command { + var configPath string + var includeDoctor bool + var includeServer bool + cmd := &cobra.Command{ + Use: "bundle ", + Short: "Write a redacted support bundle", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + bundle := collectSupportBundle(cmd.Context(), supportBundleOptions{ + ConfigPath: configPath, + IncludeDoctor: includeDoctor, + IncludeServer: includeServer, + }) + return writeSupportBundle(args[0], bundle) + }, + } + cmd.Flags().StringVar(&configPath, "config", "", "config file to include; defaults to normal StacyVM config lookup") + cmd.Flags().BoolVar(&includeDoctor, "include-doctor", false, "include local doctor checks") + cmd.Flags().BoolVar(&includeServer, "include-server", false, "include /api/v1/diagnostics from --server when reachable") + return cmd +} + +type supportBundleOptions struct { + ConfigPath string + IncludeDoctor bool + IncludeServer bool +} + +func collectSupportBundle(ctx context.Context, opts supportBundleOptions) supportBundle { + bundle := supportBundle{ + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Version: version, + Runtime: map[string]string{ + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + }, + Redactions: []string{ + "secret-like config keys", + "API keys", + "bearer tokens", + "basic auth credentials in URLs", + }, + } + + cfg, err := loadLintConfig(opts.ConfigPath) + if err != nil { + bundle.CollectionErrors = append(bundle.CollectionErrors, "config: "+err.Error()) + } else { + bundle.Config = redactConfig(cfg) + bundle.ConfigLint = redactDoctorChecks(lintConfig(cfg, true)) + } + + if opts.IncludeDoctor { + bundle.Doctor = redactDoctorChecks(runDoctor(ctx, true)) + } + if opts.IncludeServer { + diagnostics, err := fetchServerDiagnostics() + if err != nil { + bundle.CollectionErrors = append(bundle.CollectionErrors, "server_diagnostics: "+err.Error()) + } else { + bundle.ServerDiagnostics = diagnostics + } + } + return bundle +} + +func writeSupportBundle(path string, bundle supportBundle) error { + data, err := json.MarshalIndent(bundle, "", " ") + if err != nil { + return err + } + data = []byte(redactString(string(data))) + if err := os.WriteFile(path, data, 0600); err != nil { + return err + } + fmt.Printf("support bundle written: %s\n", path) + return nil +} + +func fetchServerDiagnostics() (map[string]interface{}, error) { + client := getClient() + resp, err := client.do(http.MethodGet, "/api/v1/diagnostics", nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + var diagnostics map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&diagnostics); err != nil { + return nil, err + } + return redactMap(diagnostics).(map[string]interface{}), nil +} + +func redactConfig(cfg *config.Config) map[string]interface{} { + data, err := json.Marshal(cfg) + if err != nil { + return map[string]interface{}{"error": err.Error()} + } + var raw map[string]interface{} + if err := json.Unmarshal(data, &raw); err != nil { + return map[string]interface{}{"error": err.Error()} + } + return redactMap(raw).(map[string]interface{}) +} + +func redactDoctorChecks(checks []doctorCheck) []doctorCheck { + redacted := make([]doctorCheck, len(checks)) + for i, check := range checks { + redacted[i] = doctorCheck{ + Name: check.Name, + Status: check.Status, + Message: redactString(check.Message), + Remediation: redactString(check.Remediation), + } + } + return redacted +} + +func redactMap(value interface{}) interface{} { + switch typed := value.(type) { + case map[string]interface{}: + out := make(map[string]interface{}, len(typed)) + for key, nested := range typed { + if isSecretKey(key) { + out[key] = "[REDACTED]" + continue + } + out[key] = redactMap(nested) + } + return out + case []interface{}: + out := make([]interface{}, len(typed)) + for i, nested := range typed { + out[i] = redactMap(nested) + } + return out + case string: + return redactString(typed) + default: + return typed + } +} + +func isSecretKey(key string) bool { + normalized := strings.ToLower(strings.ReplaceAll(key, "-", "_")) + for _, marker := range []string{"api_key", "apikey", "signing_key", "private_key", "key_file", "token", "secret", "password", "credential", "authorization", "auth_header"} { + if strings.Contains(normalized, marker) { + return true + } + } + return false +} + +var redactionPatterns = []*regexp.Regexp{ + regexp.MustCompile(`stacyvm-worker-v1\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+`), + regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`), + regexp.MustCompile(`(?i)(x-api-key|x-admin-api-key|api[_-]?key)\s*[:=]\s*["']?[^"',\s}]+`), + regexp.MustCompile(`(?i)(password|token|secret|signing[_-]?key|private[_-]?key)\s*[:=]\s*["']?[^"',\s}]+`), + regexp.MustCompile(`([a-z][a-z0-9+.-]*://)[^:/@\s]+:[^/@\s]+@`), + regexp.MustCompile(`(?i)sk-[a-z0-9][a-z0-9_-]{8,}`), +} + +func redactString(value string) string { + redacted := value + for _, pattern := range redactionPatterns { + redacted = pattern.ReplaceAllStringFunc(redacted, func(match string) string { + if strings.Contains(match, "://") && strings.Contains(match, "@") { + parts := pattern.FindStringSubmatch(match) + if len(parts) > 1 { + return parts[1] + "[REDACTED]@" + } + } + return "[REDACTED]" + }) + } + return redacted +} diff --git a/cmd/stacyvm/cmd_support_test.go b/cmd/stacyvm/cmd_support_test.go new file mode 100644 index 0000000..197333a --- /dev/null +++ b/cmd/stacyvm/cmd_support_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +func TestSupportBundleRedactsSecrets(t *testing.T) { + input := map[string]interface{}{ + "api_key": "sk-super-secret-value", + "nested": map[string]interface{}{ + "token": "Bearer abcdefghijklmnop", + "url": "https://user:password@example.com/path", + }, + "message": "X-Admin-API-Key: admin-secret-value", + } + + redacted := redactMap(input) + data, err := json.Marshal(redacted) + if err != nil { + t.Fatal(err) + } + body := string(data) + for _, secret := range []string{"sk-super-secret-value", "abcdefghijklmnop", "password", "admin-secret-value"} { + if strings.Contains(body, secret) { + t.Fatalf("support redaction leaked %q in %s", secret, body) + } + } + if !strings.Contains(body, "[REDACTED]") { + t.Fatalf("expected redaction marker in %s", body) + } +} + +func TestWriteSupportBundleRedactsFinalJSON(t *testing.T) { + path := t.TempDir() + "/support.json" + bundle := supportBundle{ + Version: "test", + CollectionErrors: []string{ + "server returned bearer abcdefghijklmnop", + "database url postgres://user:pass@example.com/db", + }, + } + if err := writeSupportBundle(path, bundle); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + body := string(data) + for _, secret := range []string{"abcdefghijklmnop", "user:pass"} { + if strings.Contains(body, secret) { + t.Fatalf("final support bundle leaked %q in %s", secret, body) + } + } +} diff --git a/cmd/stacyvm/cmd_upgrade.go b/cmd/stacyvm/cmd_upgrade.go new file mode 100644 index 0000000..7aa6419 --- /dev/null +++ b/cmd/stacyvm/cmd_upgrade.go @@ -0,0 +1,194 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" +) + +type upgradeRehearsalReport struct { + GeneratedAt string `json:"generated_at"` + ConfigPath string `json:"config_path,omitempty"` + DatabasePath string `json:"database_path"` + BackupOutput string `json:"backup_output"` + ConfigLint []doctorCheck `json:"config_lint"` + DatabaseChecks []doctorCheck `json:"database_checks"` + Doctor []doctorCheck `json:"doctor,omitempty"` + RecommendedSteps []string `json:"recommended_steps"` + ProductionReady bool `json:"production_ready"` + RequiresLiveCheck bool `json:"requires_live_check"` +} + +func newUpgradeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "upgrade", + Short: "Run upgrade preparation checks", + } + cmd.AddCommand(newUpgradeRehearseCmd()) + return cmd +} + +func newUpgradeRehearseCmd() *cobra.Command { + var configPath string + var dbPath string + var backupOutput string + var includeDoctor bool + cmd := &cobra.Command{ + Use: "rehearse", + Short: "Rehearse a single-node production upgrade", + RunE: func(cmd *cobra.Command, args []string) error { + report, err := runUpgradeRehearsal(cmd.Context(), upgradeRehearsalOptions{ + ConfigPath: configPath, + DatabasePath: dbPath, + BackupOutput: backupOutput, + IncludeDoctor: includeDoctor, + }) + if err != nil { + return err + } + return printUpgradeRehearsal(report) + }, + } + cmd.Flags().StringVar(&configPath, "config", "", "config file to rehearse; defaults to normal StacyVM config lookup") + cmd.Flags().StringVar(&dbPath, "database", "", "database path; defaults to configured database.path") + cmd.Flags().StringVar(&backupOutput, "backup-output", "", "intended backup output path; defaults to a timestamped file next to the database") + cmd.Flags().BoolVar(&includeDoctor, "include-doctor", false, "also run live host doctor checks") + return cmd +} + +type upgradeRehearsalOptions struct { + ConfigPath string + DatabasePath string + BackupOutput string + IncludeDoctor bool +} + +func runUpgradeRehearsal(ctx context.Context, opts upgradeRehearsalOptions) (*upgradeRehearsalReport, error) { + cfg, err := loadLintConfig(opts.ConfigPath) + if err != nil { + return nil, err + } + + dbPath := opts.DatabasePath + if dbPath == "" { + dbPath = cfg.Database.Path + } + dbAbs, err := filepath.Abs(dbPath) + if err != nil { + return nil, err + } + + backupOutput := opts.BackupOutput + if backupOutput == "" { + backupOutput = filepath.Join(filepath.Dir(dbAbs), "stacyvm-upgrade-"+time.Now().UTC().Format("20060102T150405Z")+".db") + } + backupAbs, err := filepath.Abs(backupOutput) + if err != nil { + return nil, err + } + + report := &upgradeRehearsalReport{ + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + ConfigPath: opts.ConfigPath, + DatabasePath: dbAbs, + BackupOutput: backupAbs, + ConfigLint: lintConfig(cfg, true), + DatabaseChecks: rehearseDatabaseChecks(ctx, dbAbs, backupAbs), + RecommendedSteps: upgradeSteps(dbAbs, backupAbs, opts.ConfigPath), + RequiresLiveCheck: !opts.IncludeDoctor, + } + if opts.IncludeDoctor { + report.Doctor = runDoctor(ctx, true) + report.RequiresLiveCheck = false + } + report.ProductionReady = allChecksPass(report.ConfigLint) && allChecksPass(report.DatabaseChecks) && (!opts.IncludeDoctor || allChecksPass(report.Doctor)) + return report, nil +} + +func rehearseDatabaseChecks(ctx context.Context, dbPath, backupOutput string) []doctorCheck { + var checks []doctorCheck + if _, err := os.Stat(dbPath); err != nil { + checks = append(checks, doctorCheck{Name: "database.source", Status: doctorFail, Message: err.Error(), Remediation: "Run the rehearsal on the host that owns the SQLite database, or pass --database to the active DB path."}) + } else if err := checkSQLiteIntegrity(ctx, dbPath); err != nil { + checks = append(checks, doctorCheck{Name: "database.integrity", Status: doctorFail, Message: err.Error(), Remediation: "Investigate SQLite integrity before upgrading; do not proceed until integrity_check returns ok."}) + } else { + checks = append(checks, doctorCheck{Name: "database.integrity", Status: doctorPass, Message: "ok"}) + } + + backupDir := filepath.Dir(backupOutput) + if info, err := os.Stat(backupDir); err != nil { + checks = append(checks, doctorCheck{Name: "backup.directory", Status: doctorFail, Message: err.Error(), Remediation: "Create the backup directory and ensure the StacyVM operator can write to it."}) + } else if !info.IsDir() { + checks = append(checks, doctorCheck{Name: "backup.directory", Status: doctorFail, Message: "path is not a directory", Remediation: "Choose a backup output path whose parent is a directory."}) + } else { + checks = append(checks, doctorCheck{Name: "backup.directory", Status: doctorPass, Message: backupDir}) + } + if _, err := os.Stat(backupOutput); err == nil { + checks = append(checks, doctorCheck{Name: "backup.output", Status: doctorFail, Message: "backup output already exists", Remediation: "Choose a fresh --backup-output path or run stacyvm db backup with --force intentionally."}) + } else if err != nil && !os.IsNotExist(err) { + checks = append(checks, doctorCheck{Name: "backup.output", Status: doctorFail, Message: err.Error(), Remediation: "Choose a backup output path that can be checked by the operator."}) + } else { + checks = append(checks, doctorCheck{Name: "backup.output", Status: doctorPass, Message: backupOutput}) + } + return checks +} + +func upgradeSteps(dbPath, backupOutput, configPath string) []string { + if configPath == "" { + configPath = "the active StacyVM config" + } + return []string{ + fmt.Sprintf("Run stacyvm config lint --production --file %s with the service environment loaded.", configPath), + fmt.Sprintf("Run stacyvm db backup %s --database %s before replacing binaries or images.", backupOutput, dbPath), + "Replace the stacyvm binary or update STACYVM_IMAGE.", + "Restart the service.", + "Confirm GET /api/v1/ready succeeds before routing traffic.", + "If readiness fails after upgrade, stop StacyVM and run stacyvm db restore against the pre-upgrade backup.", + } +} + +func printUpgradeRehearsal(report *upgradeRehearsalReport) error { + fmt.Println("Upgrade rehearsal") + fmt.Printf(" database: %s\n", report.DatabasePath) + fmt.Printf(" backup: %s\n", report.BackupOutput) + fmt.Println() + fmt.Println("Config lint:") + configFailures := printDoctorChecks(report.ConfigLint) + fmt.Println() + fmt.Println("Database checks:") + dbFailures := printDoctorChecks(report.DatabaseChecks) + doctorFailures := 0 + if len(report.Doctor) > 0 { + fmt.Println() + fmt.Println("Doctor checks:") + doctorFailures = printDoctorChecks(report.Doctor) + } + fmt.Println() + fmt.Println("Recommended upgrade flow:") + for i, step := range report.RecommendedSteps { + fmt.Printf(" %d. %s\n", i+1, step) + } + if report.RequiresLiveCheck { + fmt.Println() + fmt.Println("Live host checks were skipped. Run with --include-doctor or run stacyvm doctor --production on the target host before go-live.") + } + if failures := configFailures + dbFailures + doctorFailures; failures > 0 { + return fmt.Errorf("upgrade rehearsal found %d failing check(s)", failures) + } + fmt.Println() + fmt.Println("Upgrade rehearsal passed.") + return nil +} + +func allChecksPass(checks []doctorCheck) bool { + for _, check := range checks { + if check.Status == doctorFail { + return false + } + } + return true +} diff --git a/cmd/stacyvm/cmd_upgrade_test.go b/cmd/stacyvm/cmd_upgrade_test.go new file mode 100644 index 0000000..b6c0c36 --- /dev/null +++ b/cmd/stacyvm/cmd_upgrade_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestRunUpgradeRehearsalPassesWithValidInputs(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "stacyvm.db") + configPath := writeUpgradeTestConfig(t, dir, dbPath) + createTestSQLiteDB(t, dbPath) + + report, err := runUpgradeRehearsal(context.Background(), upgradeRehearsalOptions{ + ConfigPath: configPath, + DatabasePath: dbPath, + BackupOutput: filepath.Join(dir, "backup.db"), + }) + if err != nil { + t.Fatal(err) + } + if !report.ProductionReady { + t.Fatalf("ProductionReady = false, config=%v database=%v", report.ConfigLint, report.DatabaseChecks) + } + if !report.RequiresLiveCheck { + t.Fatalf("RequiresLiveCheck = false, want true when doctor is skipped") + } +} + +func TestRunUpgradeRehearsalFailsWhenBackupExists(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "stacyvm.db") + backupPath := filepath.Join(dir, "backup.db") + configPath := writeUpgradeTestConfig(t, dir, dbPath) + createTestSQLiteDB(t, dbPath) + if err := os.WriteFile(backupPath, []byte("exists"), 0600); err != nil { + t.Fatal(err) + } + + report, err := runUpgradeRehearsal(context.Background(), upgradeRehearsalOptions{ + ConfigPath: configPath, + DatabasePath: dbPath, + BackupOutput: backupPath, + }) + if err != nil { + t.Fatal(err) + } + if report.ProductionReady { + t.Fatalf("ProductionReady = true, want false") + } + if got := statusFor(report.DatabaseChecks, "backup.output"); got != doctorFail { + t.Fatalf("backup.output status = %s, want %s", got, doctorFail) + } +} + +func writeUpgradeTestConfig(t *testing.T, dir, dbPath string) string { + t.Helper() + path := filepath.Join(dir, "stacyvm.yaml") + body := `server: + cors_allowed_origins: + - "https://console.example.com" +auth: + enabled: true + api_key: "regular-api-key-with-at-least-32-bytes" + admin_api_key: "admin-api-key-with-at-least-32-bytesxx" + admin_fallback_enabled: false + admin_audit_retention: "2160h" +rate_limit: + enabled: true + requests_per_minute: 120 + burst: 60 + key_by: "api_key" +database: + path: "` + dbPath + `" +defaults: + max_sandboxes: 100 + max_sandboxes_per_owner: 10 + max_spawn_queue: 100 + default_exec_timeout: "30s" + max_exec_timeout: "10m" + max_ttl: "24h" +logging: + format: "json" +providers: + default: "docker" + docker: + enabled: true + runtime: "runc" + network_mode: "stacyvm-network" + seccomp_profile: "default" + memory: "512m" + cpus: "1" + pids_limit: 256 + user: "1000:1000" + dropped_caps: ["ALL"] + added_caps: [] +` + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + return path +} + +func createTestSQLiteDB(t *testing.T, path string) { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec("CREATE TABLE sandboxes (id TEXT PRIMARY KEY)"); err != nil { + t.Fatal(err) + } +} + +func statusFor(checks []doctorCheck, name string) doctorStatus { + for _, check := range checks { + if check.Name == name { + return check.Status + } + } + return "" +} diff --git a/cmd/stacyvm/cmd_worker.go b/cmd/stacyvm/cmd_worker.go new file mode 100644 index 0000000..dc6a13d --- /dev/null +++ b/cmd/stacyvm/cmd_worker.go @@ -0,0 +1,757 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/config" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/worker" + "github.com/rs/zerolog" + "github.com/spf13/cobra" +) + +func newWorkerCmd() *cobra.Command { + var id string + var controlPlaneURL string + var token string + var tokenFile string + var signingKeyFile string + var bootstrapAdminKey string + var bootstrapAdminKeyFile string + var bootstrapTokenTTL string + var heartbeatInterval string + var listenAddr string + var previewDomain string + var once bool + cmd := &cobra.Command{ + Use: "worker", + Short: "Start a StacyVM remote worker process", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + if id == "" { + id = cfg.Worker.ID + } + if id == "" { + hostname, _ := os.Hostname() + id = hostname + } + if controlPlaneURL == "" { + controlPlaneURL = cfg.Worker.ControlPlaneURL + } + if token == "" { + token = cfg.Auth.WorkerToken + } + var tokenFunc func() (string, error) + if tokenFile != "" { + if cmd.Flags().Changed("worker-token") { + return fmt.Errorf("worker token must be set with either --worker-token or --worker-token-file, not both") + } + token = "" + tokenFunc = fileWorkerTokenFunc(tokenFile) + } + signingKey := cfg.Auth.WorkerSigningKey + if signingKeyFile != "" { + signingKey, err = readSecretFile(signingKeyFile) + if err != nil { + return fmt.Errorf("worker signing key file: %w", err) + } + } + if heartbeatInterval == "" { + heartbeatInterval = cfg.Worker.HeartbeatInterval + } + if listenAddr == "" { + listenAddr = cfg.Worker.ListenAddr + } + if previewDomain == "" { + previewDomain = cfg.Worker.PreviewDomain + } + if previewDomain == "" { + previewDomain = cfg.Server.PreviewDomain + } + interval, err := time.ParseDuration(heartbeatInterval) + if err != nil { + return fmt.Errorf("worker heartbeat interval: %w", err) + } + logger := newCommandLogger(cfg) + registry := buildWorkerRegistry(cfg, logger, previewDomain) + // Resolve bootstrap admin key from flag or file. + if bootstrapAdminKeyFile != "" { + bootstrapAdminKey, err = readSecretFile(bootstrapAdminKeyFile) + if err != nil { + return fmt.Errorf("bootstrap admin key file: %w", err) + } + } + // Priority: explicit token > token file > bootstrap issuer > local signing key. + if tokenFunc == nil { + if bootstrapAdminKey != "" { + tokenFunc = worker.NewIssuerTokenFunc(controlPlaneURL, id, bootstrapAdminKey, bootstrapTokenTTL) + } else { + tokenFunc = signedWorkerTokenFunc(id, signingKey) + if token != "" { + tokenFunc = nil + } + } + } + rt := worker.Runtime{ + Client: worker.Client{ + BaseURL: strings.TrimRight(controlPlaneURL, "/"), + WorkerID: id, + Token: token, + TokenFunc: tokenFunc, + }, + HeartbeatInterval: interval, + ListenAddr: listenAddr, + Logger: logger, + Providers: enabledProviderNames(cfg), + Capacity: map[string]interface{}{ + "max_sandboxes": cfg.Defaults.MaxSandboxes, + "max_sandboxes_per_owner": cfg.Defaults.MaxSandboxesPerOwner, + "preview_domain": previewDomain, + }, + Registry: registry, + RPCTLS: workerTLSConfig(cfg.Worker.RPCTLS), + SigningKey: signingKey, + SigningKeys: cfg.Auth.WorkerSigningKeys, + RevokedTokenIDs: cfg.Auth.WorkerRevokedTokenIDs, + } + if once { + return rt.RunOnce(cmd.Context()) + } + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + logger.Info().Str("worker_id", id).Str("control_plane", controlPlaneURL).Msg("starting StacyVM worker") + return rt.Run(ctx) + }, + } + cmd.Flags().StringVar(&id, "id", "", "worker ID; defaults to worker.id or hostname") + cmd.Flags().StringVar(&controlPlaneURL, "control-plane", "", "control plane URL; defaults to worker.control_plane_url") + cmd.Flags().StringVar(&token, "worker-token", os.Getenv("STACYVM_AUTH_WORKER_TOKEN"), "worker bearer token; defaults to auth.worker_token") + cmd.Flags().StringVar(&tokenFile, "worker-token-file", "", "file containing the worker bearer token") + cmd.Flags().StringVar(&signingKeyFile, "worker-signing-key-file", "", "file containing the worker signing key used to derive short-lived signed tokens") + cmd.Flags().StringVar(&bootstrapAdminKey, "bootstrap-admin-key", os.Getenv("STACYVM_WORKER_BOOTSTRAP_ADMIN_KEY"), "admin API key used to fetch signed worker tokens from the control-plane issuer") + cmd.Flags().StringVar(&bootstrapAdminKeyFile, "bootstrap-admin-key-file", "", "file containing the admin API key for token issuance") + cmd.Flags().StringVar(&bootstrapTokenTTL, "bootstrap-token-ttl", "5m", "TTL for tokens fetched from the control-plane issuer (max 15m)") + cmd.Flags().StringVar(&heartbeatInterval, "heartbeat-interval", "", "worker heartbeat interval") + cmd.Flags().StringVar(&listenAddr, "listen", "", "worker RPC listen address; defaults to worker.listen_addr") + cmd.Flags().StringVar(&previewDomain, "preview-domain", "", "worker preview domain; defaults to worker.preview_domain or server.preview_domain") + cmd.Flags().BoolVar(&once, "once", false, "send one heartbeat and exit") + cmd.AddCommand(newWorkerTokenCmd()) + return cmd +} + +func newWorkerTokenCmd() *cobra.Command { + var signingKey string + var signingKeyFile string + var ttl string + var scopes []string + var audience string + var tokenID string + var notBefore string + var outputFormat string + cmd := &cobra.Command{ + Use: "token ", + Short: "Issue a signed worker token", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + if signingKey == "" { + signingKey = cfg.Auth.WorkerSigningKey + } + if signingKeyFile != "" { + if cmd.Flags().Changed("signing-key") { + return fmt.Errorf("worker signing key must be set with either --signing-key or --signing-key-file, not both") + } + signingKey, err = readSecretFile(signingKeyFile) + if err != nil { + return fmt.Errorf("worker signing key file: %w", err) + } + } + result, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: args[0], + SigningKey: signingKey, + TTL: ttl, + Scopes: scopes, + Audience: audience, + TokenID: tokenID, + NotBefore: notBefore, + Now: time.Now, + }) + if err != nil { + return err + } + switch strings.ToLower(strings.TrimSpace(outputFormat)) { + case "", "token": + _, err = fmt.Fprintln(cmd.OutOrStdout(), result.Token) + case "json": + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + err = encoder.Encode(result) + default: + err = fmt.Errorf("worker token output format must be token or json") + } + return err + }, + } + cmd.Flags().StringVar(&signingKey, "signing-key", os.Getenv("STACYVM_AUTH_WORKER_SIGNING_KEY"), "worker signing key; defaults to auth.worker_signing_key") + cmd.Flags().StringVar(&signingKeyFile, "signing-key-file", "", "file containing the worker signing key") + cmd.Flags().StringVar(&ttl, "ttl", "5m", "token lifetime") + cmd.Flags().StringVar(&audience, "audience", middleware.WorkerTokenAudienceControlPlane, "token audience: worker:control-plane or worker:rpc") + cmd.Flags().StringVar(&tokenID, "token-id", "", "explicit token id for incident-response tracking; generated when empty") + cmd.Flags().StringVar(¬Before, "not-before", "0s", "delay before token becomes valid") + cmd.Flags().StringVar(&outputFormat, "format", "token", "output format: token or json") + cmd.Flags().StringArrayVar(&scopes, "scope", nil, "worker scope to include; repeatable, defaults to all worker scopes") + cmd.AddCommand(newWorkerTokenInspectCmd()) + cmd.AddCommand(newWorkerTokenVerifyCmd()) + cmd.AddCommand(newWorkerTokenRotationPlanCmd()) + return cmd +} + +func newWorkerTokenInspectCmd() *cobra.Command { + return &cobra.Command{ + Use: "inspect ", + Short: "Inspect signed worker token claims without verifying the signature", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + result, err := inspectWorkerToken(args[0]) + if err != nil { + return err + } + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(result) + }, + } +} + +func newWorkerTokenVerifyCmd() *cobra.Command { + var signingKey string + var signingKeyFile string + var verificationKeys []string + var verificationKeyFiles []string + var audience string + var workerID string + var revokedTokenIDs []string + cmd := &cobra.Command{ + Use: "verify ", + Short: "Verify a signed worker token against signing keys and revocation settings", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + if signingKey == "" { + signingKey = cfg.Auth.WorkerSigningKey + } + if signingKeyFile != "" { + if cmd.Flags().Changed("signing-key") { + return fmt.Errorf("worker signing key must be set with either --signing-key or --signing-key-file, not both") + } + signingKey, err = readSecretFile(signingKeyFile) + if err != nil { + return fmt.Errorf("worker signing key file: %w", err) + } + } + for _, path := range verificationKeyFiles { + key, err := readSecretFile(path) + if err != nil { + return fmt.Errorf("worker verification key file: %w", err) + } + verificationKeys = append(verificationKeys, key) + } + verificationKeys = append(append([]string{}, cfg.Auth.WorkerSigningKeys...), verificationKeys...) + revokedTokenIDs = append(append([]string{}, cfg.Auth.WorkerRevokedTokenIDs...), revokedTokenIDs...) + result, err := verifyWorkerToken(workerTokenVerifyOptions{ + Token: args[0], + SigningKey: signingKey, + VerificationKey: verificationKeys, + Audience: audience, + WorkerID: workerID, + RevokedTokenIDs: revokedTokenIDs, + Now: time.Now, + }) + if err != nil { + return err + } + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(result) + }, + } + cmd.Flags().StringVar(&signingKey, "signing-key", os.Getenv("STACYVM_AUTH_WORKER_SIGNING_KEY"), "active worker signing key; defaults to auth.worker_signing_key") + cmd.Flags().StringVar(&signingKeyFile, "signing-key-file", "", "file containing the active worker signing key") + cmd.Flags().StringArrayVar(&verificationKeys, "verification-key", nil, "additional verification key accepted during rotation; repeatable") + cmd.Flags().StringArrayVar(&verificationKeyFiles, "verification-key-file", nil, "file containing an additional verification key accepted during rotation; repeatable") + cmd.Flags().StringVar(&audience, "audience", "", "expected token audience: worker:control-plane or worker:rpc") + cmd.Flags().StringVar(&workerID, "worker-id", "", "expected worker ID") + cmd.Flags().StringArrayVar(&revokedTokenIDs, "revoked-token-id", nil, "revoked token ID to reject; repeatable") + return cmd +} + +func newWorkerTokenRotationPlanCmd() *cobra.Command { + var newKeyRef string + var previousKeyRef string + var ttl string + var outputFormat string + cmd := &cobra.Command{ + Use: "rotation-plan", + Short: "Print a no-secret signed worker token rotation plan", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + result, err := workerTokenRotationPlan(workerTokenRotationPlanOptions{ + NewKeyRef: newKeyRef, + PreviousKeyRef: previousKeyRef, + TTL: ttl, + }) + if err != nil { + return err + } + switch strings.ToLower(strings.TrimSpace(outputFormat)) { + case "", "text": + _, err = fmt.Fprint(cmd.OutOrStdout(), result.Text) + case "json": + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + err = encoder.Encode(result) + default: + err = fmt.Errorf("worker token rotation-plan output format must be text or json") + } + return err + }, + } + cmd.Flags().StringVar(&newKeyRef, "new-key-ref", "auth.worker_signing_key", "operator-visible reference for the new active signing key") + cmd.Flags().StringVar(&previousKeyRef, "previous-key-ref", "auth.worker_signing_keys[0]", "operator-visible reference for the previous verification key") + cmd.Flags().StringVar(&ttl, "ttl", "5m", "maximum signed worker token lifetime to wait before removing the previous key") + cmd.Flags().StringVar(&outputFormat, "format", "text", "output format: text or json") + return cmd +} + +type workerTokenIssueOptions struct { + WorkerID string + SigningKey string + TTL string + Scopes []string + Audience string + TokenID string + NotBefore string + Now func() time.Time +} + +type workerTokenIssueResult struct { + Token string `json:"token"` + TokenID string `json:"token_id"` + WorkerID string `json:"worker_id"` + Audience string `json:"audience"` + Scopes []string `json:"scopes,omitempty"` + IssuedAt string `json:"issued_at"` + NotBefore string `json:"not_before,omitempty"` + ExpiresAt string `json:"expires_at"` +} + +type workerTokenInspectResult struct { + SignatureVerified bool `json:"signature_verified"` + WorkerID string `json:"worker_id,omitempty"` + TokenID string `json:"token_id,omitempty"` + Audience string `json:"audience,omitempty"` + Scopes []string `json:"scopes,omitempty"` + IssuedAt string `json:"issued_at,omitempty"` + NotBefore string `json:"not_before,omitempty"` + ExpiresAt string `json:"expires_at,omitempty"` +} + +type workerTokenVerifyOptions struct { + Token string + SigningKey string + VerificationKey []string + Audience string + WorkerID string + RevokedTokenIDs []string + Now func() time.Time +} + +type workerTokenRotationPlanOptions struct { + NewKeyRef string + PreviousKeyRef string + TTL string +} + +type workerTokenRotationPlanResult struct { + NewKeyRef string `json:"new_key_ref"` + PreviousKeyRef string `json:"previous_key_ref"` + MaxTokenTTL string `json:"max_token_ttl"` + Steps []string `json:"steps"` + ConfigSnippet string `json:"config_snippet"` + Validation []string `json:"validation"` + Text string `json:"text,omitempty"` +} + +func issueWorkerToken(opts workerTokenIssueOptions) (workerTokenIssueResult, error) { + workerID := strings.TrimSpace(opts.WorkerID) + signingKey := strings.TrimSpace(opts.SigningKey) + if workerID == "" { + return workerTokenIssueResult{}, fmt.Errorf("worker id is required") + } + if signingKey == "" { + return workerTokenIssueResult{}, fmt.Errorf("worker signing key is required") + } + ttl, err := time.ParseDuration(opts.TTL) + if err != nil { + return workerTokenIssueResult{}, fmt.Errorf("worker token ttl: %w", err) + } + if ttl <= 0 { + return workerTokenIssueResult{}, fmt.Errorf("worker token ttl must be positive") + } + if ttl > middleware.MaxWorkerTokenTTL { + return workerTokenIssueResult{}, fmt.Errorf("worker token ttl must be <= %s", middleware.MaxWorkerTokenTTL) + } + audience := strings.TrimSpace(opts.Audience) + if audience == "" { + audience = middleware.WorkerTokenAudienceControlPlane + } + if audience != middleware.WorkerTokenAudienceControlPlane && audience != middleware.WorkerTokenAudienceRPC { + return workerTokenIssueResult{}, fmt.Errorf("worker token audience must be %q or %q", middleware.WorkerTokenAudienceControlPlane, middleware.WorkerTokenAudienceRPC) + } + notBefore := strings.TrimSpace(opts.NotBefore) + if notBefore == "" { + notBefore = "0s" + } + notBeforeDelay, err := time.ParseDuration(notBefore) + if err != nil { + return workerTokenIssueResult{}, fmt.Errorf("worker token not-before: %w", err) + } + if notBeforeDelay < 0 { + return workerTokenIssueResult{}, fmt.Errorf("worker token not-before must be non-negative") + } + now := opts.Now + if now == nil { + now = time.Now + } + issuedAt := now().UTC() + tokenID := strings.TrimSpace(opts.TokenID) + if tokenID == "" { + tokenID, err = middleware.NewWorkerTokenID() + if err != nil { + return workerTokenIssueResult{}, fmt.Errorf("worker token id: %w", err) + } + } + notBeforeAt := issuedAt.Add(notBeforeDelay) + claims := middleware.WorkerTokenClaims{ + WorkerID: workerID, + TokenID: tokenID, + Audience: audience, + Scopes: opts.Scopes, + IssuedAt: issuedAt.Unix(), + ExpiresAt: issuedAt.Add(ttl).Unix(), + } + result := workerTokenIssueResult{ + TokenID: tokenID, + WorkerID: workerID, + Audience: audience, + Scopes: opts.Scopes, + IssuedAt: issuedAt.Format(time.RFC3339), + ExpiresAt: issuedAt.Add(ttl).Format(time.RFC3339), + } + if notBeforeDelay > 0 { + claims.NotBefore = notBeforeAt.Unix() + result.NotBefore = notBeforeAt.Format(time.RFC3339) + } + result.Token, err = middleware.SignWorkerToken(signingKey, claims) + if err != nil { + return workerTokenIssueResult{}, err + } + return result, nil +} + +func inspectWorkerToken(token string) (workerTokenInspectResult, error) { + claims, ok := middleware.DecodeWorkerTokenClaims(token) + if !ok { + return workerTokenInspectResult{}, fmt.Errorf("invalid signed worker token format") + } + return workerTokenInspectResult{ + SignatureVerified: false, + WorkerID: claims.WorkerID, + TokenID: claims.TokenID, + Audience: claims.Audience, + Scopes: claims.Scopes, + IssuedAt: unixTimeString(claims.IssuedAt), + NotBefore: unixTimeString(claims.NotBefore), + ExpiresAt: unixTimeString(claims.ExpiresAt), + }, nil +} + +func verifyWorkerToken(opts workerTokenVerifyOptions) (workerTokenInspectResult, error) { + keys := cleanStrings(append([]string{opts.SigningKey}, opts.VerificationKey...)) + if len(keys) == 0 { + return workerTokenInspectResult{}, fmt.Errorf("worker signing key is required") + } + audience := strings.TrimSpace(opts.Audience) + if audience != "" && audience != middleware.WorkerTokenAudienceControlPlane && audience != middleware.WorkerTokenAudienceRPC { + return workerTokenInspectResult{}, fmt.Errorf("worker token audience must be %q or %q", middleware.WorkerTokenAudienceControlPlane, middleware.WorkerTokenAudienceRPC) + } + now := opts.Now + if now == nil { + now = time.Now + } + var claims middleware.WorkerTokenClaims + var ok bool + for _, key := range keys { + claims, ok = middleware.VerifyWorkerTokenForAudience(key, opts.Token, audience, now().UTC()) + if ok { + break + } + } + if !ok { + return workerTokenInspectResult{}, fmt.Errorf("invalid signed worker token") + } + if expectedWorkerID := strings.TrimSpace(opts.WorkerID); expectedWorkerID != "" && claims.WorkerID != expectedWorkerID { + return workerTokenInspectResult{}, fmt.Errorf("worker token worker_id %q does not match expected worker %q", claims.WorkerID, expectedWorkerID) + } + revoked := map[string]struct{}{} + for _, id := range cleanStrings(opts.RevokedTokenIDs) { + revoked[id] = struct{}{} + } + if _, isRevoked := revoked[claims.TokenID]; claims.TokenID != "" && isRevoked { + return workerTokenInspectResult{}, fmt.Errorf("worker token %q is revoked", claims.TokenID) + } + result, _ := inspectWorkerToken(opts.Token) + result.SignatureVerified = true + return result, nil +} + +func cleanStrings(values []string) []string { + clean := make([]string, 0, len(values)) + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + clean = append(clean, value) + } + } + return clean +} + +func readSecretFile(path string) (string, error) { + path = strings.TrimSpace(path) + if path == "" { + return "", fmt.Errorf("path is required") + } + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + secret := strings.TrimSpace(string(data)) + if secret == "" { + return "", fmt.Errorf("secret file is empty") + } + return secret, nil +} + +func fileWorkerTokenFunc(path string) func() (string, error) { + return func() (string, error) { + token, err := readSecretFile(path) + if err != nil { + return "", fmt.Errorf("worker token file: %w", err) + } + return token, nil + } +} + +func workerTokenRotationPlan(opts workerTokenRotationPlanOptions) (workerTokenRotationPlanResult, error) { + newKeyRef := strings.TrimSpace(opts.NewKeyRef) + previousKeyRef := strings.TrimSpace(opts.PreviousKeyRef) + if newKeyRef == "" { + return workerTokenRotationPlanResult{}, fmt.Errorf("new key reference is required") + } + if previousKeyRef == "" { + return workerTokenRotationPlanResult{}, fmt.Errorf("previous key reference is required") + } + ttl := strings.TrimSpace(opts.TTL) + if ttl == "" { + ttl = "5m" + } + duration, err := time.ParseDuration(ttl) + if err != nil { + return workerTokenRotationPlanResult{}, fmt.Errorf("worker token rotation ttl: %w", err) + } + if duration <= 0 { + return workerTokenRotationPlanResult{}, fmt.Errorf("worker token rotation ttl must be positive") + } + if duration > middleware.MaxWorkerTokenTTL { + return workerTokenRotationPlanResult{}, fmt.Errorf("worker token rotation ttl must be <= %s", middleware.MaxWorkerTokenTTL) + } + steps := []string{ + fmt.Sprintf("Put the new active signing key at %s.", newKeyRef), + fmt.Sprintf("Move the previous active signing key into %s.", previousKeyRef), + "Restart or reload control-plane and worker processes so new tokens are minted with the new key.", + fmt.Sprintf("Wait at least %s, plus clock skew, before removing the previous verification key.", duration), + "Remove the previous verification key after old signed worker tokens have expired.", + } + configSnippet := fmt.Sprintf("auth:\n worker_signing_key: \"\"\n worker_signing_keys:\n - \"\"\n", newKeyRef, previousKeyRef) + validation := []string{ + "stacyvm config lint --production", + "stacyvm worker token --ttl " + duration.String() + " --format json", + "stacyvm worker token verify '' --worker-id --audience worker:control-plane", + } + result := workerTokenRotationPlanResult{ + NewKeyRef: newKeyRef, + PreviousKeyRef: previousKeyRef, + MaxTokenTTL: duration.String(), + Steps: steps, + ConfigSnippet: configSnippet, + Validation: validation, + } + result.Text = formatWorkerTokenRotationPlan(result) + return result, nil +} + +func formatWorkerTokenRotationPlan(result workerTokenRotationPlanResult) string { + var b strings.Builder + b.WriteString("Signed worker token rotation plan\n\n") + b.WriteString("Key references:\n") + b.WriteString(fmt.Sprintf("- new active key: %s\n", result.NewKeyRef)) + b.WriteString(fmt.Sprintf("- previous verification key: %s\n", result.PreviousKeyRef)) + b.WriteString(fmt.Sprintf("- maximum token TTL: %s\n\n", result.MaxTokenTTL)) + b.WriteString("Steps:\n") + for i, step := range result.Steps { + b.WriteString(fmt.Sprintf("%d. %s\n", i+1, step)) + } + b.WriteString("\nConfig sketch:\n") + b.WriteString(result.ConfigSnippet) + b.WriteString("\nValidation:\n") + for _, command := range result.Validation { + b.WriteString("- " + command + "\n") + } + return b.String() +} + +func unixTimeString(sec int64) string { + if sec <= 0 { + return "" + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339) +} + +func signedWorkerTokenFunc(workerID, signingKey string) func() (string, error) { + signingKey = strings.TrimSpace(signingKey) + workerID = strings.TrimSpace(workerID) + if signingKey == "" || workerID == "" { + return nil + } + return func() (string, error) { + now := time.Now().UTC() + tokenID, err := middleware.NewWorkerTokenID() + if err != nil { + return "", fmt.Errorf("worker token id: %w", err) + } + return middleware.SignWorkerToken(signingKey, middleware.WorkerTokenClaims{ + WorkerID: workerID, + TokenID: tokenID, + Audience: middleware.WorkerTokenAudienceControlPlane, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(5 * time.Minute).Unix(), + }) + } +} + +func workerTLSConfig(cfg config.WorkerRPCTLSConfig) worker.TLSConfig { + return worker.TLSConfig{ + Enabled: cfg.Enabled, + ServerCertFile: cfg.ServerCertFile, + ServerKeyFile: cfg.ServerKeyFile, + ClientCAFile: cfg.ClientCAFile, + CAFile: cfg.CAFile, + ClientCertFile: cfg.ClientCertFile, + ClientKeyFile: cfg.ClientKeyFile, + ServerName: cfg.ServerName, + InsecureSkipVerify: cfg.InsecureSkipVerify, + } +} + +func newCommandLogger(cfg *config.Config) zerolog.Logger { + var logger zerolog.Logger + if cfg.Logging.Format == "pretty" { + logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stdout}).With().Timestamp().Logger() + } else { + logger = zerolog.New(os.Stdout).With().Timestamp().Logger() + } + if level, err := zerolog.ParseLevel(cfg.Logging.Level); err == nil { + logger = logger.Level(level) + } + return logger +} + +func enabledProviderNames(cfg *config.Config) []string { + var providers []string + if cfg.Providers.Mock.Enabled { + providers = append(providers, "mock") + } + if cfg.Providers.Firecracker.Enabled { + providers = append(providers, "firecracker") + } + if cfg.Providers.Docker.Enabled { + providers = append(providers, "docker") + } + if cfg.Providers.E2B.Enabled { + providers = append(providers, "e2b") + } + if cfg.Providers.Custom.Enabled { + providers = append(providers, cfg.Providers.Custom.Name) + } + if cfg.Providers.PRoot.Enabled { + providers = append(providers, "proot") + } + return providers +} + +func buildWorkerRegistry(cfg *config.Config, logger zerolog.Logger, previewDomain string) *providers.Registry { + registry := providers.NewRegistry() + if cfg.Providers.Mock.Enabled { + registry.Register(providers.NewMockProvider()) + } + if cfg.Providers.Docker.Enabled { + docker, err := providers.NewDockerProvider(providers.DockerProviderConfig{ + Socket: cfg.Providers.Docker.Socket, + Runtime: cfg.Providers.Docker.Runtime, + DefaultImage: cfg.Providers.Docker.DefaultImage, + NetworkMode: cfg.Providers.Docker.NetworkMode, + SeccompProfile: cfg.Providers.Docker.SeccompProfile, + ReadOnlyRootfs: cfg.Providers.Docker.ReadOnlyRootfs, + Memory: cfg.Providers.Docker.Memory, + CPUs: cfg.Providers.Docker.CPUs, + PidsLimit: cfg.Providers.Docker.PidsLimit, + User: cfg.Providers.Docker.User, + DroppedCaps: cfg.Providers.Docker.DroppedCaps, + AddedCaps: cfg.Providers.Docker.AddedCaps, + Tmpfs: cfg.Providers.Docker.Tmpfs, + PoolSecurity: providers.PoolSecurityProviderConfig{ + PerUserUID: cfg.Providers.Docker.PoolSecurity.PerUserUID, + PIDNamespace: cfg.Providers.Docker.PoolSecurity.PIDNamespace, + WorkspacePermissions: cfg.Providers.Docker.PoolSecurity.WorkspacePermissions, + HidePID: cfg.Providers.Docker.PoolSecurity.HidePID, + }, + PreviewDomain: previewDomain, + }, logger) + if err != nil { + logger.Error().Err(err).Msg("failed to create worker docker provider") + } else { + registry.Register(docker) + } + } + if len(registry.List()) > 0 { + if err := registry.SetDefault(cfg.Providers.Default); err != nil { + _ = registry.SetDefault(registry.List()[0]) + } + } + return registry +} diff --git a/cmd/stacyvm/cmd_worker_test.go b/cmd/stacyvm/cmd_worker_test.go new file mode 100644 index 0000000..65d8732 --- /dev/null +++ b/cmd/stacyvm/cmd_worker_test.go @@ -0,0 +1,350 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" +) + +func TestIssueWorkerTokenSignsExpectedClaims(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + result, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: " worker-a ", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + TTL: "10m", + Scopes: []string{middleware.ScopeWorkerHeartbeat}, + TokenID: "token-id-1", + NotBefore: "2m", + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("issue worker token: %v", err) + } + + if result.TokenID != "token-id-1" || result.NotBefore != now.Add(2*time.Minute).Format(time.RFC3339) { + t.Fatalf("unexpected token issue result: %+v", result) + } + claims, ok := middleware.VerifyWorkerToken("worker-signing-key-with-at-least-32-bytes", result.Token, now.Add(3*time.Minute)) + if !ok { + t.Fatal("issued token did not verify after not-before") + } + if claims.WorkerID != "worker-a" { + t.Fatalf("worker id = %q, want worker-a", claims.WorkerID) + } + if claims.TokenID != "token-id-1" { + t.Fatalf("token id = %q, want token-id-1", claims.TokenID) + } + if claims.Audience != middleware.WorkerTokenAudienceControlPlane { + t.Fatalf("audience = %q, want %q", claims.Audience, middleware.WorkerTokenAudienceControlPlane) + } + if claims.IssuedAt != now.Unix() { + t.Fatalf("issued at = %d, want %d", claims.IssuedAt, now.Unix()) + } + if claims.ExpiresAt != now.Add(10*time.Minute).Unix() { + t.Fatalf("expires at = %d, want %d", claims.ExpiresAt, now.Add(10*time.Minute).Unix()) + } + if claims.NotBefore != now.Add(2*time.Minute).Unix() { + t.Fatalf("not before = %d, want %d", claims.NotBefore, now.Add(2*time.Minute).Unix()) + } + if len(claims.Scopes) != 1 || claims.Scopes[0] != middleware.ScopeWorkerHeartbeat { + t.Fatalf("scopes = %#v, want heartbeat scope", claims.Scopes) + } +} + +func TestIssueWorkerTokenRejectsInvalidInputs(t *testing.T) { + tests := []struct { + name string + opts workerTokenIssueOptions + }{ + {name: "missing worker", opts: workerTokenIssueOptions{SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: "5m"}}, + {name: "missing signing key", opts: workerTokenIssueOptions{WorkerID: "worker-a", TTL: "5m"}}, + {name: "bad ttl", opts: workerTokenIssueOptions{WorkerID: "worker-a", SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: "soon"}}, + {name: "zero ttl", opts: workerTokenIssueOptions{WorkerID: "worker-a", SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: "0s"}}, + {name: "too long ttl", opts: workerTokenIssueOptions{WorkerID: "worker-a", SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: (middleware.MaxWorkerTokenTTL + time.Second).String()}}, + {name: "bad audience", opts: workerTokenIssueOptions{WorkerID: "worker-a", SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: "5m", Audience: "admin"}}, + {name: "bad not before", opts: workerTokenIssueOptions{WorkerID: "worker-a", SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: "5m", NotBefore: "later"}}, + {name: "negative not before", opts: workerTokenIssueOptions{WorkerID: "worker-a", SigningKey: "worker-signing-key-with-at-least-32-bytes", TTL: "5m", NotBefore: "-1s"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, err := issueWorkerToken(tt.opts); err == nil { + t.Fatal("expected error") + } + }) + } +} + +func TestWorkerTokenCommandReadsSigningKeyFile(t *testing.T) { + keyPath := writeTestSecret(t, "worker-signing-key-with-at-least-32-bytes\n") + cmd := newWorkerTokenCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"worker-a", "--signing-key-file", keyPath, "--token-id", "token-id-1", "--format", "json"}) + + if err := cmd.Execute(); err != nil { + t.Fatalf("execute worker token command: %v", err) + } + var result workerTokenIssueResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("decode worker token json: %v", err) + } + if result.TokenID != "token-id-1" || result.WorkerID != "worker-a" { + t.Fatalf("unexpected token result: %+v", result) + } + if _, ok := middleware.VerifyWorkerToken("worker-signing-key-with-at-least-32-bytes", result.Token, time.Now().UTC()); !ok { + t.Fatal("issued token did not verify with signing key from file") + } +} + +func TestInspectWorkerTokenShowsUnverifiedClaims(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + issued, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: "worker-a", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + TTL: "5m", + Scopes: []string{middleware.ScopeWorkerHeartbeat}, + Audience: middleware.WorkerTokenAudienceRPC, + TokenID: "token-id-1", + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("issue worker token: %v", err) + } + + inspected, err := inspectWorkerToken(issued.Token) + if err != nil { + t.Fatalf("inspect worker token: %v", err) + } + if inspected.SignatureVerified { + t.Fatal("inspect result should not report signature verification") + } + if inspected.WorkerID != "worker-a" || inspected.TokenID != "token-id-1" || inspected.Audience != middleware.WorkerTokenAudienceRPC { + t.Fatalf("unexpected inspect result: %+v", inspected) + } + if inspected.IssuedAt != now.Format(time.RFC3339) || inspected.ExpiresAt != now.Add(5*time.Minute).Format(time.RFC3339) { + t.Fatalf("unexpected inspect timestamps: %+v", inspected) + } + if len(inspected.Scopes) != 1 || inspected.Scopes[0] != middleware.ScopeWorkerHeartbeat { + t.Fatalf("scopes = %#v, want heartbeat scope", inspected.Scopes) + } +} + +func TestInspectWorkerTokenRejectsInvalidFormat(t *testing.T) { + if _, err := inspectWorkerToken("not-a-worker-token"); err == nil { + t.Fatal("expected error") + } +} + +func TestVerifyWorkerTokenAcceptsRotationKey(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + issued, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: "worker-a", + SigningKey: "old-worker-signing-key-with-at-least-32-bytes", + TTL: "5m", + Audience: middleware.WorkerTokenAudienceRPC, + TokenID: "token-id-1", + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("issue worker token: %v", err) + } + + verified, err := verifyWorkerToken(workerTokenVerifyOptions{ + Token: issued.Token, + SigningKey: "new-worker-signing-key-with-at-least-32-bytes", + VerificationKey: []string{"old-worker-signing-key-with-at-least-32-bytes"}, + Audience: middleware.WorkerTokenAudienceRPC, + WorkerID: "worker-a", + Now: func() time.Time { return now.Add(time.Minute) }, + }) + if err != nil { + t.Fatalf("verify worker token: %v", err) + } + if !verified.SignatureVerified { + t.Fatal("verify result should report signature verification") + } + if verified.TokenID != "token-id-1" || verified.WorkerID != "worker-a" || verified.Audience != middleware.WorkerTokenAudienceRPC { + t.Fatalf("unexpected verify result: %+v", verified) + } +} + +func TestVerifyWorkerTokenRejectsRevokedTokenID(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + issued, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: "worker-a", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + TTL: "5m", + TokenID: "revoked-token-id", + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("issue worker token: %v", err) + } + + if _, err := verifyWorkerToken(workerTokenVerifyOptions{ + Token: issued.Token, + SigningKey: "worker-signing-key-with-at-least-32-bytes", + RevokedTokenIDs: []string{"revoked-token-id"}, + Now: func() time.Time { return now.Add(time.Minute) }, + }); err == nil { + t.Fatal("expected revoked token to fail verification") + } +} + +func TestVerifyWorkerTokenRejectsWrongAudienceAndWorker(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + issued, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: "worker-a", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + TTL: "5m", + Audience: middleware.WorkerTokenAudienceControlPlane, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("issue worker token: %v", err) + } + + if _, err := verifyWorkerToken(workerTokenVerifyOptions{ + Token: issued.Token, + SigningKey: "worker-signing-key-with-at-least-32-bytes", + Audience: middleware.WorkerTokenAudienceRPC, + Now: func() time.Time { return now.Add(time.Minute) }, + }); err == nil { + t.Fatal("expected wrong audience to fail verification") + } + if _, err := verifyWorkerToken(workerTokenVerifyOptions{ + Token: issued.Token, + SigningKey: "worker-signing-key-with-at-least-32-bytes", + WorkerID: "worker-b", + Now: func() time.Time { return now.Add(time.Minute) }, + }); err == nil { + t.Fatal("expected wrong worker to fail verification") + } +} + +func TestWorkerTokenVerifyCommandReadsSigningKeyFiles(t *testing.T) { + now := time.Now().UTC() + issued, err := issueWorkerToken(workerTokenIssueOptions{ + WorkerID: "worker-a", + SigningKey: "old-worker-signing-key-with-at-least-32-bytes", + TTL: "5m", + Audience: middleware.WorkerTokenAudienceRPC, + TokenID: "token-id-1", + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("issue worker token: %v", err) + } + activeKeyPath := writeTestSecret(t, "new-worker-signing-key-with-at-least-32-bytes") + oldKeyPath := writeTestSecret(t, "old-worker-signing-key-with-at-least-32-bytes\n") + cmd := newWorkerTokenVerifyCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{ + issued.Token, + "--signing-key-file", activeKeyPath, + "--verification-key-file", oldKeyPath, + "--worker-id", "worker-a", + "--audience", middleware.WorkerTokenAudienceRPC, + }) + + if err := cmd.Execute(); err != nil { + t.Fatalf("execute worker token verify command: %v", err) + } + var result workerTokenInspectResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("decode worker token verify json: %v", err) + } + if !result.SignatureVerified || result.TokenID != "token-id-1" { + t.Fatalf("unexpected verify result: %+v", result) + } +} + +func TestWorkerTokenRotationPlanCommandOutputsJSON(t *testing.T) { + cmd := newWorkerTokenRotationPlanCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{ + "--new-key-ref", "/run/secrets/worker-signing-key-new", + "--previous-key-ref", "/run/secrets/worker-signing-key-old", + "--ttl", "10m", + "--format", "json", + }) + + if err := cmd.Execute(); err != nil { + t.Fatalf("execute worker token rotation-plan command: %v", err) + } + var result workerTokenRotationPlanResult + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("decode rotation plan json: %v", err) + } + if result.NewKeyRef != "/run/secrets/worker-signing-key-new" || result.PreviousKeyRef != "/run/secrets/worker-signing-key-old" { + t.Fatalf("unexpected key refs: %+v", result) + } + if result.MaxTokenTTL != "10m0s" { + t.Fatalf("max token ttl = %q, want 10m0s", result.MaxTokenTTL) + } + if len(result.Steps) != 5 || len(result.Validation) != 3 || result.ConfigSnippet == "" { + t.Fatalf("unexpected rotation plan: %+v", result) + } +} + +func TestWorkerTokenRotationPlanRejectsInvalidTTL(t *testing.T) { + if _, err := workerTokenRotationPlan(workerTokenRotationPlanOptions{ + NewKeyRef: "new", + PreviousKeyRef: "old", + TTL: (middleware.MaxWorkerTokenTTL + time.Second).String(), + }); err == nil { + t.Fatal("expected overlong ttl to fail") + } +} + +func TestReadSecretFileRejectsEmptySecret(t *testing.T) { + path := writeTestSecret(t, "\n") + if _, err := readSecretFile(path); err == nil { + t.Fatal("expected empty secret file to fail") + } +} + +func TestFileWorkerTokenFuncReloadsTokenFile(t *testing.T) { + path := writeTestSecret(t, "first-token\n") + tokenFunc := fileWorkerTokenFunc(path) + + got, err := tokenFunc() + if err != nil { + t.Fatalf("read first token: %v", err) + } + if got != "first-token" { + t.Fatalf("first token = %q, want first-token", got) + } + + if err := os.WriteFile(path, []byte("second-token\n"), 0o600); err != nil { + t.Fatalf("rotate token file: %v", err) + } + got, err = tokenFunc() + if err != nil { + t.Fatalf("read rotated token: %v", err) + } + if got != "second-token" { + t.Fatalf("rotated token = %q, want second-token", got) + } +} + +func writeTestSecret(t *testing.T, value string) string { + t.Helper() + file, err := os.CreateTemp(t.TempDir(), "secret-*") + if err != nil { + t.Fatalf("create secret file: %v", err) + } + if _, err := file.WriteString(value); err != nil { + t.Fatalf("write secret file: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close secret file: %v", err) + } + return file.Name() +} diff --git a/cmd/stacyvm/main.go b/cmd/stacyvm/main.go index 242a896..2c0894d 100644 --- a/cmd/stacyvm/main.go +++ b/cmd/stacyvm/main.go @@ -26,6 +26,7 @@ func main() { root.AddCommand( newServeCmd(), + newWorkerCmd(), newSpawnCmd(), newExecCmd(), newKillCmd(), @@ -33,6 +34,11 @@ func main() { newVersionCmd(), newTUICmd(), newBuildImageCmd(), + newDoctorCmd(), + newConfigCmd(), + newDBCmd(), + newUpgradeCmd(), + newSupportCmd(), ) if err := root.Execute(); err != nil { diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..b27e4a8 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,9 @@ +STACYVM_IMAGE=ghcr.io/stacyos/stacyvm:latest +STACYVM_HOST_PORT=7423 +STACYVM_TRAEFIK_HOST_PORT=80 +STACYVM_API_KEY=change-me-generate-at-least-32-bytes +STACYVM_ADMIN_API_KEY=change-me-generate-a-separate-admin-key +STACYVM_PREVIEW_DOMAIN=localhost +STACYVM_LOG_LEVEL=info +STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db +STACYVM_DOCKER_SOCKET=/var/run/docker.sock diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..464096b --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,48 @@ +services: + stacyvm: + image: ${STACYVM_IMAGE:-ghcr.io/stacyos/stacyvm:latest} + restart: unless-stopped + working_dir: /etc/stacyvm + ports: + - "${STACYVM_HOST_PORT:-7423}:7423" + volumes: + - stacyvm-data:/var/lib/stacyvm + - ${STACYVM_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock + - ./stacyvm.production.yaml:/etc/stacyvm/stacyvm.yaml:ro + environment: + STACYVM_AUTH_API_KEY: ${STACYVM_API_KEY:?set STACYVM_API_KEY} + STACYVM_AUTH_ADMIN_API_KEY: ${STACYVM_ADMIN_API_KEY:-} + STACYVM_DATABASE_PATH: ${STACYVM_DATABASE_PATH:-/var/lib/stacyvm/stacyvm.db} + STACYVM_LOGGING_LEVEL: ${STACYVM_LOG_LEVEL:-info} + STACYVM_SERVER_PREVIEW_DOMAIN: ${STACYVM_PREVIEW_DOMAIN:-localhost} + STACYVM_PROVIDERS_DOCKER_NETWORK_MODE: stacyvm-network + networks: + - stacyvm-network + healthcheck: + test: ["CMD-SHELL", "wget -qO- --header=\"X-API-Key: $${STACYVM_AUTH_API_KEY}\" http://127.0.0.1:7423/api/v1/live >/dev/null"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + + traefik: + image: traefik:v3.6 + restart: unless-stopped + command: + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + ports: + - "${STACYVM_TRAEFIK_HOST_PORT:-80}:80" + volumes: + - ${STACYVM_DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock:ro + networks: + - stacyvm-network + +volumes: + stacyvm-data: + +networks: + stacyvm-network: + name: stacyvm-network + driver: bridge diff --git a/deploy/stacyvm.env.example b/deploy/stacyvm.env.example new file mode 100644 index 0000000..8e756ba --- /dev/null +++ b/deploy/stacyvm.env.example @@ -0,0 +1,10 @@ +STACYVM_AUTH_API_KEY=change-me-generate-at-least-32-bytes +STACYVM_AUTH_ADMIN_API_KEY=change-me-generate-a-separate-admin-key +STACYVM_AUTH_ADMIN_FALLBACK_ENABLED=false +STACYVM_AUTH_ADMIN_AUDIT_RETENTION=2160h +STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db +STACYVM_LOGGING_LEVEL=info +STACYVM_LOGGING_FORMAT=json +STACYVM_SERVER_PREVIEW_DOMAIN=localhost +STACYVM_PROVIDERS_DEFAULT=docker +STACYVM_PROVIDERS_DOCKER_NETWORK_MODE=bridge diff --git a/deploy/stacyvm.production.yaml b/deploy/stacyvm.production.yaml new file mode 100644 index 0000000..ecad1f6 --- /dev/null +++ b/deploy/stacyvm.production.yaml @@ -0,0 +1,105 @@ +server: + host: "0.0.0.0" + port: 7423 + preview_domain: "localhost" + cors_allowed_origins: + - "https://stacyvm.example.com" + +providers: + default: "docker" + + docker: + enabled: true + socket: "unix:///var/run/docker.sock" + runtime: "runc" + default_image: "alpine:latest" + network_mode: "stacyvm-network" + seccomp_profile: "default" + read_only_rootfs: false + memory: "512m" + cpus: "1" + pids_limit: 256 + user: "1000:1000" + dropped_caps: ["ALL"] + added_caps: [] + pool_security: + per_user_uid: false + pid_namespace: false + workspace_permissions: true + hidepid: false + + firecracker: + enabled: false + firecracker_path: "/usr/local/bin/firecracker" + kernel_path: "/var/lib/stacyvm/vmlinux.bin" + agent_path: "/usr/local/bin/stacyvm-agent" + data_dir: "/var/lib/stacyvm" + + e2b: + enabled: false + api_key: "" + base_url: "https://api.e2b.dev" + + custom: + enabled: false + name: "custom" + base_url: "" + api_key: "" + timeout: "60s" + + proot: + enabled: false + rootfs_path: "/var/lib/stacyvm/rootfs" + proot_binary: "proot" + workspace_base: "/var/lib/stacyvm/workspaces" + default_timeout: "60s" + max_sandboxes: 10 + max_memory_mb: 512 + max_disk_mb: 1024 + languages: ["python3", "node", "bash"] + +defaults: + ttl: "30m" + image: "alpine:latest" + memory_mb: 1024 + vcpus: 1 + disk_size_mb: 1024 + max_ttl: "24h" + default_exec_timeout: "30s" + max_exec_timeout: "10m" + max_sandboxes: 100 + max_sandboxes_per_owner: 10 + spawn_overflow: "queue" + spawn_queue_timeout: "30s" + max_spawn_queue: 100 + +auth: + enabled: true + api_key: "change-me-generate-at-least-32-bytes" + admin_api_key: "change-me-generate-a-separate-admin-key" + admin_fallback_enabled: false + admin_audit_retention: "2160h" + +rate_limit: + enabled: true + requests_per_minute: 120 + burst: 60 + key_by: "api_key" + bucket_ttl: "15m" + cleanup_interval: "1m" + +database: + path: "/var/lib/stacyvm/stacyvm.db" + +logging: + level: "info" + format: "json" + +pool: + enabled: false + max_vms: 10 + max_users_per_vm: 5 + image: "alpine:latest" + memory_mb: 2048 + vcpus: 2 + overflow: "reject" diff --git a/deploy/stacyvm.service b/deploy/stacyvm.service new file mode 100644 index 0000000..9fda38f --- /dev/null +++ b/deploy/stacyvm.service @@ -0,0 +1,29 @@ +[Unit] +Description=StacyVM sandbox API +Documentation=https://github.com/StacyOS/stacyvm +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +User=stacyvm +Group=stacyvm +SupplementaryGroups=docker +WorkingDirectory=/etc/stacyvm +EnvironmentFile=-/etc/stacyvm/stacyvm.env +Environment=STACYVM_DATABASE_PATH=/var/lib/stacyvm/stacyvm.db +ExecStart=/usr/local/bin/stacyvm serve +Restart=on-failure +RestartSec=5s +TimeoutStopSec=30s + +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/stacyvm +ReadOnlyPaths=/etc/stacyvm + +[Install] +WantedBy=multi-user.target diff --git a/docs.json b/docs.json new file mode 100644 index 0000000..5b76b21 --- /dev/null +++ b/docs.json @@ -0,0 +1,148 @@ +{ + "$schema": "https://mintlify.com/docs.json", + "theme": "mint", + "name": "StacyVM", + "description": "Deterministic execution infrastructure for autonomous software systems.", + "colors": { + "primary": "#FF7038", + "light": "#D1C2A5", + "dark": "#FF7038" + }, + "logo": { + "light": "/assets/stacy-logo-light.png", + "dark": "/assets/stacy-logo-dark.png", + "href": "https://github.com/StacyOS/stacyvm" + }, + "favicon": "/assets/stacy-mark-orange.png", + "appearance": { + "default": "system" + }, + "fonts": { + "family": "IBM Plex Sans", + "heading": { + "family": "Libre Baskerville", + "weight": 400 + } + }, + "background": { + "color": { + "light": "#F0E7DA", + "dark": "#262626" + } + }, + "styling": { + "codeblocks": { + "theme": { + "light": "github-light", + "dark": "github-dark" + } + } + }, + "icons": { + "library": "lucide" + }, + "navigation": { + "groups": [ + { + "group": "Get Started", + "pages": [ + "index", + "docs/getting-started/what-is-stacyvm", + "docs/getting-started/prerequisites", + "docs/getting-started/quickstart", + "docs/getting-started/installation", + "docs/getting-started/core-concepts" + ] + }, + { + "group": "Build With StacyVM", + "pages": [ + "docs/tutorials/code-runner", + "docs/tutorials/typescript-code-runner", + "docs/sdks/python", + "docs/sdks/typescript", + "docs/rest/sandboxes", + "docs/rest-api" + ] + }, + { + "group": "Operate", + "pages": [ + "docs/deployment", + "docs/production-readiness", + "docs/public-readiness-evidence", + "docs/public-support-matrix", + "docs/releasing" + ] + }, + { + "group": "Production Readiness", + "pages": [ + "docs/security-governance", + "docs/threat-model" + ] + }, + { + "group": "Runtime And Conformance", + "pages": [ + "docs/provider-contract", + "docs/runtime-certification", + "docs/runtime-conformance", + "docs/cluster-conformance", + "docs/remote-worker-staging", + "docs/worker-rpc-contract" + ] + }, + { + "group": "Architecture", + "pages": [ + "docs/architecture/system-overview", + "docs/admin-control-plane", + "docs/live-preview-architecture", + "docs/snapshot-restore", + "docs/enterprise-signoff-runbook" + ] + }, + { + "group": "Release Notes", + "pages": [ + "docs/releases/phase-14-worker-identity-hardening", + "docs/releases/phase-13-cluster-store-and-worker-identity", + "docs/releases/phase-12-remote-sandbox-io-routing", + "docs/releases/phase-11-remote-worker-runtime", + "docs/releases/phase-10-multi-worker-foundation", + "docs/releases/phase-9-public-self-serve-release-trust", + "docs/releases/phase-8-single-node-production", + "docs/releases/phase-7-release-candidate-hardening", + "docs/releases/phase-6-security-governance", + "docs/releases/phase-5-admin-control-plane", + "docs/releases/phase-4-production-deployment", + "docs/releases/phase-3-quotas-and-scheduling", + "docs/releases/phase-2-observability-and-ops", + "docs/releases/phase-1-foundation-hardening" + ] + } + ] + }, + "api": { + "openapi": "docs/openapi.json", + "playground": { + "display": "simple" + } + }, + "navbar": { + "links": [ + { + "label": "GitHub", + "href": "https://github.com/StacyOS/stacyvm" + }, + { + "label": "Releases", + "href": "https://github.com/StacyOS/stacyvm/releases" + } + ] + }, + "seo": { + "indexing": "navigable" + } +} diff --git a/docs/admin-control-plane.md b/docs/admin-control-plane.md new file mode 100644 index 0000000..d184f63 --- /dev/null +++ b/docs/admin-control-plane.md @@ -0,0 +1,107 @@ +# Admin Control Plane + +This guide covers the operator-facing admin surface added in Phase 5: separate admin authentication, dashboard Operations workflows, owner quota management, diagnostics, and persisted admin audit history. + +## Authentication + +StacyVM supports a regular API key and an optional separate admin API key: + +```yaml +auth: + enabled: true + api_key: "sk-client" + admin_api_key: "sk-admin" + admin_fallback_enabled: false + admin_audit_retention: "2160h" +``` + +The same values can be supplied with environment variables: + +```bash +STACYVM_AUTH_API_KEY=sk-client +STACYVM_AUTH_ADMIN_API_KEY=sk-admin +STACYVM_AUTH_ADMIN_FALLBACK_ENABLED=false +STACYVM_AUTH_ADMIN_AUDIT_RETENTION=2160h +``` + +Use `X-Admin-API-Key` for `/api/v1/admin/*` requests. If `auth.admin_api_key` is configured, the regular `auth.api_key` cannot access admin routes. If no admin key is configured, admin routes fall back to `auth.api_key` for compatibility unless `auth.admin_fallback_enabled` is set to `false`. + +Production deployments should set `auth.admin_fallback_enabled: false` and configure a dedicated `auth.admin_api_key`. + +Authenticated requests now carry a request-scoped identity with either the `api` or `admin` role. Admin identities receive both `api:*` and `admin:*` scopes; regular API identities receive `api:*`. This keeps the current API-key behavior stable while creating a typed authorization boundary for later RBAC and identity-provider integrations. + +When API-key auth is enabled, admin routes also enforce the `admin:*` scope at the route layer. This is intentionally redundant with admin-key authentication today and gives future RBAC/OIDC integrations a single policy hook to satisfy. + +For the production hardening checklist and OIDC/SSO integration plan, see [security-governance](/docs/security-governance). + +## Dashboard Setup + +Open Settings, enable API key sending, and set: + +- API Key: sent as `X-API-Key` for regular API requests. +- Admin API Key: sent as `X-Admin-API-Key` for operator routes. + +Settings are stored in browser local storage. Treat operator browsers as privileged clients and avoid sharing screenshots or profiles that may expose local configuration. + +## Operations Dashboard + +The Operations page has three tabs: + +| Tab | Purpose | +|---|---| +| Quotas | List, create, update, delete, summarize, and inspect owner quota usage. | +| Diagnostics | View redacted process, scheduler, rate-limit, build, provider, store, sandbox, and redaction data. | +| Audit | Review, filter, and export persisted admin route access logs. | + +## Owner Quotas + +Owner quotas are persisted overrides keyed by owner ID. They apply to requests that carry an owner identity through `X-User-ID` or an `owner_id` field where supported. + +Fields: + +| Field | Meaning | +|---|---| +| `max_sandboxes` | Maximum active sandboxes for the owner. `0` means no override. | +| `max_ttl` | Maximum sandbox TTL for the owner, such as `30m` or `2h`. | +| `max_exec_timeout` | Maximum exec timeout for the owner, such as `30s`. | + +Quota changes are admin operations and are recorded in the admin audit log. + +## Diagnostics + +Diagnostics are redacted by design. They include operational state useful for support and production checks while avoiding raw API keys, provider secrets, registry credentials, and environment secrets. + +Use diagnostics during incidents to confirm: + +- Store health and latency. +- Provider health and capabilities. +- Scheduler and spawn queue state. +- Rate-limit state. +- Sandbox count by state/provider. +- Quota coverage summary. + +## Admin Audit + +Admin route access is persisted in SQLite in the `admin_audit_logs` table. Records include: + +- Actor from `X-User-ID`, or `admin` when no actor header is supplied. +- When `X-User-ID` is missing on an authenticated admin request, the actor falls back to the authenticated role and key header, such as `admin:X-Admin-API-Key`. +- HTTP method and path. +- HTTP status and request duration. +- Request ID. +- Client address and user agent. +- Timestamp. + +The Audit tab supports filters for actor, method, status, and path substring. It can export the current filtered view as CSV. The API equivalent is: + +```bash +curl \ + -H "X-Admin-API-Key: $STACYVM_ADMIN_API_KEY" \ + "https://stacyvm.example.com/api/v1/admin/audit?actor=operator-a&method=PUT&format=csv" +``` + +## Storage And Retention + +Audit logs live in the main SQLite database. They are included in normal database backups. Native retention is controlled by `auth.admin_audit_retention`. + +Set the value to a Go duration such as `720h` for 30 days or `2160h` for 90 days. `0s` disables native pruning. Pruning runs after successful admin audit writes, so records older than the retention window are removed as operators use the admin control plane. diff --git a/docs/api.md b/docs/api.md deleted file mode 100644 index 53b25e0..0000000 --- a/docs/api.md +++ /dev/null @@ -1,567 +0,0 @@ -# StacyVM REST API Reference - -This document is the source of truth for the StacyVM HTTP API. The Python and TypeScript SDKs are thin wrappers over these endpoints — anything they do, you can do with `curl`. - -- **Base URL:** `http://localhost:7423/api/v1` -- **Content type:** `application/json` (request and response, except where noted) -- **OpenAPI spec:** [swagger.yaml](swagger.yaml) / [swagger.json](swagger.json) - ---- - -## Table of contents - -- [Authentication](#authentication) -- [Conventions](#conventions) -- [Errors](#errors) -- [Sandboxes](#sandboxes) -- [Files](#files) -- [Templates](#templates) -- [Providers](#providers) -- [Snapshots](#snapshots) -- [Pool](#pool) -- [System](#system) -- [Events stream](#events-stream) -- [WebSocket exec](#websocket-exec) - ---- - -## Authentication - -Two optional headers, both off by default: - -| Header | Purpose | Required when | -|---|---|---| -| `X-API-Key` | API key authentication | `auth.enabled: true` in `stacyvm.yaml` | -| `X-User-ID` | Multi-tenant pool mode user identifier | `pool.enabled: true` | - -```bash -curl -H 'X-API-Key: sk-xyz123' \ - -H 'X-User-ID: alice@example.com' \ - http://localhost:7423/api/v1/sandboxes -``` - -CORS is permissive by default (`*`). Lock it down via reverse proxy if you expose StacyVM to the open internet. - ---- - -## Conventions - -- **IDs.** Sandbox IDs look like `sb-a1b2c3d4`. Templates are addressed by `name`. -- **Durations.** All `ttl` and `timeout` fields use Go duration strings: `30s`, `5m`, `1h30m`. -- **Timestamps.** ISO 8601 UTC, e.g. `2026-05-04T10:30:00Z`. -- **File modes.** Octal strings, e.g. `"755"`, `"644"`. -- **Streaming.** `POST /sandboxes/{id}/exec` switches to NDJSON (`application/x-ndjson`) when `stream: true`. - ---- - -## Errors - -Errors return a JSON body with HTTP status reflecting the failure class: - -```json -{ - "code": "not_found", - "message": "sandbox sb-a1b2c3d4 not found" -} -``` - -| Status | Code | When | -|---|---|---| -| `400` | `bad_request` | Invalid input — missing field, malformed JSON | -| `401` | `unauthorized` | Bad / missing API key | -| `404` | `not_found` | Sandbox / template / provider does not exist | -| `409` | `conflict` | Template name already exists | -| `500` | `provider_error` | Provider failed (Docker, Firecracker, etc.) | -| `503` | `unavailable` | Pool full with `overflow: reject` | - ---- - -## Sandboxes - -### Spawn a sandbox - -``` -POST /api/v1/sandboxes -``` - -**Request body** (all fields optional, server defaults apply): -```json -{ - "image": "python:3.12", - "provider": "docker", - "memory_mb": 1024, - "vcpus": 2, - "ttl": "1h", - "metadata": { "user": "alice" } -} -``` - -**Response** `201 Created`: -```json -{ - "id": "sb-a1b2c3d4", - "state": "running", - "provider": "docker", - "image": "python:3.12", - "memory_mb": 1024, - "vcpus": 2, - "created_at": "2026-05-04T10:30:00Z", - "expires_at": "2026-05-04T11:30:00Z", - "metadata": { "user": "alice" }, - "preview_domain": "localhost" -} -``` - -### List sandboxes - -``` -GET /api/v1/sandboxes -``` - -**Response** `200 OK`: array of sandbox objects. - -### Get a sandbox - -``` -GET /api/v1/sandboxes/{id} -``` - -**Response** `200 OK` or `404 Not Found`. - -### Destroy a sandbox - -``` -DELETE /api/v1/sandboxes/{id} -``` - -**Response** `200 OK`: -```json -{ "status": "destroyed" } -``` - -### Prune expired sandboxes - -``` -DELETE /api/v1/sandboxes -``` - -**Response** `200 OK`: -```json -{ "pruned": 7 } -``` - -### Extend TTL - -``` -POST /api/v1/sandboxes/{id}/extend -``` - -**Request body**: -```json -{ "ttl": "1h" } -``` - -**Response** `200 OK`: full sandbox object with updated `expires_at`. - -### Execute a command - -``` -POST /api/v1/sandboxes/{id}/exec -``` - -**Request body**: -```json -{ - "command": "python3 -c 'print(40+2)'", - "args": ["--coverage"], - "env": { "NODE_ENV": "test" }, - "workdir": "/app", - "timeout": "30s", - "stream": false -} -``` - -**Response** `200 OK` (non-streaming): -```json -{ - "exit_code": 0, - "stdout": "42\n", - "stderr": "", - "duration": "127ms" -} -``` - -**Response** `200 OK` (streaming, `stream: true`): `application/x-ndjson` — one JSON object per line: -``` -{"stream":"stdout","data":"installing pandas...\n"} -{"stream":"stdout","data":"done\n"} -{"stream":"stderr","data":"warning: deprecated flag\n"} -``` - -### Console logs - -``` -GET /api/v1/sandboxes/{id}/logs?lines=200 -``` - -`lines` defaults to `100`. - -**Response** `200 OK`: -```json -["[init] mounting /workspace", "[init] starting agent", "..."] -``` - ---- - -## Files - -All file paths are absolute inside the sandbox. The endpoints below are scoped under `/sandboxes/{id}/files`. - -### Write a file - -``` -POST /api/v1/sandboxes/{id}/files -``` - -```json -{ "path": "/app/main.py", "content": "print('hi')", "mode": "644" } -``` - -**Response** `200 OK`: `{ "status": "written" }`. - -### Read a file - -``` -GET /api/v1/sandboxes/{id}/files?path=/app/main.py -``` - -**Response** `200 OK`: raw file contents (binary safe). The SDKs decode as UTF-8. - -### Delete a file or directory - -``` -DELETE /api/v1/sandboxes/{id}/files?path=/app/cache&recursive=true -``` - -`recursive` defaults to `false`. **Response** `200 OK`: `{ "status": "deleted" }`. - -### List a directory - -``` -GET /api/v1/sandboxes/{id}/files/list?path=/app -``` - -`path` defaults to `/`. - -**Response** `200 OK`: -```json -[ - { - "name": "main.py", - "path": "/app/main.py", - "size": 11, - "is_dir": false, - "mod_time": "2026-05-04T10:32:14Z", - "mode": "0644" - } -] -``` - -### Move / rename - -``` -POST /api/v1/sandboxes/{id}/files/move -``` - -```json -{ "old_path": "/app/main.py", "new_path": "/app/entry.py" } -``` - -**Response** `200 OK`: `{ "status": "moved" }`. - -### Change permissions - -``` -POST /api/v1/sandboxes/{id}/files/chmod -``` - -```json -{ "path": "/app/run.sh", "mode": "755" } -``` - -**Response** `200 OK`: `{ "status": "chmod applied" }`. - -### Stat - -``` -GET /api/v1/sandboxes/{id}/files/stat?path=/app/main.py -``` - -**Response** `200 OK`: a single `FileInfo` object (same shape as list). - -### Glob - -``` -GET /api/v1/sandboxes/{id}/files/glob?pattern=/app/**/*.py -``` - -**Response** `200 OK`: -```json -["/app/main.py", "/app/utils/helpers.py"] -``` - ---- - -## Templates - -### Create a template - -``` -POST /api/v1/templates -``` - -```json -{ - "name": "python-dev", - "image": "python:3.12-slim", - "memory_mb": 1024, - "vcpus": 2, - "ttl": "1h", - "provider": "docker", - "metadata": { "language": "python" } -} -``` - -**Response** `201 Created`: the template object. `409 Conflict` if name is taken. - -### List templates - -``` -GET /api/v1/templates -``` - -**Response** `200 OK`: array of templates. - -### Get a template - -``` -GET /api/v1/templates/{name} -``` - -**Response** `200 OK` or `404 Not Found`. - -### Update a template - -``` -PUT /api/v1/templates/{name} -``` - -Same body as create (without `name`). **Response** `200 OK` or `404`. - -### Delete a template - -``` -DELETE /api/v1/templates/{name} -``` - -**Response** `200 OK`: `{ "status": "deleted" }`. - -### Spawn from a template - -``` -POST /api/v1/templates/{name}/spawn -``` - -Optional override body: -```json -{ "ttl": "30m", "provider": "firecracker" } -``` - -**Response** `201 Created`: full sandbox object. - ---- - -## Providers - -### List providers - -``` -GET /api/v1/providers -``` - -**Response** `200 OK`: -```json -[ - { "name": "docker", "healthy": true, "default": true }, - { "name": "firecracker", "healthy": true, "default": false }, - { "name": "mock", "healthy": true, "default": false } -] -``` - -### Get a provider - -``` -GET /api/v1/providers/{name} -``` - -**Response** `200 OK`: -```json -{ - "name": "docker", - "healthy": true, - "default": true, - "sandbox_count": 12, - "config": { "runtime": "runc", "network_mode": "stacyvm-network" } -} -``` - -### Health-check all providers - -``` -POST /api/v1/providers/test -``` - -**Response** `200 OK`: -```json -{ "docker": true, "firecracker": true, "mock": true } -``` - ---- - -## Snapshots - -### List Firecracker snapshots - -``` -GET /api/v1/snapshots -``` - -**Response** `200 OK`: array of snapshot summaries (image name, kernel, size, created_at). - ---- - -## Pool - -### Pool status - -``` -GET /api/v1/pool/status -``` - -**Response** `200 OK` (pool enabled): -```json -{ - "enabled": true, - "vms": 3, - "max_vms": 20, - "total_users": 14, - "max_users_per_vm": 5 -} -``` - -**Response** `200 OK` (pool disabled): -```json -{ "enabled": false } -``` - ---- - -## System - -### Health - -``` -GET /api/v1/health -``` - -**Response** `200 OK`: -```json -{ "status": "ok", "version": "0.5.1", "uptime": "2h13m" } -``` - -### Metrics - -``` -GET /api/v1/metrics -``` - -**Response** `200 OK`: -```json -{ - "goroutines": 42, - "memory_alloc": 17825792, - "active_sandboxes": 12, - "total_sandboxes": 138 -} -``` - -For Prometheus-style metrics, scrape this endpoint and parse to your needs (a `/metrics` Prometheus exporter is on the roadmap). - ---- - -## Events stream - -``` -GET /api/v1/events -``` - -**Response** `200 OK` with `Content-Type: text/event-stream`. The server emits orchestrator events as Server-Sent Events: - -``` -event: sandbox.spawned -data: {"id":"sb-a1b2c3d4","provider":"docker","image":"python:3.12"} - -event: sandbox.destroyed -data: {"id":"sb-a1b2c3d4","reason":"ttl_expired"} - -event: sandbox.exec -data: {"id":"sb-a1b2c3d4","command":"python3 main.py","exit_code":0} -``` - -Use any SSE client (`EventSource` in browsers, `httpx-sse` in Python, etc.) to consume. - ---- - -## WebSocket exec - -``` -GET /api/v1/sandboxes/{id}/exec/ws -``` - -Upgrades the connection to a WebSocket for interactive command execution. Useful for terminals, REPLs, and any case where you need bi-directional I/O. - -**Client → server messages:** -```json -{ "type": "start", "command": "python3", "env": { "PYTHONUNBUFFERED": "1" } } -{ "type": "stdin", "data": "print('hi')\n" } -{ "type": "resize", "cols": 80, "rows": 24 } -{ "type": "signal", "signal": "SIGINT" } -``` - -**Server → client messages:** -```json -{ "type": "stdout", "data": "hi\n" } -{ "type": "stderr", "data": "..." } -{ "type": "exit", "exit_code": 0 } -``` - -The web dashboard uses this endpoint to power its live terminal — a concrete reference is at [`web/src/`](../web/src/). - ---- - -## SDK mapping - -If you'd rather write Python or TypeScript than `curl`, every endpoint above maps 1:1 to an SDK method: - -| Endpoint | Python | TypeScript | -|---|---|---| -| `POST /sandboxes` | `client.spawn(...)` | `client.spawn(...)` | -| `GET /sandboxes/{id}` | `client.get(id)` | `client.get(id)` | -| `POST /sandboxes/{id}/exec` | `sb.exec(cmd)` / `sb.exec_stream(cmd)` | `sb.exec(cmd)` / `sb.execStream(cmd)` | -| `POST /sandboxes/{id}/files` | `sb.write_file(path, content)` | `sb.writeFile(path, content)` | -| `GET /sandboxes/{id}/files` | `sb.read_file(path)` | `sb.readFile(path)` | -| `POST /templates/{name}/spawn` | `client.spawn_template(name)` | `client.templates.spawn(name)` | -| `GET /pool/status` | `client.pool_status()` | `client.poolStatus()` | -| `GET /health` | `client.health()` | `client.health()` | - -Full SDK docs: [Python](../sdk/python/README.md) · [TypeScript](../sdk/js/README.md). diff --git a/docs/architecture/system-overview.mdx b/docs/architecture/system-overview.mdx new file mode 100644 index 0000000..1ae042c --- /dev/null +++ b/docs/architecture/system-overview.mdx @@ -0,0 +1,228 @@ +--- +title: "System Architecture" +description: "Understand StacyVM's control plane, scheduler, workers, providers, persistence model, and sandbox lifecycle with diagrams." +--- + +This page explains StacyVM from top to bottom. Use it when you need to understand how a request moves from an SDK call to an isolated runtime and back. + +
+
+ Control plane + HTTP API, auth, quotas, scheduling, audit events, and status. +
+
+ Execution plane + Local or remote workers translate product requests into provider operations. +
+
+ Runtime plane + Docker, Firecracker, PRoot, and custom providers run the actual sandbox. +
+
+ +## High-Level Architecture + +```mermaid +flowchart TB + subgraph Clients["Clients"] + REST["REST API users"] + Py["Python SDK"] + TS["TypeScript SDK"] + Agents["AI agents and tools"] + end + + subgraph Control["StacyVM control plane"] + API["HTTP API server"] + Auth["Auth and tenant identity"] + Quota["Quota and admission checks"] + Scheduler["Scheduler"] + Audit["Audit events"] + Store["Store: SQLite or Postgres"] + Events["Events and metrics"] + end + + subgraph Runtime["Execution plane"] + LocalWorker["Local worker"] + RemoteWorker["Remote worker"] + Provider["Provider contract"] + Docker["Docker"] + Firecracker["Firecracker"] + PRoot["PRoot"] + Custom["Custom provider"] + end + + REST --> API + Py --> API + TS --> API + Agents --> API + API --> Auth + Auth --> Quota + Quota --> Scheduler + Scheduler --> Store + Scheduler --> LocalWorker + Scheduler --> RemoteWorker + LocalWorker --> Provider + RemoteWorker --> Provider + Provider --> Docker + Provider --> Firecracker + Provider --> PRoot + Provider --> Custom + API --> Audit + API --> Events + Audit --> Store + Events --> Store +``` + +## Request Flow + +When a client creates a sandbox, StacyVM validates identity and policy before touching a runtime provider. + +```mermaid +sequenceDiagram + participant Client + participant API as StacyVM API + participant Policy as Auth/Quota + participant Scheduler + participant Worker + participant Provider + participant Store + + Client->>API: POST /sandboxes + API->>Policy: authenticate and admit + Policy-->>API: allowed + API->>Scheduler: choose worker/provider + Scheduler->>Store: reserve sandbox record + Scheduler->>Worker: spawn request + Worker->>Provider: provider.Spawn + Provider-->>Worker: runtime id + Worker-->>Scheduler: sandbox running + Scheduler->>Store: persist state + API-->>Client: sandbox info +``` + +## Sandbox Lifecycle + +```mermaid +stateDiagram-v2 + [*] --> creating + creating --> running: provider spawn succeeds + creating --> error: spawn fails + running --> unhealthy: health check fails + unhealthy --> running: recovers + running --> expired: TTL reached + running --> destroying: client destroy + expired --> destroying: cleanup loop + destroying --> destroyed: provider destroy succeeds + destroying --> error: provider destroy fails + destroyed --> [*] + error --> destroying: cleanup retry +``` + +## Provider Contract + +Providers implement the runtime-specific work behind a stable product API. + +| Capability | Purpose | +| --- | --- | +| Spawn | Create an isolated runtime from an image or template. | +| Exec | Run a command and return exit code, stdout, stderr, and duration. | +| Stream | Send stdout/stderr chunks while a command is running. | +| Files | Write, read, list, stat, move, chmod, delete, and glob files. | +| Destroy | Tear down a sandbox safely and idempotently. | +| Health | Report provider availability and runtime readiness. | +| Logs | Expose provider and sandbox diagnostics. | + +## Persistence Model + +StacyVM uses a store abstraction so single-node installs can use SQLite while cluster deployments can use Postgres. + +```mermaid +erDiagram + OWNER ||--o{ SANDBOX : owns + WORKER ||--o{ SANDBOX : runs + PROVIDER ||--o{ SANDBOX : backs + TEMPLATE ||--o{ SANDBOX : spawns + SANDBOX ||--o{ EXECUTION : records + SANDBOX ||--o{ AUDIT_EVENT : emits + WORKER ||--o{ LEASE : holds + + OWNER { + string id + string api_key_hash + string quota_policy + } + + WORKER { + string id + string endpoint + string status + datetime last_heartbeat + } + + SANDBOX { + string id + string owner_id + string worker_id + string provider + string image + string state + datetime created_at + datetime expires_at + } + + EXECUTION { + string id + string sandbox_id + int exit_code + string duration + datetime created_at + } + + AUDIT_EVENT { + string id + string actor + string action + string target + datetime created_at + } +``` + +## Single-Node Mode + +Single-node mode runs the API, scheduler, local worker, provider, and store in one process on one host. This is the right starting point for internal staging and technical users. + +```mermaid +flowchart LR + Client["Client"] --> Server["stacyvm serve"] + Server --> SQLite["SQLite"] + Server --> Docker["Docker provider"] + Docker --> Sandbox["Sandbox"] +``` + +## Multi-Worker Mode + +Multi-worker mode keeps the API/control plane separate from worker nodes. The scheduler assigns sandboxes to workers, and worker RPC plus leases prevent two workers from managing the same runtime. + +```mermaid +flowchart LR + Client["Client"] --> Control["Control plane"] + Control --> Postgres["Postgres"] + Control --> W1["Worker A"] + Control --> W2["Worker B"] + W1 --> P1["Docker/Firecracker"] + W2 --> P2["Docker/PRoot"] +``` + +## Operational Boundaries + +- Use Docker for the broadest quickstart path. +- Use Firecracker only on hosts where KVM, kernel, rootfs, agent, networking, and snapshot behavior have passed certification. +- Use PRoot only after validating the real rootfs/bin setup on the target host. +- Use remote workers when you need horizontal capacity, runtime isolation by node class, or enterprise deployment boundaries. + +## Related + +- [Provider contract](/docs/provider-contract) +- [Runtime certification](/docs/runtime-certification) +- [Remote worker staging](/docs/remote-worker-staging) +- [Production readiness](/docs/production-readiness) diff --git a/docs/cluster-conformance.md b/docs/cluster-conformance.md new file mode 100644 index 0000000..2af19ad --- /dev/null +++ b/docs/cluster-conformance.md @@ -0,0 +1,126 @@ +# Cluster Conformance Matrix + +This matrix defines the minimum checks StacyVM must pass before a branch is considered production-aligned for multi-worker operation. It is intentionally stricter than the single-node deployment smoke tests because cluster mode depends on durable ownership, worker identity, leases, and store behavior remaining consistent across processes. + +## CI Coverage + +The always-on CI entrypoint is: + +```bash +scripts/ci-cluster-conformance.sh +``` + +It currently verifies: + +- SQLite passes the reusable store contract harness. +- Worker route authentication accepts per-worker credentials. +- Worker-specific credentials override the shared staging token. +- Worker route authentication accepts short-lived signed worker tokens. +- Worker route authentication rejects signed tokens scoped to the worker RPC audience. +- Worker route authentication rejects revoked signed worker token IDs. +- Worker RPC accepts short-lived signed control-plane-to-worker tokens. +- Worker RPC rejects signed tokens scoped to the control-plane route audience. +- Worker RPC rejects revoked signed worker token IDs. +- Remote spawn can route through worker RPC using signed tokens without a shared worker token. +- Worker RPC mTLS completes a real client-authenticated request using generated certificates. +- Worker lease renewal is guarded by `worker:lease`. +- A production-aligned cluster config with `auth.worker_tokens` or `auth.worker_signing_key` passes `stacyvm config lint --production`. +- Worker identity certification smoke produces a Markdown report without token values. +- Signed-token migration lint warns when shared worker tokens or invalid signing-key rotation state remain configured. +- Postgres configuration with a valid DSN passes `stacyvm config lint --production`. +- Live Postgres passes the reusable store contract when `STACYVM_POSTGRES_TEST_DSN` is set. +- Live Postgres proves one active lease holder under concurrent acquire and expired takeover attempts. +- Live Postgres proves migrations apply idempotently and record every expected schema version. +- A Postgres-backed remote worker smoke runs control plane plus worker against the mock provider. + +For host-level worker identity signoff, run: + +```bash +scripts/certify-worker-identity.sh worker-a +scripts/certify-worker-identity.sh worker-a --format markdown --output worker-identity-certification.md +``` + +The script issues a signed worker token from a secret file, inspects unverified metadata, verifies the signature and audience, confirms revoked token IDs are rejected, and generates a no-secret rotation plan. + +## Store Matrix + +| Store | Status | Required Checks | +|---|---|---| +| SQLite | Supported for single-node and internal staging | `TestSQLiteStoreContract`, migration tests, backup/restore tests | +| Postgres | Contract-backed cluster store path | `TestPostgresStoreContract`, `TestPostgresMigrationRehearsal`, Postgres migration alignment tests, lease takeover race tests, remote worker smoke, startup reconciliation with multiple workers | + +Postgres must not be marked production-ready for a deployment until it runs the same store contract suite as SQLite, passes lease race coverage, and passes the remote worker smoke in that deployment's target topology. + +## Worker Identity Matrix + +| Mode | Status | Intended Use | +|---|---|---| +| `auth.worker_token` | Supported | Local development and internal staging with a shared worker token | +| `auth.worker_tokens.` | Supported | Production-aligned staging with individually rotatable worker credentials | +| `auth.worker_signing_key` | Supported | Public or enterprise deployments that need short-lived signed worker credentials | +| `auth.worker_signing_keys` | Supported | No-downtime signing-key rotation window for old verification keys | +| Worker RPC mTLS | Supported | Enterprise deployments that require network-level worker transport identity | + +When `auth.worker_tokens` contains a worker ID, that worker must authenticate with its own token. The shared token is rejected for that worker ID. + +Signed worker tokens must use the `stacyvm-worker-v1` HMAC-SHA256 format. The signed subject must match `X-Worker-ID`, the `exp` claim must be in the future, and only worker scopes are granted. + +## Runtime Matrix + +| Runtime | Cluster Status | Notes | +|---|---|---| +| Mock | CI certified | Used for fast worker routing and control-plane smoke tests | +| Docker | Host-certified | Requires Docker daemon access and runtime certification outside the sandboxed CI path | +| gVisor/Kata | Host-certified | Requires configured Docker runtime and host-level certification | +| Firecracker | Platform-gated | Requires Linux/KVM, kernel, rootfs, and agent assets | +| PRoot | Platform-gated | Requires real rootfs/bin setup on the host | + +## Promotion Gates + +Before calling a multi-worker branch production-ready: + +1. `scripts/ci-cluster-conformance.sh` passes in CI. +2. `scripts/smoke-remote-worker.sh` passes against a real control-plane plus worker pair. +3. Runtime certification passes for every runtime advertised by the deployment. +4. Postgres passes the store contract harness. +5. Postgres lease tests prove one active holder per sandbox under concurrent acquisition, renewal, expiry, and takeover. +6. Startup reconciliation is tested against persisted sandboxes whose owning worker is online, stale, draining, offline, and missing. +7. Worker credentials are per-worker, rotated, and not shared with user or admin API credentials. + +## Current Phase 13 Position + +Phase 13 has completed: + +- Driver-based store selection. +- SQLite store contract coverage. +- Postgres-native migration definitions. +- Postgres store driver and live contract path. +- Per-worker token authentication. +- Cluster conformance CI scaffolding. + +Phase 14 starts worker identity hardening on top of that foundation: + +- HMAC-signed worker tokens. +- Signed-token config lint awareness. +- Worker runtime token derivation for heartbeat and lease renewal. +- Secret-file inputs for worker tokens and signing keys. +- No-secret signing-key rotation plan generation. +- Signed control-plane-to-worker RPC token derivation for remote worker calls. +- Worker RPC mTLS config, transport wiring, and production lint checks. +- Worker RPC mTLS conformance using generated CA, server, and client certificates. + +Phase 14 hardening additions (implemented): + +- OIDC/JWT RS256 bearer token authentication with JWKS and static public key support. +- RBAC roles: viewer, operator, admin, tenant_admin with scoped permissions. +- Tenant/project model: tenant CRUD, member role assignment, per-tenant audit log export. +- Per-tenant policy controls: image/provider/network allow-deny rules with glob pattern matching. +- Centralized signed worker token issuance via `/api/v1/admin/worker-tokens` (workers no longer need direct signing key access). +- Postgres backup via `stacyvm db pg-backup` (wraps pg_dump). +- Postgres migration rehearsal via `stacyvm db pg-rehearse` (schema version check before upgrades). +- Admin UI: Tenants page with member management and policy controls. + +Remaining enterprise production work: + +- Extend multi-worker conformance beyond the mock provider into certified Docker, gVisor/Kata, and Firecracker hosts. +- Run worker RPC mTLS smoke tests with deployment-issued certificates in the target enterprise network. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..6d1e7f5 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,201 @@ +--- +title: "Production Deployment" +description: "Deploy StacyVM as a single-node service with Docker, systemd, health checks, metrics, and release validation." +--- + +# Production Deployment + +This guide covers a single-node StacyVM deployment suitable for an internal service, staging, or a small production installation. The default production path uses the Docker provider because it works on the broadest set of hosts; Firecracker and PRoot require extra host setup and should be validated on the target platform before rollout. + +## Requirements + +- Linux host with Docker installed when using the Docker provider. +- A persistent data directory, normally `/var/lib/stacyvm`. +- A generated API key with at least 32 bytes of entropy. +- TLS and public ingress handled by a reverse proxy or load balancer in front of StacyVM. +- Explicit `server.cors_allowed_origins` for every browser origin that may call the API. +- Health checks wired to the API endpoints listed below. + +StacyVM reads config from `./stacyvm.yaml`, then `~/.stacyvm/config.yaml`, then `STACYVM_` environment variables. In production, prefer a checked-in baseline config plus environment variables or secret files for secrets and environment-specific values. Worker credentials can be mounted through `auth.worker_token_file` and `auth.worker_signing_key_file`; the loader rejects configs that set both the inline secret and its file reference. + +Before starting a single-node staging or production host, lint the final config with the same environment variables the service will use: + +```bash +STACYVM_AUTH_API_KEY=sk-live \ +STACYVM_AUTH_ADMIN_API_KEY=sk-admin \ +stacyvm config lint --production --file deploy/stacyvm.production.yaml +``` + +The lint command is deterministic and does not require Docker or KVM access. In production mode it fails wildcard CORS, missing auth, weak rate limits, relative database paths, missing sandbox caps, unsafe Docker settings, and other public-exposure risks. Use `stacyvm doctor --production` after linting when you also want live host checks for Docker, Firecracker, PRoot, database directories, and installed binaries. + +## Health and Metrics + +Use these endpoints for load balancers and monitors: + +| Endpoint | Purpose | +|---|---| +| `GET /api/v1/live` | Process liveness. Use this for simple restart checks. | +| `GET /api/v1/ready` | Readiness. Use this before routing traffic after deploys. | +| `GET /api/v1/health` | Dependency and provider health summary. | +| `GET /api/v1/metrics/prometheus` | Prometheus metrics scrape endpoint. | + +Authenticated deployments should send `X-API-Key: ` to protected API endpoints. Keep health probes scoped to your private network if they bypass auth at an upstream proxy. + +After a deploy, run the smoke script: + +```bash +STACYVM_SMOKE_URL=https://stacyvm.example.com STACYVM_API_KEY=sk-live scripts/smoke-deployment.sh +``` + +## Docker Compose + +The files in `deploy/` provide a production-oriented Compose starting point: + +- `deploy/docker-compose.yml` starts StacyVM and Traefik for live previews. +- `deploy/stacyvm.production.yaml` enables auth, explicit CORS origins, rate limiting, sandbox caps, queueing, JSON logs, and persistent SQLite state. +- `deploy/.env.example` lists the environment variables expected by the Compose file. +- `deploy/stacyvm.env.example` is the systemd environment file template. + +Use separate values for `STACYVM_API_KEY` and `STACYVM_ADMIN_API_KEY` in production. Admin routes live under `/api/v1/admin/*` and should be restricted to operator networks where possible. Replace the example `server.cors_allowed_origins` value with the exact public console/API origins for your deployment; do not expose browser clients with wildcard CORS. + +See [admin-control-plane](/docs/admin-control-plane) for admin dashboard setup, quota operations, diagnostics, audit export, and audit retention notes. See [security-governance](/docs/security-governance) for the production admin hardening checklist and OIDC/SSO integration plan. The production config keeps 90 days of admin audit history with `auth.admin_audit_retention: "2160h"` and disables admin fallback with `auth.admin_fallback_enabled: false`. + +```bash +cd deploy +cp .env.example .env +# Edit .env and replace STACYVM_API_KEY before starting. +docker compose up -d +docker compose logs -f stacyvm +``` + +For local image testing before a registry image exists: + +```bash +docker build -t stacyvm:local .. +STACYVM_IMAGE=stacyvm:local docker compose up -d +``` + +For non-invasive smoke runs on a shared host, override the published ports: + +```bash +STACYVM_IMAGE=stacyvm:local STACYVM_HOST_PORT=17426 STACYVM_TRAEFIK_HOST_PORT=18080 docker compose up -d +``` + +Then validate the API surface: + +```bash +scripts/smoke-deployment.sh http://127.0.0.1:17426 "$STACYVM_API_KEY" +``` + +Live-preview routing can be checked by spawning a sandbox that serves port `3000` and requesting Traefik with `Host: 3000-.`. + +## systemd + +Use `deploy/stacyvm.service` when running the binary directly on a Linux host. + +```bash +sudo useradd --system --home /var/lib/stacyvm --shell /usr/sbin/nologin stacyvm +sudo usermod -aG docker stacyvm +sudo install -d -o stacyvm -g stacyvm /var/lib/stacyvm +sudo install -d -m 0750 /etc/stacyvm +sudo install -o root -g stacyvm -m 0640 deploy/stacyvm.production.yaml /etc/stacyvm/stacyvm.yaml +sudo install -o root -g stacyvm -m 0640 deploy/stacyvm.env.example /etc/stacyvm/stacyvm.env +sudo install -m 0755 bin/stacyvm /usr/local/bin/stacyvm +sudo install -m 0755 bin/stacyvm-agent /usr/local/bin/stacyvm-agent +sudo install -m 0644 deploy/stacyvm.service /etc/systemd/system/stacyvm.service +``` + +Edit `/etc/stacyvm/stacyvm.env` and set real `STACYVM_AUTH_API_KEY` and `STACYVM_AUTH_ADMIN_API_KEY` values. Then enable the service: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now stacyvm +sudo systemctl status stacyvm +``` + +The included unit uses `WorkingDirectory=/etc/stacyvm` so StacyVM can load `/etc/stacyvm/stacyvm.yaml` through its current `./stacyvm.yaml` lookup path while keeping persistent database state in `/var/lib/stacyvm`. + +## Reverse Proxy + +Terminate TLS before StacyVM. A typical proxy should: + +- Forward API traffic to `http://127.0.0.1:7423`. +- Preserve `X-API-Key` headers. +- Route live preview hostnames such as `3000-sb-.` to Traefik when using Docker live previews. +- Restrict admin and metrics endpoints to trusted networks. + +Set `server.preview_domain` or `STACYVM_SERVER_PREVIEW_DOMAIN` to the domain that resolves preview subdomains to your proxy. + +## Backups + +The default store is SQLite at `/var/lib/stacyvm/stacyvm.db`. Prefer the built-in backup command because it uses SQLite's online backup path and validates the output: + +```bash +stacyvm db backup /backup/stacyvm-$(date +%Y%m%dT%H%M%SZ).db --database /var/lib/stacyvm/stacyvm.db +``` + +Restore requires the service to be stopped. The command validates the backup, creates a pre-restore safety copy of the current database, removes stale WAL/SHM sidecars, and replaces the target database: + +```bash +sudo systemctl stop stacyvm +stacyvm db restore /backup/stacyvm-20260508T120000Z.db --database /var/lib/stacyvm/stacyvm.db --yes +sudo systemctl start stacyvm +``` + +For a manual fallback: + +```bash +sudo systemctl stop stacyvm +sudo cp /var/lib/stacyvm/stacyvm.db /backup/stacyvm.db +sudo cp /var/lib/stacyvm/stacyvm.db-wal /backup/ 2>/dev/null || true +sudo cp /var/lib/stacyvm/stacyvm.db-shm /backup/ 2>/dev/null || true +sudo systemctl start stacyvm +``` + +If you run with Docker Compose, stop the service or snapshot the backing volume with your volume provider's backup tooling. + +## Upgrades + +Before changing binaries or images, rehearse the upgrade with the exact config and database path the service uses: + +```bash +stacyvm upgrade rehearse \ + --config /etc/stacyvm/stacyvm.yaml \ + --database /var/lib/stacyvm/stacyvm.db \ + --backup-output /backup/stacyvm-pre-upgrade.db +``` + +Use `--include-doctor` on the target host when you also want live provider checks before the upgrade. + +Upgrade flow: + +1. Check the release notes for config or API changes. +2. Run `stacyvm upgrade rehearse` and resolve any failing checks. +3. Back up `/var/lib/stacyvm/stacyvm.db` with `stacyvm db backup`. +4. Replace the binary or update `STACYVM_IMAGE`. +5. Restart the service. +6. Confirm `GET /api/v1/ready` succeeds before routing traffic. +7. If the upgrade fails, stop StacyVM and restore the pre-upgrade backup with `stacyvm db restore --yes`. + +## Support Bundles + +For support requests, generate a redacted bundle instead of sharing raw config, logs, or environment output: + +```bash +stacyvm support bundle /tmp/stacyvm-support.json \ + --config /etc/stacyvm/stacyvm.yaml \ + --include-doctor \ + --include-server +``` + +The bundle includes version/runtime data, redacted config shape, production config lint results, optional doctor checks, and optional `/api/v1/diagnostics` output. Secret-shaped keys, API keys, bearer tokens, and URLs with embedded credentials are redacted before the file is written. + +## Provider Notes + +Docker is the safest default for broad deployment compatibility. For stronger isolation, run Docker with gVisor (`runtime: "runsc"`) or Kata after validating the runtime on the host. + +Firecracker requires Linux/KVM, a kernel image, rootfs images, networking setup, and the `stacyvm-agent` binary available to the runtime. Keep Firecracker disabled in shared templates until a host conformance check passes. + +PRoot requires a real rootfs with the binaries your sandboxes need. Use it for restricted environments where Docker and KVM are unavailable, and validate memory/disk limits against the host because PRoot enforcement is not equivalent to VM isolation. + +Use [runtime-conformance](/docs/runtime-conformance) as the signoff checklist for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. diff --git a/docs/docs.go b/docs/docs.go index c76277f..35eaa94 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -15,6 +15,97 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/admin/audit": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return recent redacted admin route access records", + "produces": [ + "application/json" + ], + "tags": [ + "admin" + ], + "summary": "List admin audit logs", + "parameters": [ + { + "type": "integer", + "description": "Maximum number of records, capped at 500", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Actor exact match", + "name": "actor", + "in": "query" + }, + { + "type": "string", + "description": "HTTP method exact match", + "name": "method", + "in": "query" + }, + { + "type": "integer", + "description": "HTTP status exact match", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "Path substring match", + "name": "path", + "in": "query" + }, + { + "type": "string", + "description": "Response format: json or csv", + "name": "format", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.AdminAuditResponse" + } + } + } + } + } + }, + "/diagnostics": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return redacted build, store, provider, sandbox, event, and operation diagnostics", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Get diagnostics", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.DiagnosticsResponse" + } + } + } + } + }, "/events": { "get": { "security": [ @@ -34,7 +125,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Event" } } } @@ -65,6 +156,31 @@ const docTemplate = `{ } } }, + "/live": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the StacyVM API process is alive", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Liveness check", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.HealthResponse" + } + } + } + } + }, "/metrics": { "get": { "security": [ @@ -90,6 +206,31 @@ const docTemplate = `{ } } }, + "/metrics/prometheus": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format", + "produces": [ + "text/plain" + ], + "tags": [ + "system" + ], + "summary": "Get Prometheus metrics", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, "/providers": { "get": { "security": [ @@ -180,165 +321,144 @@ const docTemplate = `{ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes": { + "/quotas": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all active sandboxes", + "description": "Return all persisted owner quota overrides", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "List sandboxes", + "summary": "List owner quotas", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } } } - }, - "post": { + } + }, + "/quotas/summary": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Spawn a new sandbox with the given configuration", - "consumes": [ - "application/json" - ], + "description": "Return non-identifying counts for persisted owner quota overrides", "produces": [ "application/json" ], "tags": [ - "sandboxes" - ], - "summary": "Create a sandbox", - "parameters": [ - { - "description": "Spawn request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest" - } - } + "quotas" ], + "summary": "Get quota summary", "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" } } } - }, - "delete": { + } + }, + "/quotas/{ownerID}": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Destroy all expired sandboxes and return the count", + "description": "Return the persisted quota override for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" + ], + "summary": "Get owner quota", + "parameters": [ + { + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + } ], - "summary": "Prune sandboxes", "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.PruneResponse" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } }, - "500": { - "description": "Internal Server Error", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}": { - "get": { + }, + "put": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a sandbox by its ID", + "description": "Create or update quota overrides for an owner", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Get a sandbox", + "summary": "Save owner quota", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true + }, + { + "description": "Quota request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } } @@ -349,19 +469,19 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Destroy a sandbox and release its resources", + "description": "Delete the quota override for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Destroy a sandbox", + "summary": "Delete owner quota", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true } @@ -376,161 +496,106 @@ const docTemplate = `{ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}/exec": { - "post": { + "/quotas/{ownerID}/usage": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Run a command inside a sandbox. Set stream=true for streaming output.", - "consumes": [ - "application/json" - ], + "description": "Return active sandbox usage and effective quota for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Execute a command", + "summary": "Get owner quota usage", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true - }, - { - "description": "Exec request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest" - } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage" } } } } }, - "/sandboxes/{sandboxID}/exec/ws": { + "/ready": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Open a WebSocket connection to execute a command with streaming output", + "description": "Return whether the API is ready to serve sandbox traffic", + "produces": [ + "application/json" + ], "tags": [ - "sandboxes" - ], - "summary": "Execute via WebSocket", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - } + "system" ], + "summary": "Readiness check", "responses": { - "101": { - "description": "WebSocket upgrade" + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" + } }, - "400": { - "description": "Bad request" + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" + } } } } }, - "/sandboxes/{sandboxID}/files": { + "/sandboxes": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Read file content from a sandbox", + "description": "Return all active sandboxes", "produces": [ - "application/octet-stream" + "application/json" ], "tags": [ "sandboxes" ], - "summary": "Read a file", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "File path inside the sandbox", - "name": "path", - "in": "query", - "required": true - } - ], + "summary": "List sandboxes", "responses": { "200": { "description": "OK", "schema": { - "type": "file" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } @@ -541,7 +606,7 @@ const docTemplate = `{ "ApiKeyAuth": [] } ], - "description": "Write content to a file inside a sandbox", + "description": "Spawn a new sandbox with the given configuration", "consumes": [ "application/json" ], @@ -551,272 +616,190 @@ const docTemplate = `{ "tags": [ "sandboxes" ], - "summary": "Write a file", + "summary": "Create a sandbox", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "description": "File write request", + "description": "Spawn request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" } } ], "responses": { - "200": { - "description": "OK", + "201": { + "description": "Created", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, - "404": { - "description": "Not Found", + "429": { + "description": "Too Many Requests", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/files/list": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "List files in a directory inside a sandbox", + "description": "Destroy all expired sandboxes and return the count", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "List files", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Directory path (default: /)", - "name": "path", - "in": "query" - } - ], + "summary": "Prune sandboxes", "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo" - } - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.PruneResponse" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}/logs": { - "get": { + "/sandboxes/admission": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Retrieve console log lines from a sandbox", + "description": "Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Get console logs", + "summary": "Evaluate spawn admission", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "integer", - "description": "Number of lines to retrieve (default: 100)", - "name": "lines", - "in": "query" + "description": "Spawn request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + } } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision" } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates": { + "/sandboxes/{sandboxID}": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all registered templates", - "produces": [ - "application/json" - ], - "tags": [ - "templates" - ], - "summary": "List templates", - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - } - } - }, - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Register a new sandbox template", - "consumes": [ - "application/json" - ], + "description": "Return a sandbox by its ID", "produces": [ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Create a template", + "summary": "Get a sandbox", "parameters": [ { - "description": "Template definition", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true } ], "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - }, - "400": { - "description": "Bad Request", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, - "409": { - "description": "Conflict", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/templates/{name}": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a template by its name", + "description": "Destroy a sandbox and release its resources", "produces": [ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Get a template", + "summary": "Destroy a sandbox", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true } @@ -825,30 +808,32 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "put": { + } + }, + "/sandboxes/{sandboxID}/exec": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Update an existing template by name", + "description": "Run a command inside a sandbox. Set stream=true for streaming output.", "consumes": [ "application/json" ], @@ -856,24 +841,24 @@ const docTemplate = `{ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Update a template", + "summary": "Execute a command", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true }, { - "description": "Updated template", + "description": "Exec request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest" } } ], @@ -881,37 +866,587 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "delete": { + } + }, + "/sandboxes/{sandboxID}/exec/ws": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Delete a template by name", - "produces": [ + "description": "Open a WebSocket connection to execute a command with streaming output", + "tags": [ + "sandboxes" + ], + "summary": "Execute via WebSocket", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + } + ], + "responses": { + "101": { + "description": "WebSocket upgrade" + }, + "400": { + "description": "Bad request" + } + } + } + }, + "/sandboxes/{sandboxID}/extend": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Add additional time to a sandbox's expiration", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Extend sandbox TTL", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "description": "TTL extension", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "object", + "properties": { + "ttl": { + "type": "string" + } + } + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/files": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Read file content from a sandbox", + "produces": [ + "application/octet-stream" + ], + "tags": [ + "sandboxes" + ], + "summary": "Read a file", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "File path inside the sandbox", + "name": "path", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Write content to a file inside a sandbox", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Write a file", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "description": "File write request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.StatusResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/files/list": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "List files in a directory inside a sandbox", + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "List files", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Directory path (default: /)", + "name": "path", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/logs": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Retrieve console log lines from a sandbox", + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Get console logs", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Number of lines to retrieve (default: 100)", + "name": "lines", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/snapshots": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all pre-built VM snapshots available for fast restore", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "List snapshots", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary" + } + } + } + } + } + }, + "/templates": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all registered templates", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "List templates", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Register a new sandbox template", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Create a template", + "parameters": [ + { + "description": "Template definition", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/templates/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return a template by its name", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Get a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Update an existing template by name", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Update a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "Updated template", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete a template by name", + "produces": [ "application/json" ], "tags": [ @@ -937,26 +1472,186 @@ const docTemplate = `{ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/templates/{name}/spawn": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Create a new sandbox using a template's configuration, with optional overrides", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Spawn from template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "Optional overrides", + "name": "request", + "in": "body", + "schema": { + "$ref": "#/definitions/internal_api_routes.TemplateSpawnOverrides" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/workers": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return worker registry records and heartbeat state", + "produces": [ + "application/json" + ], + "tags": [ + "workers" + ], + "summary": "List workers", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.WorkerResponse" + } + } + } + } + } + }, + "/workers/{workerID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return one worker registry record", + "produces": [ + "application/json" + ], + "tags": [ + "workers" + ], + "summary": "Get worker", + "parameters": [ + { + "type": "string", + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.WorkerResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Remove a worker registry record", + "tags": [ + "workers" + ], + "summary": "Delete worker", + "parameters": [ + { + "type": "string", + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.StatusResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates/{name}/spawn": { + "/workers/{workerID}/heartbeat": { "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Create a new sandbox using a template's configuration, with optional overrides", + "description": "Create or update worker registry state for a worker", "consumes": [ "application/json" ], @@ -964,43 +1659,32 @@ const docTemplate = `{ "application/json" ], "tags": [ - "templates" + "workers" ], - "summary": "Spawn from template", + "summary": "Heartbeat worker", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Worker ID", + "name": "workerID", "in": "path", "required": true }, { - "description": "Optional overrides", + "description": "Worker heartbeat", "name": "request", "in": "body", + "required": true, "schema": { - "$ref": "#/definitions/internal_api_routes.TemplateSpawnOverrides" + "$ref": "#/definitions/internal_api_routes.WorkerHeartbeatRequest" } } ], "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.WorkerResponse" } } } @@ -1008,18 +1692,53 @@ const docTemplate = `{ } }, "definitions": { - "github_com_stacyvm-dev_stacyvm_internal_httputil.APIError": { + "github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats": { + "type": "object", + "properties": { + "active_buckets": { + "type": "integer" + }, + "allowed_total": { + "type": "integer" + }, + "bucket_ttl": { + "type": "string" + }, + "burst": { + "type": "integer" + }, + "cleanup_interval": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "evicted_total": { + "type": "integer" + }, + "key_by": { + "type": "string" + }, + "limited_total": { + "type": "integer" + }, + "requests_per_minute": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_httputil.APIError": { "type": "object", "properties": { "code": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode" }, "message": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode": { + "github_com_StacyOs_stacyvm_internal_httputil.ErrorCode": { "type": "string", "enum": [ "NOT_FOUND", @@ -1027,7 +1746,9 @@ const docTemplate = `{ "INTERNAL_ERROR", "UNAUTHORIZED", "CONFLICT", - "UNAVAILABLE" + "UNAVAILABLE", + "TIMEOUT", + "RESOURCE_LIMIT" ], "x-enum-varnames": [ "CodeNotFound", @@ -1035,10 +1756,12 @@ const docTemplate = `{ "CodeInternal", "CodeUnauth", "CodeConflict", - "CodeUnavailable" + "CodeUnavailable", + "CodeTimeout", + "CodeResourceLimit" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Event": { "type": "object", "properties": { "data": { @@ -1057,11 +1780,25 @@ const docTemplate = `{ "type": "string" }, "type": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventType" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats": { + "type": "object", + "properties": { + "events_total": { + "type": "integer" + }, + "history_size": { + "type": "integer" + }, + "subscribers": { + "type": "integer" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType": { + "github_com_StacyOs_stacyvm_internal_orchestrator.EventType": { "type": "string", "enum": [ "sandbox.created", @@ -1070,8 +1807,19 @@ const docTemplate = `{ "sandbox.error", "exec.started", "exec.completed", + "exec.failed", + "exec.timeout", "file.written", - "file.read" + "file.read", + "operation.failed", + "resource.limit", + "provider.failed", + "reconcile.action", + "spawn.queued", + "spawn.dequeued", + "spawn.queue_timeout", + "quota.saved", + "quota.deleted" ], "x-enum-varnames": [ "EventSandboxCreated", @@ -1080,11 +1828,22 @@ const docTemplate = `{ "EventSandboxError", "EventExecStarted", "EventExecCompleted", + "EventExecFailed", + "EventExecTimeout", "EventFileWritten", - "EventFileRead" + "EventFileRead", + "EventOperationFailed", + "EventResourceLimit", + "EventProviderFailed", + "EventReconcileAction", + "EventSpawnQueued", + "EventSpawnDequeued", + "EventSpawnQueueTimeout", + "EventQuotaSaved", + "EventQuotaDeleted" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest": { "type": "object", "properties": { "args": { @@ -1102,6 +1861,9 @@ const docTemplate = `{ "type": "string" } }, + "mode": { + "type": "string" + }, "stream": { "type": "boolean" }, @@ -1113,7 +1875,7 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult": { + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult": { "type": "object", "properties": { "duration": { @@ -1130,7 +1892,7 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo": { + "github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo": { "type": "object", "properties": { "is_dir": { @@ -1150,7 +1912,7 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest": { "type": "object", "properties": { "content": { @@ -1164,7 +1926,137 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics": { + "type": "object", + "properties": { + "failure_total": { + "type": "integer" + }, + "last_error": { + "type": "string" + }, + "last_observed_unix": { + "type": "integer" + }, + "latency_avg_ms": { + "type": "integer" + }, + "latency_count": { + "type": "integer" + }, + "latency_max_ms": { + "type": "integer" + }, + "latency_min_ms": { + "type": "integer" + }, + "latency_total_ms": { + "type": "integer" + }, + "operation": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "success_total": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo": { + "type": "object", + "properties": { + "default_exec_timeout": { + "type": "string" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_sandboxes_per_owner": { + "type": "integer" + }, + "max_spawn_queue": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_timeout": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "owner_id": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage": { + "type": "object", + "properties": { + "active_sandboxes": { + "type": "integer" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "owner_id": { + "type": "string" + }, + "quota_configured": { + "type": "boolean" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary": { + "type": "object", + "properties": { + "total": { + "type": "integer" + }, + "with_max_exec_timeout": { + "type": "integer" + }, + "with_max_sandboxes": { + "type": "integer" + }, + "with_max_ttl": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox": { "type": "object", "properties": { "created_at": { @@ -1179,55 +2071,171 @@ const docTemplate = `{ "image": { "type": "string" }, - "memory_mb": { + "memory_mb": { + "type": "integer" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "owner_id": { + "type": "string" + }, + "preview_domain": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "state": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState" + }, + "tenant_id": { + "type": "string" + }, + "vcpus": { + "type": "integer" + }, + "vm_id": { + "type": "string" + }, + "worker_id": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState": { + "type": "string", + "enum": [ + "creating", + "running", + "idle", + "unhealthy", + "expired", + "destroyed", + "error" + ], + "x-enum-varnames": [ + "StateCreating", + "StateRunning", + "StateIdle", + "StateUnhealthy", + "StateExpired", + "StateDestroyed", + "StateError" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus": { + "type": "object", + "properties": { + "admission_control": { + "type": "string" + }, + "eligible_workers": { + "type": "integer" + }, + "max_spawn_queue": { + "type": "integer" + }, + "selected_worker_id": { + "type": "string" + }, + "spawn_dequeued_total": { + "type": "integer" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_depth": { + "type": "integer" + }, + "spawn_queue_timeout": { + "type": "string" + }, + "spawn_queue_timeouts": { + "type": "integer" + }, + "spawn_queue_wait_avg": { + "type": "string" + }, + "spawn_queue_wait_avg_ms": { + "type": "integer" + }, + "spawn_queue_wait_count": { + "type": "integer" + }, + "spawn_queue_wait_max": { + "type": "string" + }, + "spawn_queue_wait_max_ms": { + "type": "integer" + }, + "spawn_queue_wait_total": { + "type": "string" + }, + "spawn_queue_wait_total_ms": { + "type": "integer" + }, + "spawn_queued_total": { + "type": "integer" + }, + "worker_id": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig": { + "type": "object", + "properties": { + "inject_at": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision": { + "type": "object", + "properties": { + "active_owner_sandboxes": { + "type": "integer" + }, + "active_sandboxes": { + "type": "integer" + }, + "allowed": { + "type": "boolean" + }, + "eligible_workers": { "type": "integer" }, - "metadata": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "max_owner_sandboxes": { + "type": "integer" }, - "provider": { + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { "type": "string" }, - "state": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState" + "queueable": { + "type": "boolean" }, - "vcpus": { - "type": "integer" - } - } - }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState": { - "type": "string", - "enum": [ - "creating", - "running", - "idle", - "destroyed", - "error" - ], - "x-enum-varnames": [ - "StateCreating", - "StateRunning", - "StateIdle", - "StateDestroyed", - "StateError" - ] - }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig": { - "type": "object", - "properties": { - "inject_at": { + "reason": { "type": "string" }, - "name": { + "selected_worker_id": { + "type": "string" + }, + "worker_reason": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest": { "type": "object", "properties": { "image": { @@ -1242,12 +2250,18 @@ const docTemplate = `{ "type": "string" } }, + "owner_id": { + "type": "string" + }, "provider": { "type": "string" }, "template": { "type": "string" }, + "tenant_id": { + "type": "string" + }, "ttl": { "type": "string" }, @@ -1256,7 +2270,7 @@ const docTemplate = `{ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Template": { "type": "object", "properties": { "allowed_hosts": { @@ -1295,7 +2309,7 @@ const docTemplate = `{ "secrets": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig" } }, "setup": { @@ -1315,6 +2329,141 @@ const docTemplate = `{ } } }, + "github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "image": { + "type": "string" + }, + "provider": { + "type": "string" + } + } + }, + "internal_api_routes.AdminAuditResponse": { + "type": "object", + "properties": { + "actor": { + "type": "string", + "example": "admin" + }, + "created_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "duration_ms": { + "type": "integer", + "example": 4 + }, + "id": { + "type": "integer", + "example": 42 + }, + "method": { + "type": "string", + "example": "PUT" + }, + "path": { + "type": "string", + "example": "/api/v1/admin/quotas/owner-a" + }, + "remote_addr": { + "type": "string", + "example": "127.0.0.1" + }, + "request_id": { + "type": "string", + "example": "req-abc123" + }, + "status": { + "type": "integer", + "example": 200 + }, + "tenant_id": { + "type": "string", + "example": "tenant-acme" + }, + "user_agent": { + "type": "string", + "example": "stacyvm-web" + } + } + }, + "internal_api_routes.DiagnosticsResponse": { + "type": "object", + "properties": { + "build": { + "type": "object", + "additionalProperties": true + }, + "events": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats" + }, + "generated_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "leases": { + "type": "object", + "additionalProperties": true + }, + "limits": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics" + } + }, + "process": { + "type": "object", + "additionalProperties": true + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "quotas": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + }, + "rate_limit": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats" + }, + "redactions": { + "type": "array", + "items": { + "type": "string" + } + }, + "remediation": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "sandboxes": { + "type": "object", + "additionalProperties": true + }, + "scheduler": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus" + }, + "store": { + "type": "object", + "additionalProperties": true + }, + "workers": { + "type": "object", + "additionalProperties": true + } + } + }, "internal_api_routes.HealthResponse": { "type": "object", "properties": { @@ -1370,6 +2519,9 @@ const docTemplate = `{ "type": "boolean", "example": true }, + "health": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + }, "healthy": { "type": "boolean", "example": true @@ -1384,20 +2536,86 @@ const docTemplate = `{ } } }, + "internal_api_routes.ProviderHealth": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "spawn", + "exec", + "files" + ] + }, + "default": { + "type": "boolean", + "example": true + }, + "error": { + "type": "string", + "example": "health check returned false" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, + "name": { + "type": "string", + "example": "docker" + }, + "runtime_count": { + "type": "integer", + "example": 2 + } + } + }, "internal_api_routes.ProviderInfo": { "type": "object", "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, "default": { "type": "boolean", "example": true }, + "error": { + "type": "string", + "example": "health check returned false" + }, "healthy": { "type": "boolean", "example": true }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, "name": { "type": "string", "example": "firecracker" + }, + "runtime_count": { + "type": "integer", + "example": 2 } } }, @@ -1410,6 +2628,37 @@ const docTemplate = `{ } } }, + "internal_api_routes.ReadinessResponse": { + "type": "object", + "properties": { + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "ready_providers": { + "type": "integer", + "example": 1 + }, + "status": { + "type": "string", + "example": "ready" + }, + "total_providers": { + "type": "integer", + "example": 2 + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + }, + "version": { + "type": "string", + "example": "1.0.0" + } + } + }, "internal_api_routes.StatusResponse": { "type": "object", "properties": { @@ -1431,6 +2680,84 @@ const docTemplate = `{ "example": "30m" } } + }, + "internal_api_routes.WorkerHeartbeatRequest": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "capacity": { + "type": "object", + "additionalProperties": true + }, + "hostname": { + "type": "string", + "example": "stacyvm-host-1" + }, + "providers": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "type": "string", + "example": "online" + } + } + }, + "internal_api_routes.WorkerResponse": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "capacity": { + "type": "object", + "additionalProperties": true + }, + "created_at": { + "type": "string", + "example": "2026-05-09T10:00:00Z" + }, + "hostname": { + "type": "string", + "example": "stacyvm-host-1" + }, + "id": { + "type": "string", + "example": "worker-local" + }, + "last_heartbeat": { + "type": "string", + "example": "2026-05-09T10:30:00Z" + }, + "providers": { + "type": "array", + "items": { + "type": "string" + } + }, + "stale": { + "type": "boolean", + "example": false + }, + "status": { + "type": "string", + "example": "online" + }, + "updated_at": { + "type": "string", + "example": "2026-05-09T10:30:00Z" + } + } } }, "securityDefinitions": { diff --git a/docs/enterprise-signoff-runbook.md b/docs/enterprise-signoff-runbook.md new file mode 100644 index 0000000..36556ca --- /dev/null +++ b/docs/enterprise-signoff-runbook.md @@ -0,0 +1,230 @@ +# Enterprise Production Signoff Runbook + +This runbook covers the evidence-collection steps that operators must run on +their own infrastructure before a StacyVM enterprise multi-worker deployment is +considered signed off for production. All automated CI gates (cluster +conformance, mTLS smoke, runtime certification) must pass first. + +## Prerequisites + +- StacyVM binary built for the target platform (`make build` or release artifact). +- A Postgres cluster accessible from the control plane. +- At least one worker host with the target runtime installed (Docker, Firecracker, etc.). +- A PKI that can issue TLS certificates (corporate CA, Vault, cert-manager, etc.). + +--- + +## 1. Worker RPC mTLS smoke with deployment-issued certificates + +CI validates mTLS with ephemeral certificates. This step proves the same path +works with your actual PKI. + +### Prerequisites + +Issue three certificates from your deployment CA: + +| File | Subject | SAN | +|---|---|---| +| `ca.crt` | Your CA certificate | — | +| `worker.crt` / `worker.key` | Worker RPC server cert | `IP:` or `DNS:` | +| `cp.crt` / `cp.key` | Control-plane client cert | `CN=stacyvm-control-plane` | + +### Run + +```bash +scripts/smoke-remote-worker.sh ./stacyvm --mtls \ + --ca-cert /path/to/ca.crt \ + --server-cert /path/to/worker.crt \ + --server-key /path/to/worker.key \ + --client-cert /path/to/cp.crt \ + --client-key /path/to/cp.key +``` + +### Expected output + +``` +==> Remote worker smoke PASSED [mTLS] + mTLS certs used: + CA: /path/to/ca.crt + server: /path/to/worker.crt + client: /path/to/cp.crt +``` + +### What it proves + +- Control plane authenticates to worker RPC over TLS (mutual auth). +- Worker presents a valid server cert signed by the deployment CA. +- Sandbox spawn, status, exec, and destroy all succeed over the mTLS channel. + +### Record + +Retain the script output (or a screenshot) as evidence. Reference it in your +change-management ticket using the format: + +``` +mTLS smoke: PASSED +Binary: stacyvm +CA: +Date: +Operator: +``` + +--- + +## 2. Runtime certification on each worker host + +Run this on **every** worker host for **every** runtime it will serve. The +report becomes the durable evidence artifact. + +### Docker / gVisor / Kata + +```bash +# Host-level checks + StacyVM integration smoke. +scripts/certify-runtime.sh docker \ + --stacyvm-url https://:7423 \ + --stacyvm-api-key "$STACYVM_API_KEY" \ + --format markdown \ + --output $(hostname)-docker-certification.md + +# Review the report. +cat $(hostname)-docker-certification.md +``` + +For gVisor or Kata, replace `docker` with `gvisor` or `kata`. The script will +check for the runtime in `docker info` and attempt `docker run --runtime=runsc`. + +### Firecracker + +```bash +export STACYVM_FIRECRACKER_KERNEL=/var/lib/stacyvm/vmlinux.bin + +scripts/certify-runtime.sh firecracker \ + --stacyvm-url https://:7423 \ + --stacyvm-api-key "$STACYVM_API_KEY" \ + --format markdown \ + --output $(hostname)-firecracker-certification.md +``` + +### Auto-start mode (no external server needed) + +If you want to certify the binary itself rather than a running cluster: + +```bash +scripts/certify-runtime.sh docker \ + --stacyvm-bin ./stacyvm \ + --format markdown \ + --output $(hostname)-docker-certification.md +``` + +### What the report covers + +| Check | Meaning | +|---|---| +| `docker.cli` | Docker CLI found in PATH | +| `docker.daemon` | Docker daemon reachable | +| `docker.seccomp` | seccomp advertised by docker info | +| `docker.run` | `docker run alpine echo ok` succeeds | +| `stacyvm.ready` | StacyVM API responds to `/api/v1/ready` | +| `stacyvm.provider_health` | Provider health endpoint returns healthy | +| `stacyvm.spawn` | Sandbox spawned via target runtime | +| `stacyvm.exec` | Command executed in sandbox (exit 0) | +| `stacyvm.destroy` | Sandbox destroyed | + +### Record + +Retain the Markdown report. Every worker host that serves production traffic +must have a report on file before go-live. Reference them in your +change-management ticket: + +``` +Runtime certification: + Host: worker-01.prod.example.com + Runtime: docker (gVisor) + Report: worker-01-docker-certification.md + Status: PASS + Date: + Operator: +``` + +--- + +## 3. Postgres migration rehearsal + +Run before every binary upgrade that includes a database schema change. + +```bash +stacyvm db pg-rehearse --dsn "$STACYVM_DATABASE_DSN" +``` + +### Expected output + +``` +connection: OK +schema_migrations: N applied — versions [1 2 3 ... N] +tables: all 16 expected tables present +pg-rehearse: PASS — schema is production-aligned +``` + +If any tables are missing, run the new binary once with +`STACYVM_DATABASE_DSN` set and the server will apply migrations automatically +on startup. Then re-run `pg-rehearse` to confirm. + +--- + +## 4. OIDC/SSO sign-off + +For deployments using `auth.oidc_enabled`, validate the configuration before +exposing to users. + +```bash +stacyvm config lint --production --file stacyvm.yaml +``` + +All `auth.oidc_*` checks must be `[PASS]`. Common failure modes: + +| Lint output | Fix | +|---|---| +| `OIDC issuer is not set` | Add `auth.oidc_issuer` pointing to your IdP | +| `no OIDC verification key configured` | Add `auth.oidc_jwks_url` or `auth.oidc_public_key_file` | +| `OIDC audience not set` | Add `auth.oidc_audience` matching your IdP's client audience | +| `no OIDC group-to-role mappings` | Add at least `auth.oidc_admin_groups` | + +Then mint a test token from your IdP and verify it is accepted: + +```bash +TOKEN="" +curl -H "Authorization: Bearer $TOKEN" https://:7423/api/v1/sandboxes +# Expected: 200 with sandbox list (empty is fine) +``` + +--- + +## 5. Worker identity certification + +```bash +scripts/certify-worker-identity.sh \ + --format markdown \ + --output worker-identity-certification.md +``` + +This verifies signed token issuance, inspection, verification, and revocation +without writing token values to the report. Retain the report alongside the +runtime certification. + +--- + +## Signoff checklist + +Copy this into your change-management ticket before go-live: + +``` +[ ] stacyvm config lint --production passes with no FAILs +[ ] stacyvm upgrade rehearse passes (binary + config + database) +[ ] stacyvm db pg-rehearse passes (if Postgres) +[ ] Worker RPC mTLS smoke with deployment-issued certs: PASSED +[ ] Runtime certification report on file for every worker host +[ ] Worker identity certification report on file for every worker ID +[ ] OIDC test token accepted by the production control plane +[ ] stacyvm doctor --production passes on the control-plane host +[ ] stacyvm support bundle generates without token/key leakage +``` diff --git a/docs/getting-started/core-concepts.mdx b/docs/getting-started/core-concepts.mdx new file mode 100644 index 0000000..3edd354 --- /dev/null +++ b/docs/getting-started/core-concepts.mdx @@ -0,0 +1,59 @@ +--- +title: "Core Concepts" +description: "Understand StacyVM sandboxes, providers, templates, files, exec sessions, previews, quotas, and cleanup." +--- + +StacyVM is small at the API boundary: you create a sandbox, run work inside it, move files in and out, and destroy it. Production behavior comes from the provider, scheduler, quotas, audit logs, and host configuration behind that boundary. + +## Sandbox + +A sandbox is an isolated runtime instance. It has an ID, state, image, provider, resource request, TTL, metadata, and optional preview domain. + +Typical states include `creating`, `running`, `unhealthy`, `expired`, `destroying`, `destroyed`, and `error`. + +## Provider + +A provider owns the actual runtime. StacyVM includes support for Docker, Firecracker, PRoot, custom providers, mock providers, and remote workers. Docker is the broadest quickstart path. Firecracker and PRoot require host-specific certification before you claim production support. + +## Image + +An image describes the filesystem and runtime available inside the sandbox. For Docker, this is a container image such as `python:3.12` or `node:20`. + +## Exec + +Exec runs a command inside an existing sandbox. Use timeouts for generated or untrusted code so a process cannot run forever. + +```json +{ + "command": "python3 /app/main.py", + "timeout": "30s" +} +``` + +## Files + +The file API lets you write source code, read outputs, list directories, move files, change modes, and delete paths inside the sandbox. Use absolute paths such as `/app/main.py`. + +## TTL + +TTL is automatic cleanup. Set a short TTL for agent tasks and still call destroy when the task completes. + +## Templates + +Templates define reusable sandbox settings such as image, provider, memory, vCPU, TTL, and metadata. Use templates when many developers or agents need the same environment. + +## Live Previews + +Live previews expose web applications running inside a sandbox through a preview URL. This is useful for coding agents that need to build and inspect web apps. + +## Quotas + +Quotas keep one user or workflow from consuming the whole host. StacyVM can track owner identity through `X-User-ID`, API key identity, and configured limits. + +## Audit Logs + +Production deployments should keep audit logs for sandbox creation, exec, file writes, destroy operations, admin changes, and registry actions. + +## Production Claims + +Do not claim a runtime is production-ready until you have run certification on the actual host class. Start with the [public support matrix](/docs/public-support-matrix), then run [runtime certification](/docs/runtime-certification). diff --git a/docs/getting-started/installation.mdx b/docs/getting-started/installation.mdx new file mode 100644 index 0000000..63db2bd --- /dev/null +++ b/docs/getting-started/installation.mdx @@ -0,0 +1,120 @@ +--- +title: "Installation" +description: "Install and run StacyVM locally from a release binary, Docker image, or source checkout." +--- + +Use this page to get a StacyVM server running before you follow the quickstart. + +## Prerequisites + +- Linux is recommended for runtime work. macOS is useful for SDK and docs development, but Docker provider behavior should be certified on Linux before public claims. +- Docker is required for the default local provider path. +- Go is required only when building from source. + +See [Prerequisites](/docs/getting-started/prerequisites) for the full local, SDK, and production checklist. + +## OS-Specific Setup + +Use the [Prerequisites OS setup](/docs/getting-started/prerequisites#os-setup) before choosing an installation mode: + +- macOS: Docker Desktop plus Go for `make serve`. +- Windows: WSL 2 with Ubuntu and Docker Desktop WSL integration. +- Linux: Docker and Go from your distribution package manager. +- Ubuntu: Docker Engine, Compose plugin, Go, Git, curl, and make. + +For individual local development from a source checkout, use the one-command setup: + +```bash +make dev +``` + +This checks the host, builds StacyVM, and starts the API server. + +You can also use the npm/npx bootstrapper: + +```bash +npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening +``` + +That command can clone the repo, install Node package dependencies, download Go modules, build StacyVM, and start the server. It does not install Docker Desktop or Go for you. + +To test the branch directly from GitHub instead of npm, use: + +```bash +npx github:StacyOS/stacyvm#phase-14-worker-identity-hardening stacyvm-setup \ + --branch phase-14-worker-identity-hardening +``` + +## Option 1: Release Binary + +Use a signed GitHub release for production-like installs. + +```bash +curl -L -o stacyvm.tar.gz \ + https://github.com/StacyOS/stacyvm/releases/latest/download/stacyvm_linux_amd64.tar.gz +tar -xzf stacyvm.tar.gz +sudo install -m 0755 stacyvm /usr/local/bin/stacyvm +``` + +Validate the release artifacts before you install them in production: + +```bash +scripts/post-release-validate.sh v0.14.4 +``` + +Replace `v0.14.4` with the release tag you plan to deploy. + +## Option 2: Docker Compose + +The repository includes a production-oriented Docker Compose starting point. + +```bash +cd deploy +cp .env.example .env +docker compose up -d +docker compose logs -f stacyvm +``` + +Edit `.env` before exposing the service. Use strong API keys, exact CORS origins, and a persistent data directory. + +## Option 3: Source Checkout + +Build from source when you are contributing to StacyVM itself. + +```bash +git clone https://github.com/StacyOS/stacyvm.git +cd stacyvm +make dev +``` + +## Configure Auth + +For local experiments you can run without auth. For any shared host, enable auth and pass `X-API-Key` from clients. + +```yaml +auth: + enabled: true + api_key: "sk_live_REPLACE_ME" +server: + cors_allowed_origins: + - "https://your-app.example.com" +``` + +## Verify The Host + +Run config validation before a deploy: + +```bash +stacyvm config lint --production --file deploy/stacyvm.production.yaml +``` + +Run live host diagnostics after the service is installed: + +```bash +stacyvm doctor --production +``` + +## Continue + +Once the server responds, follow the [quickstart](/docs/getting-started/quickstart). If setup fails, use the [troubleshooting guide](/docs/getting-started/prerequisites#troubleshooting). diff --git a/docs/getting-started/prerequisites.mdx b/docs/getting-started/prerequisites.mdx new file mode 100644 index 0000000..fd53593 --- /dev/null +++ b/docs/getting-started/prerequisites.mdx @@ -0,0 +1,453 @@ +--- +title: "Prerequisites" +description: "Check the operating system, runtime, SDK, network, and production requirements before you install StacyVM." +--- + +Use this checklist before you run StacyVM locally or deploy it for other developers. + +## Quick Checklist + +| Area | Required for local quickstart | Required for production | +| --- | --- | --- | +| Host OS | Linux recommended, macOS acceptable for SDK/docs work | Linux host certified for the runtime you claim | +| Runtime | Docker for the default provider path | Docker, Firecracker, PRoot, or remote workers certified on target hosts | +| CLI tools | `curl`, `git`, Docker CLI | `curl`, `git`, Docker CLI, service manager, backup tooling | +| Build tools | Go only when building from source | Go only for source builds; release binaries are preferred | +| SDK tools | Python 3.9+ or Node.js 18+ | Match your application language runtime | +| Network | Local port `7423` available | TLS, reverse proxy, exact CORS origins, private admin access | +| Secrets | Optional local API key | Strong API key, admin key, worker credentials where applicable | +| Persistence | Local SQLite file is fine | Persistent data directory and backup plan | + +## Local Development + +For the simplest local path, you need: + +- Docker installed and reachable by the StacyVM process. +- Port `7423` available on localhost. +- `curl` for API smoke checks. +- Python 3.9+ if you want to use the Python SDK or examples. +- Node.js 18+ if you want to use the TypeScript SDK or examples. +- Go if you want to run `make serve`, `go test ./...`, or build from source. + +If you do not have Go installed, use a release binary or Docker Compose instead of `make serve`. + +Once the repository is cloned, the one-command local setup path is: + +```bash +make dev +``` + +`make dev` checks Go, Docker, Docker daemon access, and port `7423`, then builds StacyVM and starts `./stacyvm serve`. It does not install Docker Desktop or OS packages for you; when something is missing, it prints the right OS-specific fix. + +If you prefer an npm/npx-driven setup, use: + +```bash +npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening +``` + +The npm setup command can clone StacyVM, run `npm install` for the web, SDK, and TypeScript example packages, download Go modules, build the StacyVM binary, and start the server. It still expects Docker and Go to be installed on the host. + +To test the branch directly from GitHub instead of npm, use: + +```bash +npx github:StacyOS/stacyvm#phase-14-worker-identity-hardening stacyvm-setup \ + --branch phase-14-worker-identity-hardening +``` + +## OS Setup + +Choose the setup path for the machine where you will run StacyVM. + +### macOS + +macOS is a good local development environment for the Docker provider, SDKs, and docs. Use Docker Desktop as the runtime. Firecracker is not a macOS starter path because it requires Linux/KVM. + +Install tools: + +```bash +brew install go git curl make +``` + +Install and start Docker Desktop, then verify Docker works: + +```bash +docker version +docker run --rm hello-world +``` + +Run StacyVM from source: + +```bash +git clone https://github.com/StacyOS/stacyvm.git +cd stacyvm +make dev +``` + +Check the API: + +```bash +curl http://localhost:7423/api/v1/live +``` + +Use multi-architecture images such as `python:3.12`, `node:20`, or `alpine:latest` on Apple Silicon. + +### Windows + +Use Windows with WSL 2. Run StacyVM commands inside Ubuntu on WSL, not in PowerShell, so paths, shell behavior, and Docker integration match Linux more closely. + +Install: + +- WSL 2 with Ubuntu. +- Docker Desktop with WSL integration enabled for your Ubuntu distro. +- Go, Git, curl, and make inside Ubuntu. + +Inside Ubuntu on WSL: + +```bash +sudo apt update +sudo apt install -y git curl make golang-go +docker version +docker run --rm hello-world +``` + +Run StacyVM: + +```bash +git clone https://github.com/StacyOS/stacyvm.git +cd stacyvm +make dev +``` + +If Docker commands fail inside WSL, open Docker Desktop settings and enable WSL integration for the Ubuntu distro you are using. + +### Linux + +Linux is the recommended host family for runtime work and production certification. Docker is the easiest first provider; Firecracker and PRoot require additional host-specific setup. + +Install basic tools with your distribution package manager: + +```bash +sudo apt update +sudo apt install -y git curl make golang-go docker.io +sudo systemctl enable --now docker +sudo usermod -aG docker "$USER" +``` + +Log out and back in after adding your user to the Docker group, then verify: + +```bash +docker run --rm hello-world +``` + +Run StacyVM: + +```bash +git clone https://github.com/StacyOS/stacyvm.git +cd stacyvm +make dev +``` + +### Ubuntu + +Ubuntu is the most straightforward Linux path for individual setup and single-node staging. + +Install Docker and build tools: + +```bash +sudo apt update +sudo apt install -y ca-certificates curl git make golang-go +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo tee /etc/apt/keyrings/docker.asc >/dev/null +sudo chmod a+r /etc/apt/keyrings/docker.asc +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null +sudo apt update +sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo usermod -aG docker "$USER" +``` + +Log out and back in, then verify: + +```bash +docker run --rm hello-world +docker compose version +``` + +Run StacyVM: + +```bash +git clone https://github.com/StacyOS/stacyvm.git +cd stacyvm +make dev +``` + +For production-like Ubuntu hosts, continue with [Production Deployment](/docs/deployment) after local verification. + +## Runtime Requirements + +### Docker + +Docker is the recommended first runtime because it is easiest to install and validate. + +- Docker daemon running on the host. +- Permission for the StacyVM process to create and destroy containers. +- Resource limits configured for memory, CPU, TTL, and concurrent sandboxes. +- Production config that disables unsafe Docker options. + +### Firecracker + +Use Firecracker only after host certification. + +- Linux host with KVM access. +- Firecracker binary installed. +- Kernel, rootfs, and StacyVM agent configured. +- Networking configured for your deployment model. +- Runtime certification evidence for the exact host class. + +### PRoot + +Use PRoot only after validating the real rootfs and binaries on the target host. + +- PRoot installed. +- Rootfs path configured. +- Required shell and runtime binaries available inside the rootfs. +- File and exec conformance passing for your target workload. + +## Production Requirements + +Before exposing StacyVM to other users, prepare: + +- `auth.enabled: true` with a strong API key. +- Separate admin API key for admin routes. +- Exact `server.cors_allowed_origins`; do not use wildcard CORS for public browser clients. +- Rate limits, sandbox caps, TTLs, and quotas. +- Persistent store path with backups. +- Health checks for `/api/v1/live`, `/api/v1/ready`, and `/api/v1/health`. +- Metrics scraping for `/api/v1/metrics/prometheus`. +- Audit log retention policy. +- Runtime certification for every provider you publicly claim. + +## Verify Your Setup + +### Verify The Npm Bootstrapper + +Run this flow in a fresh terminal when you want to confirm the published `stacyvm-setup` package, clone path, build path, server, and sandbox execution are all working. + +Verify the npm package: + +```bash +npm view stacyvm-setup name version bin --json +npx stacyvm-setup@latest --help +``` + +Run setup without starting the server: + +```bash +mkdir -p /tmp/stacyvm-npx-test +cd /tmp/stacyvm-npx-test + +npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening \ + --dir ./stacyvm \ + --no-start +``` + +Expected result: + +```bash +./stacyvm/stacyvm +``` + +Start StacyVM: + +```bash +cd /tmp/stacyvm-npx-test/stacyvm +./stacyvm serve +``` + +In a second terminal, check health: + +```bash +curl http://localhost:7423/api/v1/live +curl http://localhost:7423/api/v1/ready +``` + +Create a sandbox: + +```bash +curl -sS -X POST http://localhost:7423/api/v1/sandboxes \ + -H "Content-Type: application/json" \ + -d '{"image":"python:3.12","ttl":"10m"}' +``` + +Copy the returned sandbox ID, then run code inside it: + +```bash +export SANDBOX_ID="PASTE_ID_HERE" + +curl -sS -X POST "http://localhost:7423/api/v1/sandboxes/${SANDBOX_ID}/exec" \ + -H "Content-Type: application/json" \ + -d '{"command":"python3 -c \"print(40 + 2)\"","timeout":"10s"}' +``` + +The response should include: + +```json +{ + "stdout": "42\n" +} +``` + +Destroy the sandbox: + +```bash +curl -sS -X DELETE "http://localhost:7423/api/v1/sandboxes/${SANDBOX_ID}" +``` + +After that passes, you can test the full one-command path: + +```bash +cd /tmp +npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening \ + --dir ./stacyvm-full-test +``` + +### Troubleshooting + + + + Make sure you are using the published package name: + + ```bash + npm view stacyvm-setup name version bin --json + npx stacyvm-setup@latest --help + ``` + + If you are testing an unpublished branch, use the GitHub fallback: + + ```bash + npx github:StacyOS/stacyvm#phase-14-worker-identity-hardening stacyvm-setup \ + --branch phase-14-worker-identity-hardening + ``` + + + + On macOS, start Docker Desktop and wait until it is fully running: + + ```bash + open -a Docker + docker info + docker run --rm hello-world + ``` + + On Windows, run setup inside WSL 2 Ubuntu and enable Docker Desktop WSL integration for that distro. + + On Linux, start Docker and make sure your user can access it: + + ```bash + sudo systemctl enable --now docker + sudo usermod -aG docker "$USER" + ``` + + Log out and back in after changing groups. + + + + Source setup requires Go because the npm bootstrapper builds the StacyVM binary. + + macOS: + + ```bash + brew install go + ``` + + Ubuntu or Debian: + + ```bash + sudo apt update + sudo apt install -y golang-go + ``` + + + + Stop the process using port `7423`, then rerun setup. + + macOS or Linux: + + ```bash + lsof -iTCP:7423 -sTCP:LISTEN + ``` + + If you already have StacyVM running, use the existing server and continue with the [quickstart](/docs/getting-started/quickstart). + + + + Reuse the existing checkout after fixing the host issue: + + ```bash + cd /tmp/stacyvm-npx-test + npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening \ + --dir ./stacyvm \ + --no-start + ``` + + The setup command detects an existing StacyVM checkout and continues from there. + + + + You can skip Docker daemon validation: + + ```bash + npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening \ + --skip-docker-check \ + --no-start + ``` + + This only verifies setup and build behavior. Docker must be running before you create real sandboxes with the Docker provider. + + + + Use a clean npm cache directory: + + ```bash + npm_config_cache=/tmp/stacyvm-npm-cache npx stacyvm-setup@latest --help + ``` + + If npm asks for auth unexpectedly, verify your registry: + + ```bash + npm config get registry + ``` + + It should usually be `https://registry.npmjs.org/`. + + + +Run config lint before production: + +```bash +stacyvm config lint --production --file deploy/stacyvm.production.yaml +``` + +Run host diagnostics after installation: + +```bash +stacyvm doctor --production +``` + +Run a smoke check against the deployed API: + +```bash +STACYVM_SMOKE_URL=https://stacyvm.example.com \ +STACYVM_API_KEY=sk_live_REPLACE_ME \ +scripts/smoke-deployment.sh +``` + +## Continue + +- Install StacyVM with the [installation guide](/docs/getting-started/installation). +- Create your first sandbox with the [quickstart](/docs/getting-started/quickstart). +- Review public runtime claims in the [support matrix](/docs/public-support-matrix). diff --git a/docs/getting-started/quickstart.mdx b/docs/getting-started/quickstart.mdx new file mode 100644 index 0000000..7d4f56a --- /dev/null +++ b/docs/getting-started/quickstart.mdx @@ -0,0 +1,174 @@ +--- +title: "Quickstart" +description: "Create your first StacyVM sandbox, run code, move files, and clean up in a few minutes." +--- + +This guide gets you from a running StacyVM server to a verified sandbox workflow. + +## Prerequisites + +- A StacyVM server listening on `http://localhost:7423`. +- Docker installed and available to the StacyVM process when using the default Docker provider. +- An API key if `auth.enabled` is set in your config. +- Python 3.9+ or Node.js 18+ if you want to use an SDK. + +If you have not set up a host yet, start with the [Prerequisites](/docs/getting-started/prerequisites) and [Installation](/docs/getting-started/installation) guides. + +From a source checkout, the simplest local start command is: + +```bash +make dev +``` + +From npm/npx, use: + +```bash +npx stacyvm-setup@latest \ + --branch phase-14-worker-identity-hardening +``` + +To test the branch directly from GitHub instead of npm, use: + +```bash +npx github:StacyOS/stacyvm#phase-14-worker-identity-hardening stacyvm-setup \ + --branch phase-14-worker-identity-hardening +``` + +## 1. Check The Server + +```bash +curl -sS http://localhost:7423/api/v1/live +``` + +A healthy local server returns a success response. If auth is enabled, send your API key on protected routes: + +```bash +export STACYVM_API_KEY="sk_test_YOUR_API_KEY" +``` + +## 2. Create A Sandbox + + +```bash cURL +curl -sS -X POST http://localhost:7423/api/v1/sandboxes \ + -H "Content-Type: application/json" \ + -H "X-API-Key: ${STACYVM_API_KEY}" \ + -d '{"image":"python:3.12","ttl":"10m","memory_mb":512,"vcpus":1}' +``` + +```python Python +from stacyvm import Client + +client = Client( + base_url="http://localhost:7423", + api_key="sk_test_YOUR_API_KEY", +) + +sandbox = client.spawn( + image="python:3.12", + ttl="10m", + memory_mb=512, + vcpus=1, +) +print(sandbox.id) +``` + +```typescript TypeScript +import { Client } from "stacyvm"; + +const client = new Client({ + baseUrl: "http://localhost:7423", + apiKey: "sk_test_YOUR_API_KEY", +}); + +const sandbox = await client.spawn({ + image: "python:3.12", + ttl: "10m", + memory_mb: 512, + vcpus: 1, +}); +console.log(sandbox.id); +``` + + +Save the returned sandbox ID: + +```bash +export SANDBOX_ID="sb_YOUR_SANDBOX_ID" +``` + +## 3. Run Code + + +```bash cURL +curl -sS -X POST "http://localhost:7423/api/v1/sandboxes/${SANDBOX_ID}/exec" \ + -H "Content-Type: application/json" \ + -H "X-API-Key: ${STACYVM_API_KEY}" \ + -d '{"command":"python3 -c \"print(40 + 2)\"","timeout":"10s"}' +``` + +```python Python +result = sandbox.exec("python3 -c 'print(40 + 2)'", timeout="10s") +print(result.stdout) +``` + +```typescript TypeScript +const result = await sandbox.exec("python3 -c 'print(40 + 2)'", { + timeout: "10s", +}); +console.log(result.stdout); +``` + + +## 4. Move A File Into The Sandbox + + +```bash cURL +curl -sS -X PUT "http://localhost:7423/api/v1/sandboxes/${SANDBOX_ID}/files/app/main.py" \ + -H "Content-Type: application/json" \ + -H "X-API-Key: ${STACYVM_API_KEY}" \ + -d '{"content":"name = \"StacyVM\"\\nprint(f\"hello from {name}\")\\n","mode":"644"}' +``` + +```python Python +sandbox.write_file( + "/app/main.py", + 'name = "StacyVM"\nprint(f"hello from {name}")\n', +) +print(sandbox.exec("python3 /app/main.py").stdout) +``` + +```typescript TypeScript +await sandbox.writeFile( + "/app/main.py", + 'name = "StacyVM"\nprint(f"hello from {name}")\n', +); +console.log((await sandbox.exec("python3 /app/main.py")).stdout); +``` + + +## 5. Destroy The Sandbox + +Always destroy sandboxes when work is complete. TTL cleanup is a fallback, not the primary lifecycle control. + + +```bash cURL +curl -sS -X DELETE "http://localhost:7423/api/v1/sandboxes/${SANDBOX_ID}" \ + -H "X-API-Key: ${STACYVM_API_KEY}" +``` + +```python Python +sandbox.destroy() +``` + +```typescript TypeScript +await sandbox.destroy(); +``` + + +## Next Steps + +- Build the [example code runner](/docs/tutorials/code-runner). +- Learn the [core concepts](/docs/getting-started/core-concepts). +- Use the [Python SDK](/docs/sdks/python) or [TypeScript SDK](/docs/sdks/typescript). +- Prepare a host with the [production deployment guide](/docs/deployment). diff --git a/docs/getting-started/what-is-stacyvm.mdx b/docs/getting-started/what-is-stacyvm.mdx new file mode 100644 index 0000000..1bce4a9 --- /dev/null +++ b/docs/getting-started/what-is-stacyvm.mdx @@ -0,0 +1,105 @@ +--- +title: "What Is StacyVM?" +description: "Learn what StacyVM does, why teams use it, how it compares to alternatives, and where it fits in agent infrastructure." +--- + +StacyVM is self-hosted sandbox infrastructure for applications that need to run code safely, repeatedly, and observably. It gives agents and developer tools a disposable machine-like workspace with shell access, files, timeouts, quotas, previews, and audit trails. + +## The Short Version + +Your application calls StacyVM when it needs a clean runtime. StacyVM creates a sandbox, runs work inside it, streams output back, exposes files and previews, and destroys the environment when the task is done. + +```mermaid +flowchart LR + App["Your app or agent"] --> API["StacyVM API"] + API --> Sandbox["Disposable sandbox"] + Sandbox --> Result["Logs, files, exit code, preview URL"] + Result --> App +``` + +## Why StacyVM Exists + +AI coding agents, workflow engines, and developer tools increasingly need to execute generated code. Running that code directly on your host is risky. Outsourcing every execution to a cloud sandbox can be expensive, slower, or difficult to govern. + +StacyVM sits in the middle: you keep the execution plane under your control while developers get a simple API. + +## Advantages + +
+
+ Control + Your runtime, network, logs, credentials, and certification boundary stay with you. +
+
+ Speed + Agents get a simple API for shell, files, streaming output, and previews. +
+
+ Trust + TTL cleanup, quotas, audit logs, typed errors, and conformance gates are built in. +
+
+ +| Advantage | What it means | +| --- | --- | +| Self-hosted control | You decide where code runs, what network it can reach, and how logs are retained. | +| Provider flexibility | Start with Docker, then certify Firecracker, PRoot, custom providers, or remote workers when needed. | +| Developer-friendly API | Use REST, Python, or TypeScript without learning the underlying runtime provider first. | +| Production guardrails | Use TTLs, quotas, typed errors, health checks, audit logs, config linting, and runtime conformance. | +| Agent-ready workflows | Run commands, stream output, move files, and expose live previews for generated web apps. | + +## Common Use Cases + + + + Give agents a real shell and filesystem without giving them your host. + + + Run submitted snippets, scripts, tests, and generated code in short-lived environments. + + + Let users inspect web apps built inside sandboxes through preview URLs. + + + Run build, migration, analysis, or validation jobs with consistent cleanup. + + + +## Comparison + +| Approach | Best for | Tradeoff | +| --- | --- | --- | +| Direct host execution | Trusted internal scripts | Unsafe for generated or user-provided code. | +| Docker-only wrapper | Fast local prototypes | Usually lacks a stable API, quotas, audit trails, and multi-worker routing. | +| Cloud sandbox APIs | Teams that do not want to operate infrastructure | External dependency, data egress concerns, cost, and less host-level control. | +| Kubernetes jobs | Long-running platform teams | Strong orchestration, but heavier developer ergonomics for per-agent interactive execution. | +| StacyVM | Self-hosted, agent-facing sandbox infrastructure | You operate the control plane and certify the runtimes you publicly claim. | + +## Mental Model + +Think of StacyVM as a control plane in front of many possible sandbox runtimes. + +```mermaid +flowchart TB + User["Developer or agent"] --> SDK["SDK or REST client"] + SDK --> Control["StacyVM control plane"] + Control --> Policy["Auth, quotas, audit, scheduling"] + Policy --> Runtime["Runtime provider"] + Runtime --> Docker["Docker"] + Runtime --> Firecracker["Firecracker"] + Runtime --> PRoot["PRoot"] + Runtime --> Remote["Remote worker"] +``` + +## What StacyVM Is Not + +- It is not a promise that every provider is safe on every host. You must certify runtime claims. +- It is not a replacement for application-level authentication. +- It is not a general Kubernetes distribution. +- It is not a reason to run unbounded user code without TTLs, quotas, and audit logging. + +## Next Steps + +- Start with the [quickstart](/docs/getting-started/quickstart). +- Review the [system architecture](/docs/architecture/system-overview). +- Build the [Python example app](/docs/tutorials/code-runner) or [TypeScript example app](/docs/tutorials/typescript-code-runner). diff --git a/docs/openapi.json b/docs/openapi.json new file mode 100644 index 0000000..0400f51 --- /dev/null +++ b/docs/openapi.json @@ -0,0 +1,3034 @@ +{ + "openapi": "3.0.0", + "info": { + "description": "StacyVM microVM sandbox orchestrator API", + "title": "StacyVM API", + "contact": {}, + "version": "1.0" + }, + "paths": { + "/admin/audit": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return recent redacted admin route access records", + "tags": [ + "admin" + ], + "summary": "List admin audit logs", + "parameters": [ + { + "description": "Maximum number of records, capped at 500", + "name": "limit", + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "description": "Actor exact match", + "name": "actor", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "description": "HTTP method exact match", + "name": "method", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "description": "HTTP status exact match", + "name": "status", + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "description": "Path substring match", + "name": "path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "description": "Response format: json or csv", + "name": "format", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/internal_api_routes.AdminAuditResponse" + } + } + } + } + } + } + } + }, + "/diagnostics": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return redacted build, store, provider, sandbox, event, and operation diagnostics", + "tags": [ + "system" + ], + "summary": "Get diagnostics", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.DiagnosticsResponse" + } + } + } + } + } + } + }, + "/events": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Open an SSE stream for real-time sandbox and system events", + "tags": [ + "system" + ], + "summary": "Subscribe to events", + "responses": { + "200": { + "description": "OK", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Event" + } + } + } + } + } + } + }, + "/health": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return the health status, version, and uptime", + "tags": [ + "system" + ], + "summary": "Health check", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.HealthResponse" + } + } + } + } + } + } + }, + "/live": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the StacyVM API process is alive", + "tags": [ + "system" + ], + "summary": "Liveness check", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.HealthResponse" + } + } + } + } + } + } + }, + "/metrics": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return runtime metrics including sandbox count, goroutines, and memory usage", + "tags": [ + "system" + ], + "summary": "Get metrics", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.MetricsResponse" + } + } + } + } + } + } + }, + "/metrics/prometheus": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format", + "tags": [ + "system" + ], + "summary": "Get Prometheus metrics", + "responses": { + "200": { + "description": "OK", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/providers": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all registered providers with health status", + "tags": [ + "providers" + ], + "summary": "List providers", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/internal_api_routes.ProviderInfo" + } + } + } + } + } + } + } + }, + "/providers/test": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Run health checks on all registered providers", + "tags": [ + "providers" + ], + "summary": "Test providers", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } + } + } + } + } + } + } + }, + "/providers/{providerName}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return detailed information about a specific provider", + "tags": [ + "providers" + ], + "summary": "Get provider details", + "parameters": [ + { + "description": "Provider name", + "name": "providerName", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.ProviderDetail" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/quotas": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all persisted owner quota overrides", + "tags": [ + "quotas" + ], + "summary": "List owner quotas", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } + } + } + } + } + } + } + }, + "/quotas/summary": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return non-identifying counts for persisted owner quota overrides", + "tags": [ + "quotas" + ], + "summary": "Get quota summary", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + } + } + } + } + } + } + }, + "/quotas/{ownerID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return the persisted quota override for an owner", + "tags": [ + "quotas" + ], + "summary": "Get owner quota", + "parameters": [ + { + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Create or update quota overrides for an owner", + "tags": [ + "quotas" + ], + "summary": "Save owner quota", + "parameters": [ + { + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } + } + }, + "description": "Quota request", + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } + } + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete the quota override for an owner", + "tags": [ + "quotas" + ], + "summary": "Delete owner quota", + "parameters": [ + { + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.StatusResponse" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/quotas/{ownerID}/usage": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return active sandbox usage and effective quota for an owner", + "tags": [ + "quotas" + ], + "summary": "Get owner quota usage", + "parameters": [ + { + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage" + } + } + } + } + } + } + }, + "/ready": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the API is ready to serve sandbox traffic", + "tags": [ + "system" + ], + "summary": "Readiness check", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.ReadinessResponse" + } + } + } + }, + "503": { + "description": "Service Unavailable", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.ReadinessResponse" + } + } + } + } + } + } + }, + "/sandboxes": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all active sandboxes", + "tags": [ + "sandboxes" + ], + "summary": "List sandboxes", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Spawn a new sandbox with the given configuration", + "tags": [ + "sandboxes" + ], + "summary": "Create a sandbox", + "requestBody": { + "$ref": "#/components/requestBodies/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + }, + "responses": { + "201": { + "description": "Created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "429": { + "description": "Too Many Requests", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Destroy all expired sandboxes and return the count", + "tags": [ + "sandboxes" + ], + "summary": "Prune sandboxes", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.PruneResponse" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/admission": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits", + "tags": [ + "sandboxes" + ], + "summary": "Evaluate spawn admission", + "requestBody": { + "$ref": "#/components/requestBodies/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/{sandboxID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return a sandbox by its ID", + "tags": [ + "sandboxes" + ], + "summary": "Get a sandbox", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Destroy a sandbox and release its resources", + "tags": [ + "sandboxes" + ], + "summary": "Destroy a sandbox", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.StatusResponse" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/{sandboxID}/exec": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Run a command inside a sandbox. Set stream=true for streaming output.", + "tags": [ + "sandboxes" + ], + "summary": "Execute a command", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest" + } + } + }, + "description": "Exec request", + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/{sandboxID}/exec/ws": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Open a WebSocket connection to execute a command with streaming output", + "tags": [ + "sandboxes" + ], + "summary": "Execute via WebSocket", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "101": { + "description": "WebSocket upgrade" + }, + "400": { + "description": "Bad request" + } + } + } + }, + "/sandboxes/{sandboxID}/extend": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Add additional time to a sandbox's expiration", + "tags": [ + "sandboxes" + ], + "summary": "Extend sandbox TTL", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ttl": { + "type": "string" + } + } + } + } + }, + "description": "TTL extension", + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/{sandboxID}/files": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Read file content from a sandbox", + "tags": [ + "sandboxes" + ], + "summary": "Read a file", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "File path inside the sandbox", + "name": "path", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/octet-stream": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/octet-stream": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/octet-stream": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Write content to a file inside a sandbox", + "tags": [ + "sandboxes" + ], + "summary": "Write a file", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest" + } + } + }, + "description": "File write request", + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.StatusResponse" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/{sandboxID}/files/list": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "List files in a directory inside a sandbox", + "tags": [ + "sandboxes" + ], + "summary": "List files", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Directory path (default: /)", + "name": "path", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo" + } + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/sandboxes/{sandboxID}/logs": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Retrieve console log lines from a sandbox", + "tags": [ + "sandboxes" + ], + "summary": "Get console logs", + "parameters": [ + { + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Number of lines to retrieve (default: 100)", + "name": "lines", + "in": "query", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/snapshots": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all pre-built VM snapshots available for fast restore", + "tags": [ + "snapshots" + ], + "summary": "List snapshots", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary" + } + } + } + } + } + } + } + }, + "/templates": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all registered templates", + "tags": [ + "templates" + ], + "summary": "List templates", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Register a new sandbox template", + "tags": [ + "templates" + ], + "summary": "Create a template", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + }, + "description": "Template definition", + "required": true + }, + "responses": { + "201": { + "description": "Created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "409": { + "description": "Conflict", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/templates/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return a template by its name", + "tags": [ + "templates" + ], + "summary": "Get a template", + "parameters": [ + { + "description": "Template name", + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Update an existing template by name", + "tags": [ + "templates" + ], + "summary": "Update a template", + "parameters": [ + { + "description": "Template name", + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + }, + "description": "Updated template", + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete a template by name", + "tags": [ + "templates" + ], + "summary": "Delete a template", + "parameters": [ + { + "description": "Template name", + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.StatusResponse" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/templates/{name}/spawn": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Create a new sandbox using a template's configuration, with optional overrides", + "tags": [ + "templates" + ], + "summary": "Spawn from template", + "parameters": [ + { + "description": "Template name", + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.TemplateSpawnOverrides" + } + } + }, + "description": "Optional overrides" + }, + "responses": { + "201": { + "description": "Created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/workers": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return worker registry records and heartbeat state", + "tags": [ + "workers" + ], + "summary": "List workers", + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/internal_api_routes.WorkerResponse" + } + } + } + } + } + } + } + }, + "/workers/{workerID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return one worker registry record", + "tags": [ + "workers" + ], + "summary": "Get worker", + "parameters": [ + { + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.WorkerResponse" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Remove a worker registry record", + "tags": [ + "workers" + ], + "summary": "Delete worker", + "parameters": [ + { + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.StatusResponse" + } + } + } + }, + "404": { + "description": "Not Found", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + } + } + }, + "/workers/{workerID}/heartbeat": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Create or update worker registry state for a worker", + "tags": [ + "workers" + ], + "summary": "Heartbeat worker", + "parameters": [ + { + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.WorkerHeartbeatRequest" + } + } + }, + "description": "Worker heartbeat", + "required": true + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/internal_api_routes.WorkerResponse" + } + } + } + } + } + } + } + }, + "servers": [ + { + "url": "//localhost:7423/api/v1" + } + ], + "components": { + "requestBodies": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + } + } + }, + "description": "Spawn request", + "required": true + } + }, + "securitySchemes": { + "ApiKeyAuth": { + "type": "apiKey", + "name": "X-API-Key", + "in": "header" + } + }, + "schemas": { + "github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats": { + "type": "object", + "properties": { + "active_buckets": { + "type": "integer" + }, + "allowed_total": { + "type": "integer" + }, + "bucket_ttl": { + "type": "string" + }, + "burst": { + "type": "integer" + }, + "cleanup_interval": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "evicted_total": { + "type": "integer" + }, + "key_by": { + "type": "string" + }, + "limited_total": { + "type": "integer" + }, + "requests_per_minute": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_httputil.APIError": { + "type": "object", + "properties": { + "code": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode" + }, + "message": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_httputil.ErrorCode": { + "type": "string", + "enum": [ + "NOT_FOUND", + "BAD_REQUEST", + "INTERNAL_ERROR", + "UNAUTHORIZED", + "CONFLICT", + "UNAVAILABLE", + "TIMEOUT", + "RESOURCE_LIMIT" + ], + "x-enum-varnames": [ + "CodeNotFound", + "CodeBadRequest", + "CodeInternal", + "CodeUnauth", + "CodeConflict", + "CodeUnavailable", + "CodeTimeout", + "CodeResourceLimit" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.Event": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "integer" + } + }, + "id": { + "type": "string" + }, + "sandbox_id": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "type": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.EventType" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats": { + "type": "object", + "properties": { + "events_total": { + "type": "integer" + }, + "history_size": { + "type": "integer" + }, + "subscribers": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventType": { + "type": "string", + "enum": [ + "sandbox.created", + "sandbox.running", + "sandbox.destroyed", + "sandbox.error", + "exec.started", + "exec.completed", + "exec.failed", + "exec.timeout", + "file.written", + "file.read", + "operation.failed", + "resource.limit", + "provider.failed", + "reconcile.action", + "spawn.queued", + "spawn.dequeued", + "spawn.queue_timeout", + "quota.saved", + "quota.deleted" + ], + "x-enum-varnames": [ + "EventSandboxCreated", + "EventSandboxRunning", + "EventSandboxDestroyed", + "EventSandboxError", + "EventExecStarted", + "EventExecCompleted", + "EventExecFailed", + "EventExecTimeout", + "EventFileWritten", + "EventFileRead", + "EventOperationFailed", + "EventResourceLimit", + "EventProviderFailed", + "EventReconcileAction", + "EventSpawnQueued", + "EventSpawnDequeued", + "EventSpawnQueueTimeout", + "EventQuotaSaved", + "EventQuotaDeleted" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest": { + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "type": "string" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "mode": { + "type": "string" + }, + "stream": { + "type": "boolean" + }, + "timeout": { + "type": "string" + }, + "workdir": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult": { + "type": "object", + "properties": { + "duration": { + "type": "string" + }, + "exit_code": { + "type": "integer" + }, + "stderr": { + "type": "string" + }, + "stdout": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo": { + "type": "object", + "properties": { + "is_dir": { + "type": "boolean" + }, + "mod_time": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "path": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics": { + "type": "object", + "properties": { + "failure_total": { + "type": "integer" + }, + "last_error": { + "type": "string" + }, + "last_observed_unix": { + "type": "integer" + }, + "latency_avg_ms": { + "type": "integer" + }, + "latency_count": { + "type": "integer" + }, + "latency_max_ms": { + "type": "integer" + }, + "latency_min_ms": { + "type": "integer" + }, + "latency_total_ms": { + "type": "integer" + }, + "operation": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "success_total": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo": { + "type": "object", + "properties": { + "default_exec_timeout": { + "type": "string" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_sandboxes_per_owner": { + "type": "integer" + }, + "max_spawn_queue": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_timeout": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "owner_id": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage": { + "type": "object", + "properties": { + "active_sandboxes": { + "type": "integer" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "owner_id": { + "type": "string" + }, + "quota_configured": { + "type": "boolean" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary": { + "type": "object", + "properties": { + "total": { + "type": "integer" + }, + "with_max_exec_timeout": { + "type": "integer" + }, + "with_max_sandboxes": { + "type": "integer" + }, + "with_max_ttl": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "expires_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "image": { + "type": "string" + }, + "memory_mb": { + "type": "integer" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "owner_id": { + "type": "string" + }, + "preview_domain": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "state": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState" + }, + "tenant_id": { + "type": "string" + }, + "vcpus": { + "type": "integer" + }, + "vm_id": { + "type": "string" + }, + "worker_id": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState": { + "type": "string", + "enum": [ + "creating", + "running", + "idle", + "unhealthy", + "expired", + "destroyed", + "error" + ], + "x-enum-varnames": [ + "StateCreating", + "StateRunning", + "StateIdle", + "StateUnhealthy", + "StateExpired", + "StateDestroyed", + "StateError" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus": { + "type": "object", + "properties": { + "admission_control": { + "type": "string" + }, + "eligible_workers": { + "type": "integer" + }, + "max_spawn_queue": { + "type": "integer" + }, + "selected_worker_id": { + "type": "string" + }, + "spawn_dequeued_total": { + "type": "integer" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_depth": { + "type": "integer" + }, + "spawn_queue_timeout": { + "type": "string" + }, + "spawn_queue_timeouts": { + "type": "integer" + }, + "spawn_queue_wait_avg": { + "type": "string" + }, + "spawn_queue_wait_avg_ms": { + "type": "integer" + }, + "spawn_queue_wait_count": { + "type": "integer" + }, + "spawn_queue_wait_max": { + "type": "string" + }, + "spawn_queue_wait_max_ms": { + "type": "integer" + }, + "spawn_queue_wait_total": { + "type": "string" + }, + "spawn_queue_wait_total_ms": { + "type": "integer" + }, + "spawn_queued_total": { + "type": "integer" + }, + "worker_id": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig": { + "type": "object", + "properties": { + "inject_at": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision": { + "type": "object", + "properties": { + "active_owner_sandboxes": { + "type": "integer" + }, + "active_sandboxes": { + "type": "integer" + }, + "allowed": { + "type": "boolean" + }, + "eligible_workers": { + "type": "integer" + }, + "max_owner_sandboxes": { + "type": "integer" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "queueable": { + "type": "boolean" + }, + "reason": { + "type": "string" + }, + "selected_worker_id": { + "type": "string" + }, + "worker_reason": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "memory_mb": { + "type": "integer" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "owner_id": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "template": { + "type": "string" + }, + "tenant_id": { + "type": "string" + }, + "ttl": { + "type": "string" + }, + "vcpus": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.Template": { + "type": "object", + "properties": { + "allowed_hosts": { + "type": "array", + "items": { + "type": "string" + } + }, + "cpu_cores": { + "type": "integer" + }, + "created_at": { + "type": "string" + }, + "description": { + "type": "string" + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "image": { + "type": "string" + }, + "memory_mb": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "pool_size": { + "type": "integer" + }, + "secrets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig" + } + }, + "setup": { + "type": "array", + "items": { + "type": "string" + } + }, + "ttl_seconds": { + "type": "integer" + }, + "updated_at": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "image": { + "type": "string" + }, + "provider": { + "type": "string" + } + } + }, + "internal_api_routes.AdminAuditResponse": { + "type": "object", + "properties": { + "actor": { + "type": "string", + "example": "admin" + }, + "created_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "duration_ms": { + "type": "integer", + "example": 4 + }, + "id": { + "type": "integer", + "example": 42 + }, + "method": { + "type": "string", + "example": "PUT" + }, + "path": { + "type": "string", + "example": "/api/v1/admin/quotas/owner-a" + }, + "remote_addr": { + "type": "string", + "example": "127.0.0.1" + }, + "request_id": { + "type": "string", + "example": "req-abc123" + }, + "status": { + "type": "integer", + "example": 200 + }, + "tenant_id": { + "type": "string", + "example": "tenant-acme" + }, + "user_agent": { + "type": "string", + "example": "stacyvm-web" + } + } + }, + "internal_api_routes.DiagnosticsResponse": { + "type": "object", + "properties": { + "build": { + "type": "object", + "additionalProperties": true + }, + "events": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats" + }, + "generated_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "leases": { + "type": "object", + "additionalProperties": true + }, + "limits": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics" + } + }, + "process": { + "type": "object", + "additionalProperties": true + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/internal_api_routes.ProviderHealth" + } + }, + "quotas": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + }, + "rate_limit": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats" + }, + "redactions": { + "type": "array", + "items": { + "type": "string" + } + }, + "remediation": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "sandboxes": { + "type": "object", + "additionalProperties": true + }, + "scheduler": { + "$ref": "#/components/schemas/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus" + }, + "store": { + "type": "object", + "additionalProperties": true + }, + "workers": { + "type": "object", + "additionalProperties": true + } + } + }, + "internal_api_routes.HealthResponse": { + "type": "object", + "properties": { + "status": { + "type": "string", + "example": "ok" + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + }, + "version": { + "type": "string", + "example": "1.0.0" + } + } + }, + "internal_api_routes.MetricsResponse": { + "type": "object", + "properties": { + "goroutines": { + "type": "integer", + "example": 12 + }, + "memory_alloc_mb": { + "type": "integer", + "example": 64 + }, + "providers": { + "type": "integer", + "example": 2 + }, + "sandboxes_active": { + "type": "integer", + "example": 5 + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + } + } + }, + "internal_api_routes.ProviderDetail": { + "type": "object", + "properties": { + "config": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "default": { + "type": "boolean", + "example": true + }, + "health": { + "$ref": "#/components/schemas/internal_api_routes.ProviderHealth" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "name": { + "type": "string", + "example": "firecracker" + }, + "sandbox_count": { + "type": "integer", + "example": 3 + } + } + }, + "internal_api_routes.ProviderHealth": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "spawn", + "exec", + "files" + ] + }, + "default": { + "type": "boolean", + "example": true + }, + "error": { + "type": "string", + "example": "health check returned false" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, + "name": { + "type": "string", + "example": "docker" + }, + "runtime_count": { + "type": "integer", + "example": 2 + } + } + }, + "internal_api_routes.ProviderInfo": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "default": { + "type": "boolean", + "example": true + }, + "error": { + "type": "string", + "example": "health check returned false" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, + "name": { + "type": "string", + "example": "firecracker" + }, + "runtime_count": { + "type": "integer", + "example": 2 + } + } + }, + "internal_api_routes.PruneResponse": { + "type": "object", + "properties": { + "pruned": { + "type": "integer", + "example": 3 + } + } + }, + "internal_api_routes.ReadinessResponse": { + "type": "object", + "properties": { + "providers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/internal_api_routes.ProviderHealth" + } + }, + "ready_providers": { + "type": "integer", + "example": 1 + }, + "status": { + "type": "string", + "example": "ready" + }, + "total_providers": { + "type": "integer", + "example": 2 + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + }, + "version": { + "type": "string", + "example": "1.0.0" + } + } + }, + "internal_api_routes.StatusResponse": { + "type": "object", + "properties": { + "status": { + "type": "string", + "example": "destroyed" + } + } + }, + "internal_api_routes.TemplateSpawnOverrides": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "example": "firecracker" + }, + "ttl": { + "type": "string", + "example": "30m" + } + } + }, + "internal_api_routes.WorkerHeartbeatRequest": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "capacity": { + "type": "object", + "additionalProperties": true + }, + "hostname": { + "type": "string", + "example": "stacyvm-host-1" + }, + "providers": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "type": "string", + "example": "online" + } + } + }, + "internal_api_routes.WorkerResponse": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "capacity": { + "type": "object", + "additionalProperties": true + }, + "created_at": { + "type": "string", + "example": "2026-05-09T10:00:00Z" + }, + "hostname": { + "type": "string", + "example": "stacyvm-host-1" + }, + "id": { + "type": "string", + "example": "worker-local" + }, + "last_heartbeat": { + "type": "string", + "example": "2026-05-09T10:30:00Z" + }, + "providers": { + "type": "array", + "items": { + "type": "string" + } + }, + "stale": { + "type": "boolean", + "example": false + }, + "status": { + "type": "string", + "example": "online" + }, + "updated_at": { + "type": "string", + "example": "2026-05-09T10:30:00Z" + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/production-readiness.md b/docs/production-readiness.md new file mode 100644 index 0000000..3eb5bda --- /dev/null +++ b/docs/production-readiness.md @@ -0,0 +1,124 @@ +# Production Readiness Checklist + +This checklist tracks the Phase 7 release-candidate hardening work needed before StacyVM is marketed as production-ready. + +## Readiness Levels + +| Level | Target user | Current gate | +|---|---|---| +| Internal staging | StacyOS team and trusted operators | `stacyvm doctor`, CI, mock deployment smoke, documented rollback | +| Single-node production | Technical self-hosters | Docker/gVisor or Firecracker conformance, hardened auth, backup/restore drill | +| Public self-serve | Users without handholding | Signed releases, upgrade tests, support bundle, clear failure remediation | +| Enterprise/multi-worker | Infrastructure teams | Postgres, workers, durable scheduler, leases, OIDC/RBAC | + +## Phase 7 Acceptance Criteria + +- `stacyvm doctor` reports actionable local and production diagnostics. +- Docker command execution has explicit shell and argv semantics. Done in Phase 7 slice 2. +- File APIs have path traversal tests across manager scoping and provider boundaries. Done in Phase 7 final cleanup. +- Sensitive operations are covered by persisted operation audit records. Done in Phase 7 final cleanup. +- Runtime certification scripts exist for Docker, gVisor, Kata, Firecracker, and PRoot host checks. Done in Phase 7 final cleanup. +- Threat model is documented for runtime, API, admin, live-preview, pool, and registry surfaces. +- Release notes describe verified CI and known platform caveats. + +## Phase 8 Acceptance Criteria + +- SQLite backup and restore are available through the CLI with integrity checks and restore safety copies. Done in Phase 8 slice 1. +- Production config linting is available through `stacyvm config lint --production` and can run against explicit config files without requiring Docker/KVM host access. Done in Phase 8 slice 2. +- Upgrade rehearsal checks document backup, config lint, service restart, readiness validation, and rollback. Done in Phase 8 slice 3. +- Support bundle export exists and redacts secrets before sharing with maintainers. Done in Phase 8 slice 3. + +## Phase 9 Acceptance Criteria + +- Release binaries and checksums are signed through the GitHub Actions release workflow. Done in Phase 9 slice 1. +- Published container image digests are signed through the GitHub Actions release workflow. Done in Phase 9 slice 1. +- A public verification script exists for release signatures and checksums. Done in Phase 9 slice 1. +- Installer supports Sigstore verification and a fail-closed mode. Done in Phase 9 slice 1. +- Upgrade and config migration tests run in CI. Done in Phase 9 slice 2. +- Public docs expose known limitations and exact remediation paths. Done in Phase 9 slice 2. +- Public release sanity builds and checksum verification run in CI. Done in Phase 9 final polish. +- SDK parity smoke tests run in CI without requiring a live runtime. Done in Phase 9 final polish. +- GitHub issue templates request support bundle, config lint, upgrade rehearsal, runtime certification, and release verification evidence. Done in Phase 9 final polish. + +## Phase 10 Acceptance Criteria + +- Worker registration and heartbeat records are stored durably. Done in Phase 10 slice 1. +- Single-node servers self-register as the `local` worker with provider and capacity metadata. Done in Phase 10 slice 1. +- Single-node servers refresh the `local` worker heartbeat while running. Done in Phase 10 heartbeat slice. +- Read-only worker discovery is available through the normal API. Done in Phase 10 slice 1. +- Worker heartbeat and deletion are protected by the admin namespace. Done in Phase 10 slice 1. +- Diagnostics and Prometheus expose worker registry state. Done in Phase 10 slice 1. +- Sandbox records persist their owning worker ID and diagnostics expose sandbox counts by worker. Done in Phase 10 slice 2. +- Scheduler placement policy is worker-aware. Remote spawn, status, destroy, live exec streaming, files, logs, preview metadata, and conservative drain/offline ownership policy are available for workers that advertise `rpc_url`. +- Sandbox ownership is tied to worker IDs. Remote spawn/status/destroy ownership is enforced through worker RPC and persisted runtime IDs. +- Distributed leases prevent duplicate worker ownership. Remote spawn, renew, and destroy now carry lease tokens; persistence now has SQLite and Postgres store paths with Postgres lease race coverage. +- Remote worker authentication and RPC contract are implemented for heartbeat, lease renewal, spawn, status, destroy, exec, files, logs, preview metadata, and drain/offline ownership reconciliation. Shared worker tokens remain available for staging, and per-worker token mapping now supports individually rotatable worker credentials. + +## Current Release-Candidate Gates + +| Gate | Status | Notes | +|---|---|---| +| Full Go test suite | Passing | CI runs `make test`. | +| Web build | Passing | CI runs `npm run build`. | +| SDK checks | Passing | TypeScript builds, Python imports, and mock-based SDK parity smoke tests run in CI. | +| Deployment smoke | Passing | Mock-provider smoke is in CI. Docker live host certification remains external. | +| Cluster conformance | Partial | Always-on CI covers SQLite store contract, live Postgres store contract, Postgres migration rehearsal, Postgres lease concurrency, per-worker and signed worker identity, worker identity certification reporting, production cluster config lint, and Postgres-backed remote worker smoke. See `docs/cluster-conformance.md`. | +| Runtime conformance | Partial | Harness and host certification script exist; Firecracker/PRoot remain platform-gated. | +| Security posture | Strong | Admin governance, operation audit, path traversal checks, explicit exec modes, OIDC/JWT RS256+ES256 auth with RBAC, real SHA256 hash in RS256 verification, admin routes protected in OIDC-only mode, tenant/project model, per-tenant audit, policy enforcement on spawn, policy controls for providers/images/networks, and hardened centralized worker token issuer are implemented. | +| Release automation | Passing | Release workflow signs binaries, checksums, and GHCR image digests; public verifier and installer verification exist. | +| Worker registry | Near-complete | Durable worker registration, heartbeat, diagnostics, metrics, placement, ownership, leases, per-worker token auth, signed worker identity, centralized token issuance, worker RPC routing, and worker RPC mTLS wiring exist. Remaining: target-network mTLS smoke with deployment-issued certificates. | +| Enterprise/OIDC | Passing | OIDC/JWT RS256 verification, RBAC roles (viewer/operator/admin/tenant_admin), OIDC group→role mapping, tenant model, per-tenant audit, and policy enforcement are implemented. | +| Public API exposure | Passing | CORS origins are configurable through `server.cors_allowed_origins`; production config lint fails wildcard or empty CORS before public exposure. | + +## Required Before Single-Node Production + +- Production config uses distinct API and admin keys. +- `server.cors_allowed_origins` contains only exact trusted `https://` origins for public browser clients. +- `auth.admin_fallback_enabled` is `false`. +- `auth.admin_audit_retention` is set to a production window. +- Docker provider runs with explicit runtime, network mode, dropped caps, pid limit, memory, CPU, and seccomp settings. +- Firecracker hosts pass Linux/KVM conformance before being marked production. +- Backup and restore are tested against the SQLite database. +- `stacyvm config lint --production` passes with the same config and environment variables the service will use. +- `stacyvm upgrade rehearse` passes before binary/image replacement. +- Operators can generate `stacyvm support bundle` output without exposing API keys or provider secrets. +- Runtime certification artifacts are generated on the actual host with `scripts/certify-runtime.sh --format markdown --output -certification.md`. +- Operators run `stacyvm doctor --production` before go-live. + +## Required Before Public Self-Serve + +- Release artifacts are signed and checksummed. +- Upgrade and config migration tests run in CI. +- `stacyvm doctor` includes remediation links for every failure. +- Support bundle export exists and redacts secrets. +- Threat model is reviewed for each release candidate. +- Known limitations are visible in README, docs, and release notes. +- Public support expectations are documented in [public-support-matrix](/docs/public-support-matrix). +- Bug and production support issue templates ask for the same evidence required by the public support matrix. +- Public release sanity CI builds release binaries and validates checksums; real GitHub release asset verification must be repeated after each version tag is published. +- Public browser clients use explicit CORS origins; wildcard CORS must fail `stacyvm config lint --production`. +- Final public evidence is generated with `scripts/public-readiness-evidence.sh`; announcement requires a **PUBLIC SELF-SERVE READY** verdict for the release tag, target host runtime, and deployment network. + +## Required Before Enterprise/Multi-Worker + +- Postgres store implementation. Driver, migrations, contract path, migration rehearsal, lease race coverage, and mock-provider remote worker smoke exist. Backup rehearsal is available via `stacyvm db pg-rehearse` and `stacyvm db pg-backup`. Done in Phase 14. +- Worker registration and heartbeat model. Durable registry, per-worker token auth, signed worker tokens, issuer/rotation workflow, centralized token issuance via `/api/v1/admin/worker-tokens`, worker identity certification reporting, and worker RPC mTLS wiring exist. Target-network mTLS smoke with deployment-issued certificates remains pending for specific enterprise networks. +- Scheduler abstraction with placement policy. Done in Phase 10. +- Durable queue/pub-sub for lifecycle events. Covered by EventBus and persisted lease model. +- Distributed leases to prevent double ownership. Done in Phase 10. +- OIDC/SSO and RBAC implemented. RS256 JWT Bearer token validation with configurable OIDC issuer, JWKS URL, audience, and group-to-role mapping is implemented. Roles: `viewer`, `operator`, `admin`, `tenant_admin`, `worker`. Done in Phase 14. +- Tenant/project model implemented. Tenants, tenant members with RBAC roles, policy controls (image/provider/network allow-deny lists), per-tenant audit export, and admin UI management are implemented. Done in Phase 14. +- Worker RPC transport enforces [worker-rpc-contract](/docs/worker-rpc-contract). Done in Phase 11-13. + +## Phase 14 Acceptance Criteria + +- OIDC/JWT RS256 Bearer token validation with configurable issuer, JWKS URL, audience, groups claim, and group-to-role mapping. Done. +- RBAC roles beyond admin/api: viewer, operator, tenant_admin with scoped permissions. Done. +- Tenant/project model: tenant CRUD, member RBAC, per-tenant resource scoping. Done. +- Per-tenant audit export: admin and operation audit logs scoped by tenant_id. Done. +- Policy controls: per-tenant allow/deny policies for image, provider, and network resources. Done. +- Centralized worker token issuer: admin API endpoint mints signed worker tokens so workers do not need direct signing key access. Done. +- Postgres backup: `stacyvm db pg-backup` wraps pg_dump for production cluster snapshots. Done. +- Postgres migration rehearsal: `stacyvm db pg-rehearse` verifies schema state before upgrades. Done. +- Admin UI tenant management: Tenants page with member RBAC and policy management. Done. +- Worker RPC mTLS smoke with deployment-issued certificates in target enterprise network: Pending (external to code). diff --git a/docs/provider-contract.md b/docs/provider-contract.md new file mode 100644 index 0000000..850ad23 --- /dev/null +++ b/docs/provider-contract.md @@ -0,0 +1,85 @@ +# StacyVM Provider Contract + +Providers are the runtime boundary for StacyVM. The orchestrator, API, SDKs, and +dashboard must be able to treat every provider the same way, whether the runtime +is Docker, Firecracker, PRoot, E2B, a custom HTTP service, or a test double. + +The Go source of truth is `internal/providers/provider.go`. This document +spells out the behavioral contract expected by the shared provider conformance +tests. + +## Lifecycle + +- `Name` returns a stable unique identifier used in config, API responses, and + persisted sandbox records. +- `Spawn` creates a running sandbox and returns a non-empty sandbox ID. +- `Status` returns `running` for an active sandbox. +- `Destroy` tears down runtime resources. Providers may return `ErrSandboxNotFound` + if the runtime object is already gone. +- After destroy, exec and file operations must fail with either + `ErrSandboxDestroyed` or `ErrSandboxNotFound`. +- Provider implementations should be best-effort idempotent around external + cleanup. The orchestrator may call destroy during TTL reaping, explicit API + requests, or recovery flows. + +## Exec + +- `Exec` runs the requested command and returns stdout, stderr, and exit code. +- Exec mode is explicit: + - Empty mode and `shell` run `Command` through `/bin/sh -c`; `Args` are + shell-quoted and appended for backwards compatibility. + - `argv` runs `Command` directly with `Args` as literal process arguments. + Providers must not invoke a shell in this mode. +- A nonzero command exit is not a provider error. It must return an `ExecResult` + with the nonzero exit code. +- Provider errors are reserved for runtime failures: missing sandbox, provider + unavailable, transport failure, timeout, or invalid provider state. +- Context cancellation and deadlines should be honored. Deadline expiration + should map to `ErrExecTimeout` where the provider can detect it. +- `ExecStream` emits stdout/stderr chunks and closes its channel when the command + finishes or the stream fails. + +## Files + +- File paths are interpreted inside the sandbox filesystem. +- Providers must support write, read, list, delete, move, chmod, stat, and glob. +- `WriteFile` should create missing parent directories when the runtime can do so. +- File reads return an `io.ReadCloser`; callers own closing it. +- Missing sandbox errors should use `ErrSandboxNotFound` or `ErrSandboxDestroyed`. +- Missing file behavior can remain provider-specific unless an API route maps it + into a user-facing error. + +## Health + +- `Healthy` should be fast and side-effect free. +- It should return false when the runtime dependency is unreachable, for example + Docker daemon unavailable, PRoot binary missing, Firecracker binary missing, or + a custom HTTP backend failing its health endpoint. + +## Typed Errors + +Providers should use these sentinel errors from `internal/providers/errors.go`: + +- `ErrSandboxNotFound` +- `ErrSandboxDestroyed` +- `ErrProviderNotFound` +- `ErrProviderUnavailable` +- `ErrExecTimeout` +- `ErrResourceLimit` + +Wrapping is encouraged with `fmt.Errorf("context: %w", err)` so callers can use +`errors.Is`. + +## Conformance Tests + +Shared conformance tests live in +`internal/providers/provider_conformance_test.go`. + +Current coverage: + +- Mock provider +- Docker provider, when Docker is available +- Custom provider through an in-process fake HTTP backend + +Future providers should be wired into the same harness whenever their runtime +dependencies are available. diff --git a/docs/public-readiness-evidence.md b/docs/public-readiness-evidence.md new file mode 100644 index 0000000..245eb1d --- /dev/null +++ b/docs/public-readiness-evidence.md @@ -0,0 +1,88 @@ +# Public Readiness Evidence + +Use this runbook to create the final evidence bundle before announcing StacyVM as public self-serve production-ready. + +The codebase can be public-readiness complete before a release is public-launch complete. Public launch also needs proof from the actual tag, host runtimes, network, and staging environment. + +## Candidate Evidence + +For a branch or release candidate, run: + +```bash +STACYVM_AUTH_API_KEY=replace-with-32-byte-key \ +STACYVM_AUTH_ADMIN_API_KEY=replace-with-different-32-byte-key \ +scripts/public-readiness-evidence.sh --output public-readiness-evidence.md +``` + +This verifies: + +- shell syntax for public install, release, readiness, upgrade, cluster, and runtime scripts +- `stacyvm config lint --production` against the production config +- full Go test suite +- web production build +- public release sanity build and checksum verification +- upgrade and migration sanity + +The report verdict is **PUBLIC SELF-SERVE CANDIDATE** when local gates pass but tag/host-gated evidence is skipped. + +## Announcement Evidence + +After publishing a real GitHub release tag and choosing the runtime claims for launch, run the full gate: + +```bash +STACYVM_AUTH_API_KEY=replace-with-32-byte-key \ +STACYVM_AUTH_ADMIN_API_KEY=replace-with-different-32-byte-key \ +STACYVM_POST_RELEASE_VERSION=v0.0.0 \ +STACYVM_VALIDATE_INSTALLER=true \ +STACYVM_RUNTIME_CERTIFY=docker \ +STACYVM_RUN_CLUSTER_CONFORMANCE=true \ +scripts/public-readiness-evidence.sh --output public-readiness-evidence.md +``` + +Add runtimes only when the target host can certify them: + +```bash +STACYVM_RUNTIME_CERTIFY=docker,gvisor,kata,firecracker +``` + +Do not claim Firecracker production readiness from a non-Linux or non-KVM host. Do not claim PRoot as a production isolation boundary. + +The report verdict is **PUBLIC SELF-SERVE READY** only when all required local, tag, runtime, and cluster gates pass without skips. + +## GitHub-Hosted Certification Gate + +After a tag is published, maintainers can run the manual **Public Readiness Certification** workflow from GitHub Actions. Use the published tag, for example `v0.14.3`, and the runtime claim to validate on the GitHub-hosted Linux runner. + +The workflow performs: + +- `scripts/post-release-validate.sh ` +- `STACYVM_VALIDATE_INSTALLER=true` installer verify-only on Linux +- `scripts/certify-runtime.sh docker --stacyvm-bin ./stacyvm-linux-amd64` +- evidence upload as a workflow artifact + +This is acceptable proof for the GitHub-hosted Docker runtime claim. It does not replace certification on the actual production host, because Docker daemon settings, seccomp/AppArmor, image cache behavior, KVM, gVisor/Kata, and network policy are host-specific. + +## Required Attachments + +Keep these artifacts with the release: + +- `public-readiness-evidence.md` +- output from `scripts/post-release-validate.sh ` +- runtime certification Markdown for every runtime claimed publicly +- live Postgres contract evidence for cluster or multi-worker claims +- target-network worker RPC mTLS smoke output when enterprise/multi-worker support is claimed +- staging install rehearsal notes from published artifacts +- redacted support bundle generated from the staging deployment + +## Final Approval Rule + +Public announcement is allowed only when: + +- the readiness evidence verdict is **PUBLIC SELF-SERVE READY** +- release assets are signed and checksum-verified +- every public runtime claim has host certification evidence +- production config lint passes with real secrets supplied through environment or secret files +- `server.cors_allowed_origins` contains exact trusted origins, not `*` +- rollback, backup/restore, and upgrade rehearsal have been exercised in staging + +If any of those are missing, announce the build as a release candidate or technical preview instead. diff --git a/docs/public-support-matrix.md b/docs/public-support-matrix.md new file mode 100644 index 0000000..693eb7d --- /dev/null +++ b/docs/public-support-matrix.md @@ -0,0 +1,81 @@ +# Public Support Matrix + +This matrix sets expectations for public self-serve StacyVM installs. It separates generally supported workflows from host-gated runtime certification so operators know what can be used immediately, what must be validated on their own infrastructure, and what remains experimental. + +## Support Levels + +| Level | Meaning | +|---|---| +| Supported | Covered by CI, documented setup, release verification, and support-bundle workflows. | +| Host-certified | Supported after `scripts/certify-runtime.sh` passes on the target host and the report is retained. | +| Preview | Usable for evaluation, but not recommended for production workloads without maintainer review. | +| Experimental | Available for development or constrained environments; not a production isolation boundary. | +| Planned | Production design exists, but implementation is not complete enough for self-serve users. | + +## Runtime And Deployment Matrix + +| Mode | Support level | Public self-serve status | Required evidence | Limitations | +|---|---|---|---|---| +| Local mock provider | Supported | Development only | `go test ./...` or CI pass | No sandbox isolation; not a production runtime. | +| Single-node Docker/runc | Supported | Technical production with hardened config | `stacyvm config lint --production`, `stacyvm doctor --production`, support bundle | Isolation is container-based; operators must keep Docker, kernel, and seccomp policy patched. | +| Single-node Docker with gVisor | Host-certified | Recommended container hardening path | Runtime certification report for `gvisor` | Requires host runtime installation and Docker runtime wiring outside StacyVM. | +| Single-node Docker with Kata | Host-certified | VM-backed container path | Runtime certification report for `kata` | Requires host runtime installation, VM support, and capacity planning. | +| Firecracker | Host-certified | VM isolation path for Linux/KVM hosts | Runtime certification report for `firecracker` | Requires Linux, KVM, kernel/rootfs/agent assets, and host networking setup. | +| PRoot | Experimental | Development and restricted hosts only | Runtime certification report for `proot` if used | Not a VM or container isolation boundary; production use is not recommended. | +| E2B/custom provider | Preview | Integration-specific | Provider health, conformance results, and provider-specific logs | External provider availability, auth, and isolation guarantees are outside StacyVM's direct control. | +| Multi-worker cluster | Preview | Enterprise self-serve with OIDC + tenant model | Cluster conformance output, Postgres contract output, worker identity certification report, runtime certification for every worker runtime, OIDC provider configuration, and tenant policy review | Worker registry, placement, leases, Postgres store path, signed worker identity, centralized token issuance, worker RPC routing, mTLS wiring, OIDC/JWT RS256 auth, RBAC roles (viewer/operator/admin/tenant_admin), tenant model, policy controls, and per-tenant audit export are implemented. Remaining: target-network mTLS smoke with deployment-issued certificates for specific enterprise networks. | + +## Public Install Requirements + +Before treating a self-serve install as supported, operators should capture: + +- Release verification output from `scripts/verify-release.sh ` or installer output with Sigstore verification. +- `stacyvm config lint --production --file ` output with production environment variables loaded. +- Confirmation that `server.cors_allowed_origins` contains exact trusted origins, not `*`. +- `stacyvm upgrade rehearse --config --database ` output before binary or image replacement. +- `stacyvm doctor --production` output from the target host. +- `stacyvm support bundle --output support.json` when opening a support issue. +- Runtime certification output for gVisor, Kata, Firecracker, or PRoot hosts. +- Worker identity certification output from `scripts/certify-worker-identity.sh --format markdown --output worker-identity-certification.md` for multi-worker previews. +- Evidence that externally issued worker token files refresh before expiry when workers rely on `stacyvm worker --worker-token-file` instead of local signing keys. +- Public readiness evidence from `scripts/public-readiness-evidence.sh --output public-readiness-evidence.md`. + +GitHub bug and production support issue templates ask for this same evidence. Reports without the relevant artifacts may need an extra triage round before maintainers can reproduce or classify the issue. + +## Known Public Limitations + +- SQLite is the supported single-node store. Postgres-backed cluster semantics exist, but multi-worker production still requires deployment-specific Postgres contract evidence and backup/restore rehearsal. +- API keys and admin keys are the primary auth path for single-node deployments. OIDC/JWT RS256 Bearer token auth with JWKS is implemented for enterprise multi-tenant deployments (`auth.oidc_enabled`, `auth.oidc_issuer`, `auth.oidc_jwks_url`). OIDC group-to-role mapping covers viewer, operator, admin, and tenant_admin roles. +- Docker/runc is convenient and supported with hardened settings, but it is not equivalent to VM isolation. +- Firecracker production readiness is host-gated because KVM, kernel, rootfs, agent, and networking setup vary by host. +- PRoot is useful where Docker/KVM are unavailable, but it should not be presented as a production isolation boundary. +- Release signatures prove artifact provenance from the StacyVM GitHub Actions release workflow; they do not certify a host runtime. +- CORS is intentionally wildcard by default for local development, but public browser/API deployments must configure explicit trusted origins and pass production config lint. + +## Support Triage Links + +| Symptom | First remediation path | +|---|---| +| Install verification fails | [releasing](/docs/releasing) | +| Production config lint fails | [deployment](/docs/deployment) | +| Upgrade rehearsal fails | [deployment#upgrades](/docs/deployment#upgrades) | +| Runtime health fails | [runtime-certification](/docs/runtime-certification) | +| Runtime behavior differs across providers | [runtime-conformance](/docs/runtime-conformance) | +| Admin or auth hardening question | [security-governance](/docs/security-governance) | +| Operator diagnostics needed | [deployment#support-bundles](/docs/deployment#support-bundles) | + +## Post-Tag Release Verification + +CI builds release binaries and validates checksums before code is merged into a release branch. After a real version tag is published, maintainers should run the public verifier against the GitHub release assets: + +```bash +scripts/post-release-validate.sh +``` + +On a clean Linux host, also run the installer in verify-only mode with signatures required: + +```bash +STACYVM_VALIDATE_INSTALLER=true scripts/post-release-validate.sh +``` + +For a final install smoke, run `scripts/install.sh` once with default checksum verification and once with `STACYVM_REQUIRE_SIGNATURES=true` plus `cosign` installed. diff --git a/docs/releases/phase-1-foundation-hardening.md b/docs/releases/phase-1-foundation-hardening.md new file mode 100644 index 0000000..ed98ff0 --- /dev/null +++ b/docs/releases/phase-1-foundation-hardening.md @@ -0,0 +1,161 @@ +# Phase 1 Foundation Hardening Release Notes + +Date: 2026-05-08 +Branch: `feat/phase-1-foundation-hardening` +Commit: `194267e` + +## Summary + +Phase 1 focused on turning StacyVM's early provider/orchestrator foundation into a more production-ready base. The work improves error consistency, provider contracts, startup recovery, platform-aware conformance coverage, and local developer build reliability. + +This phase does not introduce a new user-facing sandbox feature. Instead, it strengthens the foundation that future phases will build on: predictable errors, safer reconciliation after restarts, clearer provider expectations, and stronger regression coverage. + +## What Changed + +### Provider Contract And Conformance + +- Added `docs/provider-contract.md` to document the required behavior for every sandbox provider. +- Added a reusable provider conformance test harness covering: + - spawn, status, and destroy lifecycle + - command execution success and non-zero exits + - streaming command output + - file write, read, stat, glob, move, chmod, and delete +- Wired conformance coverage for Mock, Docker, Custom, PRoot, and Firecracker providers. +- PRoot and Firecracker conformance tests are platform-gated so they skip locally unless the required runtime dependencies are available. + +### Typed Error Taxonomy + +- Added typed provider errors for: + - sandbox not found + - sandbox destroyed + - provider not found + - provider unavailable + - exec timeout + - resource limit +- Added typed store errors for: + - not found + - conflict +- Re-exported provider errors through the orchestrator package where API routes need stable domain-level matching. + +### API Error Handling + +- Added shared route error mapping in `internal/api/routes/errors.go`. +- Replaced string-matching error handling in sandbox, template, environment, and provider routes with typed error checks. +- Added response codes for timeout and resource-limit failures. +- API responses now map important failure classes consistently: + - `404` for missing resources and sandbox lifecycle misses + - `408` for exec timeout + - `429` for resource limits + - `503` for provider unavailability + +### Startup Reconciliation + +- Added `Manager.Reconcile(ctx)` to refresh persisted sandbox state from provider runtime state at server startup. +- Server startup now runs reconciliation before starting the manager reaper. +- Persisted sandboxes whose runtime no longer exists are marked `destroyed`. +- Persisted sandboxes whose provider is unavailable are marked `error`. +- Live provider runtimes can be restored into the manager's in-memory cache. + +### Docker Runtime Adoption + +- Docker sandboxes now include richer `stacyvm.*` labels for runtime discovery. +- Added Docker runtime inventory support through `ListRuntimeSandboxes`. +- Startup reconciliation can adopt StacyVM Docker containers that still exist but are missing from SQLite after a process restart. +- Docker missing-container cases now map to typed `ErrSandboxNotFound`. + +### Streaming Timeout Semantics + +- `Manager.ExecStream` now honors request-level timeout values. +- Streaming timeout paths emit an explicit stderr timeout chunk instead of silently closing. +- Docker, Custom, and Firecracker streaming paths now propagate timeout state more clearly. + +### macOS Build Reliability + +- The Linux-only `stacyvm-agent` entrypoint now has a Linux build tag. +- Added a non-Linux stub so `make build` and `make test` work on macOS while preserving the real Linux agent behavior. + +## Code Changes By Area + +### New Files + +- `CHANGELOG.md` +- `cmd/stacyvm-agent/main_unsupported.go` +- `docs/provider-contract.md` +- `internal/api/routes/errors.go` +- `internal/api/routes/templates_test.go` +- `internal/orchestrator/errors.go` +- `internal/providers/custom_conformance_test.go` +- `internal/providers/errors.go` +- `internal/providers/provider_conformance_test.go` +- `internal/store/errors.go` + +### Core Orchestrator + +- `internal/orchestrator/manager.go` + - Added startup reconciliation. + - Added provider runtime adoption. + - Added streaming timeout handling. +- `internal/orchestrator/manager_test.go` + - Added tests for reconciliation, runtime adoption, and streaming timeout behavior. +- `cmd/stacyvm/cmd_serve.go` + - Runs reconciliation during server startup. + +### Providers + +- `internal/providers/provider.go` + - Documented provider contract expectations. + - Added optional runtime inventory interfaces. +- `internal/providers/docker.go` + - Added StacyVM labels. + - Added runtime listing. + - Added typed not-found handling. + - Improved streaming timeout propagation. +- `internal/providers/custom.go` + - Added typed HTTP error mapping. + - Improved streaming timeout propagation. +- `internal/providers/firecracker.go` + - Added typed lifecycle behavior and streaming read deadlines. +- `internal/providers/proot.go` + - Added typed lifecycle, timeout, and resource-limit behavior. +- `internal/providers/mock.go` + - Added typed lifecycle and timeout behavior for tests. +- `internal/providers/registry.go` + - Uses typed provider-not-found errors. + +### Store + +- `internal/store/sqlite.go` + - Maps missing rows and constraint conflicts to typed store errors. +- `internal/store/sqlite_test.go` + - Adds coverage for typed store errors. + +### API Routes + +- `internal/api/routes/sandboxes.go` +- `internal/api/routes/templates.go` +- `internal/api/routes/environments.go` +- `internal/api/routes/providers.go` + +These routes now use shared typed error handling instead of string comparisons. + +## Verification + +The following checks passed: + +```sh +make test +make build +cd web && npm run build +``` + +Additional Docker provider conformance and runtime inventory checks passed with Docker daemon access. + +## Platform Notes + +- Firecracker conformance requires Linux, `/dev/kvm`, Firecracker, kernel, rootfs, and agent paths. +- PRoot conformance requires `proot` and a usable rootfs. +- The full Go integration suite uses `httptest`; local sandboxed runs need permission to bind local test sockets. + +## Impact + +Phase 1 leaves the codebase ready for Phase 2 work by making provider behavior explicit, testable, and recoverable. The next phase can focus on scalability and production operations without first untangling provider lifecycle ambiguity or inconsistent API failure behavior. diff --git a/docs/releases/phase-10-multi-worker-foundation.md b/docs/releases/phase-10-multi-worker-foundation.md new file mode 100644 index 0000000..f7826d6 --- /dev/null +++ b/docs/releases/phase-10-multi-worker-foundation.md @@ -0,0 +1,115 @@ +# Phase 10 Multi-Worker Foundation Release Notes + +Date: 2026-05-09 +Branch: `phase-10-multi-worker-foundation` + +## Summary + +Phase 10 starts the enterprise and multi-worker production track. This slice adds the durable worker registry foundation StacyVM needs before scheduler placement, worker ownership, leases, and remote worker RPC can be made production-grade. + +This is not a full distributed runtime yet. It is the first production-aligned control-plane layer for observing workers, recording heartbeats, and exposing that state through APIs, diagnostics, and metrics. + +## What Changed + +### Worker Registry Storage + +- Added a SQLite migration for the `workers` table. +- Added durable worker fields for ID, hostname, status, providers, capabilities, capacity, heartbeat timestamp, and lifecycle timestamps. +- Added store methods for saving, fetching, listing, and deleting worker records. + +### Local Worker Registration + +- The API server now registers the current process as the `local` worker at startup. +- The API server now refreshes the `local` worker heartbeat periodically while running. +- Server shutdown stops the heartbeat loop cleanly. +- The local record includes configured providers, single-node capabilities, and manager capacity limits. +- Single-node deployments now appear in the same worker registry surface that future multi-worker deployments will use. + +### Worker API + +- Added read-only worker discovery: + - `GET /api/v1/workers` + - `GET /api/v1/workers/{workerID}` +- Added admin-only worker mutations: + - `POST /api/v1/admin/workers/{workerID}/heartbeat` + - `DELETE /api/v1/admin/workers/{workerID}` +- Worker responses include a computed `stale` flag when the last heartbeat is older than the freshness window. + +### Sandbox Worker Ownership + +- Added persisted `worker_id` ownership to sandbox records. +- New and adopted local sandboxes are stamped with the active worker ID. +- Scheduler status now reports the current worker ID. +- Sandbox API responses now include `worker_id` when ownership is known. + +### Worker-Aware Scheduler Placement + +- Spawn admission now evaluates worker placement using worker status, heartbeat freshness, provider support, declared capacity, and active sandbox counts. +- Scheduler status now reports the selected worker and number of eligible workers. +- Local execution remains honest: if the scheduler would place work on a remote worker, admission reports `remote_worker_rpc_unavailable` until the worker RPC slice lands. +- Stale local worker records are no longer special-cased; real server runs keep the local worker fresh through the heartbeat loop. + +### Distributed Lease Foundation + +- Added durable lease records for resource ownership fencing. +- Added store APIs to acquire, renew, release, get, and list leases. +- Lease acquisition is holder-aware and expiry-aware: a competing worker cannot acquire an unexpired lease held by another worker. +- Lease renewals require the current holder and an unexpired lease. +- Diagnostics and Prometheus now expose lease totals so operators can inspect active and expired lease state. + +### Lease Enforcement + +- Local spawns now acquire a sandbox lease before persisting the sandbox record. +- Runtime adoption during reconciliation now acquires a sandbox lease before adopting unknown provider runtimes. +- Pool VM and pooled logical sandbox creation now acquire leases. +- Destroy now acquires or renews the local worker lease before mutating provider/runtime/store state. +- Successful destroy releases the sandbox lease. +- Wrong-holder lease tests now prevent local destroy from mutating a sandbox owned by another worker. + +### Worker RPC Contract And Auth Model + +- Added `internal/workerproto` with transport-neutral worker request and response envelopes. +- Defined contract methods for heartbeat, spawn, destroy, status, lease renewal, and shutdown. +- Mutating worker assignments require a lease token in the message contract. +- Added transport-neutral worker auth claims and initial scopes. +- Documented the worker trust boundary, suggested headers, lease fencing rules, and Postgres cluster-store guarantees in `docs/worker-rpc-contract.md`. +- Remote worker execution remains gated until a network transport enforces this contract. + +### Diagnostics And Metrics + +- Diagnostics now include worker totals, online count, stale count, unhealthy count, and worker items. +- Diagnostics now include lease totals, active count, expired count, and active leases by holder. +- Diagnostics sandbox summaries now include `by_worker` counts. +- Prometheus output now includes: + - `stacyvm_workers_total{status="total"}` + - `stacyvm_workers_total{status="online"}` + - `stacyvm_workers_total{status="stale"}` + - `stacyvm_workers_total{status="unhealthy"}` + - `stacyvm_leases_total{status="active"}` + - `stacyvm_sandboxes_by_worker_total{worker="local"}` + +### Documentation + +- Updated the changelog with Phase 10 changes. +- Updated the API reference with worker endpoints and metrics. +- Updated the README endpoint table with worker discovery. +- Updated the production readiness checklist with Phase 10 acceptance criteria. + +## Code Areas + +- `internal/store`: worker model, lease model, migrations, SQLite CRUD, and sandbox `worker_id` persistence. +- `internal/workerproto`: worker RPC contract and auth claim types. +- `internal/api/routes`: worker routes, diagnostics worker summary, and Prometheus worker metrics. +- `internal/api/server.go`: local worker startup registration, heartbeat refresh loop, and route mounting. +- `docs`: API, README, changelog, production readiness, and release notes. + +## Verification + +- `go test ./internal/store ./internal/api/routes ./internal/api` +- `scripts/check-swagger.sh` +- `go test ./...` +- `git diff --check` + +## Remaining Phase 10 Direction + +Phase 10 is complete as a foundation branch. Follow-up phases should implement the network worker daemon/transport, Postgres-backed cluster store, remote lifecycle conformance tests, and OIDC/RBAC for enterprise production. diff --git a/docs/releases/phase-11-remote-worker-runtime.md b/docs/releases/phase-11-remote-worker-runtime.md new file mode 100644 index 0000000..bd12a8b --- /dev/null +++ b/docs/releases/phase-11-remote-worker-runtime.md @@ -0,0 +1,98 @@ +# Phase 11 Remote Worker Runtime Release Notes + +Date: 2026-05-09 +Branch: `phase-11-remote-worker-runtime` + +## Summary + +Phase 11 turns the Phase 10 worker RPC contract into a real remote-worker runtime path. It adds a worker process, worker-specific authentication, worker heartbeat, inbound worker RPC, and control-plane routing for remote spawn, status, destroy, lease renewal, and drain-mode shutdown. + +This is now suitable for internal two-process staging with the mock provider. It is not yet enterprise multi-worker production: remote exec/files/logs/previews, production-grade worker identity, Postgres-backed lease semantics, and assignment handoff remain future work. + +## What Changed + +### Worker Runtime Command + +- Added `stacyvm worker` as the entrypoint for remote worker processes. +- Added `--id`, `--control-plane`, `--worker-token`, `--heartbeat-interval`, `--listen`, and `--once` flags. +- The worker ID defaults to `worker.id`, then the host name. +- `--once` sends a single heartbeat and exits for smoke tests and staging probes. +- `--listen` starts the inbound worker RPC server for control-plane-to-worker calls. + +### Worker Configuration + +- Added `worker.id`. +- Added `worker.control_plane_url`. +- Added `worker.listen_addr`. +- Added `worker.heartbeat_interval`. +- Added `worker.shutdown_timeout`. +- Added `auth.worker_token` for worker-to-control-plane authentication. +- Worker durations are validated through the normal config validation path. + +### Worker Authentication + +- Added a dedicated worker auth role and `worker:heartbeat` scope. +- Added `X-Worker-ID` and `X-Worker-Token` validation for worker endpoints. +- Worker credentials cannot authenticate regular API/admin routes. +- Worker heartbeats are rejected when the authenticated worker ID does not match the requested worker path. + +### Worker Heartbeat Transport + +- Added `internal/worker` with a heartbeat client and runtime loop. +- Added a worker-only control-plane endpoint: + - `POST /api/v1/worker/{workerID}/heartbeat` +- Added a worker-only control-plane lease renewal endpoint: + - `POST /api/v1/worker/{workerID}/leases/{resourceID}/renew` +- The endpoint persists worker host, status, provider list, capability list, capacity, and last heartbeat timestamp through the existing worker registry. + +### Worker RPC Transport + +- Added worker-side `/rpc` HTTP handling for `workerproto.Request` envelopes. +- Added worker RPC authentication with `X-Worker-ID` and `X-Worker-Token`. +- Implemented `worker.status` against the local worker provider registry. +- Implemented `worker.renew_lease` with resource, holder, and expiry validation before calling the control plane to renew the durable lease. +- Implemented worker-side `worker.spawn` with lease validation and provider-backed runtime creation. +- Added a typed worker RPC client for control-plane calls to `worker.spawn` and `worker.status`. +- Added control-plane remote spawn assignment when the scheduler selects a non-local worker with an advertised `rpc_url`. +- Remote spawn now acquires the durable sandbox lease for the selected worker before calling `worker.spawn`. +- Remote spawn persists the control-plane sandbox ID, selected `worker_id`, and provider `runtime_id`. +- Sandbox reads now refresh remote-owned sandbox state through `worker.status` using the persisted provider `runtime_id`. +- Remote status refresh updates persisted sandbox state when the worker reports a state change. +- Implemented worker-side `worker.destroy` with lease validation and provider runtime teardown. +- Added control-plane remote destroy routing for remote-owned sandboxes using persisted `worker_id` and `runtime_id`. +- Remote destroy updates sandbox state, releases the durable lease, and publishes the normal destroyed event. +- Implemented `worker.shutdown` drain behavior: the worker marks itself draining, rejects new spawn assignments, and reports `draining` on subsequent heartbeats. +- `worker.spawn` returns the control-plane sandbox ID and provider runtime ID separately, and the control plane persists that mapping for later routing. +- Destroy now uses the worker RPC path for remote-owned sandboxes. +- Added a two-process remote worker staging guide in `docs/remote-worker-staging.md`. +- Added `scripts/smoke-remote-worker.sh` to exercise control plane plus worker with the mock provider. + +## Code Areas + +- `cmd/stacyvm/cmd_worker.go`: remote worker command. +- `internal/worker`: worker heartbeat client, runtime loop, and inbound worker RPC handler. +- `internal/api/middleware/auth.go`: worker auth role, scope, and credential validation. +- `internal/api/server.go`: worker-only heartbeat route. +- `internal/api/routes/workers.go`: worker ID ownership check for heartbeat. +- `internal/config/config.go`: worker runtime and worker token configuration. +- `docs/remote-worker-staging.md`: two-process staging guide. +- `scripts/smoke-remote-worker.sh`: local mock remote-worker smoke flow. + +## Verification + +- `go test ./internal/config ./internal/api ./internal/worker ./cmd/stacyvm` +- `go test ./internal/worker ./internal/config ./cmd/stacyvm` +- `go test ./...` +- `scripts/check-swagger.sh` +- `bash -n scripts/smoke-remote-worker.sh` +- `scripts/smoke-remote-worker.sh /private/tmp/stacyvm-phase11-smoke` +- `npm run build` in `web` +- `npm run build` in `sdk/js` +- `PYTHONPYCACHEPREFIX=/private/tmp/stacyvm-pycache python3 -m compileall stacyvm` in `sdk/python` +- `scripts/ci-upgrade-migration.sh` +- `scripts/ci-public-release-sanity.sh` + +## Next Phase Direction + +- Extend worker routing beyond spawn/status/destroy to exec, files, logs, and previews. +- Add production-grade drain handoff/reassignment across workers. diff --git a/docs/releases/phase-12-remote-sandbox-io-routing.md b/docs/releases/phase-12-remote-sandbox-io-routing.md new file mode 100644 index 0000000..ebd55f6 --- /dev/null +++ b/docs/releases/phase-12-remote-sandbox-io-routing.md @@ -0,0 +1,85 @@ +# Phase 12 Remote Sandbox I/O Routing Release Notes + +Date: 2026-05-09 +Branch: `phase-12-remote-sandbox-io-routing` + +## Summary + +Phase 12 extends the Phase 11 remote worker runtime beyond lifecycle operations into sandbox I/O routing and conservative remote ownership handling. + +Non-streaming exec, live exec-stream calls, file APIs, console logs, preview metadata, and drain/offline ownership policy are now routed through remote worker ownership. + +## What Changed + +### Worker RPC Contract + +- Added the `worker.exec` method to the worker RPC contract. +- Added the `worker.exec_stream` method to the worker RPC contract. +- Added NDJSON stream transport for `worker.exec_stream`. +- Added `workerproto.ExecParams` for command, argv mode, environment, workdir, timeout, provider, sandbox ID, and provider runtime ID. +- Added `workerproto.ExecResult` for exit code, stdout, and stderr. +- Added `workerproto.ExecStreamResult` for stdout/stderr chunk delivery. +- Added worker file RPC methods for write, read, list, delete, move, chmod, stat, and glob. +- Added worker file result payloads for content, file listings, stat entries, and glob matches. +- Added `worker.logs` for remote console log retrieval. +- Added `worker.preview_domain` config and heartbeat capacity advertisement for worker-specific preview ingress. +- Added `unhealthy` and `expired` sandbox states for remote ownership reconciliation. +- Added the `worker:exec` scope constant for future token-scoped worker identity. +- Added the `worker:files` scope constant for future token-scoped worker identity. +- Added the `worker:logs` scope constant for future token-scoped worker identity. + +### Worker Runtime + +- Implemented worker-side `worker.exec` handling in the inbound RPC server. +- Implemented worker-side `worker.exec_stream` handling in the inbound RPC server. +- Worker exec resolves the provider runtime ID, runs through the worker's provider registry, and returns a typed RPC result. +- Worker exec and live exec stream honor the provided timeout string as a worker-side context deadline when present. +- Implemented worker-side file read/write/list/delete/move/chmod/stat/glob handling in the inbound RPC server. +- Implemented worker-side console log handling in the inbound RPC server. +- Added typed `RPCClient.Exec` support for control-plane calls. +- Added typed `RPCClient.ExecStream` support for control-plane calls. +- Added typed `RPCClient.ExecStreamLive` support for live NDJSON control-plane calls. +- Added typed file RPC client helpers for control-plane calls. +- Added typed logs RPC client support for control-plane calls. + +### Control Plane Routing + +- `Manager.Exec` now detects remote-owned sandboxes and routes non-streaming exec to the owning worker RPC endpoint. +- `Manager.ExecStream` now detects remote-owned sandboxes and routes live exec streams to the owning worker RPC endpoint. +- File APIs now detect remote-owned sandboxes and route through the owning worker RPC endpoint. +- Console logs now detect remote-owned sandboxes and route through the owning worker RPC endpoint. +- Remote-owned sandboxes now return the owning worker's advertised preview domain when present. +- Startup reconciliation now applies a remote worker ownership policy: + - fresh draining workers keep existing sandbox ownership and remain unavailable for new placement. + - stale, offline, or missing worker ownership marks non-expired sandboxes `unhealthy`. + - expired remote-owned sandboxes become `expired` and release their durable lease. +- Remote exec uses persisted `worker_id` and provider `runtime_id` instead of local provider state. +- Remote file APIs use persisted `worker_id` and provider `runtime_id` instead of local provider state. +- Remote logs use persisted `worker_id` and provider `runtime_id` instead of local provider state. +- Remote preview URLs use persisted worker ownership plus worker heartbeat capacity. +- Remote exec keeps the existing event, audit, metrics, timeout, and exec-log behavior. +- Remote exec stream keeps the existing manager channel API while forwarding worker NDJSON chunks as they arrive. +- Remote sandboxes no longer inherit pool-mode default workdir or file path scoping just because their provider runtime ID is stored in `VMID`. + +## Code Areas + +- `internal/workerproto/protocol.go`: `worker.exec`, `worker.exec_stream`, file RPC, and logs RPC contract/result types. +- `internal/worker/rpc.go`: worker-side exec, exec-stream, file, and logs RPC handlers. +- `internal/worker/rpc_client.go`: typed exec, live exec-stream, file, and logs RPC client methods. +- `internal/orchestrator/manager.go`: remote-owned sandbox exec, exec-stream, file, and logs routing. +- `internal/orchestrator/scheduler.go`: expired sandbox ownership exclusion from placement capacity. +- `cmd/stacyvm/cmd_worker.go`: worker preview domain capacity advertisement and Docker preview-domain wiring. +- `internal/config/config.go`: worker preview domain configuration. +- `internal/worker/rpc_test.go`: worker exec and exec-stream RPC handler coverage. +- `internal/worker/rpc_client_test.go`: typed exec, exec-stream, file, and logs client coverage. +- `internal/orchestrator/manager_test.go`: control-plane remote exec, exec-stream, file, and logs routing coverage. + +## Verification + +- `go test ./internal/workerproto ./internal/worker ./internal/orchestrator` +- `go test ./...` +- `git diff --check` + +## Remaining Direction + +- Real stateful runtime migration is still provider-dependent and should be implemented per provider through snapshot or migration capabilities rather than simulated by the control plane. diff --git a/docs/releases/phase-13-cluster-store-and-worker-identity.md b/docs/releases/phase-13-cluster-store-and-worker-identity.md new file mode 100644 index 0000000..e8357e4 --- /dev/null +++ b/docs/releases/phase-13-cluster-store-and-worker-identity.md @@ -0,0 +1,79 @@ +# Phase 13 Cluster Store And Worker Identity Release Notes + +Date: 2026-05-09 +Branch: `phase-13-cluster-store-and-worker-identity` + +## Summary + +Phase 13 starts the enterprise multi-worker production track after Phase 12 completed remote sandbox I/O routing. The checkpoints make persistence explicitly driver-based, add a reusable store contract harness, and link a Postgres-backed store path while keeping SQLite as the default supported store. + +## What Changed + +### Store Factory + +- Added `store.Open` with explicit `sqlite` and `postgres` driver handling. +- SQLite remains the default and continues to use the existing `NewSQLiteStore` implementation. +- Postgres now opens through `NewPostgresStore` with the pgx stdlib driver. +- Added factory tests for default SQLite opening and missing database configuration. + +### Postgres Migration Foundation + +- Added Postgres-native migration definitions for the current store schema. +- Introduced shared migration metadata so SQLite and Postgres migration versions can be compared directly. +- Added tests that verify Postgres migrations track SQLite migration versions. +- Added tests that verify Postgres migrations cover all store tables and avoid SQLite-only dialect tokens. +- Added a Postgres store migrator that applies those migrations through `store.Open`. +- Added live Postgres migration rehearsal coverage for idempotent migration application. + +### Store Contract Harness + +- Added a reusable store contract test harness in `internal/store`. +- Wired the contract harness to SQLite as the first concrete driver. +- Wired the contract harness to Postgres when `STACYVM_POSTGRES_TEST_DSN` is set. +- Covered sandbox lifecycle semantics, including soft-delete behavior and active-list filtering. +- Covered worker registry behavior for save, update, list, get, and delete. +- Covered lease acquisition, renewal, conflict detection, release ownership, and expired lease takeover. +- Covered live Postgres lease acquisition and expired-takeover races across multiple store connections. +- Covered exec logs, admin audit logs, operation audit logs, owner quotas, and provider configs. +- Covered templates, environment specs, environment builds, build artifacts, and registry connections. + +### Configuration + +- Added `database.driver`. +- Added `database.dsn`. +- Kept `database.path` for SQLite. +- Config validation now rejects unsupported database drivers. +- Config validation now requires `database.dsn` when `database.driver` is `postgres`. + +### CLI And Diagnostics + +- `stacyvm serve` now opens the store through the driver-based factory. +- `stacyvm config lint` accepts Postgres configs with a valid DSN. +- `stacyvm config lint --production` now distinguishes shared staging worker tokens from production per-worker credentials. + +### Worker Identity + +- Added `auth.worker_tokens` as a map of `worker_id: token`. +- Kept `auth.worker_token` for shared-token staging compatibility. +- Per-worker credentials override the shared worker token for that worker ID. +- Worker identities now receive explicit scopes for heartbeat, spawn, destroy, status, exec, files, logs, and leases. +- Worker lease renewal now requires the dedicated `worker:lease` scope. + +### Cluster Conformance + +- Added `scripts/ci-cluster-conformance.sh`. +- Added the `cluster-conformance` GitHub Actions job. +- Added the `remote-worker-postgres-smoke` GitHub Actions job. +- Added `docs/cluster-conformance.md` with store, worker identity, runtime, and promotion gates. +- CI now verifies the SQLite store contract, live Postgres store contract, live Postgres lease concurrency, live Postgres migration rehearsal, worker identity tests, production-aligned cluster config linting, and a Postgres-backed remote worker smoke. + +## Verification + +- `go test ./internal/store` +- `go test ./internal/api/middleware ./internal/api ./internal/config ./cmd/stacyvm` +- `scripts/ci-cluster-conformance.sh` + +## Next Phase 13 Direction + +- Continue worker identity hardening toward signed tokens or mTLS transport enforcement. +- Extend multi-worker conformance beyond the mock-provider smoke into Docker/gVisor/Kata/Firecracker certified hosts. diff --git a/docs/releases/phase-14-worker-identity-hardening.md b/docs/releases/phase-14-worker-identity-hardening.md new file mode 100644 index 0000000..83eb51b --- /dev/null +++ b/docs/releases/phase-14-worker-identity-hardening.md @@ -0,0 +1,158 @@ +# Phase 14 Worker Identity Hardening + +Phase 14 begins the worker identity hardening lane for production multi-worker StacyVM deployments. The goal of this slice is to move beyond static shared worker credentials while keeping the existing Phase 11-13 worker runtime compatible. + +## What Changed + +### Signed worker tokens + +- Added HMAC-SHA256 signed worker token support. +- Added the `stacyvm-worker-v1..` token format. +- Added signed token claims for: + - `worker_id` + - token ID through `jti` + - audience through `aud` + - optional worker `scopes` + - issued-at time through `iat` + - optional not-before time through `nbf` + - expiry time through `exp` +- Enforced signed-token expiry before accepting worker requests. +- Enforced signed-token not-before and issued-at validation with clock-skew tolerance. +- Enforced a 15 minute max signed worker token lifetime when `iat` is present. +- Enforced that signed `worker_id` must match the `X-Worker-ID` request header. +- Enforced token audience separation between worker-to-control-plane routes and control-plane-to-worker RPC. +- Added `auth.worker_revoked_token_ids` emergency revocation for signed worker token IDs. +- Added worker token issuer `--format json`, `--token-id`, and `--not-before` options for incident-response runbooks. +- Added `stacyvm worker token inspect ` to recover unverified signed-token metadata and `jti` values during incident response. +- Added `stacyvm worker token verify ` to validate signed tokens against active and rotation keys, expected worker IDs, expected audiences, and revoked token IDs. +- Added `stacyvm worker token rotation-plan` to print a no-secret signing-key rotation checklist, config sketch, and validation commands. +- Added worker secret file flags for token issuance, token verification, and worker runtime startup so operators can use secret-mounted files instead of command-line or environment secrets. +- Filtered signed-token scopes so tokens cannot grant user, API, or admin scopes. +- Added `stacyvm worker token ` to issue signed worker tokens from the CLI. + +### Configuration + +- Added `auth.worker_signing_key`. +- Added `auth.worker_token_file` and `auth.worker_signing_key_file` so production services can mount worker credentials from files. +- Added `auth.worker_signing_keys` for old verification keys during rotation. +- Kept `auth.worker_token` for shared-token staging compatibility. +- Kept `auth.worker_tokens` for per-worker static token migration paths. +- Updated `stacyvm config lint --production` so a strong `auth.worker_signing_key` satisfies production-aligned worker credential checks. +- Added config lint warnings when revoked signed-token IDs are configured without signed worker-token verification. +- Added config lint warnings for shared worker tokens left enabled beside signed worker tokens, duplicate rotation keys, and rotation keys that repeat the active signing key. +- Config loading now rejects ambiguous inline/file worker secret pairs such as `auth.worker_signing_key` plus `auth.worker_signing_key_file`. +- Config lint now reports whether worker token and signing-key values are file-backed or still configured inline. + +### Worker runtime + +- Added dynamic worker token generation for `stacyvm worker` heartbeat and lease-renewal calls. +- When no static `--worker-token` or `auth.worker_token` is configured, a worker can derive short-lived signed control-plane tokens from `auth.worker_signing_key`. +- Workers can read static worker tokens from `--worker-token-file` or signing keys from `--worker-signing-key-file`. +- `stacyvm worker --worker-token-file` reloads the token file for each heartbeat and lease-renewal request, so a sidecar or external issuer can rotate short-lived signed tokens without restarting the worker process. +- Worker RPC servers now accept signed control-plane-to-worker tokens. +- Control planes can mint short-lived worker RPC tokens from `auth.worker_signing_key` when no shared `auth.worker_token` is configured. +- Existing static token behavior is unchanged. + +### Rotation + +- New signed tokens are minted with `auth.worker_signing_key`. +- Old tokens can continue verifying through `auth.worker_signing_keys` during a rotation window. +- Operators can generate a concrete no-secret rollout checklist with `stacyvm worker token rotation-plan`. +- The documented rotation sequence is: + - promote the new key into `auth.worker_signing_key` + - move the old key into `auth.worker_signing_keys` + - restart or reload workers + - wait for old token TTLs to expire + - remove the old key from `auth.worker_signing_keys` + +### Worker RPC mTLS + +- Added `worker.rpc_tls` configuration for enterprise worker RPC networks. +- Added TLS server support for `stacyvm worker --listen`. +- Added mTLS client support for control-plane calls to worker RPC. +- Added worker RPC mTLS conformance that completes a real RPC call with generated CA, server, and client certificates. +- Added config lint checks for worker server certificates, client CA verification, control-plane client certificates, worker CA verification, and unsafe `insecure_skip_verify`. +- Documented how worker-side and control-plane-side certificate settings are used. + +### Documentation + +- Updated the README configuration example. +- Updated the worker RPC contract with signed-token semantics. +- Documented the issue, inspect, verify, and revoke operator runbook for worker tokens. +- Updated the API docs for worker heartbeat and lease-renewal headers. +- Updated the cluster conformance matrix to mark signed worker tokens as the public/enterprise worker identity path. +- Added `scripts/certify-worker-identity.sh` for host-level signed-token lifecycle signoff with text, JSON, or Markdown report output. +- Updated the public support matrix to reflect multi-worker signed identity, worker RPC routing, mTLS wiring, and worker identity certification evidence. +- Updated production readiness notes to reflect signed worker tokens, worker identity certification reporting, worker RPC mTLS wiring, and the remaining target-network/runtime signoff work. +- Added cluster conformance coverage for signed-token migration lint warnings. +- Added cluster conformance coverage for worker identity certification report generation. +- Updated the threat model and remote-worker staging guide so worker impersonation controls and staging guidance reflect the implemented signed-token and mTLS paths. +- Documented reloadable worker token files as the handoff path for external worker-token issuers. +- Updated deployment and configuration docs for secret-mounted worker token and signing-key files. +- Documented production lint visibility for file-backed worker credential sources. + +## Code Areas Changed + +- `internal/api/middleware`: signed token creation, verification, worker scope filtering, and worker auth config. +- `internal/api`: server worker auth wiring for `auth.worker_signing_key`. +- `internal/config`: config schema and defaults for primary and rotation worker signing keys. +- `internal/worker`: dynamic token callback support for worker heartbeat and lease renewal, plus worker RPC TLS client/server helpers. +- `internal/orchestrator`: worker RPC client TLS wiring for remote worker calls. +- `cmd/stacyvm`: `serve`, `worker`, `worker token`, and `config lint` wiring. +- `scripts`: worker identity certification smoke. +- `docs`: worker identity and conformance documentation. + +## Compatibility + +The new signed-token path is additive: + +- Existing `auth.worker_token` deployments continue to work. +- Existing inline worker secrets can be moved to `auth.worker_token_file` or `auth.worker_signing_key_file` without changing runtime behavior. +- Existing `auth.worker_tokens.` deployments continue to work. +- Signed tokens can be introduced gradually by setting `auth.worker_signing_key`. +- Workers can also consume externally issued short-lived tokens through reloadable `--worker-token-file` secrets. +- Key rotation can be introduced gradually by adding old keys to `auth.worker_signing_keys`. +- Worker RPC mTLS is opt-in; local HTTP worker RPC remains available for local development and internal staging. + +## Phase 14 Enterprise Governance (implemented) + +### OIDC/SSO and RBAC +- Added RS256 JWT Bearer token validation with configurable OIDC issuer, JWKS URL, and static public key. +- Added RBAC roles: `viewer` (read-only), `operator` (sandbox lifecycle), `tenant_admin` (per-tenant admin) beyond existing `api`/`admin`. +- Added OIDC group-to-role mapping via `auth.oidc_admin_groups`, `auth.oidc_operator_groups`, `auth.oidc_viewer_groups`. +- Added scopes: `read:*`, `operator:*`, `tenant:admin`. +- API key auth and OIDC Bearer auth coexist; existing API key deployments are unaffected. + +### Tenant/project model +- Added `tenants`, `tenant_members`, and `policies` tables (migration 11). +- Added `tenant_id` to sandboxes, admin audit logs, and operation audit logs. +- Admin routes: tenant CRUD, member RBAC (viewer/operator/admin per tenant), per-tenant audit export, per-tenant policy management. + +### Policy controls +- Added `policies` store with resource_type (image/provider/network), effect (allow/deny), glob pattern, and priority. +- Added `PolicyEnforcer` middleware that checks spawn request fields against tenant and global policies. + +### Centralized worker token issuer +- Added `POST /api/v1/admin/worker-tokens` to mint signed worker tokens with configurable TTL, audience, and scopes. +- Workers no longer need direct access to `auth.worker_signing_key` in hardened deployments. + +### Postgres operations +- Added `stacyvm db pg-backup ` wrapping `pg_dump` for production cluster snapshots. +- Added `stacyvm db pg-rehearse` for pre-upgrade schema state verification. + +### Admin UI +- Added Tenants page with tenant lifecycle, member RBAC, policy management, and per-tenant audit export. + +## Public Self-Serve Hardening Follow-Up + +- Added configurable `server.cors_allowed_origins` so public browser/API deployments can restrict CORS to exact trusted origins instead of relying on reverse-proxy-only guidance. +- Updated API server CORS handling to preserve the local wildcard default while rejecting disallowed browser preflights when explicit origins are configured. +- Updated `stacyvm config lint --production` so wildcard or empty CORS fails the public production gate. +- Updated `deploy/stacyvm.production.yaml`, deployment docs, API docs, public support matrix, and production readiness checklist with explicit CORS origin requirements. +- Extended public release sanity CI to run production config lint with environment-provided secrets before building release artifacts. +- Hardened the remote worker smoke harness so missing binaries, occupied ports, and failed spawns report actionable diagnostics instead of producing misleading release-gate failures. +- Added `scripts/public-readiness-evidence.sh` and `docs/public-readiness-evidence.md` to generate a single public announcement evidence report that distinguishes branch-ready, tag-ready, runtime-certified, and target-network-certified states. + +## Remaining Phase 14 Direction + +- Run worker RPC mTLS smoke tests with deployment-issued certificates in the target enterprise network. diff --git a/docs/releases/phase-2-observability-and-ops.md b/docs/releases/phase-2-observability-and-ops.md new file mode 100644 index 0000000..08fcb49 --- /dev/null +++ b/docs/releases/phase-2-observability-and-ops.md @@ -0,0 +1,183 @@ +# Phase 2 Observability And Ops Release Notes + +Date: 2026-05-08 +Branch: `phase-2-observability-and-ops` + +## Summary + +Phase 2 turns the Phase 1 foundation into an operable production surface. The API now exposes liveness, readiness, diagnostics, structured metrics, Prometheus scraping, richer provider health, operational audit events, and configurable runtime limits. + +The goal of this phase is to make StacyVM easier to run, debug, monitor, and safely scale before deeper multi-tenant and production deployment work. + +## What Changed + +### Liveness And Readiness + +- Added `/api/v1/live` for process liveness checks. +- Added `/api/v1/ready` for dependency readiness checks. +- Readiness now reports provider health instead of only returning a generic process status. + +### Structured Runtime Metrics + +- Added an in-process operation metrics recorder. +- Operations tracked include: + - spawn + - exec + - exec stream + - destroy + - file write, read, list, delete, move, chmod, stat, and glob +- Each operation tracks: + - success count + - failure count + - latency count + - total latency + - min, max, and average latency + - last error + - last observed timestamp +- `/api/v1/metrics` now includes sandbox, provider, event, process, runtime, and operation metrics. + +### Prometheus Metrics + +- Added `/api/v1/metrics/prometheus`. +- The Prometheus endpoint exposes: + - process uptime + - goroutines + - memory and GC metrics + - sandbox counts by state and provider + - provider health + - provider health latency + - provider runtime inventory counts + - event bus stats + - operation success/failure and latency counters + +### Operational Audit Events + +- Added event IDs for published events. +- Added operational event types: + - `exec.failed` + - `exec.timeout` + - `operation.failed` + - `resource.limit` + - `provider.failed` + - `reconcile.action` +- Manager paths now publish audit events for: + - exec failures and timeouts + - stream exec timeouts + - file operation failures + - spawn/provider/resource failures + - destroy provider failures + - reconciliation actions and provider inventory failures + +### Provider Health Detail + +- Provider health now includes: + - `latency_ms` + - `last_checked` + - `error` + - `capabilities` + - `runtime_count` when runtime inventory is supported +- Provider health detail is shared across: + - `/api/v1/ready` + - `/api/v1/metrics` + - `/api/v1/metrics/prometheus` + - `/api/v1/providers` + - `/api/v1/providers/{name}` + +### Redacted Diagnostics + +- Added `/api/v1/diagnostics`. +- Diagnostics include: + - generated timestamp + - version/build info + - GOOS/GOARCH + - uptime, goroutines, memory, and GC cycles + - store health and latency + - active operational limits + - detailed provider health + - sandbox counts by state/provider + - event bus stats + - operation metrics + - explicit redaction categories +- Diagnostics are read-only and intentionally avoid returning API keys, registry credentials, provider secrets, or environment secrets. + +### Operational Limits + +- Added configurable defaults: + - `defaults.max_ttl` + - `defaults.default_exec_timeout` + - `defaults.max_exec_timeout` + - `defaults.max_sandboxes` + - `defaults.max_sandboxes_per_owner` +- Manager now centrally enforces: + - max TTL + - max total active sandboxes + - max active sandboxes per owner + - default exec timeout + - max exec timeout +- Limit violations return typed resource-limit errors and publish `resource.limit` audit events. + +## Code Changes By Area + +### API Routes + +- `internal/api/routes/system.go` + - Added liveness, readiness, diagnostics, JSON metrics, and Prometheus metrics behavior. +- `internal/api/routes/provider_health.go` + - Added shared provider health detail collection. +- `internal/api/routes/prometheus.go` + - Added Prometheus text renderer. +- `internal/api/routes/providers.go` + - Added detailed health to provider list/detail responses. +- `internal/api/routes/system_test.go` + - Added coverage for readiness, diagnostics, metrics, and Prometheus output. + +### Orchestrator + +- `internal/orchestrator/metrics.go` + - Added operation metrics recorder. +- `internal/orchestrator/manager.go` + - Added metrics recording, audit event publishing, and operational limit enforcement. +- `internal/orchestrator/events.go` + - Added event IDs and operational event types. +- `internal/orchestrator/types.go` + - Added operational limit types. +- `internal/orchestrator/manager_test.go` + - Added tests for operation metrics, audit events, TTL limits, sandbox limits, owner limits, and exec timeout limits. + +### Config And Docs + +- `internal/config/config.go` + - Added default config fields for operational limits. +- `cmd/stacyvm/cmd_serve.go` + - Wires configured operational limits into the manager. +- `README.md` + - Documents new operational limit config. +- `docs/api.md` + - Documents liveness, readiness, diagnostics, metrics, Prometheus metrics, provider health detail, and operational event shape. +- `CHANGELOG.md` + - Adds this Phase 2 checkpoint entry. + +## Verification + +The following checks passed: + +```sh +make test +make build +cd web && npm run build +``` + +## Impact + +Phase 2 gives StacyVM the baseline visibility and guardrails needed to operate safely: + +- Operators can distinguish liveness from readiness. +- Dashboards can consume JSON or Prometheus metrics. +- Support/debug flows can use a redacted diagnostics endpoint. +- Provider health is actionable rather than a single boolean. +- Resource pressure and failure modes are visible through events. +- Runtime limits can prevent accidental overload before full multi-tenant quota systems arrive. + +## Next Phase Direction + +Phase 3 should focus on production scaling and multi-tenant control planes: persistent quotas, per-owner policy, rate limits, queueing/backpressure, distributed scheduler boundaries, and deployment/CI hardening. diff --git a/docs/releases/phase-3-quotas-and-scheduling.md b/docs/releases/phase-3-quotas-and-scheduling.md new file mode 100644 index 0000000..3cc2b03 --- /dev/null +++ b/docs/releases/phase-3-quotas-and-scheduling.md @@ -0,0 +1,152 @@ +# Phase 3 Quotas And Scheduling Release Notes + +Date: 2026-05-08 +Branch: `phase-3-quotas-and-scheduling` + +## Summary + +Phase 3 adds the first production multi-tenant control plane for StacyVM. The server now supports persisted owner quota policies, API rate limiting, spawn backpressure, scheduler visibility, quota audit events, admission preflight, and SDK helpers for the new quota and admission surfaces. + +The goal of this phase is to make StacyVM safer under shared usage and load: operators can define per-owner limits, clients can understand whether work will run or queue, and dashboards can observe queue pressure and quota coverage. + +## What Changed + +### Persistent Owner Quotas + +- Added persisted owner quota policies backed by SQLite. +- Quotas can override: + - max active sandboxes per owner + - max sandbox TTL + - max exec timeout +- Added owner quota APIs: + - `GET /api/v1/quotas` + - `GET /api/v1/quotas/summary` + - `GET /api/v1/quotas/{ownerID}` + - `PUT /api/v1/quotas/{ownerID}` + - `DELETE /api/v1/quotas/{ownerID}` + - `GET /api/v1/quotas/{ownerID}/usage` +- Owner IDs are normalized and validated before quota use. +- Quota saves and deletes emit audit events. + +### Spawn Admission And Backpressure + +- Added serialized spawn admission checks to prevent concurrent over-admission. +- Added configurable spawn overflow behavior: + - `reject` + - `queue` +- Added configurable queue controls: + - `defaults.spawn_queue_timeout` + - `defaults.max_spawn_queue` +- Queued spawn requests resume when capacity opens or owner quota changes. +- Spawn queue timeouts return typed resource-limit errors. +- Added `POST /api/v1/sandboxes/admission` for preflight admission checks without creating provider resources. + +### API Rate Limiting + +- Added optional in-memory API rate limiting. +- Supported rate-limit keys: + - owner + - API key + - IP address +- Rate-limit buckets use hashed keys so raw identifiers are not exposed in memory snapshots or metrics. +- Inactive rate-limit buckets are pruned on a configurable interval. + +### Scheduler And Quota Observability + +- Diagnostics and metrics now include scheduler state, queue depth, queue totals, queue timeouts, wait totals, wait max, and wait averages. +- Prometheus now exposes spawn queue gauges/counters and quota summary metrics. +- Added redacted quota summary counts for operators without exposing owner IDs. + +### Streaming Timeout Semantics + +- Streaming exec deadline expiry still emits `exec.timeout` and a timeout stderr chunk. +- Caller cancellation is no longer mislabeled as an exec timeout. +- Pre-stream exec limit errors now use the central API error mapper, so streaming and non-streaming exec return consistent status codes. + +### SDK Support + +- TypeScript SDK: + - Added `client.admission(...)`. + - Added `client.quotaSummary()`. + - Added `SpawnAdmissionDecision` and `QuotaSummary` types. + - Added `owner_id` on `SpawnOptions`. +- Python SDK: + - Added `Client.admission(...)` and `AsyncClient.admission(...)`. + - Added `Client.quota_summary()` and `AsyncClient.quota_summary()`. + - Added `SpawnAdmissionDecision` and `QuotaSummary` models. + - Added `owner_id` spawn parameter. + +## Code Changes By Area + +### API Routes + +- `internal/api/routes/quotas.go` + - Added owner quota CRUD, owner usage, and redacted summary routes. +- `internal/api/routes/sandboxes.go` + - Added spawn admission preflight. + - Aligned streaming exec preflight error mapping with non-streaming exec. +- `internal/api/routes/system.go` + - Added scheduler, quota, and rate-limit data to diagnostics and metrics. +- `internal/api/routes/prometheus.go` + - Added scheduler queue, quota summary, and rate-limit metrics. + +### Orchestrator + +- `internal/orchestrator/manager.go` + - Added quota enforcement, quota summary, owner usage, spawn admission decisions, queue wait/resume behavior, and refined stream timeout handling. +- `internal/orchestrator/types.go` + - Added quota, owner usage, scheduler status, and admission decision types. +- `internal/orchestrator/events.go` + - Added spawn queue and quota audit events. + +### Store And Config + +- `internal/store/migrations.go` + - Added `owner_quotas` persistence. +- `internal/store/sqlite.go` + - Added owner quota CRUD. +- `internal/config/config.go` + - Added spawn queue and API rate-limit configuration. + +### SDKs And Docs + +- `sdk/js/src/client.ts` and `sdk/js/src/types.ts` + - Added quota summary and admission helpers/types. +- `sdk/python/stacyvm/client.py`, `sdk/python/stacyvm/async_client.py`, and `sdk/python/stacyvm/models.py` + - Added quota summary and admission helpers/models. +- `docs/api.md`, `docs/swagger.yaml`, `docs/swagger.json`, and `docs/docs.go` + - Documented and regenerated the Phase 3 API surface. + +## Verification + +The following checks passed during Phase 3 closeout: + +```sh +go test ./internal/api/routes ./internal/orchestrator +make build +cd web && npm run build +make test +``` + +Full `make test` requires local socket-binding permission for `httptest` integration servers in this sandboxed environment. + +## Platform Notes + +- Docker daemon validation remains host-gated when the local sandbox cannot access Docker. +- Firecracker conformance remains Linux/KVM-gated. +- PRoot conformance remains gated on a real `proot` binary and usable rootfs. + +## Impact + +Phase 3 makes StacyVM meaningfully safer for shared usage: + +- Operators can assign persistent per-owner policy. +- Clients can preflight work before spawning provider resources. +- Burst load can queue instead of failing immediately. +- Queue pressure is observable in JSON and Prometheus metrics. +- API rate limiting protects the control plane. +- SDKs expose the new control-plane helpers directly. + +## Next Phase Direction + +Phase 4 should focus on distributed production operation: durable distributed scheduling semantics, deployment/CI hardening, runtime conformance on real platform hosts, and deeper admin workflows for quota policy management. diff --git a/docs/releases/phase-4-production-deployment.md b/docs/releases/phase-4-production-deployment.md new file mode 100644 index 0000000..e861a40 --- /dev/null +++ b/docs/releases/phase-4-production-deployment.md @@ -0,0 +1,154 @@ +# Phase 4 Production Deployment Release Notes + +Date: 2026-05-08 +Branch: `phase-4-production-deployment` + +## Summary + +Phase 4 starts turning the production control-plane work from earlier phases into repeatable shipping and deployment workflows. This checkpoint adds GitHub Actions CI coverage plus operator-facing deployment templates and runbooks for single-node production installations. + +The goal of this phase is to make StacyVM easier to validate, release, and run outside a developer laptop while keeping host-specific runtime conformance explicit. + +## What Changed + +### Continuous Integration + +- Added a GitHub Actions workflow for core project verification. +- CI now validates: + - Go tests across the repository. + - CLI build. + - Swagger/OpenAPI drift. + - Web dashboard production build. + - TypeScript SDK build. + - Python SDK package install, compile, and import. +- Stabilized the Swagger drift check for cold CI runners by downloading Go modules before invoking `swag`. + +### Production Deployment Templates + +- Added `deploy/docker-compose.yml` for a production-oriented Docker provider deployment with Traefik live-preview routing. +- Added `deploy/stacyvm.production.yaml` with production defaults for: + - API auth. + - API rate limiting. + - sandbox caps and queue backpressure. + - JSON logging. + - persistent SQLite state. + - Docker as the default provider. +- Added Compose and systemd environment templates: + - `deploy/.env.example` + - `deploy/stacyvm.env.example` +- Added `deploy/stacyvm.service` for binary-based Linux/systemd installs. + +### Release Automation + +- Added a release workflow for tag-driven and manually-dispatched releases. +- Release automation builds static Linux binary artifacts for `amd64` and `arm64`. +- Release automation publishes multi-arch container images to `ghcr.io/stacyos/stacyvm`. +- Docker image builds now accept an explicit `VERSION` build argument. +- Release artifacts now build into `dist/` instead of the repository root. +- Added `.dockerignore` to keep local build outputs and dependency directories out of release image contexts. + +### Deployment Runbook + +- Added `docs/deployment.md` covering: + - host requirements. + - Docker Compose deployment. + - systemd deployment. + - health, readiness, liveness, and Prometheus endpoints. + - reverse proxy expectations. + - SQLite backup and restore basics. + - upgrade procedure. + - Docker, Firecracker, and PRoot provider notes. +- Linked the deployment guide from the README. +- Added `scripts/smoke-deployment.sh` for liveness, health, readiness, and Prometheus deployment probes. +- Added `docs/runtime-conformance.md` with host requirements and signoff checks for Docker, gVisor, Kata, Firecracker, PRoot, E2B, and custom providers. +- Registered the mock provider in `stacyvm serve` when `providers.mock.enabled` is set, giving operators and CI a no-Docker smoke path. +- Validated the production Compose template with StacyVM, Traefik, Docker provider readiness, API smoke probes, and a port `3000` live-preview route. + +## Code Changes By Area + +### CI + +- `.github/workflows/ci.yml` + - Adds repository verification jobs for Go, Swagger, web, and SDKs. + - Opts into Node 24-based JavaScript actions to address the GitHub Actions Node 20 deprecation warning. + - Runs a mock-provider deployment smoke job against the production smoke script. +- `.github/workflows/release.yml` + - Adds binary and container image release automation. +- `scripts/check-swagger.sh` + - Downloads modules before generating docs in a temporary workspace. +- `scripts/ci-smoke-deployment.sh` + - Starts StacyVM with the mock provider and runs deployment smoke probes in CI. +- `cmd/stacyvm/cmd_serve.go` + - Registers the mock provider when enabled in config. + +### Deployment + +- `deploy/docker-compose.yml` + - Adds a reusable production Compose template. + - Allows the Traefik host port to be overridden for smoke runs. +- `deploy/stacyvm.production.yaml` + - Adds a production baseline config. +- `deploy/stacyvm.service` + - Adds a systemd unit for running the StacyVM binary. +- `deploy/.env.example` and `deploy/stacyvm.env.example` + - Add environment templates for Compose and systemd. +- `Dockerfile` + - Adds BuildKit platform args and explicit version injection for release image publishing. +- `Makefile` + - Moves release artifacts into `dist/` and keeps checksums with the artifacts. +- `.dockerignore` + - Excludes build outputs, local dependency directories, and local state files from Docker build contexts. +- `scripts/smoke-deployment.sh` + - Adds a portable post-deploy smoke test for live, health, readiness, and Prometheus metrics endpoints. + +### Docs + +- `docs/deployment.md` + - Adds the deployment guide and operator runbook. +- `docs/releasing.md` + - Adds release workflow and GHCR publishing instructions. +- `docs/runtime-conformance.md` + - Adds provider/runtime production signoff expectations. +- `README.md` + - Links the deployment guide from navigation and configuration docs. +- `CHANGELOG.md` + - Adds this Phase 4 checkpoint entry. + +## Verification + +The following checks passed during this checkpoint: + +```sh +docker compose --env-file deploy/.env.example -f deploy/docker-compose.yml config +ruby -e 'require "yaml"; YAML.load_file("deploy/docker-compose.yml"); YAML.load_file("deploy/stacyvm.production.yaml")' +git diff --check +go test ./... +cd web && npm run build +scripts/check-swagger.sh +make release-build-all VERSION=phase-4-test +scripts/smoke-deployment.sh http://127.0.0.1:7423 +scripts/ci-smoke-deployment.sh +docker build --build-arg VERSION=phase-4-compose -t stacyvm:phase-4-compose . +docker compose -p stacyvm-phase4-smoke -f deploy/docker-compose.yml up -d +scripts/smoke-deployment.sh http://127.0.0.1:17426 phase4-compose-key +curl -H 'Host: 3000-.localhost' http://127.0.0.1:18080/ +``` + +GitHub Actions has also passed for the initial Phase 4 CI workflow after the Swagger drift check stabilization. + +## Platform Notes + +- Docker Compose validation does not require daemon access, but runtime sandbox conformance still requires Docker daemon access on the host. +- Firecracker remains Linux/KVM-gated and should be rolled out only after host conformance checks pass. +- PRoot remains gated on a real `proot` binary and a rootfs with the expected sandbox tooling. + +## Phase 4 Closeout + +Phase 4 implementation is complete. Release automation, container publishing workflow, deployment smoke testing, production deployment templates, CI coverage, and runtime conformance documentation are in place. + +The remaining work is external release/platform operation: + +- Trigger a real versioned release run when the project is ready to publish binaries and the GHCR image. +- Collect real-host runtime signoffs for optional host-gated runtimes such as gVisor, Kata, Firecracker, and PRoot. + +Phase 5 can proceed from this branch without more Phase 4 code work. diff --git a/docs/releases/phase-5-admin-control-plane.md b/docs/releases/phase-5-admin-control-plane.md new file mode 100644 index 0000000..cd161d7 --- /dev/null +++ b/docs/releases/phase-5-admin-control-plane.md @@ -0,0 +1,73 @@ +# Phase 5 Admin Control Plane Release Notes + +Date: 2026-05-08 +Branch: `phase-5-admin-control-plane` + +## Summary + +Phase 5 delivers the first operator/admin control plane for StacyVM. This phase builds on Phase 3 quotas and Phase 4 production deployment by separating admin access from regular API usage and preparing the API surface for safer dashboard-driven operations. + +## What Changed + +### Admin Authentication + +- Added optional `auth.admin_api_key` config. +- Added `STACYVM_AUTH_ADMIN_API_KEY` environment variable support through the existing config loader. +- Added `X-Admin-API-Key` support for admin requests. +- Admin keys can authenticate normal API requests, but normal API keys cannot access admin routes when an admin key is configured. +- When no admin key is configured, admin routes fall back to `auth.api_key` for backwards compatibility. + +### Admin Route Namespace + +- Added `/api/v1/admin/*` operator route aliases for: + - providers + - quotas + - diagnostics + - JSON metrics + - Prometheus metrics +- Existing non-admin routes remain available for compatibility during this phase. + +### Deployment And Docs + +- Added admin key examples to production config, Compose env, systemd env, README, deployment docs, and API docs. +- Added an admin control-plane operator guide covering dashboard setup, quotas, diagnostics, audit export, and storage notes. + +### Dashboard Admin Workflows + +- Added dashboard settings for a regular API key and a separate admin API key. +- The shared web API client now sends `X-API-Key` and `X-Admin-API-Key` from browser settings. +- Provider list, provider detail, provider health tests, and JSON metrics now call `/api/v1/admin/*`. +- Provider cards now understand the backend `default`, latency, runtime count, capability, and error fields. +- Added an Operations dashboard page for owner quota management and diagnostics. +- Added dashboard workflows for quota list, save, delete, summary, owner usage checks, and redacted diagnostics. + +### Admin Audit History + +- Added persisted admin audit logs for admin route access. +- Added `/api/v1/admin/audit` to list recent redacted admin audit records. +- Records include actor, method, path, status, duration, request ID, client address, user agent, and timestamp. +- Added an Audit tab to the Operations dashboard. +- Added audit filters for actor, HTTP method, status, and path substring. +- Added CSV export for filtered audit history. +- Added `auth.admin_audit_retention` for native audit log pruning. +- Production templates keep 90 days of admin audit history with `2160h`. + +## Verification + +```sh +go test ./... +npm run build +``` + +GitHub CI passed on `phase-5-admin-control-plane` for: + +- Go tests and CLI build +- Swagger drift check +- Python SDK import check +- TypeScript SDK build +- Deployment smoke test +- Web build + +## Release Status + +Phase 5 is complete and published as the `phase-5-admin-control-plane` GitHub release. diff --git a/docs/releases/phase-6-security-governance.md b/docs/releases/phase-6-security-governance.md new file mode 100644 index 0000000..8ac404f --- /dev/null +++ b/docs/releases/phase-6-security-governance.md @@ -0,0 +1,73 @@ +# Phase 6 Security Governance Release Notes + +Date: 2026-05-08 +Branch: `phase-6-security-governance` + +## Summary + +Phase 6 adds the first security and governance layer above the Phase 5 admin control plane. This phase keeps current API-key deployments compatible while adding typed request identity metadata, route-level scope enforcement, safer audit attribution, configurable admin fallback policy, and production governance guidance for future RBAC and OIDC/SSO work. + +## What Changed + +### Request Identity Foundation + +- Added request-scoped authentication identities in the API middleware. +- Added explicit `api` and `admin` roles. +- Added initial scope metadata: + - `api:*` for regular API identities. + - `api:*` and `admin:*` for admin identities. +- Admin keys used through either supported key header are now represented as admin identities. +- Regular API keys remain regular API identities and still cannot access admin routes when `auth.admin_api_key` is configured. + +### Route-Level Scope Enforcement + +- Added a reusable `RequireScope` middleware. +- Wired authenticated admin routes through `RequireScope("admin:*")`. +- Kept unauthenticated development mode behavior unchanged when no API keys are configured. + +### Audit Attribution + +- Admin audit fallback attribution now reads the authenticated role and key header from request context when no `X-User-ID` actor is supplied. +- Fallback actors are now more specific, such as `admin:X-Admin-API-Key`. +- Existing explicit actor behavior is preserved: `X-User-ID` still wins when supplied. + +### Admin Fallback Policy + +- Added `auth.admin_fallback_enabled`. +- Kept the default as `true` for backwards compatibility. +- Production templates set it to `false` so admin routes require a dedicated `auth.admin_api_key`. + +### Production Security Guidance + +- Added [security-governance.md](../security-governance.md). +- Documented production admin posture, operator attribution, key handling, and audit retention guidance. +- Added an OIDC/SSO config and claims-mapping design that reuses the Phase 6 request identity and `RequireScope` model. +- Added a Phase 6 acceptance checklist for production deployments. + +## Compatibility + +- No deployment config changes are required. +- Existing `X-API-Key` and `X-Admin-API-Key` behavior is preserved. +- Admin route fallback to `auth.api_key` remains available by default when no separate `auth.admin_api_key` is configured. + +## Verification + +```sh +go test ./internal/api/middleware +go test ./internal/api +go test ./... +npm run build +``` + +GitHub CI passed on `phase-6-security-governance` for: + +- Go tests and CLI build +- Swagger drift check +- Python SDK import check +- TypeScript SDK build +- Deployment smoke test +- Web build + +## Release Status + +Phase 6 implementation is complete. The branch is ready for the Phase 6 GitHub release. diff --git a/docs/releases/phase-7-release-candidate-hardening.md b/docs/releases/phase-7-release-candidate-hardening.md new file mode 100644 index 0000000..4acda25 --- /dev/null +++ b/docs/releases/phase-7-release-candidate-hardening.md @@ -0,0 +1,66 @@ +# Phase 7 Release Candidate Hardening Release Notes + +Date: 2026-05-08 +Branch: `phase-7-release-candidate-hardening` + +## Summary + +Phase 7 starts the release-candidate hardening track. The goal is to move StacyVM from production-oriented foundations toward a single-node release candidate that operators can validate before trusting it with real workloads. + +## What Changed + +### Doctor Command + +- Added `stacyvm doctor`. +- Added `stacyvm doctor --production` for stricter production posture checks. +- Initial diagnostics cover: + - config loading + - API key posture + - admin key and admin fallback posture + - database path persistence + - Docker CLI and daemon availability + - Docker network and capability settings + - Firecracker binary, `/dev/kvm`, kernel, and agent paths + - PRoot binary, rootfs, and workspace base paths + +### Production Readiness + +- Added [production-readiness.md](../production-readiness.md). +- Documented readiness levels for internal staging, single-node production, public self-serve, and enterprise/multi-worker operation. +- Added Phase 7 acceptance criteria and release-candidate gates. + +### Threat Model + +- Added [threat-model.md](../threat-model.md). +- Documented assets, trust boundaries, primary threats, current mitigations, and Phase 7 security objectives. + +### Exec Semantics + +- Added explicit exec modes: + - `shell` preserves the existing `/bin/sh -c` behavior. + - `argv` runs direct process arguments without shell interpolation. +- Updated Docker, mock, PRoot, Firecracker agent protocol, custom provider passthrough, and CLI exec handling. +- Made `stacyvm exec` use argv mode by default, with `--shell` available for shell expressions. +- Added tests that verify argv payloads are treated literally. + +### Final Hardening + +- Added persisted operation audit records for sandbox lifecycle, exec, and file operations. +- Tightened pooled file path traversal behavior to reject workspace escapes instead of silently clamping paths. +- Expanded traversal tests across write, read, list, delete, move, chmod, stat, and glob operations. +- Added `scripts/certify-runtime.sh` and [runtime-certification.md](../runtime-certification.md) for Docker, gVisor/Kata, Firecracker, and PRoot host checks. +- Added remediation guidance to failing and warning `stacyvm doctor` checks. + +## Verification + +```sh +go test ./cmd/stacyvm +go test ./internal/providers +go test ./... +npm run build +scripts/check-swagger.sh +``` + +## Phase 7 Completion Status + +Phase 7 is complete from a codebase and CI perspective. Remaining production signoff is host-gated: run `stacyvm doctor --production`, `scripts/certify-runtime.sh`, and live provider conformance on the actual Linux/Docker/KVM/PRoot hosts selected for release. diff --git a/docs/releases/phase-8-single-node-production.md b/docs/releases/phase-8-single-node-production.md new file mode 100644 index 0000000..ec94296 --- /dev/null +++ b/docs/releases/phase-8-single-node-production.md @@ -0,0 +1,69 @@ +# Phase 8 Single-Node Production Release Notes + +Date: 2026-05-08 +Branch: `phase-8-single-node-production` + +## Summary + +Phase 8 moves StacyVM from release-candidate hardening toward technical self-hosted production on a single node. The phase focuses on safe SQLite backup/restore workflows, deterministic production config linting, upgrade rehearsal, and redacted support bundles because single-node operators need reliable rollback points and clear support artifacts before upgrades, config changes, and provider certification. + +## What Changed + +### Database Backup And Restore + +- Added `stacyvm db backup `. +- Added `stacyvm db restore --yes`. +- Backup uses SQLite `VACUUM INTO` and verifies the backup with `PRAGMA integrity_check`. +- Restore verifies the backup before replacing the target database. +- Restore creates a timestamped `.pre-restore-*` safety copy of the existing database. +- Restore removes stale `-wal` and `-shm` sidecars before replacing the target database. +- Added tests for backup/restore, overwrite protection, integrity validation, safety copy creation, and sidecar cleanup. + +### Production Config Linting + +- Added `stacyvm config lint`. +- Added `stacyvm config lint --production` to treat production hardening issues as failures. +- Added `--file` support so operators can lint a target config file without relying on the default lookup path. +- Linting checks authentication posture, placeholder secrets, admin key separation, admin fallback, audit retention, rate limiting, database durability, runtime caps, exec timeouts, JSON logging, and Docker hardening. +- Added deterministic lint tests that do not require Docker, KVM, or a running StacyVM server. + +### Upgrade Rehearsal + +- Added `stacyvm upgrade rehearse`. +- Rehearsal runs production config lint checks and SQLite integrity checks. +- Rehearsal validates the intended backup directory and refuses an already-existing backup output path. +- Rehearsal prints the recommended upgrade and rollback flow for single-node operators. +- Added `--include-doctor` for live host checks when running on the target host. + +### Redacted Support Bundle + +- Added `stacyvm support bundle `. +- Bundle includes version/runtime data, redacted config shape, production config lint output, optional doctor checks, and optional `/api/v1/diagnostics` output. +- Redaction covers secret-like keys, API keys, bearer tokens, token/password/secret assignments, and URLs with embedded credentials. +- Added tests to ensure final support JSON does not leak representative secrets. + +### Runtime Certification Artifacts + +- Upgraded `scripts/certify-runtime.sh` to emit host certification reports in `text`, `json`, or `markdown`. +- Added `--output` support so Docker, gVisor, Kata, Firecracker, and PRoot host checks can be attached to deployment records. +- Added stricter optional Firecracker and PRoot path checks through `STACYVM_FIRECRACKER_KERNEL`, `STACYVM_PROOT_ROOTFS`, and `STACYVM_PROOT_WORKSPACE_BASE`. +- Documented required Phase 8 signoff artifacts for target infrastructure. + +### Documentation + +- Updated deployment backup and upgrade guidance to prefer `stacyvm db backup`. +- Updated deployment and release guidance to run `stacyvm config lint --production` before staging, upgrades, and release tags. +- Added backup, config lint, upgrade rehearsal, and support bundle commands to the README command list. +- Updated runtime certification and conformance docs to require host-generated certification artifacts. + +## Verification + +```sh +go test ./cmd/stacyvm +go test ./... +stacyvm config lint --production --file deploy/stacyvm.production.yaml +``` + +## Next Phase 8 Direction + +Phase 8 is now functionally complete for single-node technical production readiness. Remaining signoff is final cleanup: run the full build/test sweep, confirm GitHub CI, and keep platform-gated Docker/gVisor/Kata/Firecracker/PRoot conformance results attached to host-specific certification. diff --git a/docs/releases/phase-9-public-self-serve-release-trust.md b/docs/releases/phase-9-public-self-serve-release-trust.md new file mode 100644 index 0000000..9abcc06 --- /dev/null +++ b/docs/releases/phase-9-public-self-serve-release-trust.md @@ -0,0 +1,87 @@ +# Phase 9 Public Self-Serve Release Trust Release Notes + +Date: 2026-05-08 +Branch: `phase-9-public-self-serve-release-trust` + +## Summary + +Phase 9 starts the public self-serve readiness track. This first slice focuses on release trust: users installing StacyVM from GitHub or GHCR should be able to verify that binaries, checksums, and container images came from the StacyVM release workflow. + +## What Changed + +### Signed Release Artifacts + +- Release binaries are signed with Sigstore keyless signing. +- `checksums.txt` is signed with Sigstore keyless signing. +- The release workflow publishes `.sig` and `.pem` files next to each signed artifact. +- The GHCR image digest is signed after the multi-arch image is published. + +### Public Verification + +- Added `scripts/verify-release.sh [amd64|arm64]`. +- The verifier downloads release binaries, checksums, signatures, and certificates. +- The verifier checks the expected GitHub Actions OIDC issuer and StacyVM release workflow identity. +- The verifier runs SHA-256 checksum verification after signature verification succeeds. + +### Installer Hardening + +- `scripts/install.sh` verifies Sigstore signatures automatically when `cosign` is installed. +- `STACYVM_REQUIRE_SIGNATURES=true` makes the installer fail closed when `cosign` is unavailable. +- The installer still verifies SHA-256 checksums. + +### Documentation + +- Added release verification instructions to the README. +- Expanded release documentation with binary, checksum, and container image verification commands. +- Added Phase 9 acceptance criteria to the production readiness checklist. +- Added a public self-serve support and limitations matrix. + +### Upgrade And Migration CI + +- Added `scripts/ci-upgrade-migration.sh`. +- Added a CI job that runs focused config, upgrade rehearsal, and SQLite migration checks. +- Added coverage for migrating a legacy v1 SQLite database through the current schema. +- Made Docker integration tests opt-in with `STACYVM_DOCKER_INTEGRATION=1` so default CI is not coupled to Docker Hub availability or runner daemon state. + +### Public Release Sanity + +- Added `scripts/ci-public-release-sanity.sh`. +- Added `scripts/post-release-validate.sh ` for the post-tag release gate. +- CI now syntax-checks public install and verification scripts. +- CI builds release binaries for supported architectures and verifies `checksums.txt`. +- Real GitHub release asset verification is now executable as a post-tag drill for each published version. + +### Diagnostics Remediation + +- Added remediation links to `/api/v1/diagnostics`. +- Diagnostics now point operators to production readiness, deployment, runtime certification, runtime conformance, release verification, support bundle, and security governance docs. + +### SDK Parity + +- Added mock-based TypeScript and Python SDK parity smoke tests. +- TypeScript spawn options now include `template`, matching Python spawn behavior. +- Python SDK now exposes `templates` and `providers()` helpers for closer TypeScript parity. + +### Support Intake + +- Added GitHub issue forms for bug reports and production support requests. +- Issue templates ask for support bundle, config lint, upgrade rehearsal, runtime certification, release verification, environment, and logs. + +## Verification + +```sh +bash -n scripts/install.sh +bash -n scripts/verify-release.sh +bash -n scripts/ci-upgrade-migration.sh +scripts/post-release-validate.sh --help +scripts/ci-upgrade-migration.sh +scripts/ci-public-release-sanity.sh +bun test +python -m unittest sdk/python/tests/test_client_parity.py +git diff --check +go test ./... +``` + +## Remaining Phase 9 Direction + +Phase 9 is now complete from a branch-readiness perspective. The only release-time follow-up is to run `scripts/post-release-validate.sh ` against the actual GitHub assets after the next real version tag is published. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..c87d63f --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,154 @@ +# Releasing StacyVM + +StacyVM releases publish two deliverables: + +- Static Linux binaries for `stacyvm` and `stacyvm-agent` under the GitHub release. +- A multi-arch container image at `ghcr.io/stacyos/stacyvm`. + +Release binaries, `checksums.txt`, and the published container image digest are +signed with Sigstore keyless signing from the GitHub Actions release workflow. + +## Release Workflow + +The release workflow lives at `.github/workflows/release.yml`. + +It runs automatically for tags that match `v*`: + +```bash +git tag v0.4.0 +git push origin v0.4.0 +``` + +It can also be started manually from GitHub Actions with: + +- `version`: release version or image tag, for example `v0.4.0`. +- `publish_image`: whether to publish the GHCR image. +- `create_release`: whether to create a GitHub release with binary artifacts. + +Tag-triggered releases always build binaries, create the GitHub release, and publish the container image. + +## Binary Artifacts + +Local release artifacts can be built with: + +```bash +make release-build-all VERSION=v0.4.0 +``` + +The command writes artifacts to `dist/`: + +- `stacyvm-linux-amd64` +- `stacyvm-agent-linux-amd64` +- `stacyvm-linux-arm64` +- `stacyvm-agent-linux-arm64` +- `checksums.txt` + +The release workflow also attaches: + +- `.sig` +- `.pem` + +for every binary and `checksums.txt`. + +## Verifying A Release + +Install `cosign`, then run: + +```bash +scripts/verify-release.sh v0.4.0 amd64 +scripts/verify-release.sh v0.4.0 arm64 +``` + +The verifier checks: + +- Sigstore certificate identity for the StacyVM release workflow. +- Sigstore certificate issuer from GitHub Actions OIDC. +- Binary and agent SHA-256 entries in `checksums.txt`. + +Manual verification for one artifact: + +```bash +cosign verify-blob stacyvm-linux-amd64 \ + --signature stacyvm-linux-amd64.sig \ + --certificate stacyvm-linux-amd64.pem \ + --certificate-identity-regexp 'https://github.com/StacyOS/stacyvm/.github/workflows/release.yml@refs/tags/v.*' \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' +``` + +## Container Image + +The release workflow publishes: + +- `ghcr.io/stacyos/stacyvm:` +- `ghcr.io/stacyos/stacyvm:latest` for `v*` tag releases + +The image digest is signed after publishing: + +```bash +cosign verify ghcr.io/stacyos/stacyvm@sha256: \ + --certificate-identity-regexp 'https://github.com/StacyOS/stacyvm/.github/workflows/release.yml@refs/tags/v.*' \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' +``` + +The Dockerfile accepts a `VERSION` build argument and uses BuildKit target platform args so the release workflow can publish `linux/amd64` and `linux/arm64` images from one workflow. + +To test the image locally before publishing: + +```bash +docker build --build-arg VERSION=dev -t stacyvm:dev . +docker run --rm stacyvm:dev version +``` + +## Preflight Checklist + +Before tagging: + +```bash +make test +make build +cd web && npm run build +scripts/check-swagger.sh +stacyvm config lint --production --file deploy/stacyvm.production.yaml +make release-build-all VERSION=v0.4.0 +``` + +When linting the production template, provide real `STACYVM_AUTH_API_KEY` and `STACYVM_AUTH_ADMIN_API_KEY` values through the environment so placeholder secrets do not pass the release gate. + +For Phase 4, also confirm the production deployment templates still render: + +```bash +docker compose --env-file deploy/.env.example -f deploy/docker-compose.yml config +``` + +After the release workflow publishes artifacts, verify both architectures: + +```bash +scripts/verify-release.sh v0.4.0 amd64 +scripts/verify-release.sh v0.4.0 arm64 +``` + +Or run the full post-release gate: + +```bash +scripts/post-release-validate.sh v0.4.0 +STACYVM_VALIDATE_INSTALLER=true scripts/post-release-validate.sh v0.4.0 +``` + +The full gate confirms that every binary, checksum, signature, and certificate +asset exists on the GitHub release, runs signature and checksum verification for +both architectures, and can exercise `scripts/install.sh` in verify-only mode on +Linux. + +For a GitHub-hosted Linux evidence bundle, run the manual **Public Readiness +Certification** workflow against the published tag. It validates the release, +runs installer verify-only, certifies the selected runtime on the runner, and +uploads the generated Markdown reports. Treat this as CI-host evidence only; +production-host runtime claims still require `scripts/certify-runtime.sh` on +the actual host. + +## Notes + +- Do not store release secrets in `stacyvm.production.yaml`; pass them through environment variables. +- Keep release notes in `docs/releases/` up to date before creating a GitHub release. +- Do not publish public self-serve releases without Sigstore signatures and checksums. +- Platform conformance for Docker, gVisor/Kata, Firecracker, and PRoot remains host-gated and should be reported separately from generic build health. diff --git a/docs/remote-worker-staging.md b/docs/remote-worker-staging.md new file mode 100644 index 0000000..d5a7df6 --- /dev/null +++ b/docs/remote-worker-staging.md @@ -0,0 +1,191 @@ +# Remote Worker Staging + +This guide runs StacyVM as two local processes: one control plane and one remote worker. Use the mock provider first so you can verify worker registration, remote spawn, remote status refresh, remote exec, and remote destroy before introducing Docker, Firecracker, or a real network boundary. + +Remote worker mode can use a shared worker token for internal staging, per-worker static tokens for migration, or signed worker tokens for production-aligned staging. It is not the full enterprise production target yet because SQLite store semantics are still single-node oriented. + +## Prerequisites + +- Built `stacyvm` binary. +- One terminal for `stacyvm serve`. +- One terminal for `stacyvm worker`. +- A random worker token shared by both processes, a per-worker token configured under `auth.worker_tokens`, or a signed-token setup using `auth.worker_signing_key`. + +## Control Plane Config + +Create `control-plane.yaml`: + +```yaml +server: + host: "127.0.0.1" + port: 7423 + +providers: + default: "mock" + mock: + enabled: true + docker: + enabled: false + firecracker: + enabled: false + +auth: + api_key: "dev-api-key-dev-api-key-dev-api-key" + admin_api_key: "dev-admin-key-dev-admin-key-dev" + worker_token: "dev-worker-token-dev-worker-token" + worker_tokens: + worker-a: "dev-worker-a-token-dev-worker-a-token" + admin_fallback_enabled: false + +database: + driver: "sqlite" + path: "/tmp/stacyvm-remote-worker-staging.db" +``` + +Start the control plane: + +```bash +STACYVM_CONFIG=control-plane.yaml stacyvm serve +``` + +## Worker Config + +Create `worker.yaml`: + +```yaml +worker: + id: "worker-a" + control_plane_url: "http://127.0.0.1:7423" + listen_addr: "127.0.0.1:7430" + preview_domain: "localhost" + heartbeat_interval: "2s" + +providers: + default: "mock" + mock: + enabled: true + docker: + enabled: false + firecracker: + enabled: false + +auth: + worker_token: "dev-worker-a-token-dev-worker-a-token" +``` + +Start the worker: + +```bash +STACYVM_CONFIG=worker.yaml stacyvm worker +``` + +## Smoke Flow + +List workers: + +```bash +curl -sS -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + http://127.0.0.1:7423/api/v1/workers +``` + +Spawn a sandbox. The scheduler should select `worker-a` once its heartbeat is fresh: + +```bash +curl -sS -X DELETE \ + -H "X-Admin-API-Key: dev-admin-key-dev-admin-key-dev" \ + http://127.0.0.1:7423/api/v1/admin/workers/local + +curl -sS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + -d '{"image":"alpine:latest","provider":"mock","ttl":"5m"}' \ + http://127.0.0.1:7423/api/v1/sandboxes +``` + +The delete call is only for this single-machine smoke flow. The control plane self-registers as `local`, and the scheduler prefers a fresh eligible local worker. Removing that transient registry record forces the next spawn to exercise `worker-a`. + +Expected response fields: + +```json +{ + "worker_id": "worker-a", + "preview_domain": "localhost", + "state": "running" +} +``` + +Get the sandbox. This refreshes status through `worker.status`: + +```bash +curl -sS -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + http://127.0.0.1:7423/api/v1/sandboxes/ +``` + +Run a command. This routes through `worker.exec` and records a normal control-plane exec log: + +```bash +curl -sS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + -d '{"command":"echo remote worker ok"}' \ + http://127.0.0.1:7423/api/v1/sandboxes//exec +``` + +Run a streamed command. This routes through `worker.exec_stream` and forwards live stdout/stderr chunks through the normal API streaming response: + +```bash +curl -sS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + -d '{"command":"printf \"line 1\\nline 2\\n\"","stream":true}' \ + http://127.0.0.1:7423/api/v1/sandboxes//exec +``` + +Write and read a file. These route through the worker file RPC methods: + +```bash +curl -sS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + -d '{"path":"/workspace/remote.txt","content":"remote file"}' \ + http://127.0.0.1:7423/api/v1/sandboxes//files + +curl -sS -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + "http://127.0.0.1:7423/api/v1/sandboxes//files?path=/workspace/remote.txt" +``` + +Fetch console logs. This routes through `worker.logs`: + +```bash +curl -sS -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + "http://127.0.0.1:7423/api/v1/sandboxes//logs?lines=50" +``` + +Destroy the sandbox. This routes through `worker.destroy`, updates state, and releases the lease: + +```bash +curl -sS -X DELETE \ + -H "X-API-Key: dev-api-key-dev-api-key-dev-api-key" \ + http://127.0.0.1:7423/api/v1/sandboxes/ +``` + +## Automated Smoke Script + +You can run the same flow with: + +```bash +scripts/smoke-remote-worker.sh ./stacyvm +``` + +The script starts both processes, waits for worker registration, spawns a mock sandbox, verifies ownership by `worker-a`, and destroys it. + +## Current Limits + +- Remote non-streaming exec is routed to remote workers. +- Remote streaming exec is routed through live NDJSON worker RPC responses. +- Remote file APIs are routed to remote workers. +- Remote logs are routed to remote workers. +- Remote preview URL metadata is routed from worker heartbeat capacity. The actual preview ingress still depends on the worker/provider ingress setup, such as Docker plus Traefik on the worker host. +- Shared worker auth is suitable for local/internal staging only. Production-aligned workers should use signed worker identity, secret-file inputs, worker identity certification output, and mTLS when worker RPC crosses a host or network boundary. +- SQLite remains a staging/single-node store. Enterprise multi-worker mode still needs Postgres-grade lease semantics. +- Worker shutdown enters drain mode and rejects new spawns. Fresh draining workers keep existing sandbox ownership; stale/offline remote owners are marked `unhealthy`, and expired remote-owned sandboxes become `expired` with their lease released. diff --git a/docs/rest-api.md b/docs/rest-api.md new file mode 100644 index 0000000..17bfb4b --- /dev/null +++ b/docs/rest-api.md @@ -0,0 +1,1225 @@ +--- +title: "REST API Reference" +description: "Use StacyVM's HTTP API to create sandboxes, run commands, manage files, inspect providers, and operate clusters." +--- + +# StacyVM REST API Reference + +This document is the source of truth for the StacyVM HTTP API. The Python and TypeScript SDKs are thin wrappers over these endpoints — anything they do, you can do with `curl`. + +- **Base URL:** `http://localhost:7423/api/v1` +- **Content type:** `application/json` (request and response, except where noted) +- **OpenAPI spec:** [swagger.yaml](https://github.com/StacyOS/stacyvm/blob/main/docs/swagger.yaml) / [swagger.json](https://github.com/StacyOS/stacyvm/blob/main/docs/swagger.json) + +--- + +## Table of contents + +- [Authentication](#authentication) +- [Conventions](#conventions) +- [Errors](#errors) +- [Admin API](#admin-api) +- [Sandboxes](#sandboxes) +- [Files](#files) +- [Templates](#templates) +- [Providers](#providers) +- [Workers](#workers) +- [Snapshots](#snapshots) +- [Pool](#pool) +- [System](#system) +- [Events stream](#events-stream) +- [WebSocket exec](#websocket-exec) + +--- + +## Authentication + +Optional headers: + +| Header | Purpose | Required when | +|---|---|---| +| `X-API-Key` | API key authentication | `auth.enabled: true` in `stacyvm.yaml` | +| `X-Admin-API-Key` | Admin API key authentication | `auth.admin_api_key` is configured and calling `/api/v1/admin/*` | +| `X-User-ID` | Multi-tenant pool mode user identifier | `pool.enabled: true` | + +```bash +curl -H 'X-API-Key: sk-xyz123' \ + -H 'X-User-ID: alice@example.com' \ + http://localhost:7423/api/v1/sandboxes +``` + +CORS is permissive by default for local development (`server.cors_allowed_origins: ["*"]`). Public deployments should set exact origins, for example: + +```yaml +server: + cors_allowed_origins: + - "https://console.example.com" +``` + +`stacyvm config lint --production` fails when CORS is left wildcard or empty. + +`X-User-ID` is trimmed when present. It must be 128 characters or fewer and cannot contain whitespace, control characters, or path separators. + +--- + +## Rate limiting + +API rate limiting is optional and disabled by default. When `rate_limit.enabled` is true, StacyVM applies an in-memory token bucket to API routes. + +```yaml +rate_limit: + enabled: true + requests_per_minute: 120 + burst: 60 + key_by: owner # owner, api_key, or ip + bucket_ttl: 15m + cleanup_interval: 1m +``` + +The default `owner` mode uses `X-User-ID` when present, then falls back to `X-API-Key`, then client IP. Limited requests return `429 Too Many Requests` with `Retry-After`, `X-RateLimit-Limit`, and `X-RateLimit-Remaining` headers. + +Rate-limit buckets store hashed identity keys internally; raw owner IDs, API keys, and IP addresses are not exposed in diagnostics or metrics. + +--- + +## Conventions + +- **IDs.** Sandbox IDs look like `sb-a1b2c3d4`. Templates are addressed by `name`. +- **Durations.** All `ttl` and `timeout` fields use Go duration strings: `30s`, `5m`, `1h30m`. +- **Timestamps.** ISO 8601 UTC, e.g. `2026-05-04T10:30:00Z`. +- **File modes.** Octal strings, e.g. `"755"`, `"644"`. +- **Streaming.** `POST /sandboxes/{id}/exec` switches to NDJSON (`application/x-ndjson`) when `stream: true`. + +--- + +## Errors + +Errors return a JSON body with HTTP status reflecting the failure class: + +```json +{ + "code": "not_found", + "message": "sandbox sb-a1b2c3d4 not found" +} +``` + +| Status | Code | When | +|---|---|---| +| `400` | `bad_request` | Invalid input — missing field, malformed JSON | +| `401` | `unauthorized` | Bad / missing API key | +| `404` | `not_found` | Sandbox / template / provider does not exist | +| `409` | `conflict` | Template name already exists | +| `429` | `resource_limit` | Quota, capacity, or API rate limit exceeded | +| `500` | `provider_error` | Provider failed (Docker, Firecracker, etc.) | +| `503` | `unavailable` | Pool full with `overflow: reject` | + +--- + +## Admin API + +StacyVM supports an optional separate admin API key: + +```yaml +auth: + api_key: "sk-client" + admin_api_key: "sk-admin" + admin_fallback_enabled: false +``` + +Use `X-Admin-API-Key` for admin requests. `X-API-Key` is still accepted when it matches the admin key. If `auth.admin_api_key` is empty, admin routes fall back to `auth.api_key` for backwards compatibility unless `auth.admin_fallback_enabled` is set to `false`. + +For dashboard setup, quota workflows, diagnostics, audit history, CSV export, and storage notes, see [admin-control-plane](/docs/admin-control-plane). + +Admin route aliases: + +| Method | Path | Purpose | +|---|---|---| +| `GET` | `/api/v1/admin/providers` | List providers with health details | +| `GET` | `/api/v1/admin/providers/{name}` | Provider detail | +| `POST` | `/api/v1/admin/providers/test` | Run provider health checks | +| `GET` | `/api/v1/admin/quotas` | List owner quota overrides | +| `GET` | `/api/v1/admin/quotas/summary` | Redacted quota coverage summary | +| `GET` | `/api/v1/admin/quotas/{ownerID}` | Get owner quota | +| `PUT` | `/api/v1/admin/quotas/{ownerID}` | Create or update owner quota | +| `GET` | `/api/v1/admin/quotas/{ownerID}/usage` | Owner usage against effective quota | +| `DELETE` | `/api/v1/admin/quotas/{ownerID}` | Delete owner quota | +| `GET` | `/api/v1/admin/diagnostics` | Redacted operational diagnostics | +| `GET` | `/api/v1/admin/metrics` | Structured JSON metrics | +| `GET` | `/api/v1/admin/metrics/prometheus` | Prometheus metrics | +| `GET` | `/api/v1/admin/audit` | Filterable admin audit history, with optional `format=csv` | + +The existing non-admin paths remain available for compatibility in this phase. + +--- + +## Sandboxes + +### Spawn a sandbox + +``` +POST /api/v1/sandboxes +``` + +**Request body** (all fields optional, server defaults apply): +```json +{ + "image": "python:3.12", + "provider": "docker", + "memory_mb": 1024, + "vcpus": 2, + "ttl": "1h", + "metadata": { "user": "alice" } +} +``` + +**Response** `201 Created`: +```json +{ + "id": "sb-a1b2c3d4", + "state": "running", + "provider": "docker", + "image": "python:3.12", + "memory_mb": 1024, + "vcpus": 2, + "created_at": "2026-05-04T10:30:00Z", + "expires_at": "2026-05-04T11:30:00Z", + "metadata": { "user": "alice" }, + "preview_domain": "localhost" +} +``` + +### Evaluate spawn admission + +``` +POST /api/v1/sandboxes/admission +``` + +Preflight a spawn request against current quota and scheduler limits without creating a sandbox. `X-User-ID` overrides `owner_id`, matching the spawn endpoint. + +**Request body**: same shape as `POST /api/v1/sandboxes`. + +**Response** `200 OK`: +```json +{ + "allowed": false, + "queueable": true, + "reason": "max_sandboxes", + "active_sandboxes": 100, + "max_sandboxes": 100, + "active_owner_sandboxes": 2, + "max_owner_sandboxes": 10, + "max_ttl": "24h0m0s" +} +``` + +`queueable` reflects the configured spawn overflow mode. Capacity denials are queueable only when `defaults.spawn_overflow` is `queue`; TTL denials are never queueable. + +### List sandboxes + +``` +GET /api/v1/sandboxes +``` + +**Response** `200 OK`: array of sandbox objects. + +### Get a sandbox + +``` +GET /api/v1/sandboxes/{id} +``` + +**Response** `200 OK` or `404 Not Found`. + +### Destroy a sandbox + +``` +DELETE /api/v1/sandboxes/{id} +``` + +**Response** `200 OK`: +```json +{ "status": "destroyed" } +``` + +### Prune expired sandboxes + +``` +DELETE /api/v1/sandboxes +``` + +**Response** `200 OK`: +```json +{ "pruned": 7 } +``` + +### Extend TTL + +``` +POST /api/v1/sandboxes/{id}/extend +``` + +**Request body**: +```json +{ "ttl": "1h" } +``` + +**Response** `200 OK`: full sandbox object with updated `expires_at`. + +### Execute a command + +``` +POST /api/v1/sandboxes/{id}/exec +``` + +**Request body**: +```json +{ + "command": "python3 -c 'print(40+2)'", + "args": ["--coverage"], + "env": { "NODE_ENV": "test" }, + "workdir": "/app", + "timeout": "30s", + "stream": false +} +``` + +**Response** `200 OK` (non-streaming): +```json +{ + "exit_code": 0, + "stdout": "42\n", + "stderr": "", + "duration": "127ms" +} +``` + +**Response** `200 OK` (streaming, `stream: true`): `application/x-ndjson` — one JSON object per line: +``` +{"stream":"stdout","data":"installing pandas...\n"} +{"stream":"stdout","data":"done\n"} +{"stream":"stderr","data":"warning: deprecated flag\n"} +``` + +### Console logs + +``` +GET /api/v1/sandboxes/{id}/logs?lines=200 +``` + +`lines` defaults to `100`. + +**Response** `200 OK`: +```json +["[init] mounting /workspace", "[init] starting agent", "..."] +``` + +--- + +## Files + +All file paths are absolute inside the sandbox. The endpoints below are scoped under `/sandboxes/{id}/files`. + +### Write a file + +``` +POST /api/v1/sandboxes/{id}/files +``` + +```json +{ "path": "/app/main.py", "content": "print('hi')", "mode": "644" } +``` + +**Response** `200 OK`: `{ "status": "written" }`. + +### Read a file + +``` +GET /api/v1/sandboxes/{id}/files?path=/app/main.py +``` + +**Response** `200 OK`: raw file contents (binary safe). The SDKs decode as UTF-8. + +### Delete a file or directory + +``` +DELETE /api/v1/sandboxes/{id}/files?path=/app/cache&recursive=true +``` + +`recursive` defaults to `false`. **Response** `200 OK`: `{ "status": "deleted" }`. + +### List a directory + +``` +GET /api/v1/sandboxes/{id}/files/list?path=/app +``` + +`path` defaults to `/`. + +**Response** `200 OK`: +```json +[ + { + "name": "main.py", + "path": "/app/main.py", + "size": 11, + "is_dir": false, + "mod_time": "2026-05-04T10:32:14Z", + "mode": "0644" + } +] +``` + +### Move / rename + +``` +POST /api/v1/sandboxes/{id}/files/move +``` + +```json +{ "old_path": "/app/main.py", "new_path": "/app/entry.py" } +``` + +**Response** `200 OK`: `{ "status": "moved" }`. + +### Change permissions + +``` +POST /api/v1/sandboxes/{id}/files/chmod +``` + +```json +{ "path": "/app/run.sh", "mode": "755" } +``` + +**Response** `200 OK`: `{ "status": "chmod applied" }`. + +### Stat + +``` +GET /api/v1/sandboxes/{id}/files/stat?path=/app/main.py +``` + +**Response** `200 OK`: a single `FileInfo` object (same shape as list). + +### Glob + +``` +GET /api/v1/sandboxes/{id}/files/glob?pattern=/app/**/*.py +``` + +**Response** `200 OK`: +```json +["/app/main.py", "/app/utils/helpers.py"] +``` + +--- + +## Templates + +### Create a template + +``` +POST /api/v1/templates +``` + +```json +{ + "name": "python-dev", + "image": "python:3.12-slim", + "memory_mb": 1024, + "vcpus": 2, + "ttl": "1h", + "provider": "docker", + "metadata": { "language": "python" } +} +``` + +**Response** `201 Created`: the template object. `409 Conflict` if name is taken. + +### List templates + +``` +GET /api/v1/templates +``` + +**Response** `200 OK`: array of templates. + +### Get a template + +``` +GET /api/v1/templates/{name} +``` + +**Response** `200 OK` or `404 Not Found`. + +### Update a template + +``` +PUT /api/v1/templates/{name} +``` + +Same body as create (without `name`). **Response** `200 OK` or `404`. + +### Delete a template + +``` +DELETE /api/v1/templates/{name} +``` + +**Response** `200 OK`: `{ "status": "deleted" }`. + +### Spawn from a template + +``` +POST /api/v1/templates/{name}/spawn +``` + +Optional override body: +```json +{ "ttl": "30m", "provider": "firecracker" } +``` + +**Response** `201 Created`: full sandbox object. + +--- + +## Quotas + +Owner quotas are persisted overrides for per-owner sandbox and runtime limits. They apply when requests include an owner via `X-User-ID` or `owner_id`. + +Owner IDs are trimmed and must be 128 characters or fewer. They cannot contain whitespace, control characters, or path separators. Quota durations must use whole-second Go duration strings; use `0s` or omit a duration to inherit the global default. + +### List owner quotas + +``` +GET /api/v1/quotas +``` + +**Response** `200 OK`: +```json +[ + { + "owner_id": "team-a", + "max_sandboxes": 5, + "max_ttl": "2h0m0s", + "max_exec_timeout": "1m0s", + "created_at": "2026-05-08T10:30:00Z", + "updated_at": "2026-05-08T10:30:00Z" + } +] +``` + +### Get quota summary + +``` +GET /api/v1/quotas/summary +``` + +Returns redacted policy coverage counts without exposing owner IDs. + +**Response** `200 OK`: +```json +{ + "total": 2, + "with_max_sandboxes": 1, + "with_max_ttl": 1, + "with_max_exec_timeout": 1 +} +``` + +### Save owner quota + +``` +PUT /api/v1/quotas/{ownerID} +``` + +**Request**: +```json +{ + "max_sandboxes": 5, + "max_ttl": "2h", + "max_exec_timeout": "1m" +} +``` + +**Response** `200 OK`: full owner quota object. + +Invalid owner IDs, negative sandbox counts, malformed durations, sub-second durations, and fractional-second durations return `400 Bad Request`. + +### Get owner usage + +``` +GET /api/v1/quotas/{ownerID}/usage +``` + +**Response** `200 OK`: +```json +{ + "owner_id": "team-a", + "active_sandboxes": 3, + "max_sandboxes": 5, + "max_ttl": "2h0m0s", + "max_exec_timeout": "1m0s", + "quota_configured": true +} +``` + +### Delete owner quota + +``` +DELETE /api/v1/quotas/{ownerID} +``` + +**Response** `200 OK`: `{ "status": "deleted" }`. + +--- + +## Providers + +### List providers + +``` +GET /api/v1/providers +``` + +**Response** `200 OK`: +```json +[ + { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "exec_stream", "files", "console", "health", "runtime_inventory", "container"], + "runtime_count": 4 + }, + { + "name": "firecracker", + "healthy": false, + "default": false, + "latency_ms": 1, + "last_checked": "2026-05-08T10:30:00Z", + "error": "health check returned false", + "capabilities": ["spawn", "exec", "exec_stream", "files", "console", "health", "snapshots", "microvm", "vsock_agent"] + } +] +``` + +### Get a provider + +``` +GET /api/v1/providers/{name} +``` + +**Response** `200 OK`: +```json +{ + "name": "docker", + "healthy": true, + "default": true, + "sandbox_count": 12, + "health": { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "files", "runtime_inventory", "container"], + "runtime_count": 4 + }, + "config": { "runtime": "runc", "network_mode": "stacyvm-network" } +} +``` + +### Health-check all providers + +``` +POST /api/v1/providers/test +``` + +**Response** `200 OK`: +```json +{ "docker": true, "firecracker": true, "mock": true } +``` + +--- + +## Workers + +Worker registry endpoints expose the control-plane view of StacyVM workers. In single-node mode the API server registers itself as the `local` worker at startup. Remote workers heartbeat through `/api/v1/worker/*` using worker credentials. Admins can still manage registry records under `/api/v1/admin/workers/*`. + +### List workers + +``` +GET /api/v1/workers +``` + +**Response** `200 OK`: +```json +[ + { + "id": "local", + "hostname": "stacyvm-host-1", + "status": "online", + "providers": ["docker", "mock"], + "capabilities": ["api", "single_node", "spawn", "exec", "files"], + "capacity": { "max_sandboxes": 100, "max_sandboxes_per_owner": 10 }, + "last_heartbeat": "2026-05-08T10:30:00Z", + "created_at": "2026-05-08T10:00:00Z", + "updated_at": "2026-05-08T10:30:00Z", + "stale": false + } +] +``` + +### Get a worker + +``` +GET /api/v1/workers/{workerID} +``` + +**Response** `200 OK`: one worker object. + +### Heartbeat a worker + +``` +POST /api/v1/worker/{workerID}/heartbeat +``` + +Required headers: + +```text +X-Worker-ID: worker-a +X-Worker-Token: +``` + +**Request**: +```json +{ + "hostname": "worker-a.internal", + "status": "online", + "providers": ["docker"], + "capabilities": ["spawn", "exec", "files"], + "capacity": { "max_sandboxes": 50, "max_sandboxes_per_owner": 5 } +} +``` + +**Response** `200 OK`: updated worker object. + +Admin heartbeat aliases remain available at `/api/v1/admin/workers/{workerID}/heartbeat` for controlled registry repair and test setup. + +### Renew a worker lease + +``` +POST /api/v1/worker/{workerID}/leases/{resourceID}/renew +``` + +Required headers: + +```text +X-Worker-ID: worker-a +X-Worker-Token: +``` + +`auth.worker_token` is the shared staging token. For production-aligned worker identity, configure `auth.worker_signing_key` for short-lived signed worker tokens or `auth.worker_tokens.` for individually rotatable static credentials during migration. + +**Request**: +```json +{ "ttl": "30s" } +``` + +**Response** `200 OK`: +```json +{ + "lease": { + "resource_id": "sb-abc123", + "holder_id": "worker-a", + "generation": 4, + "expires_at": "2026-05-09T10:31:00Z" + } +} +``` + +### Delete a worker + +``` +DELETE /api/v1/admin/workers/{workerID} +``` + +**Response** `200 OK`: +```json +{ "status": "deleted" } +``` + +--- + +## Snapshots + +### List Firecracker snapshots + +``` +GET /api/v1/snapshots +``` + +**Response** `200 OK`: array of snapshot summaries (image name, kernel, size, created_at). + +--- + +## Pool + +### Pool status + +``` +GET /api/v1/pool/status +``` + +**Response** `200 OK` (pool enabled): +```json +{ + "enabled": true, + "vms": 3, + "max_vms": 20, + "total_users": 14, + "max_users_per_vm": 5 +} +``` + +**Response** `200 OK` (pool disabled): +```json +{ "enabled": false } +``` + +--- + +## System + +### Health + +``` +GET /api/v1/health +``` + +**Response** `200 OK`: +```json +{ "status": "ok", "version": "0.5.1", "uptime": "2h13m" } +``` + +### Liveness + +``` +GET /api/v1/live +``` + +**Response** `200 OK`: +```json +{ "status": "alive", "version": "0.5.1", "uptime": "2h13m" } +``` + +Use this endpoint for process liveness checks. It only confirms that the API process is responding. + +### Readiness + +``` +GET /api/v1/ready +``` + +**Response** `200 OK`: +```json +{ + "status": "ready", + "version": "0.5.1", + "uptime": "2h13m", + "ready_providers": 1, + "total_providers": 2, + "providers": [ + { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "files", "runtime_inventory", "container"], + "runtime_count": 4 + }, + { + "name": "firecracker", + "healthy": false, + "default": false, + "latency_ms": 1, + "last_checked": "2026-05-08T10:30:00Z", + "error": "health check returned false", + "capabilities": ["spawn", "exec", "files", "snapshots", "microvm", "vsock_agent"] + } + ] +} +``` + +**Response** `503 Service Unavailable` when no configured provider is healthy. + +### Diagnostics + +``` +GET /api/v1/diagnostics +``` + +**Response** `200 OK`: +```json +{ + "generated_at": "2026-05-08T10:30:00Z", + "build": { + "version": "0.5.1", + "goos": "linux", + "goarch": "amd64" + }, + "process": { + "uptime": "2h13m", + "goroutines": 42, + "memory": { + "alloc": 17825792, + "sys": 71303168, + "heap_alloc": 17825792, + "gc_cycles": 8 + } + }, + "store": { + "healthy": true, + "latency_ms": 1 + }, + "limits": { + "max_sandboxes": 100, + "max_sandboxes_per_owner": 10, + "default_exec_timeout": "30s", + "max_exec_timeout": "10m0s", + "max_ttl": "24h0m0s", + "spawn_overflow": "queue", + "spawn_queue_timeout": "30s", + "max_spawn_queue": 100 + }, + "scheduler": { + "spawn_overflow": "queue", + "spawn_queue_depth": 3, + "max_spawn_queue": 100, + "spawn_queue_timeout": "30s", + "admission_control": "worker_aware_local", + "spawn_queued_total": 18, + "spawn_dequeued_total": 16, + "spawn_queue_timeouts": 2, + "spawn_queue_wait_count": 18, + "spawn_queue_wait_total": "1m42s", + "spawn_queue_wait_max": "12s", + "spawn_queue_wait_avg": "5.666s", + "spawn_queue_wait_total_ms": 102000, + "spawn_queue_wait_max_ms": 12000, + "spawn_queue_wait_avg_ms": 5666, + "worker_id": "local", + "selected_worker_id": "local", + "eligible_workers": 1 + }, + "quotas": { + "total": 8, + "with_max_sandboxes": 6, + "with_max_ttl": 4, + "with_max_exec_timeout": 3 + }, + "rate_limit": { + "enabled": true, + "requests_per_minute": 120, + "burst": 60, + "key_by": "owner", + "active_buckets": 14, + "allowed_total": 9132, + "limited_total": 27, + "evicted_total": 4, + "bucket_ttl": "15m0s", + "cleanup_interval": "1m0s" + }, + "providers": [ + { + "name": "docker", + "healthy": true, + "default": true, + "latency_ms": 3, + "last_checked": "2026-05-08T10:30:00Z", + "capabilities": ["spawn", "exec", "files", "runtime_inventory", "container"], + "runtime_count": 4 + } + ], + "workers": { + "total": 1, + "online": 1, + "stale": 0, + "unhealthy": 0, + "items": [ + { + "id": "local", + "hostname": "stacyvm-host-1", + "status": "online", + "providers": ["docker", "mock"], + "capabilities": ["api", "single_node", "spawn", "exec", "files"], + "capacity": { "max_sandboxes": 100, "max_sandboxes_per_owner": 10 }, + "last_heartbeat": "2026-05-08T10:30:00Z", + "created_at": "2026-05-08T10:00:00Z", + "updated_at": "2026-05-08T10:30:00Z", + "stale": false + } + ] + }, + "leases": { + "total": 1, + "active": 1, + "expired": 0, + "by_holder": { "local": 1 } + }, + "sandboxes": { + "total": 138, + "active": 12, + "by_state": { "running": 12, "destroyed": 126 }, + "by_provider": { "docker": 90, "firecracker": 48 }, + "by_worker": { "local": 138 } + }, + "events": { + "subscribers": 2, + "history_size": 1000, + "events_total": 2401 + }, + "operations": [], + "remediation": { + "admin_control_plane": "docs/admin-control-plane.md", + "deployment": "docs/deployment.md", + "production_readiness": "docs/production-readiness.md", + "public_support_matrix": "docs/public-support-matrix.md", + "release_verification": "docs/releasing.md", + "runtime_certification": "docs/runtime-certification.md", + "runtime_conformance": "docs/runtime-conformance.md", + "security_governance": "docs/security-governance.md", + "support_bundle": "docs/deployment.md#support-bundles", + "upgrade_and_rollback": "docs/deployment.md#upgrade-rehearsal-and-rollback" + }, + "redactions": ["provider secrets", "registry credentials", "environment secrets", "API keys"] +} +``` + +Diagnostics are read-only and intentionally redacted. Use this endpoint for support bundles, incident debugging, and deployment sanity checks. The `remediation` object points operators to the first public document to use when a diagnostics area needs follow-up. + +### Metrics + +``` +GET /api/v1/metrics +``` + +**Response** `200 OK`: +```json +{ + "uptime": "2h13m", + "goroutines": 42, + "memory_alloc": 17825792, + "memory_sys": 71303168, + "memory_heap_alloc": 17825792, + "gc_cycles": 8, + "sandboxes": { + "total": 138, + "active": 12, + "by_state": { "running": 12, "destroyed": 126 }, + "by_provider": { "docker": 90, "firecracker": 48 }, + "by_worker": { "local": 138 } + }, + "providers": { + "total": 2, + "healthy": 1, + "items": [ + { "name": "docker", "healthy": true, "default": true }, + { "name": "firecracker", "healthy": false, "default": false } + ] + }, + "workers": { + "total": 1, + "online": 1, + "stale": 0, + "unhealthy": 0, + "items": [ + { + "id": "local", + "hostname": "stacyvm-host-1", + "status": "online", + "providers": ["docker", "mock"], + "capabilities": ["api", "single_node", "spawn", "exec", "files"], + "capacity": { "max_sandboxes": 100, "max_sandboxes_per_owner": 10 }, + "last_heartbeat": "2026-05-08T10:30:00Z", + "created_at": "2026-05-08T10:00:00Z", + "updated_at": "2026-05-08T10:30:00Z", + "stale": false + } + ] + }, + "leases": { + "total": 1, + "active": 1, + "expired": 0, + "by_holder": { "local": 1 } + }, + "events": { + "subscribers": 2, + "history_size": 1000, + "events_total": 2401 + }, + "scheduler": { + "spawn_overflow": "queue", + "spawn_queue_depth": 3, + "max_spawn_queue": 100, + "spawn_queue_timeout": "30s", + "admission_control": "worker_aware_local", + "spawn_queued_total": 18, + "spawn_dequeued_total": 16, + "spawn_queue_timeouts": 2, + "spawn_queue_wait_count": 18, + "spawn_queue_wait_total": "1m42s", + "spawn_queue_wait_max": "12s", + "spawn_queue_wait_avg": "5.666s", + "spawn_queue_wait_total_ms": 102000, + "spawn_queue_wait_max_ms": 12000, + "spawn_queue_wait_avg_ms": 5666, + "worker_id": "local", + "selected_worker_id": "local", + "eligible_workers": 1 + }, + "quotas": { + "total": 8, + "with_max_sandboxes": 6, + "with_max_ttl": 4, + "with_max_exec_timeout": 3 + }, + "rate_limit": { + "enabled": true, + "requests_per_minute": 120, + "burst": 60, + "key_by": "owner", + "active_buckets": 14, + "allowed_total": 9132, + "limited_total": 27, + "evicted_total": 4, + "bucket_ttl": "15m0s", + "cleanup_interval": "1m0s" + }, + "operations": [ + { + "operation": "exec", + "provider": "docker", + "success_total": 482, + "failure_total": 7, + "latency_count": 489, + "latency_total_ms": 39120, + "latency_min_ms": 3, + "latency_max_ms": 2500, + "latency_avg_ms": 80 + } + ] +} +``` + +### Prometheus metrics + +``` +GET /api/v1/metrics/prometheus +``` + +**Response** `200 OK`: +```text +# HELP stacyvm_uptime_seconds StacyVM API process uptime in seconds. +# TYPE stacyvm_uptime_seconds gauge +stacyvm_uptime_seconds 7980 +# HELP stacyvm_provider_healthy Provider health status where 1 is healthy and 0 is unhealthy. +# TYPE stacyvm_provider_healthy gauge +stacyvm_provider_healthy{provider="docker",default="true"} 1 +stacyvm_spawn_queue_depth 3 +stacyvm_spawn_queue_wait_milliseconds_count 18 +stacyvm_owner_quotas_total 8 +stacyvm_rate_limit_blocked_total 27 +stacyvm_workers_total{status="total"} 1 +stacyvm_workers_total{status="online"} 1 +stacyvm_workers_total{status="stale"} 0 +stacyvm_workers_total{status="unhealthy"} 0 +stacyvm_leases_total{status="total"} 1 +stacyvm_leases_total{status="active"} 1 +stacyvm_leases_total{status="expired"} 0 +stacyvm_sandboxes_by_worker_total{worker="local"} 138 +stacyvm_operation_success_total{operation="exec",provider="docker"} 482 +stacyvm_operation_failure_total{operation="exec",provider="docker"} 7 +``` + +Use this endpoint for Prometheus-compatible scraping of runtime, provider, worker, sandbox, event, and operation metrics. + +--- + +## Events stream + +``` +GET /api/v1/events +``` + +**Response** `200 OK` with `Content-Type: text/event-stream`. The server emits orchestrator events as Server-Sent Events: + +``` +data: {"id":"evt-1","type":"sandbox.created","sandbox_id":"sb-a1b2c3d4","timestamp":"2026-05-08T10:30:00Z"} + +data: {"id":"evt-2","type":"exec.timeout","sandbox_id":"sb-a1b2c3d4","timestamp":"2026-05-08T10:31:00Z","data":{"operation":"exec","provider":"docker","error":"exec timeout: sb-a1b2c3d4"}} + +data: {"id":"evt-3","type":"reconcile.action","sandbox_id":"sb-a1b2c3d4","timestamp":"2026-05-08T10:32:00Z","data":{"action":"adopted_runtime","provider":"docker","image":"python:3.12"}} +``` + +Common event types include: + +- `sandbox.created`, `sandbox.running`, `sandbox.destroyed`, `sandbox.error` +- `exec.started`, `exec.completed`, `exec.failed`, `exec.timeout` +- `file.written`, `file.read` +- `operation.failed`, `resource.limit`, `provider.failed`, `reconcile.action` +- `spawn.queued`, `spawn.dequeued`, `spawn.queue_timeout` +- `quota.saved`, `quota.deleted` + +Use any SSE client (`EventSource` in browsers, `httpx-sse` in Python, etc.) to consume. + +--- + +## WebSocket exec + +``` +GET /api/v1/sandboxes/{id}/exec/ws +``` + +Upgrades the connection to a WebSocket for interactive command execution. Useful for terminals, REPLs, and any case where you need bi-directional I/O. + +**Client → server messages:** +```json +{ "type": "start", "command": "python3", "env": { "PYTHONUNBUFFERED": "1" } } +{ "type": "stdin", "data": "print('hi')\n" } +{ "type": "resize", "cols": 80, "rows": 24 } +{ "type": "signal", "signal": "SIGINT" } +``` + +**Server → client messages:** +```json +{ "type": "stdout", "data": "hi\n" } +{ "type": "stderr", "data": "..." } +{ "type": "exit", "exit_code": 0 } +``` + +The web dashboard uses this endpoint to power its live terminal — a concrete reference is at [`web/src/`](https://github.com/StacyOS/stacyvm/tree/main/web/src). + +--- + +## SDK mapping + +If you'd rather write Python or TypeScript than `curl`, every endpoint above maps 1:1 to an SDK method: + +| Endpoint | Python | TypeScript | +|---|---|---| +| `POST /sandboxes` | `client.spawn(...)` | `client.spawn(...)` | +| `GET /sandboxes/{id}` | `client.get(id)` | `client.get(id)` | +| `POST /sandboxes/{id}/exec` | `sb.exec(cmd)` / `sb.exec_stream(cmd)` | `sb.exec(cmd)` / `sb.execStream(cmd)` | +| `POST /sandboxes/{id}/files` | `sb.write_file(path, content)` | `sb.writeFile(path, content)` | +| `GET /sandboxes/{id}/files` | `sb.read_file(path)` | `sb.readFile(path)` | +| `POST /templates/{name}/spawn` | `client.spawn_template(name)` | `client.templates.spawn(name)` | +| `GET /pool/status` | `client.pool_status()` | `client.poolStatus()` | +| `GET /health` | `client.health()` | `client.health()` | + +Full SDK docs: [Python](/docs/sdks/python) · [TypeScript](/docs/sdks/typescript). diff --git a/docs/rest/sandboxes.mdx b/docs/rest/sandboxes.mdx new file mode 100644 index 0000000..79f69c9 --- /dev/null +++ b/docs/rest/sandboxes.mdx @@ -0,0 +1,214 @@ +--- +title: "Sandboxes API" +description: "Create sandboxes, execute commands, manage files, and destroy runtime instances through the StacyVM REST API." +--- + +Use the sandboxes API when you want direct HTTP access or when you are building your own SDK. + +## Prerequisites + +- A running StacyVM server. +- `X-API-Key` when auth is enabled. +- `X-User-ID` when you want explicit tenant attribution. + +## Create A Sandbox + + +```bash cURL +curl -sS -X POST http://localhost:7423/api/v1/sandboxes \ + -H "Content-Type: application/json" \ + -H "X-API-Key: sk_test_YOUR_API_KEY" \ + -H "X-User-ID: user_123" \ + -d '{ + "image": "python:3.12", + "provider": "docker", + "memory_mb": 512, + "vcpus": 1, + "ttl": "10m", + "metadata": {"purpose": "quickstart"} + }' +``` + +```javascript JavaScript +const response = await fetch("http://localhost:7423/api/v1/sandboxes", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-API-Key": "sk_test_YOUR_API_KEY", + "X-User-ID": "user_123", + }, + body: JSON.stringify({ + image: "python:3.12", + provider: "docker", + memory_mb: 512, + vcpus: 1, + ttl: "10m", + metadata: { purpose: "quickstart" }, + }), +}); + +if (!response.ok) { + throw new Error(await response.text()); +} + +const sandbox = await response.json(); +console.log(sandbox.id); +``` + +```python Python +import requests + +response = requests.post( + "http://localhost:7423/api/v1/sandboxes", + headers={ + "Content-Type": "application/json", + "X-API-Key": "sk_test_YOUR_API_KEY", + "X-User-ID": "user_123", + }, + json={ + "image": "python:3.12", + "provider": "docker", + "memory_mb": 512, + "vcpus": 1, + "ttl": "10m", + "metadata": {"purpose": "quickstart"}, + }, + timeout=30, +) +response.raise_for_status() +print(response.json()["id"]) +``` + + + + Runtime image to start, such as `python:3.12` for Docker. + + + + Provider override. Omit this to use the server default. + + + + Requested memory limit in megabytes. + + + + Requested virtual CPU count. + + + + Auto-destroy duration using Go duration syntax, such as `10m` or `1h30m`. + + + + String key-value labels stored with the sandbox. + + +## Success Response + +```json +{ + "id": "sb_a1b2c3d4", + "state": "running", + "provider": "docker", + "image": "python:3.12", + "memory_mb": 512, + "vcpus": 1, + "created_at": "2026-05-10T10:00:00Z", + "expires_at": "2026-05-10T10:10:00Z", + "metadata": { + "purpose": "quickstart" + } +} +``` + + + Unique sandbox ID used by exec, file, preview, and destroy endpoints. + + + + Current lifecycle state, usually `running` after a successful create. + + + + UTC timestamp when TTL cleanup should destroy the sandbox. + + +## Execute A Command + + +```bash cURL +curl -sS -X POST http://localhost:7423/api/v1/sandboxes/sb_a1b2c3d4/exec \ + -H "Content-Type: application/json" \ + -H "X-API-Key: sk_test_YOUR_API_KEY" \ + -d '{"command":"python3 -c \"print(40 + 2)\"","timeout":"10s"}' +``` + +```javascript JavaScript +const response = await fetch( + "http://localhost:7423/api/v1/sandboxes/sb_a1b2c3d4/exec", + { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-API-Key": "sk_test_YOUR_API_KEY", + }, + body: JSON.stringify({ + command: 'python3 -c "print(40 + 2)"', + timeout: "10s", + }), + }, +); + +if (!response.ok) { + throw new Error(await response.text()); +} + +console.log(await response.json()); +``` + +```python Python +import requests + +response = requests.post( + "http://localhost:7423/api/v1/sandboxes/sb_a1b2c3d4/exec", + headers={ + "Content-Type": "application/json", + "X-API-Key": "sk_test_YOUR_API_KEY", + }, + json={ + "command": 'python3 -c "print(40 + 2)"', + "timeout": "10s", + }, + timeout=30, +) +response.raise_for_status() +print(response.json()["stdout"]) +``` + + +## Common Error + +```json +{ + "code": "not_found", + "message": "sandbox sb_a1b2c3d4 not found" +} +``` + +The most common causes are an expired TTL, a sandbox that was already destroyed, or an ID from another environment. + +## Destroy A Sandbox + +```bash +curl -sS -X DELETE http://localhost:7423/api/v1/sandboxes/sb_a1b2c3d4 \ + -H "X-API-Key: sk_test_YOUR_API_KEY" +``` + +Destroy is idempotent. It is safe to call during cleanup even if TTL cleanup already ran. + +## Related + +- [Quickstart](/docs/getting-started/quickstart) +- [Full REST API reference](/docs/rest-api) +- [Core concepts](/docs/getting-started/core-concepts) diff --git a/docs/runtime-certification.md b/docs/runtime-certification.md new file mode 100644 index 0000000..2f2dddf --- /dev/null +++ b/docs/runtime-certification.md @@ -0,0 +1,77 @@ +# Runtime Certification + +Phase 7 treats runtime certification as a required host-level check before a +provider is marked production-ready. + +Run dependency checks: + +```sh +scripts/certify-runtime.sh all +scripts/certify-runtime.sh docker +scripts/certify-runtime.sh firecracker +scripts/certify-runtime.sh proot +``` + +Generate a durable artifact for release or host signoff: + +```sh +scripts/certify-runtime.sh docker --format markdown --output docker-certification.md +scripts/certify-runtime.sh firecracker --format json --output firecracker-certification.json +``` + +For Phase 14 worker identity signoff, run the signed-token lifecycle smoke: + +```sh +scripts/certify-worker-identity.sh worker-a +scripts/certify-worker-identity.sh worker-a --format markdown --output worker-identity-certification.md +``` + +This verifies token issue, inspect, verify, revocation rejection, and rotation-plan generation using secret files. Set `STACYVM_WORKER_SIGNING_KEY_FILE`, `STACYVM_OLD_WORKER_SIGNING_KEY_FILE`, `STACYVM_WORKER_IDENTITY_AUDIENCE`, or `STACYVM_WORKER_IDENTITY_TTL` to point at deployment-specific values. + +Workers that receive signed tokens from an external issuer can run with `stacyvm worker --worker-token-file /run/secrets/stacyvm-worker-token`. The worker reloads that file for every heartbeat and lease-renewal request, allowing a sidecar to replace short-lived token files before expiry without a worker restart. + +The script exits non-zero when any required check fails. Warnings are included in +the artifact but do not fail the command. Attach the generated artifact to the +release checklist, support ticket, or infrastructure change record for the host +being certified. + +For Firecracker and PRoot, set optional paths to make host validation stricter: + +```sh +STACYVM_FIRECRACKER_KERNEL=/var/lib/stacyvm/vmlinux.bin \ + scripts/certify-runtime.sh firecracker --format markdown --output firecracker-certification.md + +STACYVM_PROOT_ROOTFS=/var/lib/stacyvm/rootfs \ +STACYVM_PROOT_WORKSPACE_BASE=/var/lib/stacyvm/workspaces \ + scripts/certify-runtime.sh proot --format markdown --output proot-certification.md +``` + +## Certification Matrix + +| Runtime | Checks | Production signoff | +|---|---|---| +| Docker | CLI, daemon reachability, seccomp visibility | Pass on target host, then run provider conformance with Docker enabled | +| gVisor | Docker daemon reachability and runtime discovery for `runsc`/gVisor | Pass discovery and run Docker provider with runtime configured | +| Kata | Docker daemon reachability and runtime discovery for Kata | Pass discovery and run Docker provider with runtime configured | +| Firecracker | Binary, `/dev/kvm`, optional kernel path | Pass on Linux/KVM host with configured kernel/rootfs/agent | +| PRoot | `proot` binary, optional rootfs/workspace paths | Pass with configured rootfs and workspace base | + +`stacyvm doctor --production` remains the operator-facing readiness command. +Docker provider integration tests are opt-in to keep default CI independent of Docker Hub and host daemon state. Run them on a certified Docker host with `STACYVM_DOCKER_INTEGRATION=1 make test`. +The certification script is the lower-level host check for runtime dependencies +that may not exist in CI or on developer laptops. + +## Required Phase 8 Signoff Artifacts + +Before calling a single-node host production-ready, collect: + +- `stacyvm config lint --production --file ` +- `stacyvm upgrade rehearse --config --database --backup-output ` +- `stacyvm doctor --production` +- `scripts/certify-runtime.sh --format markdown --output -certification.md` +- `scripts/certify-worker-identity.sh --format markdown --output worker-identity-certification.md` +- Provider conformance or smoke output for the configured runtime. + +Store these artifacts with the deployment record. Do not treat a runtime as +certified because CI passed on another host; runtime certification is per-host +and depends on kernel, daemon, KVM, rootfs, and installed runtime state. diff --git a/docs/runtime-conformance.md b/docs/runtime-conformance.md new file mode 100644 index 0000000..11898db --- /dev/null +++ b/docs/runtime-conformance.md @@ -0,0 +1,200 @@ +# Runtime Conformance Matrix + +This matrix describes what operators should validate before treating a StacyVM runtime provider as production-ready on a host class. The shared provider contract is documented in `docs/provider-contract.md`; this guide focuses on deployment conformance. + +## Summary + +| Runtime | Host requirement | Production status | Required validation | +|---|---|---|---| +| Docker with `runc` | Docker daemon and socket access | Default broad-compatibility path | Provider health, lifecycle, exec, files, live preview, reconciliation | +| Docker with gVisor `runsc` | Docker daemon plus installed `runsc` runtime | Stronger container isolation | Same as Docker plus runtime selection and syscall compatibility | +| Docker with Kata | Docker daemon plus installed Kata runtime and virtualization support | VM-backed container isolation | Same as Docker plus nested virtualization/runtime availability | +| Firecracker | Linux, `/dev/kvm`, Firecracker binary, kernel, rootfs, networking, `stacyvm-agent` | Highest-isolation target | Full lifecycle and file/exec conformance on real Linux/KVM host | +| PRoot | `proot` binary, rootfs with expected tools, writable workspace base | Restricted-host fallback | Lifecycle, exec, files, limits, and rootfs language/tool availability | +| E2B | E2B API key and network access | Hybrid/cloud burst option | API reachability, lifecycle, exec, files, and failure mapping | +| Custom | Reachable provider HTTP service | Bring-your-own runtime | Contract conformance against the custom backend | + +## Baseline Checks + +Run these checks for every runtime: + +```bash +make test +scripts/smoke-deployment.sh http://127.0.0.1:7423 "$STACYVM_API_KEY" +curl -fsS -H "X-API-Key: $STACYVM_API_KEY" http://127.0.0.1:7423/api/v1/providers +curl -fsS -H "X-API-Key: $STACYVM_API_KEY" http://127.0.0.1:7423/api/v1/ready +``` + +For a deployed service, use `STACYVM_SMOKE_URL` instead of positional arguments: + +```bash +STACYVM_SMOKE_URL=https://stacyvm.example.com STACYVM_API_KEY=sk-live scripts/smoke-deployment.sh +``` + +## Docker + +Required host state: + +- Docker daemon is running. +- StacyVM can access the configured Docker socket. +- The sandbox network exists when `providers.docker.network_mode` is a named network. +- Traefik or another reverse proxy can reach sandbox containers for live preview. + +Recommended validation: + +```bash +docker info +docker network inspect stacyvm-network +STACYVM_DOCKER_INTEGRATION=1 STACYVM_PROVIDERS_DEFAULT=docker make test +``` + +Runtime behavior to verify: + +- `GET /api/v1/providers/docker` reports healthy. +- Spawn an `alpine:latest` sandbox. +- Execute `echo ok`. +- Write, read, list, move, chmod, stat, glob, and delete a file. +- Destroy the sandbox. +- Restart StacyVM and confirm orphaned StacyVM containers reconcile correctly. + +## Docker gVisor + +Required host state: + +- Docker daemon is running. +- `runsc` is installed and registered as a Docker runtime. +- StacyVM config sets `providers.docker.runtime: "runsc"`. + +Recommended validation: + +```bash +docker info | grep -A5 Runtimes +docker run --rm --runtime=runsc alpine:latest echo ok +``` + +Runtime behavior to verify: + +- Docker provider health remains healthy with `runtime=runsc`. +- Basic spawn, exec, file operations, destroy, and live preview still pass. +- Workloads that need unusual syscalls are tested explicitly because gVisor changes syscall behavior. + +## Docker Kata + +Required host state: + +- Kata runtime is installed and registered with Docker. +- Host supports the virtualization mode required by the Kata installation. +- StacyVM config sets `providers.docker.runtime` to the registered Kata runtime name. + +Recommended validation: + +```bash +docker info | grep -A5 Runtimes +docker run --rm --runtime=kata-runtime alpine:latest echo ok +``` + +Runtime behavior to verify: + +- Docker provider health remains healthy with the Kata runtime. +- Spawn, exec, file operations, destroy, and live preview pass. +- Cold-start latency and memory overhead are measured against operator SLOs. + +## Firecracker + +Required host state: + +- Linux host with `/dev/kvm` available. +- Firecracker binary installed and executable. +- Kernel image exists at `providers.firecracker.kernel_path`. +- Rootfs image exists for the requested sandbox image or template. +- `stacyvm-agent` is available at `providers.firecracker.agent_path`. +- Networking setup permits guest communication. + +Recommended validation: + +```bash +test -e /dev/kvm +firecracker --version +test -f /var/lib/stacyvm/vmlinux.bin +test -x /usr/local/bin/stacyvm-agent +``` + +Runtime behavior to verify: + +- `GET /api/v1/providers/firecracker` reports healthy. +- Full provider conformance passes on the Linux/KVM host. +- Snapshot restore paths work for prepared rootfs images. +- Destroy cleans up processes, sockets, tap devices, and temporary runtime files. +- Reconciliation correctly handles stale persisted sandboxes after a StacyVM restart. + +## PRoot + +Required host state: + +- `proot` binary is installed. +- Rootfs exists at `providers.proot.rootfs_path`. +- Workspace base is writable by the StacyVM process. +- Rootfs contains the languages and binaries advertised by `providers.proot.languages`. + +Recommended validation: + +```bash +proot --version +test -d /var/lib/stacyvm/rootfs +test -w /var/lib/stacyvm/workspaces +``` + +Runtime behavior to verify: + +- `GET /api/v1/providers/proot` reports healthy. +- Basic lifecycle, exec, and file operations pass against the real rootfs. +- Configured memory and disk caps are understood as operational controls, not VM-grade isolation. +- Rootfs language availability matches templates and SDK examples. + +## E2B And Custom Providers + +Required host state: + +- Outbound network access to the provider. +- API keys configured through environment variables or a secret manager. +- Provider-specific base URL configured. + +Runtime behavior to verify: + +- Provider health returns actionable errors when credentials or network are wrong. +- Lifecycle, exec, streaming exec, files, and destroy match `docs/provider-contract.md`. +- Provider errors map to typed StacyVM errors instead of leaking backend-specific response bodies. + +## Signoff Template + +Use this checklist before marking a runtime production-ready: + +```text +Runtime: +Host OS/kernel: +StacyVM version: +Config file: +Provider health endpoint: +Smoke script result: +Lifecycle conformance: +Exec conformance: +File conformance: +Streaming conformance: +Live preview: +Restart reconciliation: +Known host caveats: +Owner/signoff: +Date: +``` + +For an auditable host artifact, generate the signoff scaffold directly: + +```bash +scripts/certify-runtime.sh docker --format markdown --output docker-certification.md +scripts/certify-runtime.sh firecracker --format markdown --output firecracker-certification.md +scripts/certify-runtime.sh proot --format markdown --output proot-certification.md +``` + +The generated report includes host metadata, dependency checks, overall status, +and an operator signoff section. Attach provider conformance logs and smoke +script output next to that artifact for final production approval. diff --git a/docs/sdks/python.mdx b/docs/sdks/python.mdx new file mode 100644 index 0000000..bda96f4 --- /dev/null +++ b/docs/sdks/python.mdx @@ -0,0 +1,123 @@ +--- +title: "Python SDK" +description: "Use the StacyVM Python SDK to create sandboxes, run commands, stream output, manage files, and clean up safely." +--- + +The Python SDK is the fastest path for Python services, agent backends, notebooks, and workflow runners. + +## Prerequisites + +- Python 3.9+. +- A running StacyVM server. +- An API key when auth is enabled. + +## Install + +```bash +pip install stacyvm +``` + +## Connect + +```python +import os +from stacyvm import Client + +client = Client( + base_url=os.getenv("STACYVM_URL", "http://localhost:7423"), + api_key=os.getenv("STACYVM_API_KEY"), + user_id="user_123", + timeout=60.0, +) +``` + + + StacyVM server URL. Defaults to `http://localhost:7423`. + + + + API key sent as `X-API-Key` when server auth is enabled. + + + + Tenant or owner identity sent as `X-User-ID` for quota and audit attribution. + + + + Per-request HTTP timeout in seconds. + + +## Run A Task + +```python +from stacyvm import Client + +client = Client( + base_url="http://localhost:7423", + api_key="sk_test_YOUR_API_KEY", +) + +with client.spawn(image="python:3.12", ttl="10m") as sandbox: + sandbox.write_file("/app/main.py", "print(sum([10, 20, 12]))\n") + result = sandbox.exec("python3 /app/main.py", timeout="10s") + + if result.exit_code != 0: + raise RuntimeError(result.stderr) + + print(result.stdout) +``` + + + Process exit code returned by the runtime provider. + + + + Captured standard output. + + + + Captured standard error. + + + + Provider-reported execution duration. + + +## Stream Output + +```python +import sys + +for chunk in sandbox.exec_stream("python3 -u /app/main.py"): + if chunk.stream == "stdout": + print(chunk.data, end="") + else: + print(chunk.data, end="", file=sys.stderr) +``` + +## Handle Errors + +```python +from stacyvm import Client, ProviderError, SandboxNotFound + +client = Client("http://localhost:7423", api_key="sk_test_YOUR_API_KEY") + +try: + sandbox = client.spawn(image="python:3.12", ttl="10m") + result = sandbox.exec("python3 /missing.py", timeout="10s") +except SandboxNotFound as exc: + print(f"sandbox disappeared: {exc}") +except ProviderError as exc: + print(f"runtime provider failed: {exc}") +finally: + try: + sandbox.destroy() + except Exception: + pass +``` + +## Related + +- [Quickstart](/docs/getting-started/quickstart) +- [Example code runner](/docs/tutorials/code-runner) +- [REST sandboxes](/docs/rest/sandboxes) diff --git a/docs/sdks/typescript.mdx b/docs/sdks/typescript.mdx new file mode 100644 index 0000000..f9e8764 --- /dev/null +++ b/docs/sdks/typescript.mdx @@ -0,0 +1,123 @@ +--- +title: "TypeScript SDK" +description: "Use the StacyVM TypeScript SDK from Node.js services, agent backends, and developer tools." +--- + +The TypeScript SDK works in Node.js 18+ and exposes the same sandbox lifecycle as the REST API. + +## Prerequisites + +- Node.js 18+. +- A running StacyVM server. +- An API key when auth is enabled. + +## Install + +```bash +npm install stacyvm +``` + +## Connect + +```typescript +import { Client } from "stacyvm"; + +const client = new Client({ + baseUrl: process.env.STACYVM_URL ?? "http://localhost:7423", + apiKey: process.env.STACYVM_API_KEY, + userId: "user_123", + timeout: 60_000, +}); +``` + + + StacyVM server URL. Defaults to `http://localhost:7423`. + + + + API key sent as `X-API-Key` when server auth is enabled. + + + + Tenant or owner identity sent as `X-User-ID` for quota and audit attribution. + + + + Per-request HTTP timeout in milliseconds. + + +## Run A Task + +```typescript +import { Client } from "stacyvm"; + +const client = new Client({ + baseUrl: "http://localhost:7423", + apiKey: "sk_test_YOUR_API_KEY", +}); + +await client.withSandbox({ image: "node:20", ttl: "10m" }, async (sandbox) => { + await sandbox.writeFile("/app/main.js", "console.log(40 + 2);\n"); + const result = await sandbox.exec("node /app/main.js", { timeout: "10s" }); + + if (result.exit_code !== 0) { + throw new Error(result.stderr); + } + + console.log(result.stdout); +}); +``` + + + Process exit code returned by the runtime provider. + + + + Captured standard output. + + + + Captured standard error. + + + + Provider-reported execution duration. + + +## Stream Output + +```typescript +for await (const chunk of sandbox.execStream("npm test")) { + if (chunk.stream === "stdout") { + process.stdout.write(chunk.data); + } else { + process.stderr.write(chunk.data); + } +} +``` + +## Handle Errors + +```typescript +import { Client, ForgevmError } from "stacyvm"; + +const client = new Client("http://localhost:7423"); + +try { + await client.withSandbox({ image: "node:20", ttl: "10m" }, async (sandbox) => { + await sandbox.exec("node /missing.js", { timeout: "10s" }); + }); +} catch (error) { + if (error instanceof ForgevmError) { + console.error(error.message); + } else { + throw error; + } +} +``` + +## Related + +- [Quickstart](/docs/getting-started/quickstart) +- [TypeScript example code runner](/docs/tutorials/typescript-code-runner) +- [REST sandboxes](/docs/rest/sandboxes) diff --git a/docs/security-governance.md b/docs/security-governance.md new file mode 100644 index 0000000..33aa466 --- /dev/null +++ b/docs/security-governance.md @@ -0,0 +1,102 @@ +# Security Governance + +This guide captures the Phase 6 security posture for production StacyVM operators and the planned shape of future external identity integration. + +## Production Admin Posture + +Use separate credentials for regular API clients and admin operators: + +```yaml +auth: + enabled: true + api_key: "sk-client" + admin_api_key: "sk-admin" + admin_fallback_enabled: false + admin_audit_retention: "2160h" +``` + +Production deployments should also set the equivalent environment variables: + +```bash +STACYVM_AUTH_API_KEY=sk-client +STACYVM_AUTH_ADMIN_API_KEY=sk-admin +STACYVM_AUTH_ADMIN_FALLBACK_ENABLED=false +STACYVM_AUTH_ADMIN_AUDIT_RETENTION=2160h +``` + +Keep admin routes under `/api/v1/admin/*` behind trusted networks or a reverse proxy allowlist. The admin dashboard should only be used from managed operator browsers because its settings are stored in browser local storage. + +## Operator Attribution + +Admin audit records use `X-User-ID` as the operator actor when it is supplied. Set it to a stable human or service identity such as `operator-a`, `sre-oncall`, or `ops-bot`. + +When `X-User-ID` is missing, authenticated admin requests fall back to role and key-header attribution such as `admin:X-Admin-API-Key`. This is useful for debugging, but production operators should still send explicit actors for accountability. + +## Key Handling + +- Generate `auth.api_key` and `auth.admin_api_key` independently with at least 32 bytes of entropy. +- Keep keys in environment-specific secret storage rather than checked-in config. +- Rotate the admin key after operator offboarding, dashboard sharing incidents, or suspected local browser compromise. +- Prefer short-lived deployment access to the host and avoid copying admin keys into issue trackers, screenshots, or shared terminals. +- Keep `auth.admin_fallback_enabled: false` in production so regular API keys never become admin credentials. + +## Audit Retention + +Audit logs are stored in SQLite with the rest of StacyVM state. `auth.admin_audit_retention` controls native pruning after successful admin audit writes. + +Recommended starting points: + +| Environment | Retention | +|---|---| +| Local development | `0s` | +| Staging | `720h` | +| Production | `2160h` | + +Back up the SQLite database before upgrades and before reducing retention windows. + +## OIDC/SSO Groundwork + +Phase 6 keeps API-key behavior as the implemented authentication mechanism. Future OIDC/SSO support should fit into the request identity model added in this phase instead of bypassing it. + +Proposed config shape: + +```yaml +auth: + oidc: + enabled: false + issuer_url: "https://idp.example.com" + client_id: "stacyvm" + audience: "stacyvm-api" + admin_groups: + - "stacyvm-admins" + api_groups: + - "stacyvm-users" + actor_claim: "email" + groups_claim: "groups" +``` + +Expected claim mapping: + +| Claim/Input | StacyVM identity | +|---|---| +| Admin group membership | `admin` role with `api:*` and `admin:*` scopes | +| API group membership | `api` role with `api:*` scope | +| Actor claim | Audit actor, replacing the need for user-supplied `X-User-ID` | +| Subject claim | Stable fallback identity when the actor claim is absent | + +Implementation boundaries: + +- Validate issuer, audience, expiry, and signature before creating an `AuthIdentity`. +- Reuse `RequireScope` for authorization decisions. +- Keep API-key auth available for service accounts and break-glass access. +- Make dashboard SSO optional and separate from API key support. +- Record the identity source in audit attribution so operators can distinguish API-key and OIDC-originated admin actions. + +## Phase 6 Acceptance Checklist + +- `auth.admin_api_key` is configured separately from `auth.api_key`. +- `auth.admin_fallback_enabled` is `false` in production. +- Admin ingress is restricted to trusted networks or authenticated upstreams. +- Operators send `X-User-ID` until OIDC supplies actor claims. +- `auth.admin_audit_retention` is set to a production retention window. +- Backups include the SQLite database before retention or upgrade changes. diff --git a/docs/swagger.json b/docs/swagger.json index fb171e5..f39bc83 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -9,6 +9,97 @@ "host": "localhost:7423", "basePath": "/api/v1", "paths": { + "/admin/audit": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return recent redacted admin route access records", + "produces": [ + "application/json" + ], + "tags": [ + "admin" + ], + "summary": "List admin audit logs", + "parameters": [ + { + "type": "integer", + "description": "Maximum number of records, capped at 500", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Actor exact match", + "name": "actor", + "in": "query" + }, + { + "type": "string", + "description": "HTTP method exact match", + "name": "method", + "in": "query" + }, + { + "type": "integer", + "description": "HTTP status exact match", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "Path substring match", + "name": "path", + "in": "query" + }, + { + "type": "string", + "description": "Response format: json or csv", + "name": "format", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.AdminAuditResponse" + } + } + } + } + } + }, + "/diagnostics": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return redacted build, store, provider, sandbox, event, and operation diagnostics", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Get diagnostics", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.DiagnosticsResponse" + } + } + } + } + }, "/events": { "get": { "security": [ @@ -28,7 +119,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Event" } } } @@ -59,6 +150,31 @@ } } }, + "/live": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return whether the StacyVM API process is alive", + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Liveness check", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.HealthResponse" + } + } + } + } + }, "/metrics": { "get": { "security": [ @@ -84,6 +200,31 @@ } } }, + "/metrics/prometheus": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format", + "produces": [ + "text/plain" + ], + "tags": [ + "system" + ], + "summary": "Get Prometheus metrics", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, "/providers": { "get": { "security": [ @@ -174,165 +315,144 @@ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes": { + "/quotas": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all active sandboxes", + "description": "Return all persisted owner quota overrides", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "List sandboxes", + "summary": "List owner quotas", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } } } - }, - "post": { + } + }, + "/quotas/summary": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Spawn a new sandbox with the given configuration", - "consumes": [ - "application/json" - ], + "description": "Return non-identifying counts for persisted owner quota overrides", "produces": [ "application/json" ], "tags": [ - "sandboxes" - ], - "summary": "Create a sandbox", - "parameters": [ - { - "description": "Spawn request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest" - } - } + "quotas" ], + "summary": "Get quota summary", "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" } } } - }, - "delete": { + } + }, + "/quotas/{ownerID}": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Destroy all expired sandboxes and return the count", + "description": "Return the persisted quota override for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" + ], + "summary": "Get owner quota", + "parameters": [ + { + "type": "string", + "description": "Owner ID", + "name": "ownerID", + "in": "path", + "required": true + } ], - "summary": "Prune sandboxes", "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/internal_api_routes.PruneResponse" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } }, - "500": { - "description": "Internal Server Error", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}": { - "get": { + }, + "put": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a sandbox by its ID", + "description": "Create or update quota overrides for an owner", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Get a sandbox", + "summary": "Save owner quota", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true + }, + { + "description": "Quota request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" + } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota" } } } @@ -343,19 +463,19 @@ "ApiKeyAuth": [] } ], - "description": "Destroy a sandbox and release its resources", + "description": "Delete the quota override for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Destroy a sandbox", + "summary": "Delete owner quota", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true } @@ -370,161 +490,106 @@ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}/exec": { - "post": { + "/quotas/{ownerID}/usage": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Run a command inside a sandbox. Set stream=true for streaming output.", - "consumes": [ - "application/json" - ], + "description": "Return active sandbox usage and effective quota for an owner", "produces": [ "application/json" ], "tags": [ - "sandboxes" + "quotas" ], - "summary": "Execute a command", + "summary": "Get owner quota usage", "parameters": [ { "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", + "description": "Owner ID", + "name": "ownerID", "in": "path", "required": true - }, - { - "description": "Exec request", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest" - } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage" } } } } }, - "/sandboxes/{sandboxID}/exec/ws": { + "/ready": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Open a WebSocket connection to execute a command with streaming output", + "description": "Return whether the API is ready to serve sandbox traffic", + "produces": [ + "application/json" + ], "tags": [ - "sandboxes" - ], - "summary": "Execute via WebSocket", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - } + "system" ], + "summary": "Readiness check", "responses": { - "101": { - "description": "WebSocket upgrade" + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" + } }, - "400": { - "description": "Bad request" + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_routes.ReadinessResponse" + } } } } }, - "/sandboxes/{sandboxID}/files": { + "/sandboxes": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Read file content from a sandbox", + "description": "Return all active sandboxes", "produces": [ - "application/octet-stream" + "application/json" ], "tags": [ "sandboxes" ], - "summary": "Read a file", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "File path inside the sandbox", - "name": "path", - "in": "query", - "required": true - } - ], + "summary": "List sandboxes", "responses": { "200": { "description": "OK", "schema": { - "type": "file" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } @@ -535,7 +600,7 @@ "ApiKeyAuth": [] } ], - "description": "Write content to a file inside a sandbox", + "description": "Spawn a new sandbox with the given configuration", "consumes": [ "application/json" ], @@ -545,272 +610,190 @@ "tags": [ "sandboxes" ], - "summary": "Write a file", + "summary": "Create a sandbox", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "description": "File write request", + "description": "Spawn request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" } } ], "responses": { - "200": { - "description": "OK", + "201": { + "description": "Created", "schema": { - "$ref": "#/definitions/internal_api_routes.StatusResponse" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, - "404": { - "description": "Not Found", + "429": { + "description": "Too Many Requests", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/sandboxes/{sandboxID}/files/list": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "List files in a directory inside a sandbox", + "description": "Destroy all expired sandboxes and return the count", "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "List files", - "parameters": [ - { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Directory path (default: /)", - "name": "path", - "in": "query" - } - ], + "summary": "Prune sandboxes", "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo" - } - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.PruneResponse" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/sandboxes/{sandboxID}/logs": { - "get": { + "/sandboxes/admission": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Retrieve console log lines from a sandbox", + "description": "Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ "sandboxes" ], - "summary": "Get console logs", + "summary": "Evaluate spawn admission", "parameters": [ { - "type": "string", - "description": "Sandbox ID", - "name": "sandboxID", - "in": "path", - "required": true - }, - { - "type": "integer", - "description": "Number of lines to retrieve (default: 100)", - "name": "lines", - "in": "query" + "description": "Spawn request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest" + } } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision" } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates": { + "/sandboxes/{sandboxID}": { "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return all registered templates", - "produces": [ - "application/json" - ], - "tags": [ - "templates" - ], - "summary": "List templates", - "responses": { - "200": { - "description": "OK", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - } - } - }, - "post": { - "security": [ - { - "ApiKeyAuth": [] - } - ], - "description": "Register a new sandbox template", - "consumes": [ - "application/json" - ], + "description": "Return a sandbox by its ID", "produces": [ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Create a template", + "summary": "Get a sandbox", "parameters": [ { - "description": "Template definition", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true } ], "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - }, - "400": { - "description": "Bad Request", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" } }, - "409": { - "description": "Conflict", + "404": { + "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - } - }, - "/templates/{name}": { - "get": { + }, + "delete": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Return a template by its name", + "description": "Destroy a sandbox and release its resources", "produces": [ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Get a template", + "summary": "Destroy a sandbox", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true } @@ -819,30 +802,32 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/internal_api_routes.StatusResponse" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "put": { + } + }, + "/sandboxes/{sandboxID}/exec": { + "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Update an existing template by name", + "description": "Run a command inside a sandbox. Set stream=true for streaming output.", "consumes": [ "application/json" ], @@ -850,24 +835,24 @@ "application/json" ], "tags": [ - "templates" + "sandboxes" ], - "summary": "Update a template", + "summary": "Execute a command", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Sandbox ID", + "name": "sandboxID", "in": "path", "required": true }, { - "description": "Updated template", + "description": "Exec request", "name": "request", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest" } } ], @@ -875,37 +860,587 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template" - } - }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult" } }, "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } - }, - "delete": { + } + }, + "/sandboxes/{sandboxID}/exec/ws": { + "get": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Delete a template by name", - "produces": [ + "description": "Open a WebSocket connection to execute a command with streaming output", + "tags": [ + "sandboxes" + ], + "summary": "Execute via WebSocket", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + } + ], + "responses": { + "101": { + "description": "WebSocket upgrade" + }, + "400": { + "description": "Bad request" + } + } + } + }, + "/sandboxes/{sandboxID}/extend": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Add additional time to a sandbox's expiration", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Extend sandbox TTL", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "description": "TTL extension", + "name": "request", + "in": "body", + "required": true, + "schema": { + "type": "object", + "properties": { + "ttl": { + "type": "string" + } + } + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/files": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Read file content from a sandbox", + "produces": [ + "application/octet-stream" + ], + "tags": [ + "sandboxes" + ], + "summary": "Read a file", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "File path inside the sandbox", + "name": "path", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Write content to a file inside a sandbox", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Write a file", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "description": "File write request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.StatusResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/files/list": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "List files in a directory inside a sandbox", + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "List files", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Directory path (default: /)", + "name": "path", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/sandboxes/{sandboxID}/logs": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Retrieve console log lines from a sandbox", + "produces": [ + "application/json" + ], + "tags": [ + "sandboxes" + ], + "summary": "Get console logs", + "parameters": [ + { + "type": "string", + "description": "Sandbox ID", + "name": "sandboxID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Number of lines to retrieve (default: 100)", + "name": "lines", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/snapshots": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all pre-built VM snapshots available for fast restore", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "List snapshots", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary" + } + } + } + } + } + }, + "/templates": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return all registered templates", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "List templates", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Register a new sandbox template", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Create a template", + "parameters": [ + { + "description": "Template definition", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/templates/{name}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return a template by its name", + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Get a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "put": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Update an existing template by name", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Update a template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "Updated template", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Delete a template by name", + "produces": [ "application/json" ], "tags": [ @@ -931,26 +1466,186 @@ "404": { "description": "Not Found", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/templates/{name}/spawn": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Create a new sandbox using a template's configuration, with optional overrides", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "templates" + ], + "summary": "Spawn from template", + "parameters": [ + { + "type": "string", + "description": "Template name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "Optional overrides", + "name": "request", + "in": "body", + "schema": { + "$ref": "#/definitions/internal_api_routes.TemplateSpawnOverrides" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } }, "500": { "description": "Internal Server Error", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + } + }, + "/workers": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return worker registry records and heartbeat state", + "produces": [ + "application/json" + ], + "tags": [ + "workers" + ], + "summary": "List workers", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.WorkerResponse" + } + } + } + } + } + }, + "/workers/{workerID}": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Return one worker registry record", + "produces": [ + "application/json" + ], + "tags": [ + "workers" + ], + "summary": "Get worker", + "parameters": [ + { + "type": "string", + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.WorkerResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "Remove a worker registry record", + "tags": [ + "workers" + ], + "summary": "Delete worker", + "parameters": [ + { + "type": "string", + "description": "Worker ID", + "name": "workerID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/internal_api_routes.StatusResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError" } } } } }, - "/templates/{name}/spawn": { + "/workers/{workerID}/heartbeat": { "post": { "security": [ { "ApiKeyAuth": [] } ], - "description": "Create a new sandbox using a template's configuration, with optional overrides", + "description": "Create or update worker registry state for a worker", "consumes": [ "application/json" ], @@ -958,43 +1653,32 @@ "application/json" ], "tags": [ - "templates" + "workers" ], - "summary": "Spawn from template", + "summary": "Heartbeat worker", "parameters": [ { "type": "string", - "description": "Template name", - "name": "name", + "description": "Worker ID", + "name": "workerID", "in": "path", "required": true }, { - "description": "Optional overrides", + "description": "Worker heartbeat", "name": "request", "in": "body", + "required": true, "schema": { - "$ref": "#/definitions/internal_api_routes.TemplateSpawnOverrides" + "$ref": "#/definitions/internal_api_routes.WorkerHeartbeatRequest" } } ], "responses": { - "201": { - "description": "Created", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" - } - }, - "500": { - "description": "Internal Server Error", + "200": { + "description": "OK", "schema": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError" + "$ref": "#/definitions/internal_api_routes.WorkerResponse" } } } @@ -1002,18 +1686,53 @@ } }, "definitions": { - "github_com_stacyvm-dev_stacyvm_internal_httputil.APIError": { + "github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats": { + "type": "object", + "properties": { + "active_buckets": { + "type": "integer" + }, + "allowed_total": { + "type": "integer" + }, + "bucket_ttl": { + "type": "string" + }, + "burst": { + "type": "integer" + }, + "cleanup_interval": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "evicted_total": { + "type": "integer" + }, + "key_by": { + "type": "string" + }, + "limited_total": { + "type": "integer" + }, + "requests_per_minute": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_httputil.APIError": { "type": "object", "properties": { "code": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode" }, "message": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode": { + "github_com_StacyOs_stacyvm_internal_httputil.ErrorCode": { "type": "string", "enum": [ "NOT_FOUND", @@ -1021,7 +1740,9 @@ "INTERNAL_ERROR", "UNAUTHORIZED", "CONFLICT", - "UNAVAILABLE" + "UNAVAILABLE", + "TIMEOUT", + "RESOURCE_LIMIT" ], "x-enum-varnames": [ "CodeNotFound", @@ -1029,10 +1750,12 @@ "CodeInternal", "CodeUnauth", "CodeConflict", - "CodeUnavailable" + "CodeUnavailable", + "CodeTimeout", + "CodeResourceLimit" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Event": { "type": "object", "properties": { "data": { @@ -1051,11 +1774,25 @@ "type": "string" }, "type": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventType" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats": { + "type": "object", + "properties": { + "events_total": { + "type": "integer" + }, + "history_size": { + "type": "integer" + }, + "subscribers": { + "type": "integer" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType": { + "github_com_StacyOs_stacyvm_internal_orchestrator.EventType": { "type": "string", "enum": [ "sandbox.created", @@ -1064,8 +1801,19 @@ "sandbox.error", "exec.started", "exec.completed", + "exec.failed", + "exec.timeout", "file.written", - "file.read" + "file.read", + "operation.failed", + "resource.limit", + "provider.failed", + "reconcile.action", + "spawn.queued", + "spawn.dequeued", + "spawn.queue_timeout", + "quota.saved", + "quota.deleted" ], "x-enum-varnames": [ "EventSandboxCreated", @@ -1074,11 +1822,22 @@ "EventSandboxError", "EventExecStarted", "EventExecCompleted", + "EventExecFailed", + "EventExecTimeout", "EventFileWritten", - "EventFileRead" + "EventFileRead", + "EventOperationFailed", + "EventResourceLimit", + "EventProviderFailed", + "EventReconcileAction", + "EventSpawnQueued", + "EventSpawnDequeued", + "EventSpawnQueueTimeout", + "EventQuotaSaved", + "EventQuotaDeleted" ] }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest": { "type": "object", "properties": { "args": { @@ -1096,6 +1855,9 @@ "type": "string" } }, + "mode": { + "type": "string" + }, "stream": { "type": "boolean" }, @@ -1107,7 +1869,7 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult": { + "github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult": { "type": "object", "properties": { "duration": { @@ -1124,7 +1886,7 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo": { + "github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo": { "type": "object", "properties": { "is_dir": { @@ -1144,7 +1906,7 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest": { "type": "object", "properties": { "content": { @@ -1158,7 +1920,137 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox": { + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics": { + "type": "object", + "properties": { + "failure_total": { + "type": "integer" + }, + "last_error": { + "type": "string" + }, + "last_observed_unix": { + "type": "integer" + }, + "latency_avg_ms": { + "type": "integer" + }, + "latency_count": { + "type": "integer" + }, + "latency_max_ms": { + "type": "integer" + }, + "latency_min_ms": { + "type": "integer" + }, + "latency_total_ms": { + "type": "integer" + }, + "operation": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "success_total": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo": { + "type": "object", + "properties": { + "default_exec_timeout": { + "type": "string" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_sandboxes_per_owner": { + "type": "integer" + }, + "max_spawn_queue": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_timeout": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "owner_id": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage": { + "type": "object", + "properties": { + "active_sandboxes": { + "type": "integer" + }, + "max_exec_timeout": { + "type": "string" + }, + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { + "type": "string" + }, + "owner_id": { + "type": "string" + }, + "quota_configured": { + "type": "boolean" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary": { + "type": "object", + "properties": { + "total": { + "type": "integer" + }, + "with_max_exec_timeout": { + "type": "integer" + }, + "with_max_sandboxes": { + "type": "integer" + }, + "with_max_ttl": { + "type": "integer" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox": { "type": "object", "properties": { "created_at": { @@ -1173,55 +2065,171 @@ "image": { "type": "string" }, - "memory_mb": { + "memory_mb": { + "type": "integer" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "owner_id": { + "type": "string" + }, + "preview_domain": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "state": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState" + }, + "tenant_id": { + "type": "string" + }, + "vcpus": { + "type": "integer" + }, + "vm_id": { + "type": "string" + }, + "worker_id": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState": { + "type": "string", + "enum": [ + "creating", + "running", + "idle", + "unhealthy", + "expired", + "destroyed", + "error" + ], + "x-enum-varnames": [ + "StateCreating", + "StateRunning", + "StateIdle", + "StateUnhealthy", + "StateExpired", + "StateDestroyed", + "StateError" + ] + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus": { + "type": "object", + "properties": { + "admission_control": { + "type": "string" + }, + "eligible_workers": { + "type": "integer" + }, + "max_spawn_queue": { + "type": "integer" + }, + "selected_worker_id": { + "type": "string" + }, + "spawn_dequeued_total": { + "type": "integer" + }, + "spawn_overflow": { + "type": "string" + }, + "spawn_queue_depth": { + "type": "integer" + }, + "spawn_queue_timeout": { + "type": "string" + }, + "spawn_queue_timeouts": { + "type": "integer" + }, + "spawn_queue_wait_avg": { + "type": "string" + }, + "spawn_queue_wait_avg_ms": { + "type": "integer" + }, + "spawn_queue_wait_count": { + "type": "integer" + }, + "spawn_queue_wait_max": { + "type": "string" + }, + "spawn_queue_wait_max_ms": { + "type": "integer" + }, + "spawn_queue_wait_total": { + "type": "string" + }, + "spawn_queue_wait_total_ms": { + "type": "integer" + }, + "spawn_queued_total": { + "type": "integer" + }, + "worker_id": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig": { + "type": "object", + "properties": { + "inject_at": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision": { + "type": "object", + "properties": { + "active_owner_sandboxes": { + "type": "integer" + }, + "active_sandboxes": { + "type": "integer" + }, + "allowed": { + "type": "boolean" + }, + "eligible_workers": { "type": "integer" }, - "metadata": { - "type": "object", - "additionalProperties": { - "type": "string" - } + "max_owner_sandboxes": { + "type": "integer" }, - "provider": { + "max_sandboxes": { + "type": "integer" + }, + "max_ttl": { "type": "string" }, - "state": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState" + "queueable": { + "type": "boolean" }, - "vcpus": { - "type": "integer" - } - } - }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState": { - "type": "string", - "enum": [ - "creating", - "running", - "idle", - "destroyed", - "error" - ], - "x-enum-varnames": [ - "StateCreating", - "StateRunning", - "StateIdle", - "StateDestroyed", - "StateError" - ] - }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig": { - "type": "object", - "properties": { - "inject_at": { + "reason": { "type": "string" }, - "name": { + "selected_worker_id": { + "type": "string" + }, + "worker_reason": { "type": "string" } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest": { + "github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest": { "type": "object", "properties": { "image": { @@ -1236,12 +2244,18 @@ "type": "string" } }, + "owner_id": { + "type": "string" + }, "provider": { "type": "string" }, "template": { "type": "string" }, + "tenant_id": { + "type": "string" + }, "ttl": { "type": "string" }, @@ -1250,7 +2264,7 @@ } } }, - "github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template": { + "github_com_StacyOs_stacyvm_internal_orchestrator.Template": { "type": "object", "properties": { "allowed_hosts": { @@ -1289,7 +2303,7 @@ "secrets": { "type": "array", "items": { - "$ref": "#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig" + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig" } }, "setup": { @@ -1309,6 +2323,141 @@ } } }, + "github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "image": { + "type": "string" + }, + "provider": { + "type": "string" + } + } + }, + "internal_api_routes.AdminAuditResponse": { + "type": "object", + "properties": { + "actor": { + "type": "string", + "example": "admin" + }, + "created_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "duration_ms": { + "type": "integer", + "example": 4 + }, + "id": { + "type": "integer", + "example": 42 + }, + "method": { + "type": "string", + "example": "PUT" + }, + "path": { + "type": "string", + "example": "/api/v1/admin/quotas/owner-a" + }, + "remote_addr": { + "type": "string", + "example": "127.0.0.1" + }, + "request_id": { + "type": "string", + "example": "req-abc123" + }, + "status": { + "type": "integer", + "example": 200 + }, + "tenant_id": { + "type": "string", + "example": "tenant-acme" + }, + "user_agent": { + "type": "string", + "example": "stacyvm-web" + } + } + }, + "internal_api_routes.DiagnosticsResponse": { + "type": "object", + "properties": { + "build": { + "type": "object", + "additionalProperties": true + }, + "events": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats" + }, + "generated_at": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "leases": { + "type": "object", + "additionalProperties": true + }, + "limits": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics" + } + }, + "process": { + "type": "object", + "additionalProperties": true + }, + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "quotas": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary" + }, + "rate_limit": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats" + }, + "redactions": { + "type": "array", + "items": { + "type": "string" + } + }, + "remediation": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "sandboxes": { + "type": "object", + "additionalProperties": true + }, + "scheduler": { + "$ref": "#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus" + }, + "store": { + "type": "object", + "additionalProperties": true + }, + "workers": { + "type": "object", + "additionalProperties": true + } + } + }, "internal_api_routes.HealthResponse": { "type": "object", "properties": { @@ -1364,6 +2513,9 @@ "type": "boolean", "example": true }, + "health": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + }, "healthy": { "type": "boolean", "example": true @@ -1378,20 +2530,86 @@ } } }, + "internal_api_routes.ProviderHealth": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "spawn", + "exec", + "files" + ] + }, + "default": { + "type": "boolean", + "example": true + }, + "error": { + "type": "string", + "example": "health check returned false" + }, + "healthy": { + "type": "boolean", + "example": true + }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, + "name": { + "type": "string", + "example": "docker" + }, + "runtime_count": { + "type": "integer", + "example": 2 + } + } + }, "internal_api_routes.ProviderInfo": { "type": "object", "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, "default": { "type": "boolean", "example": true }, + "error": { + "type": "string", + "example": "health check returned false" + }, "healthy": { "type": "boolean", "example": true }, + "last_checked": { + "type": "string", + "example": "2026-05-08T10:30:00Z" + }, + "latency_ms": { + "type": "integer", + "example": 3 + }, "name": { "type": "string", "example": "firecracker" + }, + "runtime_count": { + "type": "integer", + "example": 2 } } }, @@ -1404,6 +2622,37 @@ } } }, + "internal_api_routes.ReadinessResponse": { + "type": "object", + "properties": { + "providers": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_routes.ProviderHealth" + } + }, + "ready_providers": { + "type": "integer", + "example": 1 + }, + "status": { + "type": "string", + "example": "ready" + }, + "total_providers": { + "type": "integer", + "example": 2 + }, + "uptime": { + "type": "string", + "example": "2h30m15s" + }, + "version": { + "type": "string", + "example": "1.0.0" + } + } + }, "internal_api_routes.StatusResponse": { "type": "object", "properties": { @@ -1425,6 +2674,84 @@ "example": "30m" } } + }, + "internal_api_routes.WorkerHeartbeatRequest": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "capacity": { + "type": "object", + "additionalProperties": true + }, + "hostname": { + "type": "string", + "example": "stacyvm-host-1" + }, + "providers": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "type": "string", + "example": "online" + } + } + }, + "internal_api_routes.WorkerResponse": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + } + }, + "capacity": { + "type": "object", + "additionalProperties": true + }, + "created_at": { + "type": "string", + "example": "2026-05-09T10:00:00Z" + }, + "hostname": { + "type": "string", + "example": "stacyvm-host-1" + }, + "id": { + "type": "string", + "example": "worker-local" + }, + "last_heartbeat": { + "type": "string", + "example": "2026-05-09T10:30:00Z" + }, + "providers": { + "type": "array", + "items": { + "type": "string" + } + }, + "stale": { + "type": "boolean", + "example": false + }, + "status": { + "type": "string", + "example": "online" + }, + "updated_at": { + "type": "string", + "example": "2026-05-09T10:30:00Z" + } + } } }, "securityDefinitions": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index bb86c2a..a153cf3 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,13 +1,36 @@ basePath: /api/v1 definitions: - github_com_stacyvm-dev_stacyvm_internal_httputil.APIError: + github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats: + properties: + active_buckets: + type: integer + allowed_total: + type: integer + bucket_ttl: + type: string + burst: + type: integer + cleanup_interval: + type: string + enabled: + type: boolean + evicted_total: + type: integer + key_by: + type: string + limited_total: + type: integer + requests_per_minute: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_httputil.APIError: properties: code: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.ErrorCode' message: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_httputil.ErrorCode: + github_com_StacyOs_stacyvm_internal_httputil.ErrorCode: enum: - NOT_FOUND - BAD_REQUEST @@ -15,6 +38,8 @@ definitions: - UNAUTHORIZED - CONFLICT - UNAVAILABLE + - TIMEOUT + - RESOURCE_LIMIT type: string x-enum-varnames: - CodeNotFound @@ -23,7 +48,9 @@ definitions: - CodeUnauth - CodeConflict - CodeUnavailable - github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event: + - CodeTimeout + - CodeResourceLimit + github_com_StacyOs_stacyvm_internal_orchestrator.Event: properties: data: items: @@ -36,9 +63,18 @@ definitions: timestamp: type: string type: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventType' type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.EventType: + github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats: + properties: + events_total: + type: integer + history_size: + type: integer + subscribers: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.EventType: enum: - sandbox.created - sandbox.running @@ -46,8 +82,19 @@ definitions: - sandbox.error - exec.started - exec.completed + - exec.failed + - exec.timeout - file.written - file.read + - operation.failed + - resource.limit + - provider.failed + - reconcile.action + - spawn.queued + - spawn.dequeued + - spawn.queue_timeout + - quota.saved + - quota.deleted type: string x-enum-varnames: - EventSandboxCreated @@ -56,9 +103,20 @@ definitions: - EventSandboxError - EventExecStarted - EventExecCompleted + - EventExecFailed + - EventExecTimeout - EventFileWritten - EventFileRead - github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest: + - EventOperationFailed + - EventResourceLimit + - EventProviderFailed + - EventReconcileAction + - EventSpawnQueued + - EventSpawnDequeued + - EventSpawnQueueTimeout + - EventQuotaSaved + - EventQuotaDeleted + github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest: properties: args: items: @@ -70,6 +128,8 @@ definitions: additionalProperties: type: string type: object + mode: + type: string stream: type: boolean timeout: @@ -77,7 +137,7 @@ definitions: workdir: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult: + github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult: properties: duration: type: string @@ -88,7 +148,7 @@ definitions: stdout: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo: + github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo: properties: is_dir: type: boolean @@ -101,7 +161,7 @@ definitions: size: type: integer type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest: + github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest: properties: content: type: string @@ -110,7 +170,92 @@ definitions: path: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox: + github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics: + properties: + failure_total: + type: integer + last_error: + type: string + last_observed_unix: + type: integer + latency_avg_ms: + type: integer + latency_count: + type: integer + latency_max_ms: + type: integer + latency_min_ms: + type: integer + latency_total_ms: + type: integer + operation: + type: string + provider: + type: string + success_total: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo: + properties: + default_exec_timeout: + type: string + max_exec_timeout: + type: string + max_sandboxes: + type: integer + max_sandboxes_per_owner: + type: integer + max_spawn_queue: + type: integer + max_ttl: + type: string + spawn_overflow: + type: string + spawn_queue_timeout: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota: + properties: + created_at: + type: string + max_exec_timeout: + type: string + max_sandboxes: + type: integer + max_ttl: + type: string + owner_id: + type: string + updated_at: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage: + properties: + active_sandboxes: + type: integer + max_exec_timeout: + type: string + max_sandboxes: + type: integer + max_ttl: + type: string + owner_id: + type: string + quota_configured: + type: boolean + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary: + properties: + total: + type: integer + with_max_exec_timeout: + type: integer + with_max_sandboxes: + type: integer + with_max_ttl: + type: integer + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox: properties: created_at: type: string @@ -126,18 +271,30 @@ definitions: additionalProperties: type: string type: object + owner_id: + type: string + preview_domain: + type: string provider: type: string state: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState' + tenant_id: + type: string vcpus: type: integer + vm_id: + type: string + worker_id: + type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.SandboxState: + github_com_StacyOs_stacyvm_internal_orchestrator.SandboxState: enum: - creating - running - idle + - unhealthy + - expired - destroyed - error type: string @@ -145,16 +302,82 @@ definitions: - StateCreating - StateRunning - StateIdle + - StateUnhealthy + - StateExpired - StateDestroyed - StateError - github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig: + github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus: + properties: + admission_control: + type: string + eligible_workers: + type: integer + max_spawn_queue: + type: integer + selected_worker_id: + type: string + spawn_dequeued_total: + type: integer + spawn_overflow: + type: string + spawn_queue_depth: + type: integer + spawn_queue_timeout: + type: string + spawn_queue_timeouts: + type: integer + spawn_queue_wait_avg: + type: string + spawn_queue_wait_avg_ms: + type: integer + spawn_queue_wait_count: + type: integer + spawn_queue_wait_max: + type: string + spawn_queue_wait_max_ms: + type: integer + spawn_queue_wait_total: + type: string + spawn_queue_wait_total_ms: + type: integer + spawn_queued_total: + type: integer + worker_id: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig: properties: inject_at: type: string name: type: string type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest: + github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision: + properties: + active_owner_sandboxes: + type: integer + active_sandboxes: + type: integer + allowed: + type: boolean + eligible_workers: + type: integer + max_owner_sandboxes: + type: integer + max_sandboxes: + type: integer + max_ttl: + type: string + queueable: + type: boolean + reason: + type: string + selected_worker_id: + type: string + worker_reason: + type: string + type: object + github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest: properties: image: type: string @@ -164,16 +387,20 @@ definitions: additionalProperties: type: string type: object + owner_id: + type: string provider: type: string template: type: string + tenant_id: + type: string ttl: type: string vcpus: type: integer type: object - github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template: + github_com_StacyOs_stacyvm_internal_orchestrator.Template: properties: allowed_hosts: items: @@ -199,7 +426,7 @@ definitions: type: integer secrets: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SecretConfig' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SecretConfig' type: array setup: items: @@ -212,6 +439,101 @@ definitions: version: type: integer type: object + github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary: + properties: + created_at: + type: string + image: + type: string + provider: + type: string + type: object + internal_api_routes.AdminAuditResponse: + properties: + actor: + example: admin + type: string + created_at: + example: "2026-05-08T10:30:00Z" + type: string + duration_ms: + example: 4 + type: integer + id: + example: 42 + type: integer + method: + example: PUT + type: string + path: + example: /api/v1/admin/quotas/owner-a + type: string + remote_addr: + example: 127.0.0.1 + type: string + request_id: + example: req-abc123 + type: string + status: + example: 200 + type: integer + tenant_id: + example: tenant-acme + type: string + user_agent: + example: stacyvm-web + type: string + type: object + internal_api_routes.DiagnosticsResponse: + properties: + build: + additionalProperties: true + type: object + events: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.EventBusStats' + generated_at: + example: "2026-05-08T10:30:00Z" + type: string + leases: + additionalProperties: true + type: object + limits: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationalLimitsInfo' + operations: + items: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OperationMetrics' + type: array + process: + additionalProperties: true + type: object + providers: + items: + $ref: '#/definitions/internal_api_routes.ProviderHealth' + type: array + quotas: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary' + rate_limit: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_api_middleware.RateLimitStats' + redactions: + items: + type: string + type: array + remediation: + additionalProperties: + type: string + type: object + sandboxes: + additionalProperties: true + type: object + scheduler: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SchedulerStatus' + store: + additionalProperties: true + type: object + workers: + additionalProperties: true + type: object + type: object internal_api_routes.HealthResponse: properties: status: @@ -251,6 +573,8 @@ definitions: default: example: true type: boolean + health: + $ref: '#/definitions/internal_api_routes.ProviderHealth' healthy: example: true type: boolean @@ -261,17 +585,65 @@ definitions: example: 3 type: integer type: object + internal_api_routes.ProviderHealth: + properties: + capabilities: + example: + - spawn + - exec + - files + items: + type: string + type: array + default: + example: true + type: boolean + error: + example: health check returned false + type: string + healthy: + example: true + type: boolean + last_checked: + example: "2026-05-08T10:30:00Z" + type: string + latency_ms: + example: 3 + type: integer + name: + example: docker + type: string + runtime_count: + example: 2 + type: integer + type: object internal_api_routes.ProviderInfo: properties: + capabilities: + items: + type: string + type: array default: example: true type: boolean + error: + example: health check returned false + type: string healthy: example: true type: boolean + last_checked: + example: "2026-05-08T10:30:00Z" + type: string + latency_ms: + example: 3 + type: integer name: example: firecracker type: string + runtime_count: + example: 2 + type: integer type: object internal_api_routes.PruneResponse: properties: @@ -279,6 +651,28 @@ definitions: example: 3 type: integer type: object + internal_api_routes.ReadinessResponse: + properties: + providers: + items: + $ref: '#/definitions/internal_api_routes.ProviderHealth' + type: array + ready_providers: + example: 1 + type: integer + status: + example: ready + type: string + total_providers: + example: 2 + type: integer + uptime: + example: 2h30m15s + type: string + version: + example: 1.0.0 + type: string + type: object internal_api_routes.StatusResponse: properties: status: @@ -294,6 +688,61 @@ definitions: example: 30m type: string type: object + internal_api_routes.WorkerHeartbeatRequest: + properties: + capabilities: + items: + type: string + type: array + capacity: + additionalProperties: true + type: object + hostname: + example: stacyvm-host-1 + type: string + providers: + items: + type: string + type: array + status: + example: online + type: string + type: object + internal_api_routes.WorkerResponse: + properties: + capabilities: + items: + type: string + type: array + capacity: + additionalProperties: true + type: object + created_at: + example: "2026-05-09T10:00:00Z" + type: string + hostname: + example: stacyvm-host-1 + type: string + id: + example: worker-local + type: string + last_heartbeat: + example: "2026-05-09T10:30:00Z" + type: string + providers: + items: + type: string + type: array + stale: + example: false + type: boolean + status: + example: online + type: string + updated_at: + example: "2026-05-09T10:30:00Z" + type: string + type: object host: localhost:7423 info: contact: {} @@ -301,6 +750,64 @@ info: title: StacyVM API version: "1.0" paths: + /admin/audit: + get: + description: Return recent redacted admin route access records + parameters: + - description: Maximum number of records, capped at 500 + in: query + name: limit + type: integer + - description: Actor exact match + in: query + name: actor + type: string + - description: HTTP method exact match + in: query + name: method + type: string + - description: HTTP status exact match + in: query + name: status + type: integer + - description: Path substring match + in: query + name: path + type: string + - description: 'Response format: json or csv' + in: query + name: format + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/internal_api_routes.AdminAuditResponse' + type: array + security: + - ApiKeyAuth: [] + summary: List admin audit logs + tags: + - admin + /diagnostics: + get: + description: Return redacted build, store, provider, sandbox, event, and operation + diagnostics + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.DiagnosticsResponse' + security: + - ApiKeyAuth: [] + summary: Get diagnostics + tags: + - system /events: get: description: Open an SSE stream for real-time sandbox and system events @@ -310,15 +817,30 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Event' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Event' + security: + - ApiKeyAuth: [] + summary: Subscribe to events + tags: + - system + /health: + get: + description: Return the health status, version, and uptime + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.HealthResponse' security: - ApiKeyAuth: [] - summary: Subscribe to events + summary: Health check tags: - system - /health: + /live: get: - description: Return the health status, version, and uptime + description: Return whether the StacyVM API process is alive produces: - application/json responses: @@ -328,7 +850,7 @@ paths: $ref: '#/definitions/internal_api_routes.HealthResponse' security: - ApiKeyAuth: [] - summary: Health check + summary: Liveness check tags: - system /metrics: @@ -347,6 +869,22 @@ paths: summary: Get metrics tags: - system + /metrics/prometheus: + get: + description: Return runtime, provider, sandbox, event, and operation metrics + in Prometheus text format + produces: + - text/plain + responses: + "200": + description: OK + schema: + type: string + security: + - ApiKeyAuth: [] + summary: Get Prometheus metrics + tags: + - system /providers: get: description: Return all registered providers with health status @@ -383,7 +921,7 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get provider details @@ -406,6 +944,155 @@ paths: summary: Test providers tags: - providers + /quotas: + get: + description: Return all persisted owner quota overrides + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + type: array + security: + - ApiKeyAuth: [] + summary: List owner quotas + tags: + - quotas + /quotas/{ownerID}: + delete: + description: Delete the quota override for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.StatusResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Delete owner quota + tags: + - quotas + get: + description: Return the persisted quota override for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Get owner quota + tags: + - quotas + put: + consumes: + - application/json + description: Create or update quota overrides for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + - description: Quota request + in: body + name: request + required: true + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerQuota' + security: + - ApiKeyAuth: [] + summary: Save owner quota + tags: + - quotas + /quotas/{ownerID}/usage: + get: + description: Return active sandbox usage and effective quota for an owner + parameters: + - description: Owner ID + in: path + name: ownerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.OwnerUsage' + security: + - ApiKeyAuth: [] + summary: Get owner quota usage + tags: + - quotas + /quotas/summary: + get: + description: Return non-identifying counts for persisted owner quota overrides + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.QuotaSummary' + security: + - ApiKeyAuth: [] + summary: Get quota summary + tags: + - quotas + /ready: + get: + description: Return whether the API is ready to serve sandbox traffic + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.ReadinessResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_routes.ReadinessResponse' + security: + - ApiKeyAuth: [] + summary: Readiness check + tags: + - system /sandboxes: delete: description: Destroy all expired sandboxes and return the count @@ -419,7 +1106,7 @@ paths: "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Prune sandboxes @@ -434,12 +1121,12 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' type: array "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: List sandboxes @@ -455,22 +1142,26 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.SpawnRequest' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest' produces: - application/json responses: "201": description: Created schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "429": + description: Too Many Requests + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Create a sandbox @@ -495,11 +1186,11 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Destroy a sandbox @@ -519,15 +1210,15 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get a sandbox @@ -549,22 +1240,22 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecRequest' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecRequest' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.ExecResult' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.ExecResult' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Execute a command @@ -590,6 +1281,50 @@ paths: summary: Execute via WebSocket tags: - sandboxes + /sandboxes/{sandboxID}/extend: + post: + consumes: + - application/json + description: Add additional time to a sandbox's expiration + parameters: + - description: Sandbox ID + in: path + name: sandboxID + required: true + type: string + - description: TTL extension + in: body + name: request + required: true + schema: + properties: + ttl: + type: string + type: object + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' + "400": + description: Bad Request + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Extend sandbox TTL + tags: + - sandboxes /sandboxes/{sandboxID}/files: get: description: Read file content from a sandbox @@ -614,15 +1349,15 @@ paths: "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Read a file @@ -643,7 +1378,7 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileWriteRequest' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileWriteRequest' produces: - application/json responses: @@ -654,15 +1389,15 @@ paths: "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Write a file @@ -688,16 +1423,16 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.FileInfo' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.FileInfo' type: array "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: List files @@ -728,16 +1463,66 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get console logs tags: - sandboxes + /sandboxes/admission: + post: + consumes: + - application/json + description: Return whether a spawn request would be allowed, queued, or denied + by quota and scheduler limits + parameters: + - description: Spawn request + in: body + name: request + required: true + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.SpawnAdmissionDecision' + "400": + description: Bad Request + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Evaluate spawn admission + tags: + - sandboxes + /snapshots: + get: + description: Return all pre-built VM snapshots available for fast restore + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_providers.SnapshotSummary' + type: array + security: + - ApiKeyAuth: [] + summary: List snapshots + tags: + - snapshots /templates: get: description: Return all registered templates @@ -748,12 +1533,12 @@ paths: description: OK schema: items: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' type: array "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: List templates @@ -769,26 +1554,26 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' produces: - application/json responses: "201": description: Created schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "409": description: Conflict schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Create a template @@ -813,11 +1598,11 @@ paths: "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Delete a template @@ -837,15 +1622,15 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Get a template @@ -866,26 +1651,26 @@ paths: name: request required: true schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' produces: - application/json responses: "200": description: OK schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Template' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Template' "400": description: Bad Request schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Update a template @@ -914,20 +1699,113 @@ paths: "201": description: Created schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_orchestrator.Sandbox' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_orchestrator.Sandbox' "404": description: Not Found schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' "500": description: Internal Server Error schema: - $ref: '#/definitions/github_com_stacyvm-dev_stacyvm_internal_httputil.APIError' + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' security: - ApiKeyAuth: [] summary: Spawn from template tags: - templates + /workers: + get: + description: Return worker registry records and heartbeat state + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/internal_api_routes.WorkerResponse' + type: array + security: + - ApiKeyAuth: [] + summary: List workers + tags: + - workers + /workers/{workerID}: + delete: + description: Remove a worker registry record + parameters: + - description: Worker ID + in: path + name: workerID + required: true + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.StatusResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Delete worker + tags: + - workers + get: + description: Return one worker registry record + parameters: + - description: Worker ID + in: path + name: workerID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.WorkerResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/github_com_StacyOs_stacyvm_internal_httputil.APIError' + security: + - ApiKeyAuth: [] + summary: Get worker + tags: + - workers + /workers/{workerID}/heartbeat: + post: + consumes: + - application/json + description: Create or update worker registry state for a worker + parameters: + - description: Worker ID + in: path + name: workerID + required: true + type: string + - description: Worker heartbeat + in: body + name: request + required: true + schema: + $ref: '#/definitions/internal_api_routes.WorkerHeartbeatRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/internal_api_routes.WorkerResponse' + security: + - ApiKeyAuth: [] + summary: Heartbeat worker + tags: + - workers securityDefinitions: ApiKeyAuth: in: header diff --git a/docs/threat-model.md b/docs/threat-model.md new file mode 100644 index 0000000..68a6b47 --- /dev/null +++ b/docs/threat-model.md @@ -0,0 +1,65 @@ +# Threat Model + +This threat model is the Phase 7 baseline. It focuses on StacyVM as a self-hosted sandbox control plane with local providers, live previews, SDK access, and an admin dashboard. + +## Assets + +- Host access to Docker, KVM, Firecracker, PRoot, and filesystem paths. +- Sandbox filesystem contents and user workspaces. +- API and admin API keys. +- SQLite database, audit logs, quotas, templates, environments, and registry metadata. +- Live-preview traffic and routing metadata. +- Registry credentials and build artifacts. + +## Trust Boundaries + +| Boundary | Risk | +|---|---| +| API client to StacyVM server | Unauthorized lifecycle, file, exec, or admin operations | +| StacyVM server to provider runtime | Container escape, VM misconfiguration, stale runtime ownership | +| Sandbox to host filesystem | Path traversal, workspace breakout, shared pool leakage | +| Live preview proxy to sandbox | Host header abuse, accidental exposure, cross-tenant preview routing | +| Admin dashboard to API | Key leakage from browser storage, overbroad operator access | +| Registry/environment builder | Secret leakage, malicious image build inputs, supply-chain drift | + +## Primary Threats + +| Threat | Current mitigation | Remaining work | +|---|---|---| +| Regular API key accesses admin routes | Admin key separation, `admin:*` scope enforcement, OIDC/JWT RS256 Bearer auth with group-to-role mapping (viewer/operator/admin/tenant_admin), and per-resource policy enforcement for image/provider/network controls | Expand per-route policy tests for every provider type | +| Missing operator attribution | `X-User-ID`, admin fallback attribution, and OIDC `sub`/`email` claims injected into `AuthIdentity` and written to audit records | None; OIDC actor claims implemented | +| Sandbox file path traversal | Manager pool scoping rejects traversal; Docker/PRoot/provider tests cover traversal cases | Continue platform conformance on live runtimes | +| Shell command injection | Explicit shell/argv execution modes; argv mode avoids shell interpolation | Expand SDK examples and conformance tests for every provider | +| Docker container escape | Dropped caps/seccomp/resource config supported | Harden defaults and certify gVisor/Kata | +| Stale runtime after restart | Startup reconciliation | Distributed leases for multi-worker | +| Worker impersonation | Worker RPC contract separates worker identity from user/admin identity; signed worker tokens enforce worker ID, token ID, audience, expiry, revocation, and worker-only scopes; worker RPC mTLS is wired for transport identity; centralized token issuance via `/api/v1/admin/worker-tokens` removes the need for workers to hold the signing key directly | Target-network mTLS smoke with deployment-issued certificates | +| Audit gaps | Admin audit and operation audit persisted for sandbox lifecycle, exec, and file operations | Extend operation audit to every env/registry mutation route | +| Live preview exposure | Traefik label routing and docs | Host allowlist and preview auth options | +| Secret leakage in diagnostics | Redaction in diagnostics | Support bundle redaction tests | +| Single-node database loss | SQLite backup docs | Backup/restore test automation | + +## Phase 7 Security Objectives + +- Make production misconfiguration visible through `stacyvm doctor --production`. +- Remove ambiguous command execution semantics before recommending public workloads. +- Increase file API path traversal coverage. Done for manager scoping and provider boundaries. +- Extend persisted audit beyond admin routes. Done for sandbox lifecycle, exec, and file operations. +- Convert runtime conformance docs into repeatable host checks. Done with `scripts/certify-runtime.sh`. + +## Non-Goals For Phase 7 + +- Multi-worker scheduling. +- Full OIDC/SSO implementation. +- Postgres store. +- Enterprise RBAC. + +Those belonged to later production stages after the single-node release candidate was hardened. + +## Phase 14 Security Additions + +- OIDC/JWT RS256 Bearer token auth with JWKS and configurable issuer, audience, and group-to-role mapping. +- RBAC roles: viewer, operator, admin, tenant_admin with scoped permission sets. +- Tenant/project model: resource isolation per tenant, per-tenant audit export. +- Policy controls: image, provider, and network allow-deny enforcement at spawn time. +- Centralized worker token issuance: workers obtain signed tokens from the control plane without holding the signing key. +- Sandbox tenant scoping: List and Get enforce tenant boundaries for OIDC-authenticated callers. diff --git a/docs/tutorials/code-runner.mdx b/docs/tutorials/code-runner.mdx new file mode 100644 index 0000000..34dbb74 --- /dev/null +++ b/docs/tutorials/code-runner.mdx @@ -0,0 +1,110 @@ +--- +title: "Example App: Code Runner" +description: "Build a small FastAPI application that runs submitted Python code inside disposable StacyVM sandboxes." +--- + +This example shows the complete developer loop: receive code, create a sandbox, write a file, execute it, return output, and always destroy the sandbox. + +## Prerequisites + +- A running StacyVM server. +- Docker provider access on the StacyVM host. +- Python 3.9+. +- A StacyVM API key if auth is enabled. + +The source lives in [`examples/code-runner-python`](https://github.com/StacyOS/stacyvm/tree/phase-14-worker-identity-hardening/examples/code-runner-python). + +## Install + +```bash +cd examples/code-runner-python +python3 -m venv .venv +. .venv/bin/activate +pip install -r requirements.txt +``` + +## Configure + +```bash +export STACYVM_URL="http://localhost:7423" +export STACYVM_API_KEY="sk_test_YOUR_API_KEY" +export STACYVM_IMAGE="python:3.12" +``` + +## Run The App + +```bash +uvicorn app:app --reload --port 8080 +``` + +## Submit Code + + +```bash cURL +curl -sS -X POST http://localhost:8080/run-python \ + -H "Content-Type: application/json" \ + -d '{"code":"print(sum([10, 20, 12]))","timeout":"10s"}' +``` + +```javascript JavaScript +const response = await fetch("http://localhost:8080/run-python", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + code: "print(sum([10, 20, 12]))", + timeout: "10s", + }), +}); + +if (!response.ok) { + throw new Error(await response.text()); +} + +console.log(await response.json()); +``` + +```python Python +import requests + +response = requests.post( + "http://localhost:8080/run-python", + json={ + "code": "print(sum([10, 20, 12]))", + "timeout": "10s", + }, + timeout=30, +) +response.raise_for_status() +print(response.json()) +``` + + +## Response + +```json +{ + "exit_code": 0, + "stdout": "42\n", + "stderr": "", + "duration": "120ms" +} +``` + +## Why This Pattern Works + +- The sandbox gets a short TTL so idle work is cleaned up. +- The app destroys the sandbox in a `finally` block. +- User code is written to a file instead of interpolated into a shell command. +- Runtime failures are returned as normal execution results. +- Provider failures return a clear API error. + +## Production Notes + +Before exposing a code runner to users, add authentication to your app, per-user quotas, request size limits, audit logging, and runtime certification for the host you are using. + +## Related + +- [Quickstart](/docs/getting-started/quickstart) +- [Python SDK](/docs/sdks/python) +- [TypeScript example app](/docs/tutorials/typescript-code-runner) +- [Production deployment](/docs/deployment) diff --git a/docs/tutorials/typescript-code-runner.mdx b/docs/tutorials/typescript-code-runner.mdx new file mode 100644 index 0000000..9ff3e87 --- /dev/null +++ b/docs/tutorials/typescript-code-runner.mdx @@ -0,0 +1,99 @@ +--- +title: "Example App: TypeScript Code Runner" +description: "Build an Express application that runs JavaScript code inside disposable StacyVM sandboxes with the TypeScript SDK." +--- + +This example uses the TypeScript SDK to build a small HTTP API. Each request gets a fresh sandbox, writes submitted code to `/app/main.js`, runs it with Node.js, returns the result, and destroys the sandbox. + +## Prerequisites + +- Node.js 18+. +- A running StacyVM server. +- Docker provider access on the StacyVM host. +- A StacyVM API key if auth is enabled. + +The source lives in [`examples/code-runner-typescript`](https://github.com/StacyOS/stacyvm/tree/phase-14-worker-identity-hardening/examples/code-runner-typescript). + +## Install + +```bash +cd examples/code-runner-typescript +npm install +``` + +## Configure + +```bash +export STACYVM_URL="http://localhost:7423" +export STACYVM_API_KEY="sk_test_YOUR_API_KEY" +export STACYVM_IMAGE="node:20" +``` + +## Run The App + +```bash +npm run dev +``` + +## Submit Code + + +```bash cURL +curl -sS -X POST http://localhost:8081/run-javascript \ + -H "Content-Type: application/json" \ + -d '{"code":"console.log([10, 20, 12].reduce((a, b) => a + b, 0));","timeout":"10s"}' +``` + +```javascript JavaScript +const response = await fetch("http://localhost:8081/run-javascript", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + code: "console.log([10, 20, 12].reduce((a, b) => a + b, 0));", + timeout: "10s", + }), +}); + +if (!response.ok) { + throw new Error(await response.text()); +} + +console.log(await response.json()); +``` + +```python Python +import requests + +response = requests.post( + "http://localhost:8081/run-javascript", + json={ + "code": "console.log([10, 20, 12].reduce((a, b) => a + b, 0));", + "timeout": "10s", + }, + timeout=30, +) +response.raise_for_status() +print(response.json()) +``` + + +## Response + +```json +{ + "exit_code": 0, + "stdout": "42\n", + "stderr": "", + "duration": "120ms" +} +``` + +## Production Notes + +Before exposing this service, add authentication, request size limits, per-user quota attribution, audit logging, and host runtime certification. + +## Related + +- [TypeScript SDK](/docs/sdks/typescript) +- [REST sandboxes](/docs/rest/sandboxes) +- [System architecture](/docs/architecture/system-overview) diff --git a/docs/worker-rpc-contract.md b/docs/worker-rpc-contract.md new file mode 100644 index 0000000..8e7efe9 --- /dev/null +++ b/docs/worker-rpc-contract.md @@ -0,0 +1,245 @@ +# Worker RPC Contract + +Phase 10 defined the control-plane to worker contract. Phase 11 wired that contract into a real worker runtime: `stacyvm worker` can authenticate to the control plane, submit heartbeat state through a worker-only HTTP endpoint, and expose an optional inbound RPC server with `--listen`. + +Phase 12 starts remote sandbox I/O routing. Remote spawn, status, destroy, lease renewal, shutdown/drain, non-streaming exec, live exec-stream calls, file APIs, and console logs now use the worker RPC transport when the scheduler selects a non-local worker that advertises `rpc_url`. + +## Contract Package + +The Go contract lives in `internal/workerproto`. + +Core envelope: + +```json +{ + "id": "req-123", + "method": "worker.spawn", + "worker_id": "worker-a", + "lease": { + "resource_id": "sb-abc123", + "holder_id": "worker-a", + "generation": 4, + "expires_at": "2026-05-09T10:31:00Z" + }, + "params": {} +} +``` + +Supported methods: + +| Method | Direction | Lease required | Purpose | +|---|---|---:|---| +| `worker.heartbeat` | worker to control plane | No | Report liveness, providers, capabilities, and capacity. | +| `worker.spawn` | control plane to worker | Yes | Assign sandbox creation to the selected worker. | +| `worker.destroy` | control plane to worker | Yes | Assign sandbox teardown to the owning worker. | +| `worker.status` | control plane to worker | No | Ask a worker for runtime state. | +| `worker.exec` | control plane to worker | No | Run a non-streaming command in an owned runtime. | +| `worker.exec_stream` | control plane to worker | No | Run a command and stream stdout/stderr chunks. | +| `worker.file_write` | control plane to worker | No | Write file content in an owned runtime. | +| `worker.file_read` | control plane to worker | No | Read file content from an owned runtime. | +| `worker.file_list` | control plane to worker | No | List files in an owned runtime. | +| `worker.file_delete` | control plane to worker | No | Delete a file or directory in an owned runtime. | +| `worker.file_move` | control plane to worker | No | Move or rename a file in an owned runtime. | +| `worker.file_chmod` | control plane to worker | No | Change file mode in an owned runtime. | +| `worker.file_stat` | control plane to worker | No | Stat a file in an owned runtime. | +| `worker.file_glob` | control plane to worker | No | Evaluate a glob pattern in an owned runtime. | +| `worker.logs` | control plane to worker | No | Return console log lines from an owned runtime. | +| `worker.renew_lease` | control plane to worker | Yes | Confirm continued ownership and renew fencing. | +| `worker.shutdown` | control plane to worker | No | Drain or stop a worker process. | + +## Lease Fencing + +Every mutating lifecycle assignment must include a lease token: + +- `resource_id` is the sandbox ID. +- `holder_id` must match the selected worker. +- `generation` is incremented whenever ownership is acquired or renewed. +- `expires_at` defines when another worker may take over. + +Workers must reject mutating work if the lease holder does not match their own worker ID or if the lease is expired. The control plane must renew leases before long-running operations cross the expiry window. + +## Worker Authentication + +Worker identity must be separate from user and admin identity. + +Recommended transport headers for the future network worker: + +| Header | Purpose | +|---|---| +| `X-Worker-ID` | Stable worker ID that must match the token subject. | +| `X-Worker-Token` or `Authorization: Bearer ` | Worker token signed by the control plane or trusted issuer. | +| `X-Request-ID` | Request correlation across control plane and worker logs. | + +Validated worker tokens should produce `workerproto.AuthClaims`: + +- `worker_id` +- `scopes` +- `expires` + +Initial scopes: + +- `worker:heartbeat` +- `worker:spawn` +- `worker:destroy` +- `worker:status` +- `worker:exec` +- `worker:files` +- `worker:logs` +- `worker:lease` + +Workers must not accept user API keys or admin API keys for worker RPC. Control-plane admin access and worker execution access are separate trust boundaries. + +Current control-plane worker authentication accepts either: + +- `auth.worker_token` as a shared staging token. +- `auth.worker_token_file` as a secret-mounted file containing the shared staging token. +- `auth.worker_tokens.` as a per-worker token map for production-aligned staging. +- `auth.worker_signing_key` for HMAC-SHA256 signed worker tokens using the `stacyvm-worker-v1..` format. +- `auth.worker_signing_key_file` as a secret-mounted file containing the active signing key. +- `auth.worker_signing_keys` as additional verification keys accepted during signing-key rotation. + +When a worker has an entry in `auth.worker_tokens`, that worker-specific token takes precedence and the shared token is rejected for that worker ID. This keeps legacy staging configs compatible while giving production deployments individually rotatable worker credentials. + +Signed worker token payloads are base64url JSON claims with `worker_id`, `jti`, `aud`, optional worker scopes, `iat`, optional `nbf`, and `exp`. The authenticated `X-Worker-ID` must match the signed `worker_id`, expired or not-yet-valid tokens are rejected, and any non-worker scopes are ignored. Tokens with `iat` are capped at a 15 minute lifetime with 30 seconds of clock skew tolerance. `stacyvm worker` can derive short-lived heartbeat and lease-renewal tokens with `aud=worker:control-plane` from `auth.worker_signing_key` when no static `--worker-token` or `auth.worker_token` is provided. Operators can also issue a token explicitly with `stacyvm worker token --ttl 5m --format json` to capture the token ID and expiry metadata. + +The same signed token format is accepted by worker RPC servers for control-plane-to-worker calls, but RPC tokens use `aud=worker:rpc`. When the control plane has `auth.worker_signing_key` and no shared `auth.worker_token`, it mints short-lived RPC-audience tokens for the target worker before calling `/rpc`. This lets remote spawn, status, exec, file, log, preview, and destroy routing avoid static shared worker RPC credentials. + +Issued tokens include a `jti` token ID. During an incident, add a compromised token ID to `auth.worker_revoked_token_ids`; both worker-to-control-plane routes and control-plane-to-worker RPC reject matching signed tokens. + +Token incident-response runbook: + +```bash +stacyvm worker token worker-a --signing-key-file /run/secrets/stacyvm-worker-signing-key --ttl 5m --format json +stacyvm worker token inspect '' +stacyvm worker token verify '' --signing-key-file /run/secrets/stacyvm-worker-signing-key --worker-id worker-a --audience worker:control-plane +stacyvm worker token rotation-plan --new-key-ref /run/secrets/stacyvm-worker-signing-key-new --previous-key-ref /run/secrets/stacyvm-worker-signing-key-old --ttl 5m +``` + +`stacyvm worker token inspect` decodes token metadata without verifying the signature. Use it to recover `worker_id`, `jti`, `aud`, and expiry metadata from an already-captured token before adding the `jti` value to `auth.worker_revoked_token_ids`; do not treat inspected claims as authenticated identity. `stacyvm worker token verify` validates the signature against `auth.worker_signing_key`, accepts `auth.worker_signing_keys` during rotation, applies optional `--worker-id` and `--audience` expectations, and rejects configured revoked token IDs. + +For production services, prefer `auth.worker_token_file`, `auth.worker_signing_key_file`, `--worker-token-file`, `--worker-signing-key-file`, `--signing-key-file`, and `--verification-key-file` with secret-mounted files over passing long-lived worker secrets directly in YAML, shell history, or environment variables. The config loader rejects ambiguous pairs such as `auth.worker_signing_key` plus `auth.worker_signing_key_file`, and `stacyvm config lint --production` reports whether worker token and signing-key values came from secret file references or inline config. `stacyvm worker --worker-token-file` reloads the token file for each heartbeat and lease-renewal request, so an external issuer or sidecar can rotate short-lived signed worker tokens without restarting the worker process. + +`stacyvm worker token rotation-plan` emits a no-secret checklist, config sketch, and validation commands for a two-key rotation window. No-downtime signing-key rotation uses this sequence: + +1. Set the new key as `auth.worker_signing_key`. +2. Move the previous key into `auth.worker_signing_keys`. +3. Restart or reload workers so they mint with the new key. +4. Wait until all old worker tokens have expired. +5. Remove the old key from `auth.worker_signing_keys`. + +`stacyvm config lint --production` warns when the active signing key is repeated in `auth.worker_signing_keys`, when duplicate rotation keys are configured, or when a shared `auth.worker_token` remains configured beside signed worker tokens. + +## Worker RPC mTLS + +Signed worker tokens authenticate the worker identity at the application layer. Enterprise deployments should also protect worker RPC transport with mTLS when worker RPC crosses a host or network boundary. + +Worker RPC TLS is opt-in through `worker.rpc_tls`: + +```yaml +worker: + listen_addr: "0.0.0.0:7430" + rpc_tls: + enabled: true + server_cert_file: "/etc/stacyvm/tls/worker.crt" + server_key_file: "/etc/stacyvm/tls/worker.key" + client_ca_file: "/etc/stacyvm/tls/control-plane-ca.crt" + ca_file: "/etc/stacyvm/tls/worker-ca.crt" + client_cert_file: "/etc/stacyvm/tls/control-plane.crt" + client_key_file: "/etc/stacyvm/tls/control-plane.key" + server_name: "worker-a.internal" + insecure_skip_verify: false +``` + +On worker nodes, `server_cert_file` and `server_key_file` serve the inbound `/rpc` endpoint. When `client_ca_file` is set, the worker requires and verifies a client certificate from the control plane. + +On control-plane nodes, `ca_file` verifies worker server certificates, `client_cert_file` and `client_key_file` present the control-plane client identity, and `server_name` pins the expected worker certificate name when DNS or advertised `rpc_url` hostnames differ. + +`insecure_skip_verify` exists only for throwaway local tests and should fail production config lint. + +Current Phase 11 heartbeat endpoint: + +```text +POST /api/v1/worker/{workerID}/heartbeat +``` + +The endpoint requires `X-Worker-ID` plus `X-Worker-Token` and rejects requests where the authenticated worker ID differs from the `{workerID}` path. + +Current Phase 11 worker RPC endpoint: + +```text +POST /rpc +``` + +Run it with: + +```bash +stacyvm worker --listen 127.0.0.1:7430 +``` + +The endpoint accepts `workerproto.Request` envelopes, requires the same worker headers, and currently implements `worker.status`, `worker.exec`, `worker.exec_stream`, file operations, `worker.logs`, `worker.renew_lease`, `worker.spawn`, `worker.destroy`, and `worker.shutdown`. + +For `worker.spawn`, the request carries a control-plane `sandbox_id` and the response returns both that ID and the provider `runtime_id`. The control plane should persist that mapping before routing later status, exec, file, or destroy operations to the owning worker. + +Remote workers advertise their control-plane callback endpoint through heartbeat capacity: + +```json +{ + "capacity": { + "max_sandboxes": 10, + "rpc_url": "http://worker-a.internal:7430", + "preview_domain": "worker-a.preview.example.com" + } +} +``` + +When the scheduler selects a non-local worker with `rpc_url` and signed worker RPC tokens or a static worker token are configured, the control plane acquires the sandbox lease for that worker, calls `worker.spawn`, persists the selected `worker_id`, and stores the returned provider runtime ID for later routing. + +Sandbox reads use the persisted `worker_id` and provider `runtime_id` to call `worker.status` on the owning worker. If the worker reports a changed state, the control plane updates its stored sandbox state. If the worker is temporarily unreachable, the control plane keeps serving the cached record and logs the refresh failure at debug level. + +Remote destroy uses the same persisted ownership tuple. The control plane fetches the durable sandbox lease, presents it to `worker.destroy`, updates sandbox state to `destroyed`, and releases the lease after the worker confirms teardown. + +Remote non-streaming exec uses the same persisted ownership tuple without acquiring a new lifecycle lease. The control plane sends command, argv mode, environment, workdir, timeout, provider, sandbox ID, and provider runtime ID to `worker.exec`. The worker runs the command against its local provider registry and returns exit code, stdout, and stderr. The control plane still writes normal exec logs and emits the same audit, event, metric, and timeout behavior used by local exec. + +Remote exec stream uses `worker.exec_stream` with `X-Worker-Stream: ndjson`. The worker flushes each stdout/stderr chunk as an NDJSON `workerproto.Response`, and the control plane exposes those chunks through the manager's existing stream channel API. Clients that do not request NDJSON can still receive the buffered `ExecStreamResult` response shape. + +Remote file APIs use the same ownership tuple. The control plane validates/scopes paths, then sends the provider runtime ID and requested file operation to the owning worker. Dedicated remote sandboxes are not treated as pool sandboxes just because `VMID` stores the provider runtime ID; pool workspace scoping remains local-pool only. + +Remote console logs use `worker.logs` with the persisted provider runtime ID, so workers read logs from the runtime they actually own instead of the control-plane sandbox ID. + +Remote preview metadata uses `capacity.preview_domain` from the owning worker. The control plane returns that domain on remote-owned sandboxes so SDKs and the dashboard build URLs for the worker or cluster ingress that can actually reach the runtime. If a worker does not advertise a preview domain, the control plane falls back to `server.preview_domain`. + +`worker.shutdown` is a drain signal. After receiving it, the worker rejects new `worker.spawn` assignments and reports `draining` in future heartbeats, which keeps it out of scheduler placement. Existing sandboxes keep their worker ownership while the worker is fresh and draining. + +Startup reconciliation applies a conservative remote ownership policy: + +- Fresh draining workers keep existing sandbox ownership. +- Stale, offline, or missing workers cause non-expired remote-owned sandboxes to become `unhealthy`. +- Expired remote-owned sandboxes become `expired` and release their durable lease. +- The control plane does not pretend to migrate a stateful runtime to another worker. Real reassignment requires provider-level snapshot or migration support. + +Current Phase 11 control-plane lease renewal endpoint: + +```text +POST /api/v1/worker/{workerID}/leases/{resourceID}/renew +``` + +The worker RPC handler validates the presented lease token before calling this endpoint. The control plane only renews unexpired leases held by the authenticated worker. + +## Cluster Store Semantics + +SQLite remains suitable for single-node and local development. Enterprise multi-worker mode should use Postgres or another store with equivalent guarantees. + +Required lease guarantees: + +- Atomic acquire by `resource_id`. +- Acquire succeeds when no lease exists, the lease is expired, or the same holder renews ownership. +- Acquire fails when a different holder owns an unexpired lease. +- Renew succeeds only for the current holder and only before expiry. +- Release succeeds only for the current holder. +- Concurrent acquire attempts must serialize on the lease row. + +In Postgres terms, lease acquire should be implemented with a unique key on `resource_id`, transactional upsert semantics, and row-level contention safety. Clock skew must be bounded because expiry is time-based. + +## Current Limits + +Remote placement returns `remote_worker_rpc_unavailable` unless the selected worker advertises `rpc_url` and the control plane can authenticate to worker RPC with either short-lived signed RPC-audience tokens or a static worker token. Postgres-backed cluster storage and signed production worker identity are wired into the current transport; deployment certification still needs the target host/runtime conformance checks listed in [cluster-conformance](/docs/cluster-conformance). diff --git a/examples/code-runner-python/README.md b/examples/code-runner-python/README.md new file mode 100644 index 0000000..a13e54e --- /dev/null +++ b/examples/code-runner-python/README.md @@ -0,0 +1,48 @@ +# StacyVM Python Code Runner Example + +This example is a small FastAPI application that runs submitted Python code in a disposable StacyVM sandbox. + +## Prerequisites + +- Python 3.9+ +- A running StacyVM server +- Docker provider access on the StacyVM host +- `STACYVM_API_KEY` when server auth is enabled + +## Install + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r requirements.txt +``` + +## Run + +```bash +export STACYVM_URL="http://localhost:7423" +export STACYVM_API_KEY="sk_test_YOUR_API_KEY" +export STACYVM_IMAGE="python:3.12" +uvicorn app:app --reload --port 8080 +``` + +## Try It + +```bash +curl -sS -X POST http://localhost:8080/run-python \ + -H "Content-Type: application/json" \ + -d '{"code":"print(sum([10, 20, 12]))","timeout":"10s"}' +``` + +Expected response: + +```json +{ + "exit_code": 0, + "stdout": "42\n", + "stderr": "", + "duration": "120ms" +} +``` + +Use this as a starting point, not as a public service as-is. Add app authentication, request limits, audit logging, and quota attribution before exposing it to users. diff --git a/examples/code-runner-python/app.py b/examples/code-runner-python/app.py new file mode 100644 index 0000000..97698cc --- /dev/null +++ b/examples/code-runner-python/app.py @@ -0,0 +1,64 @@ +import os +from typing import Annotated + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field +from stacyvm import Client, ProviderError + + +class RunPythonRequest(BaseModel): + code: Annotated[str, Field(min_length=1, max_length=50_000)] + timeout: str = "10s" + + +class RunPythonResponse(BaseModel): + exit_code: int + stdout: str + stderr: str + duration: str + + +app = FastAPI(title="StacyVM Code Runner") + + +def stacy_client() -> Client: + return Client( + base_url=os.getenv("STACYVM_URL", "http://localhost:7423"), + api_key=os.getenv("STACYVM_API_KEY"), + user_id=os.getenv("STACYVM_USER_ID", "example-code-runner"), + timeout=60.0, + ) + + +@app.post("/run-python", response_model=RunPythonResponse) +def run_python(request: RunPythonRequest) -> RunPythonResponse: + image = os.getenv("STACYVM_IMAGE", "python:3.12") + sandbox = None + + try: + client = stacy_client() + sandbox = client.spawn( + image=image, + ttl="2m", + memory_mb=512, + vcpus=1, + metadata={"example": "code-runner-python"}, + ) + sandbox.write_file("/app/main.py", request.code) + result = sandbox.exec("python3 /app/main.py", timeout=request.timeout) + return RunPythonResponse( + exit_code=result.exit_code, + stdout=result.stdout, + stderr=result.stderr, + duration=result.duration, + ) + except ProviderError as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + finally: + if sandbox is not None: + try: + sandbox.destroy() + except Exception: + pass diff --git a/examples/code-runner-python/requirements.txt b/examples/code-runner-python/requirements.txt new file mode 100644 index 0000000..da0bfa9 --- /dev/null +++ b/examples/code-runner-python/requirements.txt @@ -0,0 +1,4 @@ +fastapi>=0.110,<1 +pydantic>=2,<3 +stacyvm>=0.1,<1 +uvicorn[standard]>=0.29,<1 diff --git a/examples/code-runner-typescript/README.md b/examples/code-runner-typescript/README.md new file mode 100644 index 0000000..64ec389 --- /dev/null +++ b/examples/code-runner-typescript/README.md @@ -0,0 +1,46 @@ +# StacyVM TypeScript Code Runner Example + +This example is a small Express application that runs submitted JavaScript code in a disposable StacyVM sandbox. + +## Prerequisites + +- Node.js 18+ +- A running StacyVM server +- Docker provider access on the StacyVM host +- `STACYVM_API_KEY` when server auth is enabled + +## Install + +```bash +npm install +``` + +## Run + +```bash +export STACYVM_URL="http://localhost:7423" +export STACYVM_API_KEY="sk_test_YOUR_API_KEY" +export STACYVM_IMAGE="node:20" +npm run dev +``` + +## Try It + +```bash +curl -sS -X POST http://localhost:8081/run-javascript \ + -H "Content-Type: application/json" \ + -d '{"code":"console.log([10, 20, 12].reduce((a, b) => a + b, 0));","timeout":"10s"}' +``` + +Expected response: + +```json +{ + "exit_code": 0, + "stdout": "42\n", + "stderr": "", + "duration": "120ms" +} +``` + +Use this as a starting point, not as a public service as-is. Add app authentication, request limits, audit logging, and quota attribution before exposing it to users. diff --git a/examples/code-runner-typescript/package-lock.json b/examples/code-runner-typescript/package-lock.json new file mode 100644 index 0000000..3b25e39 --- /dev/null +++ b/examples/code-runner-typescript/package-lock.json @@ -0,0 +1,1544 @@ +{ + "name": "stacyvm-code-runner-typescript", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stacyvm-code-runner-typescript", + "dependencies": { + "express": "^4.18.3", + "stacyvm": "^0.1.0", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.11.30", + "tsx": "^4.7.1", + "typescript": "^5.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "^1" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.8", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", + "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.19.40", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.40.tgz", + "integrity": "sha512-xxx6M2IpSTnnKcR0cMvIiohkiCx20/oRPtWGbenFygKCGl3zqUzdNjQ/1V4solq1LU+dgv0nQzeGOuqkqZGg0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "<1" + } + }, + "node_modules/@types/serve-static/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/stacyvm": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/stacyvm/-/stacyvm-0.1.0.tgz", + "integrity": "sha512-4aZNXHKd3kWhg2ZjwkxWEnOgLPfCxdPvM3Szyr+Low0LyilCgNm3seIxG9hMX1DtRLD4AjJ/qgbtUACG1ddsLA==", + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/examples/code-runner-typescript/package.json b/examples/code-runner-typescript/package.json new file mode 100644 index 0000000..bedddcd --- /dev/null +++ b/examples/code-runner-typescript/package.json @@ -0,0 +1,20 @@ +{ + "name": "stacyvm-code-runner-typescript", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx src/server.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "express": "^4.18.3", + "stacyvm": "^0.1.0", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.11.30", + "tsx": "^4.7.1", + "typescript": "^5.4.0" + } +} diff --git a/examples/code-runner-typescript/src/server.ts b/examples/code-runner-typescript/src/server.ts new file mode 100644 index 0000000..371c78f --- /dev/null +++ b/examples/code-runner-typescript/src/server.ts @@ -0,0 +1,67 @@ +import express from "express"; +import { z } from "zod"; +import { Client, ProviderError } from "stacyvm"; + +const requestSchema = z.object({ + code: z.string().min(1).max(50_000), + timeout: z.string().default("10s"), +}); + +const app = express(); +app.use(express.json({ limit: "64kb" })); + +function stacyClient(): Client { + return new Client({ + baseUrl: process.env.STACYVM_URL ?? "http://localhost:7423", + apiKey: process.env.STACYVM_API_KEY, + userId: process.env.STACYVM_USER_ID ?? "example-code-runner-typescript", + timeout: 60_000, + }); +} + +app.post("/run-javascript", async (request, response) => { + const parsed = requestSchema.safeParse(request.body); + if (!parsed.success) { + response.status(400).json({ error: parsed.error.flatten() }); + return; + } + + const client = stacyClient(); + const image = process.env.STACYVM_IMAGE ?? "node:20"; + const sandbox = await client.spawn({ + image, + ttl: "2m", + memory_mb: 512, + vcpus: 1, + metadata: { example: "code-runner-typescript" }, + }); + + try { + await sandbox.writeFile("/app/main.js", parsed.data.code); + const result = await sandbox.exec("node /app/main.js", { + timeout: parsed.data.timeout, + }); + + response.json({ + exit_code: result.exit_code, + stdout: result.stdout, + stderr: result.stderr, + duration: result.duration, + }); + } catch (error) { + if (error instanceof ProviderError) { + response.status(502).json({ error: error.message }); + return; + } + + const message = error instanceof Error ? error.message : "unknown error"; + response.status(500).json({ error: message }); + } finally { + await sandbox.destroy().catch(() => undefined); + } +}); + +const port = Number(process.env.PORT ?? 8081); +app.listen(port, () => { + console.log(`StacyVM TypeScript code runner listening on :${port}`); +}); diff --git a/examples/code-runner-typescript/tsconfig.json b/examples/code-runner-typescript/tsconfig.json new file mode 100644 index 0000000..759f800 --- /dev/null +++ b/examples/code-runner-typescript/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"] +} diff --git a/go.mod b/go.mod index 19bc755..2644e3d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/go-chi/chi/v5 v5.2.1 github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.9.2 github.com/mdlayher/vsock v1.2.1 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.10.1 @@ -48,6 +49,9 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index f69f8dc..25ece13 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,14 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -176,6 +184,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= diff --git a/index.md b/index.md new file mode 100644 index 0000000..b44b668 --- /dev/null +++ b/index.md @@ -0,0 +1,97 @@ +--- +title: "StacyVM" +description: "Self-hosted sandbox infrastructure for running agent code in isolated, disposable environments." +--- + +# StacyVM + +StacyVM gives your applications disposable execution environments for AI agents, code runners, browser previews, and automation workflows. You keep control of the runtime, network, credentials, and audit trail while developers get a simple API and SDK. + + + + Understand the product, use cases, advantages, and where it fits. + + + Check the host, runtime, CLI, SDK, and production requirements before you start. + + + Start a local server, create a sandbox, run code, and destroy it. + + + See the control plane, scheduler, workers, providers, store, and sandbox lifecycle. + + + Build Python and TypeScript services that run code inside StacyVM. + + + Use StacyVM from Python services, jobs, and agent frameworks. + + + Use StacyVM from Node.js, web backends, and TypeScript agents. + + + +## What You Can Build + +- Run untrusted or generated code in short-lived sandboxes. +- Give coding agents a filesystem, shell, and live preview URL without exposing your host. +- Route sandbox work across Docker, Firecracker, PRoot, local providers, or remote workers. +- Track quotas, audit events, runtime health, and production readiness evidence. +- Integrate through REST, Python, or TypeScript. + +## First Sandbox + + +```bash cURL +curl -sS -X POST http://localhost:7423/api/v1/sandboxes \ + -H "Content-Type: application/json" \ + -H "X-API-Key: sk_test_YOUR_API_KEY" \ + -d '{"image":"python:3.12","ttl":"10m"}' +``` + +```python Python +from stacyvm import Client + +client = Client( + base_url="http://localhost:7423", + api_key="sk_test_YOUR_API_KEY", +) + +with client.spawn(image="python:3.12", ttl="10m") as sandbox: + result = sandbox.exec("python3 -c 'print(40 + 2)'") + print(result.stdout) +``` + +```typescript TypeScript +import { Client } from "stacyvm"; + +const client = new Client({ + baseUrl: "http://localhost:7423", + apiKey: "sk_test_YOUR_API_KEY", +}); + +await client.withSandbox({ image: "node:20", ttl: "10m" }, async (sandbox) => { + const result = await sandbox.exec("node -e 'console.log(40 + 2)'"); + console.log(result.stdout); +}); +``` + + +## Recommended Path + + + + Review [prerequisites](/docs/getting-started/prerequisites), then follow the [installation guide](/docs/getting-started/installation) for local Docker, binary, or source builds. + + + Use the [quickstart](/docs/getting-started/quickstart) to validate spawn, exec, files, and cleanup. + + + Choose the [Python SDK](/docs/sdks/python), [TypeScript SDK](/docs/sdks/typescript), or [REST API](/docs/rest/sandboxes). + + + Use the [deployment guide](/docs/deployment), [support matrix](/docs/public-support-matrix), and [runtime certification](/docs/runtime-certification) before making public runtime claims. + + + +For public deployments, only claim support for runtimes you have certified on the target host. diff --git a/internal/agentproto/protocol.go b/internal/agentproto/protocol.go index 6946b9d..0c89075 100644 --- a/internal/agentproto/protocol.go +++ b/internal/agentproto/protocol.go @@ -38,7 +38,7 @@ type Response struct { // StreamResponse is sent from agent to host for exec_stream, one per chunk. type StreamResponse struct { ID string `json:"id"` - Stream string `json:"stream,omitempty"` // "stdout" or "stderr" + Stream string `json:"stream,omitempty"` // "stdout" or "stderr" Data string `json:"data,omitempty"` Error string `json:"error,omitempty"` Done bool `json:"done,omitempty"` @@ -54,6 +54,7 @@ type PingResult struct { type ExecParams struct { Command string `json:"command"` Args []string `json:"args,omitempty"` + Mode string `json:"mode,omitempty"` WorkDir string `json:"work_dir,omitempty"` Env map[string]string `json:"env,omitempty"` } diff --git a/internal/api/auth_matrix_test.go b/internal/api/auth_matrix_test.go new file mode 100644 index 0000000..6097e90 --- /dev/null +++ b/internal/api/auth_matrix_test.go @@ -0,0 +1,413 @@ +package api + +// Auth matrix regression tests. +// +// Covers: API-key only, OIDC only, mixed, admin route access, worker tokens. +// Every case tests both the accept path and the reject path so regressions in +// either direction are caught immediately. + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/store" +) + +// ── RSA JWT helpers ────────────────────────────────────────────────────────── + +func matrixGenRSAKey(t *testing.T) *rsa.PrivateKey { + t.Helper() + k, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + return k +} + +// matrixPEM returns the PKIX PEM encoding of an RSA public key. +func matrixPEM(t *testing.T, key *rsa.PrivateKey) string { + t.Helper() + der, err := x509.MarshalPKIXPublicKey(&key.PublicKey) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})) +} + +func matrixMintJWT(t *testing.T, key *rsa.PrivateKey, claims map[string]any) string { + t.Helper() + hdr := base64.RawURLEncoding.EncodeToString(matrixJSON(t, map[string]string{ + "alg": "RS256", "kid": "k1", "typ": "JWT", + })) + pay := base64.RawURLEncoding.EncodeToString(matrixJSON(t, claims)) + signed := hdr + "." + pay + h := sha256.Sum256([]byte(signed)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, h[:]) + if err != nil { + t.Fatal(err) + } + return signed + "." + base64.RawURLEncoding.EncodeToString(sig) +} + +func matrixJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return b +} + +func matrixDo(t *testing.T, srv *Server, method, path string, headers map[string]string) int { + t.Helper() + req := httptest.NewRequest(method, path, nil) + for k, v := range headers { + req.Header.Set(k, v) + } + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + return rr.Code +} + +func matrixDoJSON(t *testing.T, srv *Server, method, path, body string, headers map[string]string) int { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + return rr.Code +} + +func matrixOIDC(key *rsa.PrivateKey, t *testing.T, + adminGroups, operatorGroups, viewerGroups []string) middleware.OIDCConfig { + return middleware.OIDCConfig{ + Issuer: "https://idp.test", + Audience: "stacyvm", + PublicKeyPEM: matrixPEM(t, key), + GroupsClaim: "groups", + AdminGroups: adminGroups, + OperatorGroups: operatorGroups, + ViewerGroups: viewerGroups, + } +} + +func jwtClaims(key *rsa.PrivateKey, t *testing.T, groups []string) string { + now := time.Now() + claims := map[string]any{ + "sub": "u1", "iss": "https://idp.test", "aud": "stacyvm", + "exp": now.Add(5 * time.Minute).Unix(), "iat": now.Unix(), + } + if len(groups) > 0 { + claims["groups"] = groups + } + return matrixMintJWT(t, key, claims) +} + +// ── 1. API-key only ────────────────────────────────────────────────────────── + +func TestAuthMatrix_APIKeyOnly_NormalRoute(t *testing.T) { + const apiKey = "api-key-32-bytes-long-enough-!!" + srv := setupTestServer(t, ServerConfig{Addr: "127.0.0.1:0", APIKey: apiKey, Version: "test"}) + + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"X-API-Key": apiKey}); code != http.StatusOK { + t.Errorf("valid API key: want 200, got %d", code) + } + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", nil); code != http.StatusUnauthorized { + t.Errorf("missing API key: want 401, got %d", code) + } + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"X-API-Key": "wrong"}); code != http.StatusUnauthorized { + t.Errorf("wrong API key: want 401, got %d", code) + } +} + +func TestAuthMatrix_APIKeyOnly_AdminRoute(t *testing.T) { + const ( + apiKey = "api-key-32-bytes-long-enough-!!" + adminKey = "admin-key-32-bytes-long-enough-!" + ) + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", APIKey: apiKey, AdminAPIKey: adminKey, Version: "test", + }) + + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/admin/diagnostics", + map[string]string{"X-Admin-API-Key": adminKey}); code != http.StatusOK { + t.Errorf("valid admin key: want 200, got %d", code) + } + // Regular API key must not reach admin routes. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/admin/diagnostics", + map[string]string{"X-API-Key": apiKey}); code != http.StatusForbidden { + t.Errorf("regular key on admin route: want 403, got %d", code) + } + // No key at all → AuthAny returns 401 (unauthenticated) before reaching admin routes. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/admin/diagnostics", nil); code != http.StatusUnauthorized { + t.Errorf("no key on admin route: want 401, got %d", code) + } +} + +// ── 2. OIDC only ───────────────────────────────────────────────────────────── + +func TestAuthMatrix_OIDCOnly_NormalRoute(t *testing.T) { + key := matrixGenRSAKey(t) + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", Version: "test", + OIDC: matrixOIDC(key, t, []string{"admins"}, nil, nil), + }) + + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer " + jwtClaims(key, t, nil)}); code != http.StatusOK { + t.Errorf("valid OIDC bearer: want 200, got %d", code) + } + // No bearer in OIDC-only mode — anonymous identity has no scopes, 403. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", nil); code != http.StatusForbidden { + t.Errorf("no bearer in OIDC-only mode: want 403, got %d", code) + } + // Malformed token — rejected before scope check. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer not.a.jwt"}); code != http.StatusUnauthorized { + t.Errorf("malformed bearer: want 401, got %d", code) + } +} + +func TestAuthMatrix_OIDCOnly_ViewerCannotSpawn(t *testing.T) { + key := matrixGenRSAKey(t) + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", Version: "test", + OIDC: matrixOIDC(key, t, []string{"admins"}, nil, []string{"viewers"}), + }) + + viewerToken := jwtClaims(key, t, []string{"viewers"}) + // Viewer can list (read:*). + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer " + viewerToken}); code != http.StatusOK { + t.Errorf("viewer list: want 200, got %d", code) + } + // Viewer cannot spawn (requires api:*). + if code := matrixDo(t, srv, http.MethodPost, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer " + viewerToken}); code != http.StatusForbidden { + t.Errorf("viewer spawn: want 403, got %d", code) + } +} + +func TestAuthMatrix_OIDCOnly_AdminRouteRequiresAdminRole(t *testing.T) { + key := matrixGenRSAKey(t) + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", Version: "test", + OIDC: matrixOIDC(key, t, []string{"admins"}, nil, nil), + }) + + // Anonymous must be blocked — this was the bug: admin routes were open in OIDC-only mode. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/admin/diagnostics", nil); code != http.StatusForbidden { + t.Errorf("anonymous on admin route (OIDC-only): want 403, got %d — regression: admin route was unprotected", code) + } + // Non-admin OIDC user must be blocked. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/admin/diagnostics", + map[string]string{"Authorization": "Bearer " + jwtClaims(key, t, nil)}); code != http.StatusForbidden { + t.Errorf("non-admin OIDC on admin route: want 403, got %d", code) + } + // Admin-group OIDC user must pass. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/admin/diagnostics", + map[string]string{"Authorization": "Bearer " + jwtClaims(key, t, []string{"admins"})}); code != http.StatusOK { + t.Errorf("admin OIDC on admin route: want 200, got %d", code) + } +} + +// ── 3. Mixed: OIDC + API key ───────────────────────────────────────────────── + +func TestAuthMatrix_Mixed_EitherAuthWorks(t *testing.T) { + key := matrixGenRSAKey(t) + const apiKey = "api-key-32-bytes-long-enough-!!" + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", APIKey: apiKey, Version: "test", + OIDC: matrixOIDC(key, t, nil, nil, nil), + }) + + // API key works. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"X-API-Key": apiKey}); code != http.StatusOK { + t.Errorf("API key in mixed mode: want 200, got %d", code) + } + // Valid OIDC bearer works. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer " + jwtClaims(key, t, nil)}); code != http.StatusOK { + t.Errorf("OIDC bearer in mixed mode: want 200, got %d", code) + } + // Invalid bearer must be rejected — not silently downgraded to anonymous. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer not.a.jwt"}); code != http.StatusUnauthorized { + t.Errorf("invalid bearer in mixed mode: want 401, got %d", code) + } + // No auth at all → rejected. + if code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", nil); code != http.StatusUnauthorized { + t.Errorf("no auth in mixed mode: want 401, got %d", code) + } +} + +// ── 4. Worker tokens must not be treated as OIDC Bearer tokens ─────────────── + +func TestAuthMatrix_WorkerToken_NotAcceptedOnAPIRoute(t *testing.T) { + const apiKey = "api-key-32-bytes-long-enough-!!" + signingKey := "worker-signing-key-32-bytes-long!" + // Configure both API key (so auth is active) and a worker signing key. + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", + APIKey: apiKey, + WorkerSigningKey: signingKey, + Version: "test", + }) + + workerToken, err := middleware.SignWorkerToken(signingKey, middleware.WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: middleware.WorkerTokenAudienceControlPlane, + ExpiresAt: time.Now().Add(5 * time.Minute).Unix(), + IssuedAt: time.Now().Unix(), + }) + if err != nil { + t.Fatal(err) + } + + // Worker token starts with "stacyvm-worker-v1" — bearerToken() excludes it + // so it never reaches the OIDC verifier. AuthAny also won't accept it as an + // API key. Result: 401 (not 200 that would indicate the token was accepted). + code := matrixDo(t, srv, http.MethodGet, "/api/v1/sandboxes", + map[string]string{"Authorization": "Bearer " + workerToken}) + if code == http.StatusOK { + t.Error("worker token was accepted on a regular API route — bearerToken() must exclude stacyvm-worker-v1 tokens") + } +} + +// ── 5. Worker token issuer: non-worker scopes rejected ─────────────────────── + +func TestAuthMatrix_TokenIssuer_RejectsNonWorkerScopes(t *testing.T) { + const adminKey = "admin-key-32-bytes-long-enough-!" + const signingKey = "worker-signing-key-32-bytes-long!" + srv := setupTestServer(t, ServerConfig{ + Addr: "127.0.0.1:0", + AdminAPIKey: adminKey, + WorkerSigningKey: signingKey, + Version: "test", + }) + + body := `{"worker_id":"worker-a","ttl":"5m","scopes":["admin:*"]}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/worker-tokens", + mustBodyReader(t, body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Admin-API-Key", adminKey) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + + if rr.Code != http.StatusBadRequest { + t.Errorf("non-worker scope in token request: want 400, got %d: %s", rr.Code, rr.Body.String()) + } + + // Worker scope is allowed. + body2 := `{"worker_id":"worker-a","ttl":"5m","scopes":["worker:spawn"]}` + req2 := httptest.NewRequest(http.MethodPost, "/api/v1/admin/worker-tokens", + mustBodyReader(t, body2)) + req2.Header.Set("Content-Type", "application/json") + req2.Header.Set("X-Admin-API-Key", adminKey) + rr2 := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr2, req2) + + if rr2.Code != http.StatusOK { + t.Errorf("worker scope in token request: want 200, got %d: %s", rr2.Code, rr2.Body.String()) + } +} + +// ── 6. Tenant policy enforcement on real sandbox routes ───────────────────── + +func TestAuthMatrix_TenantPolicyBlocksSandboxSpawn(t *testing.T) { + const apiKey = "api-key-32-bytes-long-enough-!!" + srv, st := setupTestServerWithStore(t, ServerConfig{ + Addr: "127.0.0.1:0", + APIKey: apiKey, + Version: "test", + }) + if err := st.CreatePolicy(t.Context(), &store.PolicyRecord{ + ID: "pol-deny-acme-image", + TenantID: "tenant-acme", + ResourceType: "image", + Effect: "deny", + Pattern: "blocked:*", + Priority: 1, + }); err != nil { + t.Fatalf("create tenant policy: %v", err) + } + + headers := map[string]string{ + "X-API-Key": apiKey, + "X-Tenant-ID": "tenant-acme", + } + blockedBody := `{"image":"blocked:latest","provider":"mock","ttl":"1m"}` + if code := matrixDoJSON(t, srv, http.MethodPost, "/api/v1/sandboxes", blockedBody, headers); code != http.StatusForbidden { + t.Fatalf("tenant-denied image spawn status = %d, want %d", code, http.StatusForbidden) + } + + allowedBody := `{"image":"allowed:latest","provider":"mock","ttl":"1m"}` + if code := matrixDoJSON(t, srv, http.MethodPost, "/api/v1/sandboxes", allowedBody, headers); code != http.StatusCreated { + t.Fatalf("tenant-allowed image spawn status = %d, want %d", code, http.StatusCreated) + } +} + +func TestAuthMatrix_TenantPolicyDoesNotLeakAcrossTenantsButGlobalDoes(t *testing.T) { + const apiKey = "api-key-32-bytes-long-enough-!!" + srv, st := setupTestServerWithStore(t, ServerConfig{ + Addr: "127.0.0.1:0", + APIKey: apiKey, + Version: "test", + }) + if err := st.CreatePolicy(t.Context(), &store.PolicyRecord{ + ID: "pol-deny-acme-image", + TenantID: "tenant-acme", + ResourceType: "image", + Effect: "deny", + Pattern: "blocked:*", + Priority: 1, + }); err != nil { + t.Fatalf("create tenant policy: %v", err) + } + if err := st.CreatePolicy(t.Context(), &store.PolicyRecord{ + ID: "pol-deny-global-network", + ResourceType: "network", + Effect: "deny", + Pattern: "host", + Priority: 1, + }); err != nil { + t.Fatalf("create global policy: %v", err) + } + + otherTenantHeaders := map[string]string{ + "X-API-Key": apiKey, + "X-Tenant-ID": "tenant-other", + } + blockedForAcmeBody := `{"image":"blocked:latest","provider":"mock","ttl":"1m"}` + if code := matrixDoJSON(t, srv, http.MethodPost, "/api/v1/sandboxes", blockedForAcmeBody, otherTenantHeaders); code != http.StatusCreated { + t.Fatalf("tenant-specific image policy leaked to other tenant: status = %d, want %d", code, http.StatusCreated) + } + + globalBlockedBody := `{"image":"allowed:latest","provider":"mock","network_mode":"host","ttl":"1m"}` + if code := matrixDoJSON(t, srv, http.MethodPost, "/api/v1/sandboxes", globalBlockedBody, otherTenantHeaders); code != http.StatusForbidden { + t.Fatalf("global network policy status = %d, want %d", code, http.StatusForbidden) + } +} + +func mustBodyReader(t *testing.T, body string) *strings.Reader { + t.Helper() + return strings.NewReader(body) +} diff --git a/internal/api/middleware/admin_audit.go b/internal/api/middleware/admin_audit.go new file mode 100644 index 0000000..1ae65e7 --- /dev/null +++ b/internal/api/middleware/admin_audit.go @@ -0,0 +1,103 @@ +package middleware + +import ( + "context" + "net" + "net/http" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/store" + "github.com/rs/zerolog" +) + +type adminAuditStore interface { + CreateAdminAudit(ctx context.Context, rec *store.AdminAuditRecord) error + DeleteAdminAuditBefore(ctx context.Context, before time.Time) (int64, error) +} + +type auditResponseWriter struct { + http.ResponseWriter + status int +} + +func (rw *auditResponseWriter) WriteHeader(status int) { + rw.status = status + rw.ResponseWriter.WriteHeader(status) +} + +func AdminAudit(st adminAuditStore, logger zerolog.Logger, retention time.Duration) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if st == nil { + next.ServeHTTP(w, r) + return + } + + start := time.Now() + rw := &auditResponseWriter{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(rw, r) + + identity := AuthIdentityFromContext(r.Context()) + rec := &store.AdminAuditRecord{ + Actor: actorFromRequest(r), + Method: r.Method, + Path: r.URL.Path, + Status: rw.status, + DurationMS: time.Since(start).Milliseconds(), + RequestID: GetRequestID(r.Context()), + RemoteAddr: clientAddr(r), + UserAgent: r.UserAgent(), + TenantID: identity.TenantID, + CreatedAt: time.Now().UTC(), + } + if err := st.CreateAdminAudit(r.Context(), rec); err != nil { + logger.Warn().Err(err).Str("path", r.URL.Path).Msg("failed to write admin audit log") + return + } + if retention > 0 { + before := rec.CreatedAt.Add(-retention) + deleted, err := st.DeleteAdminAuditBefore(r.Context(), before) + if err != nil { + logger.Warn().Err(err).Msg("failed to prune admin audit logs") + } else if deleted > 0 { + logger.Debug().Int64("deleted", deleted).Dur("retention", retention).Msg("pruned admin audit logs") + } + } + }) + } +} + +func actorFromRequest(r *http.Request) string { + if actor := strings.TrimSpace(r.Header.Get("X-User-ID")); actor != "" { + return actor + } + identity := AuthIdentityFromContext(r.Context()) + if identity.Email != "" { + return identity.Email + } + if identity.Subject != "" { + return identity.Subject + } + if identity.Role != AuthRoleAnonymous && identity.Header != "" { + return string(identity.Role) + ":" + identity.Header + } else if identity.Role != AuthRoleAnonymous { + return string(identity.Role) + } + return "admin" +} + +func clientAddr(r *http.Request) string { + if forwardedFor := r.Header.Get("X-Forwarded-For"); forwardedFor != "" { + parts := strings.Split(forwardedFor, ",") + return strings.TrimSpace(parts[0]) + } + if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); realIP != "" { + return realIP + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + return host + } + return r.RemoteAddr +} diff --git a/internal/api/middleware/admin_audit_test.go b/internal/api/middleware/admin_audit_test.go new file mode 100644 index 0000000..0253535 --- /dev/null +++ b/internal/api/middleware/admin_audit_test.go @@ -0,0 +1,105 @@ +package middleware + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/store" + "github.com/rs/zerolog" +) + +type memoryAuditStore struct { + records []*store.AdminAuditRecord + deleted int64 +} + +func (s *memoryAuditStore) CreateAdminAudit(ctx context.Context, rec *store.AdminAuditRecord) error { + s.records = append(s.records, rec) + return nil +} + +func (s *memoryAuditStore) DeleteAdminAuditBefore(ctx context.Context, before time.Time) (int64, error) { + var kept []*store.AdminAuditRecord + for _, rec := range s.records { + if rec.CreatedAt.Before(before) { + s.deleted++ + continue + } + kept = append(kept, rec) + } + s.records = kept + return s.deleted, nil +} + +func TestAdminAuditPrunesWithRetention(t *testing.T) { + st := &memoryAuditStore{ + records: []*store.AdminAuditRecord{ + {Path: "/old", CreatedAt: time.Now().Add(-2 * time.Hour)}, + }, + } + handler := AdminAudit(st, zerolog.Nop(), time.Hour)(okHandler()) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if st.deleted != 1 { + t.Fatalf("deleted = %d, want 1", st.deleted) + } + if len(st.records) != 1 || st.records[0].Path != "/api/v1/admin/diagnostics" { + t.Fatalf("unexpected audit records after prune: %+v", st.records) + } +} + +func TestAdminAuditUsesAuthenticatedHeaderWhenActorHeaderMissing(t *testing.T) { + st := &memoryAuditStore{} + handler := AdminAudit(st, zerolog.Nop(), 0)(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/quotas", nil) + req = req.WithContext(WithAuthIdentity(req.Context(), AuthIdentity{ + Role: AuthRoleAdmin, + Header: "X-Admin-API-Key", + })) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if len(st.records) != 1 { + t.Fatalf("records = %d, want 1", len(st.records)) + } + if st.records[0].Actor != "admin:X-Admin-API-Key" { + t.Fatalf("actor = %q, want admin:X-Admin-API-Key", st.records[0].Actor) + } +} + +func TestAdminAuditUserIDOverridesAuthenticatedHeader(t *testing.T) { + st := &memoryAuditStore{} + handler := AdminAudit(st, zerolog.Nop(), 0)(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/quotas", nil) + req.Header.Set("X-User-ID", "operator-a") + req = req.WithContext(WithAuthIdentity(req.Context(), AuthIdentity{ + Role: AuthRoleAdmin, + Header: "X-Admin-API-Key", + })) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if len(st.records) != 1 { + t.Fatalf("records = %d, want 1", len(st.records)) + } + if st.records[0].Actor != "operator-a" { + t.Fatalf("actor = %q, want operator-a", st.records[0].Actor) + } +} diff --git a/internal/api/middleware/auth.go b/internal/api/middleware/auth.go index b3695cb..88b69fb 100644 --- a/internal/api/middleware/auth.go +++ b/internal/api/middleware/auth.go @@ -1,22 +1,119 @@ package middleware import ( + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" "crypto/subtle" + "encoding/base64" + "encoding/hex" "encoding/json" + "errors" "net/http" + "strings" + "time" ) +type AuthRole string + +const ( + AuthRoleAnonymous AuthRole = "anonymous" + AuthRoleViewer AuthRole = "viewer" // read-only sandbox/template list + AuthRoleAPI AuthRole = "api" // standard API access (spawn/exec/files) + AuthRoleOperator AuthRole = "operator" // API + template/environment management + AuthRoleAdmin AuthRole = "admin" // operator + quotas/workers/provider config + AuthRoleTenantAdmin AuthRole = "tenant_admin" // admin within a specific tenant + AuthRoleWorker AuthRole = "worker" + + ScopeAPI = "api:*" + ScopeAdmin = "admin:*" + ScopeRead = "read:*" + ScopeOperator = "operator:*" + ScopeTenantAdmin = "tenant:admin" + ScopeWorkerHeartbeat = "worker:heartbeat" + ScopeWorkerSpawn = "worker:spawn" + ScopeWorkerDestroy = "worker:destroy" + ScopeWorkerStatus = "worker:status" + ScopeWorkerExec = "worker:exec" + ScopeWorkerFiles = "worker:files" + ScopeWorkerLogs = "worker:logs" + ScopeWorkerLease = "worker:lease" +) + +type AuthIdentity struct { + Role AuthRole + Header string + WorkerID string + Scopes []string + // OIDC-populated fields + Subject string + Email string + TenantID string + Groups []string +} + +type authIdentityContextKey struct{} + +const workerSignedTokenPrefix = "stacyvm-worker-v1" + +var errInvalidWorkerTokenClaims = errors.New("invalid worker token claims") + +const ( + WorkerTokenAudienceControlPlane = "worker:control-plane" + WorkerTokenAudienceRPC = "worker:rpc" + MaxWorkerTokenTTL = 15 * time.Minute + WorkerTokenClockSkew = 30 * time.Second +) + +type WorkerAuthConfig struct { + SharedToken string + WorkerTokens map[string]string + SigningKey string + SigningKeys []string + RevokedTokenIDs []string + Now func() time.Time +} + +type WorkerTokenClaims struct { + WorkerID string `json:"worker_id"` + TokenID string `json:"jti,omitempty"` + Audience string `json:"aud,omitempty"` + Scopes []string `json:"scopes,omitempty"` + ExpiresAt int64 `json:"exp"` + IssuedAt int64 `json:"iat,omitempty"` + NotBefore int64 `json:"nbf,omitempty"` +} + func Auth(apiKey string) func(http.Handler) http.Handler { + return AuthAny(apiKey) +} + +func AuthAny(apiKeys ...string) func(http.Handler) http.Handler { + candidates := make([]authCandidate, 0, len(apiKeys)) + for i, key := range apiKeys { + role := AuthRoleAPI + if i > 0 { + role = AuthRoleAdmin + } + candidates = append(candidates, authCandidate{Key: key, Role: role}) + } return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if apiKey == "" { + if len(nonEmptyKeys(apiKeys...)) == 0 { next.ServeHTTP(w, r) return } - key := r.Header.Get("X-API-Key") + // If OIDC already established identity via Bearer token, honour it + // and skip the API-key check so mixed OIDC+API-key deployments work. + if existing := AuthIdentityFromContext(r.Context()); existing.Header == "Authorization" { + next.ServeHTTP(w, r) + return + } - if subtle.ConstantTimeCompare([]byte(key), []byte(apiKey)) != 1 { + identity, ok := authenticateRequest(r, candidates...) + if !ok { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]string{ @@ -26,7 +123,433 @@ func Auth(apiKey string) func(http.Handler) http.Handler { return } + r = r.WithContext(WithAuthIdentity(r.Context(), identity)) + next.ServeHTTP(w, r) + }) + } +} + +func AdminAuth(adminAPIKey, fallbackAPIKey string, fallbackEnabled bool) func(http.Handler) http.Handler { + candidates := []authCandidate{{Key: adminAPIKey, Role: AuthRoleAdmin}} + if adminAPIKey == "" && fallbackEnabled { + adminAPIKey = fallbackAPIKey + candidates = []authCandidate{{Key: fallbackAPIKey, Role: AuthRoleAdmin}} + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // If OIDC already established identity via Bearer token, skip the + // API-key admin check — RequireScope(ScopeAdmin) handles authorisation. + // We specifically check Header == "Authorization" so API-key identities + // (Header "X-API-Key") still go through AdminAuth for role promotion. + if existing := AuthIdentityFromContext(r.Context()); existing.Header == "Authorization" { + next.ServeHTTP(w, r) + return + } + + // No admin API key configured (OIDC-only deployment): do not grant + // anonymous access — fall through to RequireScope which will reject. + if adminAPIKey == "" { + next.ServeHTTP(w, r) + return + } + + identity, ok := authenticateRequest(r, candidates...) + if !ok { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + json.NewEncoder(w).Encode(map[string]string{ + "code": "FORBIDDEN", + "message": "admin API key required", + }) + return + } + + r = r.WithContext(WithAuthIdentity(r.Context(), identity)) next.ServeHTTP(w, r) }) } } + +func WorkerAuth(workerToken string) func(http.Handler) http.Handler { + return WorkerAuthWithTokens(workerToken, nil) +} + +func WorkerAuthWithTokens(sharedWorkerToken string, workerTokens map[string]string) func(http.Handler) http.Handler { + return WorkerAuthWithConfig(WorkerAuthConfig{ + SharedToken: sharedWorkerToken, + WorkerTokens: workerTokens, + }) +} + +func WorkerAuthWithConfig(cfg WorkerAuthConfig) func(http.Handler) http.Handler { + cleanWorkerTokens := normalizeWorkerTokens(cfg.WorkerTokens) + sharedWorkerToken := strings.TrimSpace(cfg.SharedToken) + signingKeys := normalizeSigningKeys(cfg.SigningKey, cfg.SigningKeys) + revokedTokenIDs := normalizeRevokedTokenIDs(cfg.RevokedTokenIDs) + now := cfg.Now + if now == nil { + now = time.Now + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if sharedWorkerToken == "" && len(cleanWorkerTokens) == 0 && len(signingKeys) == 0 { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(map[string]string{ + "code": "UNAVAILABLE", + "message": "worker token is not configured", + }) + return + } + workerID := strings.TrimSpace(r.Header.Get("X-Worker-ID")) + token, header := workerTokenFromRequest(r) + scopes, ok := validateWorkerCredentials(workerID, token, sharedWorkerToken, cleanWorkerTokens, signingKeys, revokedTokenIDs, now) + if workerID == "" || !ok { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{ + "code": "UNAUTHORIZED", + "message": "invalid or missing worker credentials", + }) + return + } + r = r.WithContext(WithAuthIdentity(r.Context(), AuthIdentity{ + Role: AuthRoleWorker, + Header: header, + WorkerID: workerID, + Scopes: scopes, + })) + next.ServeHTTP(w, r) + }) + } +} + +func WithAuthIdentity(ctx context.Context, identity AuthIdentity) context.Context { + return context.WithValue(ctx, authIdentityContextKey{}, identity) +} + +func AuthIdentityFromContext(ctx context.Context) AuthIdentity { + identity, ok := ctx.Value(authIdentityContextKey{}).(AuthIdentity) + if !ok { + return AuthIdentity{Role: AuthRoleAnonymous} + } + return identity +} + +func (i AuthIdentity) HasScope(scope string) bool { + for _, candidate := range i.Scopes { + if candidate == scope { + return true + } + } + return false +} + +func RequireScope(scope string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + identity := AuthIdentityFromContext(r.Context()) + if !identity.HasScope(scope) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + json.NewEncoder(w).Encode(map[string]string{ + "code": "FORBIDDEN", + "message": "required authorization scope missing", + }) + return + } + + next.ServeHTTP(w, r) + }) + } +} + +type authCandidate struct { + Key string + Role AuthRole +} + +func authenticateRequest(r *http.Request, candidates ...authCandidate) (AuthIdentity, bool) { + for _, header := range []string{"X-Admin-API-Key", "X-API-Key"} { + candidate := r.Header.Get(header) + if candidate == "" { + continue + } + for _, authCandidate := range candidates { + if authCandidate.Key == "" { + continue + } + if subtle.ConstantTimeCompare([]byte(candidate), []byte(authCandidate.Key)) == 1 { + // Extract tenant hint from header if present (API key users may still scope to a tenant). + tenantID := strings.TrimSpace(r.Header.Get("X-Tenant-ID")) + return AuthIdentity{ + Role: authCandidate.Role, + Header: header, + Scopes: scopesForRole(authCandidate.Role), + TenantID: tenantID, + }, true + } + } + } + return AuthIdentity{}, false +} + +func scopesForRole(role AuthRole) []string { + switch role { + case AuthRoleAdmin: + return []string{ScopeRead, ScopeAPI, ScopeOperator, ScopeAdmin} + case AuthRoleTenantAdmin: + return []string{ScopeRead, ScopeAPI, ScopeOperator, ScopeTenantAdmin} + case AuthRoleOperator: + return []string{ScopeRead, ScopeAPI, ScopeOperator} + case AuthRoleAPI: + return []string{ScopeRead, ScopeAPI} + case AuthRoleViewer: + return []string{ScopeRead} + case AuthRoleWorker: + return []string{ + ScopeWorkerHeartbeat, + ScopeWorkerSpawn, + ScopeWorkerDestroy, + ScopeWorkerStatus, + ScopeWorkerExec, + ScopeWorkerFiles, + ScopeWorkerLogs, + ScopeWorkerLease, + } + default: + return nil + } +} + +func workerTokenFromRequest(r *http.Request) (string, string) { + if token := strings.TrimSpace(r.Header.Get("X-Worker-Token")); token != "" { + return token, "X-Worker-Token" + } + const prefix = "Bearer " + authz := strings.TrimSpace(r.Header.Get("Authorization")) + if len(authz) > len(prefix) && strings.EqualFold(authz[:len(prefix)], prefix) { + return strings.TrimSpace(authz[len(prefix):]), "Authorization" + } + return "", "" +} + +func SignWorkerToken(signingKey string, claims WorkerTokenClaims) (string, error) { + signingKey = strings.TrimSpace(signingKey) + claims.WorkerID = strings.TrimSpace(claims.WorkerID) + if signingKey == "" || claims.WorkerID == "" || claims.ExpiresAt <= 0 { + return "", errInvalidWorkerTokenClaims + } + payload, err := json.Marshal(claims) + if err != nil { + return "", err + } + payloadB64 := base64.RawURLEncoding.EncodeToString(payload) + signedPart := workerSignedTokenPrefix + "." + payloadB64 + signature := signWorkerToken(signingKey, signedPart) + return signedPart + "." + signature, nil +} + +func NewWorkerTokenID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} + +func VerifyWorkerToken(signingKey, token string, now time.Time) (WorkerTokenClaims, bool) { + return VerifyWorkerTokenForAudience(signingKey, token, "", now) +} + +// DecodeWorkerTokenClaims decodes signed worker token metadata without verifying +// the signature. Callers must not use the returned claims as authenticated identity. +func DecodeWorkerTokenClaims(token string) (WorkerTokenClaims, bool) { + token = strings.TrimSpace(token) + parts := strings.Split(token, ".") + if len(parts) != 3 || parts[0] != workerSignedTokenPrefix { + return WorkerTokenClaims{}, false + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return WorkerTokenClaims{}, false + } + var claims WorkerTokenClaims + if err := json.Unmarshal(payload, &claims); err != nil { + return WorkerTokenClaims{}, false + } + claims.WorkerID = strings.TrimSpace(claims.WorkerID) + claims.TokenID = strings.TrimSpace(claims.TokenID) + claims.Audience = strings.TrimSpace(claims.Audience) + return claims, true +} + +func VerifyWorkerTokenForAudience(signingKey, token, audience string, now time.Time) (WorkerTokenClaims, bool) { + signingKey = strings.TrimSpace(signingKey) + token = strings.TrimSpace(token) + if signingKey == "" || token == "" { + return WorkerTokenClaims{}, false + } + parts := strings.Split(token, ".") + if len(parts) != 3 || parts[0] != workerSignedTokenPrefix { + return WorkerTokenClaims{}, false + } + signedPart := parts[0] + "." + parts[1] + expectedSignature := signWorkerToken(signingKey, signedPart) + if subtle.ConstantTimeCompare([]byte(parts[2]), []byte(expectedSignature)) != 1 { + return WorkerTokenClaims{}, false + } + claims, ok := DecodeWorkerTokenClaims(token) + if !ok { + return WorkerTokenClaims{}, false + } + if claims.WorkerID == "" || claims.ExpiresAt <= 0 || !now.Before(time.Unix(claims.ExpiresAt, 0)) { + return WorkerTokenClaims{}, false + } + if claims.NotBefore > 0 && now.Add(WorkerTokenClockSkew).Before(time.Unix(claims.NotBefore, 0)) { + return WorkerTokenClaims{}, false + } + if claims.IssuedAt > 0 && now.Add(WorkerTokenClockSkew).Before(time.Unix(claims.IssuedAt, 0)) { + return WorkerTokenClaims{}, false + } + if claims.IssuedAt > 0 && time.Unix(claims.ExpiresAt, 0).Sub(time.Unix(claims.IssuedAt, 0)) > MaxWorkerTokenTTL { + return WorkerTokenClaims{}, false + } + if audience = strings.TrimSpace(audience); audience != "" && claims.Audience != "" && claims.Audience != audience { + return WorkerTokenClaims{}, false + } + return claims, true +} + +func validateWorkerCredentials(workerID, token, sharedWorkerToken string, workerTokens map[string]string, signingKeys []string, revokedTokenIDs map[string]struct{}, now func() time.Time) ([]string, bool) { + if token == "" || workerID == "" { + return nil, false + } + if claims, ok := verifyWorkerTokenWithAnyKey(signingKeys, token, WorkerTokenAudienceControlPlane, now().UTC()); ok { + if claims.WorkerID != workerID { + return nil, false + } + if isWorkerTokenRevoked(claims, revokedTokenIDs) { + return nil, false + } + scopes := normalizeScopes(claims.Scopes) + if len(scopes) == 0 { + scopes = scopesForRole(AuthRoleWorker) + } + return scopes, true + } + if validWorkerToken(workerID, token, sharedWorkerToken, workerTokens) { + return scopesForRole(AuthRoleWorker), true + } + return nil, false +} + +func isWorkerTokenRevoked(claims WorkerTokenClaims, revokedTokenIDs map[string]struct{}) bool { + if len(revokedTokenIDs) == 0 || claims.TokenID == "" { + return false + } + _, ok := revokedTokenIDs[claims.TokenID] + return ok +} + +func verifyWorkerTokenWithAnyKey(signingKeys []string, token, audience string, now time.Time) (WorkerTokenClaims, bool) { + for _, signingKey := range signingKeys { + if claims, ok := VerifyWorkerTokenForAudience(signingKey, token, audience, now); ok { + return claims, true + } + } + return WorkerTokenClaims{}, false +} + +func signWorkerToken(signingKey, signedPart string) string { + mac := hmac.New(sha256.New, []byte(signingKey)) + mac.Write([]byte(signedPart)) + return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +func normalizeScopes(scopes []string) []string { + if len(scopes) == 0 { + return nil + } + allowed := map[string]struct{}{} + for _, scope := range scopesForRole(AuthRoleWorker) { + allowed[scope] = struct{}{} + } + cleaned := make([]string, 0, len(scopes)) + for _, scope := range scopes { + scope = strings.TrimSpace(scope) + if _, ok := allowed[scope]; ok { + cleaned = append(cleaned, scope) + } + } + return cleaned +} + +func normalizeSigningKeys(primary string, additional []string) []string { + seen := map[string]struct{}{} + keys := make([]string, 0, len(additional)+1) + for _, key := range append([]string{primary}, additional...) { + key = strings.TrimSpace(key) + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + keys = append(keys, key) + } + return keys +} + +func normalizeRevokedTokenIDs(ids []string) map[string]struct{} { + if len(ids) == 0 { + return nil + } + revoked := make(map[string]struct{}, len(ids)) + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + revoked[id] = struct{}{} + } + return revoked +} + +func validWorkerToken(workerID, token, sharedWorkerToken string, workerTokens map[string]string) bool { + if token == "" { + return false + } + if expected, ok := workerTokens[workerID]; ok { + return subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1 + } + sharedWorkerToken = strings.TrimSpace(sharedWorkerToken) + return sharedWorkerToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(sharedWorkerToken)) == 1 +} + +func normalizeWorkerTokens(workerTokens map[string]string) map[string]string { + if len(workerTokens) == 0 { + return nil + } + cleaned := make(map[string]string, len(workerTokens)) + for workerID, token := range workerTokens { + workerID = strings.TrimSpace(workerID) + token = strings.TrimSpace(token) + if workerID == "" || token == "" { + continue + } + cleaned[workerID] = token + } + return cleaned +} + +func nonEmptyKeys(apiKeys ...string) []string { + keys := make([]string, 0, len(apiKeys)) + for _, key := range apiKeys { + if key != "" { + keys = append(keys, key) + } + } + return keys +} diff --git a/internal/api/middleware/auth_test.go b/internal/api/middleware/auth_test.go new file mode 100644 index 0000000..614f58b --- /dev/null +++ b/internal/api/middleware/auth_test.go @@ -0,0 +1,528 @@ +package middleware + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestAuthAnyAcceptsPrimaryOrAdminKey(t *testing.T) { + var got AuthIdentity + handler := AuthAny("primary-key", "admin-key")(identityHandler(&got)) + + tests := []struct { + name string + header string + key string + want int + wantRole AuthRole + wantHead string + }{ + {name: "primary", header: "X-API-Key", key: "primary-key", want: http.StatusOK, wantRole: AuthRoleAPI, wantHead: "X-API-Key"}, + {name: "admin via api header", header: "X-API-Key", key: "admin-key", want: http.StatusOK, wantRole: AuthRoleAdmin, wantHead: "X-API-Key"}, + {name: "admin via admin header", header: "X-Admin-API-Key", key: "admin-key", want: http.StatusOK, wantRole: AuthRoleAdmin, wantHead: "X-Admin-API-Key"}, + {name: "wrong", header: "X-API-Key", key: "wrong", want: http.StatusUnauthorized}, + {name: "missing", want: http.StatusUnauthorized}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got = AuthIdentity{} + req := httptest.NewRequest(http.MethodGet, "/", nil) + if tt.header != "" { + req.Header.Set(tt.header, tt.key) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.want, w.Body.String()) + } + if tt.want != http.StatusOK { + return + } + if got.Role != tt.wantRole { + t.Fatalf("role = %q, want %q", got.Role, tt.wantRole) + } + if got.Header != tt.wantHead { + t.Fatalf("header = %q, want %q", got.Header, tt.wantHead) + } + }) + } +} + +func TestAdminAuthRequiresAdminKeyWhenConfigured(t *testing.T) { + var got AuthIdentity + handler := AdminAuth("admin-key", "primary-key", true)(identityHandler(&got)) + + tests := []struct { + name string + header string + key string + want int + wantHead string + }{ + {name: "admin via admin header", header: "X-Admin-API-Key", key: "admin-key", want: http.StatusOK, wantHead: "X-Admin-API-Key"}, + {name: "admin via api header", header: "X-API-Key", key: "admin-key", want: http.StatusOK, wantHead: "X-API-Key"}, + {name: "primary rejected", header: "X-API-Key", key: "primary-key", want: http.StatusForbidden}, + {name: "missing", want: http.StatusForbidden}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got = AuthIdentity{} + req := httptest.NewRequest(http.MethodGet, "/", nil) + if tt.header != "" { + req.Header.Set(tt.header, tt.key) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.want, w.Body.String()) + } + if tt.want != http.StatusOK { + return + } + if got.Role != AuthRoleAdmin { + t.Fatalf("role = %q, want %q", got.Role, AuthRoleAdmin) + } + if got.Header != tt.wantHead { + t.Fatalf("header = %q, want %q", got.Header, tt.wantHead) + } + if !got.HasScope(ScopeAdmin) || !got.HasScope(ScopeAPI) { + t.Fatalf("admin identity scopes = %#v, want admin and api scopes", got.Scopes) + } + }) + } +} + +func TestAdminAuthFallsBackToPrimaryKey(t *testing.T) { + handler := AdminAuth("", "primary-key", true)(okHandler()) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("X-API-Key", "primary-key") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestAdminAuthCanDisablePrimaryKeyFallback(t *testing.T) { + handler := AdminAuth("", "primary-key", false)(okHandler()) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("X-API-Key", "primary-key") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d when no admin key is configured: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestAuthIdentityFromContextDefaultsToAnonymous(t *testing.T) { + identity := AuthIdentityFromContext(context.Background()) + if identity.Role != AuthRoleAnonymous { + t.Fatalf("role = %q, want %q", identity.Role, AuthRoleAnonymous) + } +} + +func TestWorkerAuthWithPerWorkerToken(t *testing.T) { + var got AuthIdentity + handler := WorkerAuthWithTokens("shared-token", map[string]string{ + "worker-a": "worker-a-token", + })(identityHandler(&got)) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-a-token") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got.Role != AuthRoleWorker || got.WorkerID != "worker-a" { + t.Fatalf("identity = %+v, want worker-a worker identity", got) + } + if !got.HasScope(ScopeWorkerHeartbeat) || !got.HasScope(ScopeWorkerLease) { + t.Fatalf("worker identity scopes = %#v, want heartbeat and lease scopes", got.Scopes) + } +} + +func TestWorkerAuthPerWorkerTokenOverridesSharedToken(t *testing.T) { + handler := WorkerAuthWithTokens("shared-token", map[string]string{ + "worker-a": "worker-a-token", + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "shared-token") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d when shared token is used for a worker-specific credential: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthFallsBackToSharedTokenForUnmappedWorker(t *testing.T) { + handler := WorkerAuthWithTokens("shared-token", map[string]string{ + "worker-a": "worker-a-token", + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-b") + req.Header.Set("Authorization", "Bearer shared-token") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d for unmapped worker using shared token: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestWorkerAuthAcceptsSignedWorkerToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + Scopes: []string{ScopeWorkerHeartbeat, ScopeWorkerLease, "admin:*"}, + ExpiresAt: now.Add(time.Minute).Unix(), + IssuedAt: now.Add(-time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + + var got AuthIdentity + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(identityHandler(&got)) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got.Role != AuthRoleWorker || got.WorkerID != "worker-a" || got.Header != "Authorization" { + t.Fatalf("identity = %+v, want signed worker-a identity from Authorization", got) + } + if !got.HasScope(ScopeWorkerHeartbeat) || !got.HasScope(ScopeWorkerLease) { + t.Fatalf("worker identity scopes = %#v, want signed worker scopes", got.Scopes) + } + if got.HasScope(ScopeAdmin) { + t.Fatalf("worker identity scopes = %#v, signed token should not grant non-worker scopes", got.Scopes) + } +} + +func TestWorkerAuthRejectsSignedWorkerTokenForWrongWorker(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + ExpiresAt: now.Add(time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-b") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d for signed worker token with mismatched worker ID: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthRejectsExpiredSignedWorkerToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + ExpiresAt: now.Add(-time.Second).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d for expired signed worker token: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthRejectsNotYetValidSignedWorkerToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + IssuedAt: now.Unix(), + NotBefore: now.Add(2 * time.Minute).Unix(), + ExpiresAt: now.Add(5 * time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d for not-yet-valid signed worker token: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthAllowsSmallSignedTokenClockSkew(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + IssuedAt: now.Add(10 * time.Second).Unix(), + NotBefore: now.Add(10 * time.Second).Unix(), + ExpiresAt: now.Add(5 * time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d for signed worker token inside clock skew: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestWorkerAuthRejectsSignedWorkerTokenExceedingMaxTTL(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(MaxWorkerTokenTTL + time.Second).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d for signed worker token exceeding max TTL: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthRejectsRevokedSignedWorkerToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + TokenID: "revoked-token-id", + Audience: WorkerTokenAudienceControlPlane, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(5 * time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + RevokedTokenIDs: []string{"revoked-token-id"}, + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d for revoked signed worker token: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthRejectsRPCAudienceSignedWorkerToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("0123456789abcdef0123456789abcdef", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceRPC, + ExpiresAt: now.Add(time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d for RPC-audience token on control-plane route: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerAuthKeepsStaticTokenFallbackWithSigningKey(t *testing.T) { + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SharedToken: "shared-token", + SigningKey: "0123456789abcdef0123456789abcdef", + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "shared-token") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d for static fallback with signing key configured: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestWorkerAuthAcceptsSignedWorkerTokenFromRotationKey(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + token, err := SignWorkerToken("old-worker-signing-key-with-at-least-32-bytes", WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: WorkerTokenAudienceControlPlane, + ExpiresAt: now.Add(time.Minute).Unix(), + }) + if err != nil { + t.Fatalf("sign worker token: %v", err) + } + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "new-worker-signing-key-with-at-least-32-bytes", + SigningKeys: []string{"old-worker-signing-key-with-at-least-32-bytes"}, + Now: func() time.Time { return now }, + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", token) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d for token signed by rotation key: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestWorkerAuthSigningKeyCountsAsConfiguredCredentials(t *testing.T) { + handler := WorkerAuthWithConfig(WorkerAuthConfig{ + SigningKey: "0123456789abcdef0123456789abcdef", + })(okHandler()) + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "invalid-token") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d when signing key is configured but token is invalid: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestRequireScopeAllowsMatchingScope(t *testing.T) { + handler := RequireScope(ScopeAdmin)(okHandler()) + req := httptest.NewRequest(http.MethodGet, "/", nil) + req = req.WithContext(WithAuthIdentity(req.Context(), AuthIdentity{ + Role: AuthRoleAdmin, + Scopes: []string{ScopeAPI, ScopeAdmin}, + })) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestRequireScopeRejectsMissingScope(t *testing.T) { + handler := RequireScope(ScopeAdmin)(okHandler()) + + tests := []struct { + name string + identity AuthIdentity + }{ + {name: "regular api identity", identity: AuthIdentity{Role: AuthRoleAPI, Scopes: []string{ScopeAPI}}}, + {name: "anonymous identity", identity: AuthIdentity{}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req = req.WithContext(WithAuthIdentity(req.Context(), tt.identity)) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusForbidden, w.Body.String()) + } + }) + } +} + +func okHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) +} + +func identityHandler(dst *AuthIdentity) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + *dst = AuthIdentityFromContext(r.Context()) + w.WriteHeader(http.StatusOK) + }) +} diff --git a/internal/api/middleware/jwt.go b/internal/api/middleware/jwt.go new file mode 100644 index 0000000..9152e15 --- /dev/null +++ b/internal/api/middleware/jwt.go @@ -0,0 +1,506 @@ +package middleware + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "crypto/sha256" + "crypto/sha512" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/http" + "strings" + "sync" + "time" +) + +// OIDCConfig holds OIDC/JWT verification settings. +// Supports RS256 (RSA), ES256/ES384/ES512 (ECDSA) via JWKS or static PEM key. +type OIDCConfig struct { + Issuer string + Audience string + JWKSUrl string + PublicKeyPEM string + GroupsClaim string + TenantClaim string + AdminGroups []string + OperatorGroups []string + ViewerGroups []string + Now func() time.Time +} + +type JWTClaims struct { + Subject string `json:"sub"` + Issuer string `json:"iss"` + Audience aud `json:"aud"` + Expiry int64 `json:"exp"` + IssuedAt int64 `json:"iat"` + Email string `json:"email"` + Name string `json:"name"` + Groups []string `json:"-"` // extracted via GroupsClaim + TenantID string `json:"-"` // extracted via TenantClaim + Extra map[string]json.RawMessage +} + +// aud handles both string and []string audience in JWT. +type aud []string + +func (a *aud) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err == nil { + *a = []string{s} + return nil + } + var ss []string + if err := json.Unmarshal(b, &ss); err != nil { + return err + } + *a = ss + return nil +} + +// publicKey holds either an RSA or EC public key from a JWKS or PEM. +type publicKey struct { + rsa *rsa.PublicKey + ec *ecdsa.PublicKey +} + +type jwksCache struct { + mu sync.RWMutex + keys map[string]publicKey + fetched time.Time + ttl time.Duration + url string +} + +func newJWKSCache(url string) *jwksCache { + return &jwksCache{url: url, ttl: 5 * time.Minute, keys: map[string]publicKey{}} +} + +func (c *jwksCache) get(kid string, now time.Time) (publicKey, error) { + c.mu.RLock() + if now.Before(c.fetched.Add(c.ttl)) { + key, ok := c.keys[kid] + c.mu.RUnlock() + if ok { + return key, nil + } + return publicKey{}, fmt.Errorf("jwks: unknown kid %q", kid) + } + c.mu.RUnlock() + + keys, err := fetchJWKS(c.url) + if err != nil { + return publicKey{}, err + } + + c.mu.Lock() + c.keys = keys + c.fetched = now + c.mu.Unlock() + + key, ok := keys[kid] + if !ok { + return publicKey{}, fmt.Errorf("jwks: unknown kid %q", kid) + } + return key, nil +} + +type jwksResponse struct { + Keys []jwk `json:"keys"` +} + +type jwk struct { + Kid string `json:"kid"` + Kty string `json:"kty"` + Alg string `json:"alg"` + Use string `json:"use"` + // RSA fields + N string `json:"n"` + E string `json:"e"` + // EC fields + Crv string `json:"crv"` + X string `json:"x"` + Y string `json:"y"` +} + +func fetchJWKS(url string) (map[string]publicKey, error) { + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Get(url) + if err != nil { + return nil, fmt.Errorf("fetching jwks: %w", err) + } + defer resp.Body.Close() + + var jwks jwksResponse + if err := json.NewDecoder(resp.Body).Decode(&jwks); err != nil { + return nil, fmt.Errorf("decoding jwks: %w", err) + } + + keys := make(map[string]publicKey, len(jwks.Keys)) + for _, k := range jwks.Keys { + switch k.Kty { + case "RSA": + pub, err := rsaPublicKeyFromJWK(k) + if err != nil { + continue + } + keys[k.Kid] = publicKey{rsa: pub} + case "EC": + pub, err := ecPublicKeyFromJWK(k) + if err != nil { + continue + } + keys[k.Kid] = publicKey{ec: pub} + } + } + return keys, nil +} + +func rsaPublicKeyFromJWK(k jwk) (*rsa.PublicKey, error) { + nBytes, err := base64.RawURLEncoding.DecodeString(k.N) + if err != nil { + return nil, err + } + eBytes, err := base64.RawURLEncoding.DecodeString(k.E) + if err != nil { + return nil, err + } + n := new(big.Int).SetBytes(nBytes) + e := int(new(big.Int).SetBytes(eBytes).Int64()) + return &rsa.PublicKey{N: n, E: e}, nil +} + +func ecPublicKeyFromJWK(k jwk) (*ecdsa.PublicKey, error) { + var curve elliptic.Curve + switch k.Crv { + case "P-256": + curve = elliptic.P256() + case "P-384": + curve = elliptic.P384() + case "P-521": + curve = elliptic.P521() + default: + return nil, fmt.Errorf("unsupported EC curve %q", k.Crv) + } + xBytes, err := base64.RawURLEncoding.DecodeString(k.X) + if err != nil { + return nil, err + } + yBytes, err := base64.RawURLEncoding.DecodeString(k.Y) + if err != nil { + return nil, err + } + return &ecdsa.PublicKey{ + Curve: curve, + X: new(big.Int).SetBytes(xBytes), + Y: new(big.Int).SetBytes(yBytes), + }, nil +} + +func parsePublicKeyPEM(pemData string) (publicKey, error) { + block, _ := pem.Decode([]byte(pemData)) + if block == nil { + return publicKey{}, errors.New("oidc: invalid PEM block") + } + switch block.Type { + case "PUBLIC KEY": + key, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return publicKey{}, err + } + switch k := key.(type) { + case *rsa.PublicKey: + return publicKey{rsa: k}, nil + case *ecdsa.PublicKey: + return publicKey{ec: k}, nil + default: + return publicKey{}, fmt.Errorf("oidc: unsupported public key type %T", key) + } + case "CERTIFICATE": + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return publicKey{}, err + } + switch k := cert.PublicKey.(type) { + case *rsa.PublicKey: + return publicKey{rsa: k}, nil + case *ecdsa.PublicKey: + return publicKey{ec: k}, nil + default: + return publicKey{}, fmt.Errorf("oidc: unsupported certificate public key type %T", cert.PublicKey) + } + default: + return publicKey{}, fmt.Errorf("oidc: unsupported PEM block type %q", block.Type) + } +} + +// OIDCAuth returns middleware that validates OIDC Bearer JWTs and injects +// AuthIdentity into the request context alongside the existing API-key path. +func OIDCAuth(cfg OIDCConfig) func(http.Handler) http.Handler { + now := cfg.Now + if now == nil { + now = time.Now + } + + var staticKey publicKey + if cfg.PublicKeyPEM != "" { + k, err := parsePublicKeyPEM(cfg.PublicKeyPEM) + if err == nil { + staticKey = k + } + } + + var cache *jwksCache + if cfg.JWKSUrl != "" { + cache = newJWKSCache(cfg.JWKSUrl) + } + + adminGroups := groupSet(cfg.AdminGroups) + operatorGroups := groupSet(cfg.OperatorGroups) + viewerGroups := groupSet(cfg.ViewerGroups) + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Only intercept Bearer tokens; fall through for X-API-Key. + token := bearerToken(r) + if token == "" { + next.ServeHTTP(w, r) + return + } + + claims, err := verifyJWT(token, cfg.Issuer, cfg.Audience, staticKey, cache, now()) + if err != nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{ + "code": "UNAUTHORIZED", + "message": "invalid or expired JWT: " + err.Error(), + }) + return + } + + // Extract groups from configured claim. + if cfg.GroupsClaim != "" && cfg.GroupsClaim != "groups" { + if raw, ok := claims.Extra[cfg.GroupsClaim]; ok { + json.Unmarshal(raw, &claims.Groups) + } + } + // Extract tenant from configured claim. + if cfg.TenantClaim != "" { + if raw, ok := claims.Extra[cfg.TenantClaim]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + claims.TenantID = s + } + } + } + + role := oidcRole(claims.Groups, adminGroups, operatorGroups, viewerGroups) + identity := AuthIdentity{ + Role: role, + Header: "Authorization", + Scopes: scopesForRole(role), + Subject: claims.Subject, + Email: claims.Email, + TenantID: claims.TenantID, + Groups: claims.Groups, + } + r = r.WithContext(WithAuthIdentity(r.Context(), identity)) + next.ServeHTTP(w, r) + }) + } +} + +func bearerToken(r *http.Request) string { + const prefix = "Bearer " + v := r.Header.Get("Authorization") + if len(v) > len(prefix) && strings.EqualFold(v[:len(prefix)], prefix) { + candidate := strings.TrimSpace(v[len(prefix):]) + // Exclude signed worker tokens so they go through WorkerAuth, not OIDC. + if !strings.HasPrefix(candidate, workerSignedTokenPrefix+".") { + return candidate + } + } + return "" +} + +func verifyJWT(token, issuer, audience string, staticKey publicKey, cache *jwksCache, now time.Time) (*JWTClaims, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return nil, errors.New("malformed JWT") + } + + headerBytes, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return nil, errors.New("malformed JWT header") + } + var header struct { + Alg string `json:"alg"` + Kid string `json:"kid"` + } + if err := json.Unmarshal(headerBytes, &header); err != nil { + return nil, errors.New("malformed JWT header") + } + + sigBytes, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil { + return nil, errors.New("malformed JWT signature") + } + + signed := parts[0] + "." + parts[1] + + // Resolve the key to use for verification. + var key publicKey + if staticKey.rsa != nil || staticKey.ec != nil { + key = staticKey + } else if cache != nil { + key, err = cache.get(header.Kid, now) + if err != nil { + return nil, err + } + } else { + return nil, errors.New("no OIDC public key or JWKS URL configured") + } + + // Verify signature based on algorithm. + switch header.Alg { + case "RS256": + if key.rsa == nil { + return nil, fmt.Errorf("algorithm RS256 requires an RSA key, got EC") + } + h := sha256.Sum256([]byte(signed)) + if err := rsa.VerifyPKCS1v15(key.rsa, crypto.SHA256, h[:], sigBytes); err != nil { + return nil, errors.New("JWT signature verification failed") + } + case "ES256": + if key.ec == nil { + return nil, fmt.Errorf("algorithm ES256 requires an EC key, got RSA") + } + h := sha256.Sum256([]byte(signed)) + if !ecdsa.VerifyASN1(key.ec, h[:], sigBytes) { + return nil, errors.New("JWT signature verification failed") + } + case "ES384": + if key.ec == nil { + return nil, fmt.Errorf("algorithm ES384 requires an EC key, got RSA") + } + h := sha512.Sum384([]byte(signed)) + if !ecdsa.VerifyASN1(key.ec, h[:], sigBytes) { + return nil, errors.New("JWT signature verification failed") + } + case "ES512": + if key.ec == nil { + return nil, fmt.Errorf("algorithm ES512 requires an EC key, got RSA") + } + h := sha512.Sum512([]byte(signed)) + if !ecdsa.VerifyASN1(key.ec, h[:], sigBytes) { + return nil, errors.New("JWT signature verification failed") + } + default: + return nil, fmt.Errorf("unsupported JWT algorithm %q; supported: RS256, ES256, ES384, ES512", header.Alg) + } + + payloadBytes, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return nil, errors.New("malformed JWT payload") + } + + var raw map[string]json.RawMessage + if err := json.Unmarshal(payloadBytes, &raw); err != nil { + return nil, errors.New("malformed JWT payload") + } + + claims := &JWTClaims{Extra: raw} + for k, v := range raw { + switch k { + case "sub": + json.Unmarshal(v, &claims.Subject) + case "iss": + json.Unmarshal(v, &claims.Issuer) + case "aud": + json.Unmarshal(v, &claims.Audience) + case "exp": + json.Unmarshal(v, &claims.Expiry) + case "iat": + json.Unmarshal(v, &claims.IssuedAt) + case "email": + json.Unmarshal(v, &claims.Email) + case "name": + json.Unmarshal(v, &claims.Name) + case "groups": + json.Unmarshal(v, &claims.Groups) + } + } + + if claims.Expiry <= 0 || !now.Before(time.Unix(claims.Expiry, 0)) { + return nil, errors.New("JWT is expired") + } + if issuer != "" && claims.Issuer != issuer { + return nil, fmt.Errorf("JWT issuer %q does not match expected %q", claims.Issuer, issuer) + } + if audience != "" { + found := false + for _, a := range claims.Audience { + if a == audience { + found = true + break + } + } + if !found { + return nil, fmt.Errorf("JWT audience does not contain %q", audience) + } + } + + return claims, nil +} + +func groupSet(groups []string) map[string]struct{} { + m := make(map[string]struct{}, len(groups)) + for _, g := range groups { + if g = strings.TrimSpace(g); g != "" { + m[g] = struct{}{} + } + } + return m +} + +func oidcRole(groups []string, adminGroups, operatorGroups, viewerGroups map[string]struct{}) AuthRole { + for _, g := range groups { + if _, ok := adminGroups[g]; ok { + return AuthRoleAdmin + } + } + for _, g := range groups { + if _, ok := operatorGroups[g]; ok { + return AuthRoleOperator + } + } + for _, g := range groups { + if _, ok := viewerGroups[g]; ok { + return AuthRoleViewer + } + } + // Default to API role when authenticated but no group matches. + return AuthRoleAPI +} + +// TenantIDFromContext returns the tenant ID stored in context, if any. +func TenantIDFromContext(ctx context.Context) string { + id, _ := ctx.Value(tenantIDContextKey{}).(string) + return id +} + +type tenantIDContextKey struct{} + +// WithTenantID injects a tenant ID into context. +func WithTenantID(ctx context.Context, tenantID string) context.Context { + return context.WithValue(ctx, tenantIDContextKey{}, tenantID) +} diff --git a/internal/api/middleware/jwt_test.go b/internal/api/middleware/jwt_test.go new file mode 100644 index 0000000..42b4a81 --- /dev/null +++ b/internal/api/middleware/jwt_test.go @@ -0,0 +1,438 @@ +package middleware + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/store" +) + +// generateTestRSAKey creates a 2048-bit RSA key for testing. +func generateTestRSAKey(t *testing.T) *rsa.PrivateKey { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate RSA key: %v", err) + } + return key +} + +// mintTestJWT signs a JWT with RS256 using the provided key and claims. +func mintTestJWT(t *testing.T, key *rsa.PrivateKey, claims map[string]any) string { + t.Helper() + header := base64.RawURLEncoding.EncodeToString(mustJSON(t, map[string]string{"alg": "RS256", "kid": "test-kid", "typ": "JWT"})) + payload := base64.RawURLEncoding.EncodeToString(mustJSON(t, claims)) + signed := header + "." + payload + h := sha256.Sum256([]byte(signed)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, h[:]) + if err != nil { + t.Fatalf("sign JWT: %v", err) + } + return signed + "." + base64.RawURLEncoding.EncodeToString(sig) +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal json: %v", err) + } + return b +} + +// testJWKSCache returns a jwksCache pre-loaded with the given RSA key, bypassing HTTP. +func testJWKSCache(key *rsa.PrivateKey) *jwksCache { + return &jwksCache{ + keys: map[string]publicKey{"test-kid": {rsa: &key.PublicKey}}, + fetched: time.Now().Add(time.Hour), + ttl: time.Hour, + } +} + +// testOIDCMiddleware builds an OIDCAuth-equivalent middleware using a pre-loaded JWKS +// cache, allowing RS256 JWT tests without a real HTTP JWKS server. +func testOIDCMiddleware(key *rsa.PrivateKey, cfg OIDCConfig) func(http.Handler) http.Handler { + if cfg.Now == nil { + cfg.Now = time.Now + } + cache := testJWKSCache(key) + adminGroups := groupSet(cfg.AdminGroups) + operatorGroups := groupSet(cfg.OperatorGroups) + viewerGroups := groupSet(cfg.ViewerGroups) + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := bearerToken(r) + if token == "" { + next.ServeHTTP(w, r) + return + } + claims, err := verifyJWT(token, cfg.Issuer, cfg.Audience, publicKey{}, cache, cfg.Now()) + if err != nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{"code": "UNAUTHORIZED", "message": err.Error()}) + return + } + role := oidcRole(claims.Groups, adminGroups, operatorGroups, viewerGroups) + identity := AuthIdentity{ + Role: role, + Header: "Authorization", + Scopes: scopesForRole(role), + Subject: claims.Subject, + Email: claims.Email, + TenantID: claims.TenantID, + Groups: claims.Groups, + } + r = r.WithContext(WithAuthIdentity(r.Context(), identity)) + next.ServeHTTP(w, r) + }) + } +} + +func echoIdentityHandler(w http.ResponseWriter, r *http.Request) { + identity := AuthIdentityFromContext(r.Context()) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "role": string(identity.Role), + "subject": identity.Subject, + "email": identity.Email, + "tenant_id": identity.TenantID, + "groups": identity.Groups, + }) +} + +func doOIDCRequest(t *testing.T, mw func(http.Handler) http.Handler, token string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/", nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rr := httptest.NewRecorder() + mw(http.HandlerFunc(echoIdentityHandler)).ServeHTTP(rr, req) + return rr +} + +func TestOIDCAuth_ValidToken_GetsAPIRole(t *testing.T) { + key := generateTestRSAKey(t) + now := time.Now() + token := mintTestJWT(t, key, map[string]any{ + "sub": "user-123", + "iss": "https://idp.example.com", + "aud": "stacyvm", + "exp": now.Add(5 * time.Minute).Unix(), + "iat": now.Unix(), + "email": "alice@example.com", + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + }), token) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rr.Code, rr.Body.String()) + } + var resp map[string]any + json.NewDecoder(rr.Body).Decode(&resp) + if resp["role"] != "api" { + t.Errorf("expected api role, got %v", resp["role"]) + } + if resp["subject"] != "user-123" { + t.Errorf("expected subject user-123, got %v", resp["subject"]) + } + if resp["email"] != "alice@example.com" { + t.Errorf("expected email alice@example.com, got %v", resp["email"]) + } +} + +func TestOIDCAuth_AdminGroup_GetsAdminRole(t *testing.T) { + key := generateTestRSAKey(t) + now := time.Now() + token := mintTestJWT(t, key, map[string]any{ + "sub": "admin-user", + "iss": "https://idp.example.com", + "aud": "stacyvm", + "exp": now.Add(5 * time.Minute).Unix(), + "iat": now.Unix(), + "groups": []string{"stacyvm-admins", "engineers"}, + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + AdminGroups: []string{"stacyvm-admins"}, + }), token) + + var resp map[string]any + json.NewDecoder(rr.Body).Decode(&resp) + if resp["role"] != "admin" { + t.Errorf("expected admin role, got %v", resp["role"]) + } +} + +func TestOIDCAuth_OperatorGroup_GetsOperatorRole(t *testing.T) { + key := generateTestRSAKey(t) + now := time.Now() + token := mintTestJWT(t, key, map[string]any{ + "sub": "op-user", + "iss": "https://idp.example.com", + "aud": "stacyvm", + "exp": now.Add(5 * time.Minute).Unix(), + "iat": now.Unix(), + "groups": []string{"stacyvm-operators"}, + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + AdminGroups: []string{"stacyvm-admins"}, + OperatorGroups: []string{"stacyvm-operators"}, + }), token) + + var resp map[string]any + json.NewDecoder(rr.Body).Decode(&resp) + if resp["role"] != "operator" { + t.Errorf("expected operator role, got %v", resp["role"]) + } +} + +func TestOIDCAuth_ExpiredToken_Returns401(t *testing.T) { + key := generateTestRSAKey(t) + token := mintTestJWT(t, key, map[string]any{ + "sub": "user", + "iss": "https://idp.example.com", + "aud": "stacyvm", + "exp": time.Now().Add(-1 * time.Minute).Unix(), + "iat": time.Now().Add(-10 * time.Minute).Unix(), + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + }), token) + + if rr.Code != http.StatusUnauthorized { + t.Errorf("expected 401 for expired token, got %d", rr.Code) + } +} + +func TestOIDCAuth_WrongIssuer_Returns401(t *testing.T) { + key := generateTestRSAKey(t) + now := time.Now() + token := mintTestJWT(t, key, map[string]any{ + "sub": "user", + "iss": "https://evil.com", + "aud": "stacyvm", + "exp": now.Add(5 * time.Minute).Unix(), + "iat": now.Unix(), + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + }), token) + + if rr.Code != http.StatusUnauthorized { + t.Errorf("expected 401 for wrong issuer, got %d", rr.Code) + } +} + +func TestOIDCAuth_WrongAudience_Returns401(t *testing.T) { + key := generateTestRSAKey(t) + now := time.Now() + token := mintTestJWT(t, key, map[string]any{ + "sub": "user", + "iss": "https://idp.example.com", + "aud": "other-service", + "exp": now.Add(5 * time.Minute).Unix(), + "iat": now.Unix(), + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + }), token) + + if rr.Code != http.StatusUnauthorized { + t.Errorf("expected 401 for wrong audience, got %d", rr.Code) + } +} + +func TestOIDCAuth_WrongSigningKey_Returns401(t *testing.T) { + key1 := generateTestRSAKey(t) + key2 := generateTestRSAKey(t) + now := time.Now() + // Token signed with key1, but middleware has key2. + token := mintTestJWT(t, key1, map[string]any{ + "sub": "user", + "iss": "https://idp.example.com", + "aud": "stacyvm", + "exp": now.Add(5 * time.Minute).Unix(), + "iat": now.Unix(), + }) + + rr := doOIDCRequest(t, testOIDCMiddleware(key2, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + }), token) + + if rr.Code != http.StatusUnauthorized { + t.Errorf("expected 401 for wrong signing key, got %d", rr.Code) + } +} + +func TestOIDCAuth_NoBearerToken_FallsThrough(t *testing.T) { + key := generateTestRSAKey(t) + mw := testOIDCMiddleware(key, OIDCConfig{ + Issuer: "https://idp.example.com", + Audience: "stacyvm", + }) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + rr := httptest.NewRecorder() + called := false + mw(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + w.WriteHeader(http.StatusOK) + })).ServeHTTP(rr, req) + + if !called { + t.Error("expected downstream handler to be called when no Bearer token present") + } +} + +func TestBearerToken_WorkerSignedTokenExcluded(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("Authorization", "Bearer stacyvm-worker-v1.payload.signature") + got := bearerToken(req) + if got != "" { + t.Errorf("bearerToken should return empty for worker-signed tokens, got %q", got) + } +} + +func TestBearerToken_NormalBearerExtracted(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("Authorization", "Bearer eyJfoo.bar.baz") + got := bearerToken(req) + if got != "eyJfoo.bar.baz" { + t.Errorf("unexpected token: %q", got) + } +} + +func TestOIDCRole_Priority(t *testing.T) { + adminSet := groupSet([]string{"admins"}) + opSet := groupSet([]string{"operators"}) + viewerSet := groupSet([]string{"viewers"}) + + cases := []struct { + groups []string + want AuthRole + }{ + {[]string{"admins", "operators"}, AuthRoleAdmin}, // admin wins when both match + {[]string{"operators", "viewers"}, AuthRoleOperator}, // operator beats viewer + {[]string{"viewers"}, AuthRoleViewer}, + {[]string{"other"}, AuthRoleAPI}, // no match → default api + {nil, AuthRoleAPI}, + } + for _, tc := range cases { + got := oidcRole(tc.groups, adminSet, opSet, viewerSet) + if got != tc.want { + t.Errorf("oidcRole(%v) = %q, want %q", tc.groups, got, tc.want) + } + } +} + +// TestVerifyJWT_RS256UsesSHA256 proves the verifier uses crypto.SHA256, not hash=0. +// A token signed with hash=0 must be rejected; only SHA256-signed tokens pass. +func TestVerifyJWT_RS256UsesSHA256(t *testing.T) { + key := generateTestRSAKey(t) + now := time.Now() + + // Build a JWT header+payload, then sign with hash=0 (wrong). + header := base64.RawURLEncoding.EncodeToString(mustJSON(t, map[string]string{"alg": "RS256", "kid": "test-kid", "typ": "JWT"})) + payload := base64.RawURLEncoding.EncodeToString(mustJSON(t, map[string]any{ + "sub": "user", "exp": now.Add(5 * time.Minute).Unix(), "iat": now.Unix(), + })) + signed := header + "." + payload + h := sha256.Sum256([]byte(signed)) + + // Sign with hash=0 — this is NOT real RS256. + sigWrong, err := rsa.SignPKCS1v15(rand.Reader, key, 0, h[:]) + if err != nil { + t.Fatal(err) + } + wrongToken := signed + "." + base64.RawURLEncoding.EncodeToString(sigWrong) + + cache := testJWKSCache(key) + _, errWrong := verifyJWT(wrongToken, "", "", publicKey{}, cache, now) + if errWrong == nil { + t.Error("expected verification failure for token signed with hash=0; got nil error — RS256 is not using crypto.SHA256") + } + + // Sign correctly with crypto.SHA256 — must be accepted. + sigRight, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, h[:]) + if err != nil { + t.Fatal(err) + } + rightToken := signed + "." + base64.RawURLEncoding.EncodeToString(sigRight) + _, errRight := verifyJWT(rightToken, "", "", publicKey{}, cache, now) + if errRight != nil { + t.Errorf("expected valid RS256 token to pass, got: %v", errRight) + } +} + +func TestAudUnmarshal_StringAndArray(t *testing.T) { + var a1 aud + if err := json.Unmarshal([]byte(`"stacyvm"`), &a1); err != nil { + t.Fatal(err) + } + if len(a1) != 1 || a1[0] != "stacyvm" { + t.Errorf("string aud parse failed: %v", a1) + } + + var a2 aud + if err := json.Unmarshal([]byte(`["stacyvm","other"]`), &a2); err != nil { + t.Fatal(err) + } + if len(a2) != 2 { + t.Errorf("array aud parse failed: %v", a2) + } +} + +func TestPolicyPermits(t *testing.T) { + pol := func(effect, pattern string) *store.PolicyRecord { + return &store.PolicyRecord{Effect: effect, Pattern: pattern, Priority: 10} + } + + tests := []struct { + name string + value string + policies []*store.PolicyRecord + want bool + }{ + {"allow exact", "alpine:3.19", []*store.PolicyRecord{pol("allow", "alpine:3.19")}, true}, + {"allow glob", "alpine:3.19", []*store.PolicyRecord{pol("allow", "alpine:*")}, true}, + {"allow wildcard", "anything", []*store.PolicyRecord{pol("allow", "*")}, true}, + {"deny exact", "ubuntu:latest", []*store.PolicyRecord{pol("deny", "ubuntu:latest")}, false}, + {"deny glob", "ubuntu:22.04", []*store.PolicyRecord{pol("deny", "ubuntu:*")}, false}, + {"no policy default permit", "anything", nil, true}, + {"deny before allow", "bad:img", []*store.PolicyRecord{pol("deny", "bad:img"), pol("allow", "bad:img")}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := policyPermits(tc.value, tc.policies) + if got != tc.want { + t.Errorf("policyPermits(%q) = %v, want %v", tc.value, got, tc.want) + } + }) + } +} diff --git a/internal/api/middleware/policy.go b/internal/api/middleware/policy.go new file mode 100644 index 0000000..f377482 --- /dev/null +++ b/internal/api/middleware/policy.go @@ -0,0 +1,120 @@ +package middleware + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "path/filepath" + + "github.com/StacyOs/stacyvm/internal/store" +) + +type policyStore interface { + ListPolicies(ctx context.Context, query store.PolicyQuery) ([]*store.PolicyRecord, error) +} + +// PolicyEnforcer checks spawn requests against tenant/global policies for +// provider, image, and network_mode fields in the JSON body. +// It buffers the request body so downstream handlers can still read it. +func PolicyEnforcer(st policyStore) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || st == nil { + next.ServeHTTP(w, r) + return + } + + identity := AuthIdentityFromContext(r.Context()) + tenantID := identity.TenantID + + // Read the full body into a buffer so we can inspect it and + // restore it for the downstream handler. + rawBody, err := io.ReadAll(io.LimitReader(r.Body, 2<<20)) // 2 MiB cap + if err != nil { + next.ServeHTTP(w, r) + return + } + // Always restore the body regardless of what we do next. + r.Body = io.NopCloser(bytes.NewReader(rawBody)) + + var body map[string]any + if err := json.Unmarshal(rawBody, &body); err != nil { + // Non-JSON or empty body — pass through without enforcement. + next.ServeHTTP(w, r) + return + } + // Body is already restored above; no context injection needed. + + checks := []struct { + key string + resourceType string + }{ + {"image", "image"}, + {"provider", "provider"}, + {"network_mode", "network"}, + } + + for _, check := range checks { + val, _ := body[check.key].(string) + if val == "" { + continue + } + policies, err := st.ListPolicies(r.Context(), store.PolicyQuery{ + TenantID: tenantID, + ResourceType: check.resourceType, + }) + if err != nil || len(policies) == 0 { + continue + } + if !policyPermits(val, policies) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + json.NewEncoder(w).Encode(map[string]string{ + "code": "FORBIDDEN", + "message": check.resourceType + " \"" + val + "\" is not permitted by policy", + }) + return + } + } + + next.ServeHTTP(w, r) + }) + } +} + +// policyPermits returns true when the value is allowed by the ordered policy list. +// Rules: deny policies take precedence over allow within the same priority tier. +// If no allow policy matches, default is to deny. +func policyPermits(value string, policies []*store.PolicyRecord) bool { + for _, p := range policies { + matched, _ := filepath.Match(p.Pattern, value) + if !matched { + // Also support exact match. + matched = p.Pattern == value || p.Pattern == "*" + } + if matched { + if p.Effect == "deny" { + return false + } + if p.Effect == "allow" { + return true + } + } + } + // No policy matched — default permit (policies are opt-in restrictions). + return true +} + +type decodedBodyKey struct{} + +func withDecodedBody(ctx context.Context, body map[string]any) context.Context { + return context.WithValue(ctx, decodedBodyKey{}, body) +} + +// DecodedBodyFromContext returns the pre-decoded JSON body if PolicyEnforcer ran. +func DecodedBodyFromContext(ctx context.Context) (map[string]any, bool) { + v, ok := ctx.Value(decodedBodyKey{}).(map[string]any) + return v, ok +} diff --git a/internal/api/middleware/ratelimit.go b/internal/api/middleware/ratelimit.go new file mode 100644 index 0000000..336c5a0 --- /dev/null +++ b/internal/api/middleware/ratelimit.go @@ -0,0 +1,235 @@ +package middleware + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +type RateLimitConfig struct { + Enabled bool + RequestsPerMinute int + Burst int + KeyBy string + BucketTTL time.Duration + CleanupInterval time.Duration + Now func() time.Time +} + +type rateBucket struct { + tokens float64 + lastRefill time.Time + lastSeen time.Time +} + +type RateLimiter struct { + mu sync.Mutex + buckets map[string]*rateBucket + rate float64 + requestsPerMinute int + burst float64 + keyBy string + now func() time.Time + disabled bool + allowedTotal uint64 + limitedTotal uint64 + evictedTotal uint64 + bucketTTL time.Duration + cleanupInterval time.Duration + lastCleanup time.Time +} + +type RateLimitStats struct { + Enabled bool `json:"enabled"` + RequestsPerMinute int `json:"requests_per_minute"` + Burst int `json:"burst"` + KeyBy string `json:"key_by"` + ActiveBuckets int `json:"active_buckets"` + AllowedTotal uint64 `json:"allowed_total"` + LimitedTotal uint64 `json:"limited_total"` + EvictedTotal uint64 `json:"evicted_total"` + BucketTTL string `json:"bucket_ttl"` + CleanupInterval string `json:"cleanup_interval"` +} + +func NewRateLimiter(cfg RateLimitConfig) *RateLimiter { + if cfg.RequestsPerMinute < 0 { + cfg.RequestsPerMinute = 0 + } + if cfg.Burst <= 0 { + cfg.Burst = cfg.RequestsPerMinute + } + if cfg.Now == nil { + cfg.Now = time.Now + } + if cfg.BucketTTL == 0 { + cfg.BucketTTL = 15 * time.Minute + } + if cfg.CleanupInterval == 0 { + cfg.CleanupInterval = time.Minute + } + keyBy := strings.TrimSpace(strings.ToLower(cfg.KeyBy)) + if keyBy == "" { + keyBy = "owner" + } + return &RateLimiter{ + buckets: make(map[string]*rateBucket), + rate: float64(cfg.RequestsPerMinute) / 60.0, + requestsPerMinute: cfg.RequestsPerMinute, + burst: float64(cfg.Burst), + keyBy: keyBy, + now: cfg.Now, + disabled: !cfg.Enabled || cfg.RequestsPerMinute == 0, + bucketTTL: cfg.BucketTTL, + cleanupInterval: cfg.CleanupInterval, + } +} + +func RateLimit(cfg RateLimitConfig) func(http.Handler) http.Handler { + return NewRateLimiter(cfg).Middleware +} + +func (rl *RateLimiter) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if rl.disabled { + next.ServeHTTP(w, r) + return + } + + allowed, remaining, retryAfter := rl.allow(rl.key(r)) + w.Header().Set("X-RateLimit-Limit", strconv.Itoa(int(rl.burst))) + w.Header().Set("X-RateLimit-Remaining", strconv.Itoa(remaining)) + if !allowed { + seconds := int(math.Ceil(retryAfter.Seconds())) + if seconds < 1 { + seconds = 1 + } + w.Header().Set("Retry-After", strconv.Itoa(seconds)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _ = json.NewEncoder(w).Encode(map[string]string{ + "code": "RESOURCE_LIMIT", + "message": fmt.Sprintf("rate limit exceeded; retry after %ds", seconds), + }) + return + } + + next.ServeHTTP(w, r) + }) +} + +func (rl *RateLimiter) allow(key string) (bool, int, time.Duration) { + now := rl.now() + + rl.mu.Lock() + defer rl.mu.Unlock() + + rl.cleanupExpiredLocked(now) + + bucket := rl.buckets[key] + if bucket == nil { + bucket = &rateBucket{tokens: rl.burst, lastRefill: now} + rl.buckets[key] = bucket + } + + elapsed := now.Sub(bucket.lastRefill).Seconds() + if elapsed > 0 { + bucket.tokens = math.Min(rl.burst, bucket.tokens+elapsed*rl.rate) + bucket.lastRefill = now + } + bucket.lastSeen = now + + if bucket.tokens >= 1 { + bucket.tokens-- + rl.allowedTotal++ + return true, int(math.Floor(bucket.tokens)), 0 + } + + needed := 1 - bucket.tokens + retryAfter := time.Duration(math.Ceil(needed/rl.rate)) * time.Second + rl.limitedTotal++ + return false, 0, retryAfter +} + +func (rl *RateLimiter) Stats() RateLimitStats { + if rl == nil { + return RateLimitStats{} + } + rl.mu.Lock() + defer rl.mu.Unlock() + return RateLimitStats{ + Enabled: !rl.disabled, + RequestsPerMinute: rl.requestsPerMinute, + Burst: int(rl.burst), + KeyBy: rl.keyBy, + ActiveBuckets: len(rl.buckets), + AllowedTotal: rl.allowedTotal, + LimitedTotal: rl.limitedTotal, + EvictedTotal: rl.evictedTotal, + BucketTTL: rl.bucketTTL.String(), + CleanupInterval: rl.cleanupInterval.String(), + } +} + +func (rl *RateLimiter) cleanupExpiredLocked(now time.Time) { + if rl.bucketTTL <= 0 || rl.cleanupInterval <= 0 { + return + } + if !rl.lastCleanup.IsZero() && now.Sub(rl.lastCleanup) < rl.cleanupInterval { + return + } + rl.lastCleanup = now + for key, bucket := range rl.buckets { + if now.Sub(bucket.lastSeen) > rl.bucketTTL { + delete(rl.buckets, key) + rl.evictedTotal++ + } + } +} + +func (rl *RateLimiter) key(r *http.Request) string { + switch rl.keyBy { + case "api_key": + if apiKey := strings.TrimSpace(r.Header.Get("X-API-Key")); apiKey != "" { + return bucketKey("api_key", apiKey) + } + case "ip": + return bucketKey("ip", clientIP(r)) + default: + if ownerID := strings.TrimSpace(r.Header.Get("X-User-ID")); ownerID != "" { + return bucketKey("owner", ownerID) + } + if apiKey := strings.TrimSpace(r.Header.Get("X-API-Key")); apiKey != "" { + return bucketKey("api_key", apiKey) + } + } + return bucketKey("ip", clientIP(r)) +} + +func bucketKey(kind, value string) string { + sum := sha256.Sum256([]byte(kind + ":" + value)) + return kind + ":" + hex.EncodeToString(sum[:]) +} + +func clientIP(r *http.Request) string { + if forwardedFor := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); forwardedFor != "" { + parts := strings.Split(forwardedFor, ",") + return strings.TrimSpace(parts[0]) + } + if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); realIP != "" { + return realIP + } + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + return host + } + return r.RemoteAddr +} diff --git a/internal/api/middleware/ratelimit_test.go b/internal/api/middleware/ratelimit_test.go new file mode 100644 index 0000000..1c571ff --- /dev/null +++ b/internal/api/middleware/ratelimit_test.go @@ -0,0 +1,248 @@ +package middleware + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestRateLimitByOwner(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + KeyBy: "owner", + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "owner-a") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("first owner-a status = %d", w.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "owner-a") + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusTooManyRequests { + t.Fatalf("second owner-a status = %d", w.Code) + } + if w.Header().Get("Retry-After") == "" { + t.Fatal("expected Retry-After header") + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "owner-b") + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("owner-b status = %d", w.Code) + } +} + +func TestRateLimitRefills(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.10:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("first status = %d", w.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.10:5000" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusTooManyRequests { + t.Fatalf("second status = %d", w.Code) + } + + now = now.Add(time.Second) + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.10:5000" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("refilled status = %d", w.Code) + } +} + +func TestRateLimitDisabled(t *testing.T) { + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: false, + RequestsPerMinute: 1, + Burst: 1, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + for i := 0; i < 3; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent { + t.Fatalf("request %d status = %d", i, w.Code) + } + } +} + +func TestRateLimitErrorBody(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + for i := 0; i < 2; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.20:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if i == 1 { + var body map[string]string + if err := json.NewDecoder(w.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body["code"] != "RESOURCE_LIMIT" { + t.Fatalf("unexpected body: %+v", body) + } + } + } +} + +func TestRateLimitStats(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + KeyBy: "ip", + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + for i := 0; i < 2; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.30:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + } + + stats := limiter.Stats() + if !stats.Enabled || stats.RequestsPerMinute != 60 || stats.Burst != 1 || stats.KeyBy != "ip" { + t.Fatalf("unexpected config stats: %+v", stats) + } + if stats.ActiveBuckets != 1 || stats.AllowedTotal != 1 || stats.LimitedTotal != 1 { + t.Fatalf("unexpected counters: %+v", stats) + } + if stats.BucketTTL == "" || stats.CleanupInterval == "" { + t.Fatalf("expected cleanup settings in stats: %+v", stats) + } +} + +func TestRateLimitEvictsInactiveBuckets(t *testing.T) { + now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC) + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 1, + KeyBy: "ip", + BucketTTL: time.Minute, + CleanupInterval: time.Second, + Now: func() time.Time { return now }, + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.40:5000" + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if limiter.Stats().ActiveBuckets != 1 { + t.Fatalf("expected one active bucket: %+v", limiter.Stats()) + } + + now = now.Add(2 * time.Minute) + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.41:5000" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + stats := limiter.Stats() + if stats.ActiveBuckets != 1 || stats.EvictedTotal != 1 { + t.Fatalf("unexpected cleanup stats: %+v", stats) + } +} + +func TestRateLimitBucketKeysDoNotStoreRawIdentity(t *testing.T) { + limiter := NewRateLimiter(RateLimitConfig{ + Enabled: true, + RequestsPerMinute: 60, + Burst: 10, + KeyBy: "owner", + }) + + handler := limiter.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.Header.Set("X-User-ID", "sensitive-owner") + req.Header.Set("X-API-Key", "sk-sensitive") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + + req = httptest.NewRequest(http.MethodGet, "/api/v1/sandboxes", nil) + req.RemoteAddr = "203.0.113.77:5000" + limiter.keyBy = "ip" + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + + limiter.mu.Lock() + defer limiter.mu.Unlock() + for key := range limiter.buckets { + if strings.Contains(key, "sensitive-owner") || strings.Contains(key, "sk-sensitive") || strings.Contains(key, "203.0.113.77") { + t.Fatalf("bucket key contains raw identity: %q", key) + } + parts := strings.Split(key, ":") + if len(parts) != 2 || len(parts[1]) != 64 { + t.Fatalf("bucket key is not typed sha256 form: %q", key) + } + } +} diff --git a/internal/api/routes/admin_audit.go b/internal/api/routes/admin_audit.go new file mode 100644 index 0000000..243485a --- /dev/null +++ b/internal/api/routes/admin_audit.go @@ -0,0 +1,114 @@ +package routes + +import ( + "context" + "encoding/csv" + "net/http" + "strconv" + "strings" + + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/store" +) + +type adminAuditStore interface { + ListAdminAudit(ctx context.Context, query store.AdminAuditQuery) ([]*store.AdminAuditRecord, error) +} + +type AdminAuditRoutes struct { + store adminAuditStore +} + +func NewAdminAuditRoutes(st adminAuditStore) *AdminAuditRoutes { + return &AdminAuditRoutes{store: st} +} + +// List returns recent admin audit log entries. +// +// @Summary List admin audit logs +// @Description Return recent redacted admin route access records +// @Tags admin +// @Produce json +// @Param limit query int false "Maximum number of records, capped at 500" +// @Param actor query string false "Actor exact match" +// @Param method query string false "HTTP method exact match" +// @Param status query int false "HTTP status exact match" +// @Param path query string false "Path substring match" +// @Param format query string false "Response format: json or csv" +// @Success 200 {array} AdminAuditResponse +// @Security ApiKeyAuth +// @Router /admin/audit [get] +func (a *AdminAuditRoutes) List(w http.ResponseWriter, r *http.Request) { + if a.store == nil { + httputil.WriteJSON(w, http.StatusOK, []*store.AdminAuditRecord{}) + return + } + query := store.AdminAuditQuery{Limit: 100} + if raw := r.URL.Query().Get("limit"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "limit must be a positive integer") + return + } + query.Limit = parsed + } + query.Actor = strings.TrimSpace(r.URL.Query().Get("actor")) + query.Method = strings.ToUpper(strings.TrimSpace(r.URL.Query().Get("method"))) + query.PathLike = strings.TrimSpace(r.URL.Query().Get("path")) + if raw := r.URL.Query().Get("status"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "status must be a positive integer") + return + } + query.Status = parsed + } + records, err := a.store.ListAdminAudit(r.Context(), query) + if err != nil { + writeRouteError(w, err) + return + } + if records == nil { + records = []*store.AdminAuditRecord{} + } + if strings.EqualFold(r.URL.Query().Get("format"), "csv") { + writeAdminAuditCSV(w, records) + return + } + httputil.WriteJSON(w, http.StatusOK, records) +} + +func writeAdminAuditCSV(w http.ResponseWriter, records []*store.AdminAuditRecord) { + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="stacyvm-admin-audit.csv"`) + w.WriteHeader(http.StatusOK) + + writer := csv.NewWriter(w) + _ = writer.Write([]string{ + "id", + "created_at", + "actor", + "method", + "path", + "status", + "duration_ms", + "request_id", + "remote_addr", + "user_agent", + }) + for _, rec := range records { + _ = writer.Write([]string{ + strconv.FormatInt(rec.ID, 10), + rec.CreatedAt.UTC().Format("2006-01-02T15:04:05Z07:00"), + rec.Actor, + rec.Method, + rec.Path, + strconv.Itoa(rec.Status), + strconv.FormatInt(rec.DurationMS, 10), + rec.RequestID, + rec.RemoteAddr, + rec.UserAgent, + }) + } + writer.Flush() +} diff --git a/internal/api/routes/environments.go b/internal/api/routes/environments.go index 5386f0e..dea9a85 100644 --- a/internal/api/routes/environments.go +++ b/internal/api/routes/environments.go @@ -181,11 +181,7 @@ func (e *EnvironmentRoutes) CreateSpec(w http.ResponseWriter, r *http.Request) { UpdatedAt: now, } if err := e.store.CreateEnvironmentSpec(r.Context(), rec); err != nil { - if strings.Contains(err.Error(), "UNIQUE constraint") { - httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, "spec name already exists for this owner") - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -216,11 +212,7 @@ func (e *EnvironmentRoutes) GetSpec(w http.ResponseWriter, r *http.Request) { specID := chi.URLParam(r, "specID") rec, err := e.store.GetEnvironmentSpec(r.Context(), specID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, toSpecResponse(rec)) @@ -230,11 +222,7 @@ func (e *EnvironmentRoutes) Suggestions(w http.ResponseWriter, r *http.Request) specID := chi.URLParam(r, "specID") rec, err := e.store.GetEnvironmentSpec(r.Context(), specID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -265,11 +253,7 @@ func (e *EnvironmentRoutes) StartBuild(w http.ResponseWriter, r *http.Request) { spec, err := e.store.GetEnvironmentSpec(r.Context(), req.SpecID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -327,11 +311,7 @@ func (e *EnvironmentRoutes) GetBuild(w http.ResponseWriter, r *http.Request) { buildID := chi.URLParam(r, "buildID") resp, err := e.getBuildResponse(r, buildID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, resp) @@ -394,11 +374,7 @@ func (e *EnvironmentRoutes) CancelBuild(w http.ResponseWriter, r *http.Request) buildID := chi.URLParam(r, "buildID") build, err := e.store.GetEnvironmentBuild(r.Context(), buildID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -432,11 +408,7 @@ func (e *EnvironmentRoutes) SpawnConfig(w http.ResponseWriter, r *http.Request) buildID := chi.URLParam(r, "buildID") build, err := e.store.GetEnvironmentBuild(r.Context(), buildID) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -507,11 +479,7 @@ func (e *EnvironmentRoutes) SaveRegistryConnection(w http.ResponseWriter, r *htt UpdatedAt: now, } if err := e.store.SaveRegistryConnection(r.Context(), rec); err != nil { - if strings.Contains(err.Error(), "UNIQUE constraint") { - httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, "registry connection already exists") - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, toRegistryConnectionResponse(rec)) @@ -538,11 +506,7 @@ func (e *EnvironmentRoutes) ListRegistryConnections(w http.ResponseWriter, r *ht func (e *EnvironmentRoutes) DeleteRegistryConnection(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "connectionID") if err := e.store.DeleteRegistryConnection(r.Context(), id); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) diff --git a/internal/api/routes/environments_test.go b/internal/api/routes/environments_test.go index 1f4e5e5..cac5555 100644 --- a/internal/api/routes/environments_test.go +++ b/internal/api/routes/environments_test.go @@ -109,6 +109,40 @@ func TestEnvironmentFlow_CreateBuildSpawnConfig(t *testing.T) { } } +func TestEnvironmentCreateSpecDuplicateReturnsConflict(t *testing.T) { + r := setupEnvTestRouter(t) + + specReq := map[string]any{ + "owner_id": "user-dupe", + "name": "same-name", + "base_image": "python:3.12-slim", + } + body, _ := json.Marshal(specReq) + for i := 0; i < 2; i++ { + req := httptest.NewRequest("POST", "/api/v1/environments/specs", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if i == 0 && w.Code != http.StatusCreated { + t.Fatalf("first create: expected 201, got %d: %s", w.Code, w.Body.String()) + } + if i == 1 && w.Code != http.StatusConflict { + t.Fatalf("second create: expected 409, got %d: %s", w.Code, w.Body.String()) + } + } +} + +func TestEnvironmentGetSpecMissingReturnsNotFound(t *testing.T) { + r := setupEnvTestRouter(t) + + req := httptest.NewRequest("GET", "/api/v1/environments/specs/spec-nope", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusNotFound { + t.Fatalf("expected 404, got %d: %s", w.Code, w.Body.String()) + } +} + func TestEnvironmentBuildCancel(t *testing.T) { r := setupEnvTestRouter(t) diff --git a/internal/api/routes/errors.go b/internal/api/routes/errors.go new file mode 100644 index 0000000..3d5b65a --- /dev/null +++ b/internal/api/routes/errors.go @@ -0,0 +1,32 @@ +package routes + +import ( + "errors" + "net/http" + + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/store" +) + +func writeRouteError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, orchestrator.ErrInvalidInput): + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, err.Error()) + case errors.Is(err, orchestrator.ErrSandboxNotFound), + errors.Is(err, orchestrator.ErrSandboxDestroyed), + errors.Is(err, orchestrator.ErrProviderNotFound), + errors.Is(err, store.ErrNotFound): + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) + case errors.Is(err, store.ErrConflict): + httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, err.Error()) + case errors.Is(err, orchestrator.ErrExecTimeout): + httputil.WriteError(w, http.StatusRequestTimeout, httputil.CodeTimeout, err.Error()) + case errors.Is(err, orchestrator.ErrResourceLimit): + httputil.WriteError(w, http.StatusTooManyRequests, httputil.CodeResourceLimit, err.Error()) + case errors.Is(err, orchestrator.ErrProviderUnavailable): + httputil.WriteError(w, http.StatusServiceUnavailable, httputil.CodeUnavailable, err.Error()) + default: + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + } +} diff --git a/internal/api/routes/prometheus.go b/internal/api/routes/prometheus.go new file mode 100644 index 0000000..db6f275 --- /dev/null +++ b/internal/api/routes/prometheus.go @@ -0,0 +1,160 @@ +package routes + +import ( + "fmt" + "io" + "sort" + + "github.com/StacyOs/stacyvm/internal/orchestrator" +) + +func writePrometheusMetrics(w io.Writer, metrics systemMetricsSnapshot) { + writePrometheusHelp(w, "stacyvm_uptime_seconds", "StacyVM API process uptime in seconds.") + fmt.Fprintf(w, "stacyvm_uptime_seconds %d\n", int64(metrics.uptime.Seconds())) + + writePrometheusHelp(w, "stacyvm_runtime_goroutines", "Current number of goroutines.") + fmt.Fprintf(w, "stacyvm_runtime_goroutines %d\n", metrics.goroutines) + + writePrometheusHelp(w, "stacyvm_runtime_memory_alloc_bytes", "Current allocated memory in bytes.") + fmt.Fprintf(w, "stacyvm_runtime_memory_alloc_bytes %d\n", metrics.memoryAlloc) + + writePrometheusHelp(w, "stacyvm_runtime_memory_sys_bytes", "Total memory obtained from the OS in bytes.") + fmt.Fprintf(w, "stacyvm_runtime_memory_sys_bytes %d\n", metrics.memorySys) + + writePrometheusHelp(w, "stacyvm_runtime_gc_cycles_total", "Total completed GC cycles.") + fmt.Fprintf(w, "stacyvm_runtime_gc_cycles_total %d\n", metrics.gcCycles) + + writePrometheusHelp(w, "stacyvm_sandboxes_total", "Current sandbox records by state and provider.") + states := sortedKeys(metrics.sandboxesByState) + for _, state := range states { + fmt.Fprintf(w, "stacyvm_sandboxes_total{state=%q} %d\n", state, metrics.sandboxesByState[state]) + } + providers := sortedKeys(metrics.sandboxesByProvider) + for _, provider := range providers { + fmt.Fprintf(w, "stacyvm_sandboxes_by_provider_total{provider=%q} %d\n", provider, metrics.sandboxesByProvider[provider]) + } + workers := sortedKeys(metrics.sandboxesByWorker) + for _, worker := range workers { + fmt.Fprintf(w, "stacyvm_sandboxes_by_worker_total{worker=%q} %d\n", worker, metrics.sandboxesByWorker[worker]) + } + + writePrometheusHelp(w, "stacyvm_provider_healthy", "Provider health status where 1 is healthy and 0 is unhealthy.") + writePrometheusHelp(w, "stacyvm_provider_health_latency_milliseconds", "Provider health check latency in milliseconds.") + writePrometheusHelp(w, "stacyvm_provider_runtime_sandboxes", "Runtime sandboxes discovered directly from providers.") + for _, provider := range metrics.providerHealth { + healthy := 0 + if provider.Healthy { + healthy = 1 + } + fmt.Fprintf(w, "stacyvm_provider_healthy{provider=%q,default=%q} %d\n", provider.Name, boolLabel(provider.Default), healthy) + fmt.Fprintf(w, "stacyvm_provider_health_latency_milliseconds{provider=%q} %d\n", provider.Name, provider.LatencyMS) + if provider.RuntimeCount != nil { + fmt.Fprintf(w, "stacyvm_provider_runtime_sandboxes{provider=%q} %d\n", provider.Name, *provider.RuntimeCount) + } + } + + if metrics.workerSummary != nil { + writePrometheusHelp(w, "stacyvm_workers_total", "Registered worker count by status bucket.") + for _, key := range []string{"total", "online", "stale", "unhealthy"} { + fmt.Fprintf(w, "stacyvm_workers_total{status=%q} %d\n", key, intMetric(metrics.workerSummary[key])) + } + } + if metrics.leaseSummary != nil { + writePrometheusHelp(w, "stacyvm_leases_total", "Durable lease count by status bucket.") + for _, key := range []string{"total", "active", "expired"} { + fmt.Fprintf(w, "stacyvm_leases_total{status=%q} %d\n", key, intMetric(metrics.leaseSummary[key])) + } + } + + writePrometheusHelp(w, "stacyvm_events_total", "Total events published by the in-process event bus.") + fmt.Fprintf(w, "stacyvm_events_total %d\n", metrics.eventStats.EventsTotal) + writePrometheusHelp(w, "stacyvm_event_subscribers", "Current event stream subscriber count.") + fmt.Fprintf(w, "stacyvm_event_subscribers %d\n", metrics.eventStats.Subscribers) + writePrometheusHelp(w, "stacyvm_event_history_size", "Current event history item count.") + fmt.Fprintf(w, "stacyvm_event_history_size %d\n", metrics.eventStats.HistorySize) + + writePrometheusHelp(w, "stacyvm_spawn_queue_depth", "Current number of spawn requests waiting for capacity.") + fmt.Fprintf(w, "stacyvm_spawn_queue_depth %d\n", metrics.schedulerStatus.SpawnQueueDepth) + writePrometheusHelp(w, "stacyvm_spawn_queue_capacity", "Configured maximum number of queued spawn requests.") + fmt.Fprintf(w, "stacyvm_spawn_queue_capacity %d\n", metrics.schedulerStatus.MaxSpawnQueue) + writePrometheusHelp(w, "stacyvm_spawn_queue_enqueued_total", "Total spawn requests admitted into the capacity wait queue.") + fmt.Fprintf(w, "stacyvm_spawn_queue_enqueued_total %d\n", metrics.schedulerStatus.SpawnQueuedTotal) + writePrometheusHelp(w, "stacyvm_spawn_queue_dequeued_total", "Total spawn requests released from the capacity wait queue.") + fmt.Fprintf(w, "stacyvm_spawn_queue_dequeued_total %d\n", metrics.schedulerStatus.SpawnDequeuedTotal) + writePrometheusHelp(w, "stacyvm_spawn_queue_timeout_total", "Total spawn requests that timed out while waiting in the capacity queue.") + fmt.Fprintf(w, "stacyvm_spawn_queue_timeout_total %d\n", metrics.schedulerStatus.SpawnQueueTimeouts) + writePrometheusHelp(w, "stacyvm_spawn_queue_wait_milliseconds_sum", "Total observed spawn queue wait time in milliseconds.") + fmt.Fprintf(w, "stacyvm_spawn_queue_wait_milliseconds_sum %d\n", metrics.schedulerStatus.SpawnQueueWaitTotalMS) + writePrometheusHelp(w, "stacyvm_spawn_queue_wait_milliseconds_count", "Total observed spawn queue wait samples.") + fmt.Fprintf(w, "stacyvm_spawn_queue_wait_milliseconds_count %d\n", metrics.schedulerStatus.SpawnQueueWaitCount) + writePrometheusHelp(w, "stacyvm_spawn_queue_wait_milliseconds_max", "Maximum observed spawn queue wait time in milliseconds.") + fmt.Fprintf(w, "stacyvm_spawn_queue_wait_milliseconds_max %d\n", metrics.schedulerStatus.SpawnQueueWaitMaxMS) + + writePrometheusHelp(w, "stacyvm_owner_quotas_total", "Total configured owner quota policies.") + fmt.Fprintf(w, "stacyvm_owner_quotas_total %d\n", metrics.quotaSummary.Total) + writePrometheusHelp(w, "stacyvm_owner_quota_overrides_total", "Total configured owner quota overrides by override type.") + fmt.Fprintf(w, "stacyvm_owner_quota_overrides_total{type=%q} %d\n", "max_sandboxes", metrics.quotaSummary.WithMaxSandboxes) + fmt.Fprintf(w, "stacyvm_owner_quota_overrides_total{type=%q} %d\n", "max_ttl", metrics.quotaSummary.WithMaxTTL) + fmt.Fprintf(w, "stacyvm_owner_quota_overrides_total{type=%q} %d\n", "max_exec_timeout", metrics.quotaSummary.WithMaxExecTimeout) + + writePrometheusHelp(w, "stacyvm_rate_limit_allowed_total", "Total API requests allowed by the in-process rate limiter.") + fmt.Fprintf(w, "stacyvm_rate_limit_allowed_total %d\n", metrics.rateLimitStats.AllowedTotal) + writePrometheusHelp(w, "stacyvm_rate_limit_blocked_total", "Total API requests blocked by the in-process rate limiter.") + fmt.Fprintf(w, "stacyvm_rate_limit_blocked_total %d\n", metrics.rateLimitStats.LimitedTotal) + writePrometheusHelp(w, "stacyvm_rate_limit_evicted_buckets_total", "Total inactive rate-limit buckets evicted from memory.") + fmt.Fprintf(w, "stacyvm_rate_limit_evicted_buckets_total %d\n", metrics.rateLimitStats.EvictedTotal) + writePrometheusHelp(w, "stacyvm_rate_limit_active_buckets", "Current number of active rate-limit buckets.") + fmt.Fprintf(w, "stacyvm_rate_limit_active_buckets %d\n", metrics.rateLimitStats.ActiveBuckets) + + writeOperationMetrics(w, metrics.operationMetrics) +} + +func writeOperationMetrics(w io.Writer, operationMetrics []orchestrator.OperationMetrics) { + writePrometheusHelp(w, "stacyvm_operation_success_total", "Total successful operations by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_failure_total", "Total failed operations by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_latency_milliseconds_sum", "Total operation latency in milliseconds by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_latency_milliseconds_count", "Total observed operation latency samples by operation and provider.") + writePrometheusHelp(w, "stacyvm_operation_latency_milliseconds_max", "Maximum observed operation latency in milliseconds by operation and provider.") + for _, metric := range operationMetrics { + labels := fmt.Sprintf("operation=%q,provider=%q", metric.Operation, metric.Provider) + fmt.Fprintf(w, "stacyvm_operation_success_total{%s} %d\n", labels, metric.SuccessTotal) + fmt.Fprintf(w, "stacyvm_operation_failure_total{%s} %d\n", labels, metric.FailureTotal) + fmt.Fprintf(w, "stacyvm_operation_latency_milliseconds_sum{%s} %d\n", labels, metric.LatencyTotalMS) + fmt.Fprintf(w, "stacyvm_operation_latency_milliseconds_count{%s} %d\n", labels, metric.LatencyCount) + fmt.Fprintf(w, "stacyvm_operation_latency_milliseconds_max{%s} %d\n", labels, metric.LatencyMaxMS) + } +} + +func writePrometheusHelp(w io.Writer, name, help string) { + fmt.Fprintf(w, "# HELP %s %s\n", name, help) + fmt.Fprintf(w, "# TYPE %s gauge\n", name) +} + +func sortedKeys(values map[string]int) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func boolLabel(value bool) string { + if value { + return "true" + } + return "false" +} + +func intMetric(value interface{}) int { + switch v := value.(type) { + case int: + return v + case int64: + return int(v) + case float64: + return int(v) + default: + return 0 + } +} diff --git a/internal/api/routes/provider_health.go b/internal/api/routes/provider_health.go new file mode 100644 index 0000000..af188da --- /dev/null +++ b/internal/api/routes/provider_health.go @@ -0,0 +1,82 @@ +package routes + +import ( + "context" + "time" + + "github.com/StacyOs/stacyvm/internal/providers" +) + +func collectProviderHealth(ctx context.Context, registry *providers.Registry) []ProviderHealth { + names := registry.List() + out := make([]ProviderHealth, 0, len(names)) + defaultProvider := registry.Default() + + for _, name := range names { + start := time.Now() + checkedAt := time.Now().UTC() + item := ProviderHealth{ + Name: name, + Default: name == defaultProvider, + LastChecked: checkedAt.Format(time.RFC3339), + Capabilities: []string{"spawn", "exec", "exec_stream", "files", "console", "health"}, + } + + prov, err := registry.Get(name) + if err != nil { + item.Healthy = false + item.Error = err.Error() + item.LatencyMS = time.Since(start).Milliseconds() + out = append(out, item) + continue + } + + item.Healthy = prov.Healthy(ctx) + item.LatencyMS = time.Since(start).Milliseconds() + if !item.Healthy { + item.Error = "health check returned false" + } + item.Capabilities = append(item.Capabilities, providerCapabilities(prov)...) + + if lister, ok := prov.(providers.RuntimeSandboxLister); ok { + runtimes, err := lister.ListRuntimeSandboxes(ctx) + if err != nil { + if item.Error == "" { + item.Error = "runtime inventory: " + err.Error() + } + } else { + count := len(runtimes) + item.RuntimeCount = &count + } + } + + out = append(out, item) + } + + return out +} + +func providerCapabilities(prov providers.Provider) []string { + capabilities := make([]string, 0, 4) + if _, ok := prov.(providers.RuntimeSandboxLister); ok { + capabilities = append(capabilities, "runtime_inventory") + } + if _, ok := prov.(providers.SnapshotLister); ok { + capabilities = append(capabilities, "snapshots") + } + switch prov.(type) { + case *providers.FirecrackerProvider: + capabilities = append(capabilities, "microvm", "vsock_agent") + case *providers.DockerProvider: + capabilities = append(capabilities, "container") + case *providers.PRootProvider: + capabilities = append(capabilities, "userspace_isolation") + case *providers.CustomProvider: + capabilities = append(capabilities, "remote_http") + case *providers.E2BProvider: + capabilities = append(capabilities, "remote_e2b") + case *providers.MockProvider: + capabilities = append(capabilities, "test_provider") + } + return capabilities +} diff --git a/internal/api/routes/providers.go b/internal/api/routes/providers.go index 1752085..902c132 100644 --- a/internal/api/routes/providers.go +++ b/internal/api/routes/providers.go @@ -4,9 +4,9 @@ import ( "context" "net/http" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/providers" + "github.com/go-chi/chi/v5" ) type sandboxCounter interface { @@ -32,9 +32,14 @@ func (p *ProviderRoutes) Routes() chi.Router { // ProviderInfo is the summary info for a provider. type ProviderInfo struct { - Name string `json:"name" example:"firecracker"` - Healthy bool `json:"healthy" example:"true"` - Default bool `json:"default" example:"true"` + Name string `json:"name" example:"firecracker"` + Healthy bool `json:"healthy" example:"true"` + Default bool `json:"default" example:"true"` + LatencyMS int64 `json:"latency_ms" example:"3"` + LastChecked string `json:"last_checked" example:"2026-05-08T10:30:00Z"` + Error string `json:"error,omitempty" example:"health check returned false"` + Capabilities []string `json:"capabilities"` + RuntimeCount *int `json:"runtime_count,omitempty" example:"2"` } // ProviderDetail is the detailed info for a provider. @@ -43,6 +48,7 @@ type ProviderDetail struct { Healthy bool `json:"healthy" example:"true"` Default bool `json:"default" example:"true"` SandboxCount int `json:"sandbox_count" example:"3"` + Health ProviderHealth `json:"health"` Config map[string]string `json:"config"` } @@ -56,19 +62,18 @@ type ProviderDetail struct { // @Security ApiKeyAuth // @Router /providers [get] func (p *ProviderRoutes) List(w http.ResponseWriter, r *http.Request) { - names := p.registry.List() - infos := make([]ProviderInfo, 0, len(names)) - dflt := p.registry.Default() - - for _, name := range names { - prov, err := p.registry.Get(name) - if err != nil { - continue - } + health := collectProviderHealth(r.Context(), p.registry) + infos := make([]ProviderInfo, 0, len(health)) + for _, item := range health { infos = append(infos, ProviderInfo{ - Name: name, - Healthy: prov.Healthy(r.Context()), - Default: name == dflt, + Name: item.Name, + Healthy: item.Healthy, + Default: item.Default, + LatencyMS: item.LatencyMS, + LastChecked: item.LastChecked, + Error: item.Error, + Capabilities: item.Capabilities, + RuntimeCount: item.RuntimeCount, }) } @@ -117,7 +122,7 @@ func (p *ProviderRoutes) Detail(w http.ResponseWriter, r *http.Request) { prov, err := p.registry.Get(name) if err != nil { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "provider not found") + writeRouteError(w, err) return } @@ -149,12 +154,20 @@ func (p *ProviderRoutes) Detail(w http.ResponseWriter, r *http.Request) { if p.counter != nil { count = p.counter.CountByProvider(r.Context(), name) } + health := ProviderHealth{Name: name, Healthy: prov.Healthy(r.Context()), Default: name == dflt} + for _, item := range collectProviderHealth(r.Context(), p.registry) { + if item.Name == name { + health = item + break + } + } httputil.WriteJSON(w, http.StatusOK, ProviderDetail{ Name: name, - Healthy: prov.Healthy(r.Context()), + Healthy: health.Healthy, Default: name == dflt, SandboxCount: count, + Health: health, Config: cfg, }) } diff --git a/internal/api/routes/quotas.go b/internal/api/routes/quotas.go new file mode 100644 index 0000000..fc90a88 --- /dev/null +++ b/internal/api/routes/quotas.go @@ -0,0 +1,166 @@ +package routes + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/go-chi/chi/v5" +) + +type quotaManager interface { + ListOwnerQuotas(ctx context.Context) ([]*orchestrator.OwnerQuota, error) + QuotaSummary(ctx context.Context) (orchestrator.QuotaSummary, error) + GetOwnerQuota(ctx context.Context, ownerID string) (*orchestrator.OwnerQuota, error) + SaveOwnerQuota(ctx context.Context, quota orchestrator.OwnerQuota) (*orchestrator.OwnerQuota, error) + DeleteOwnerQuota(ctx context.Context, ownerID string) error + OwnerUsage(ctx context.Context, ownerID string) (*orchestrator.OwnerUsage, error) +} + +type QuotaRoutes struct { + manager quotaManager +} + +func NewQuotaRoutes(manager quotaManager) *QuotaRoutes { + return &QuotaRoutes{manager: manager} +} + +func (q *QuotaRoutes) Routes() chi.Router { + r := chi.NewRouter() + r.Get("/", q.List) + r.Get("/summary", q.Summary) + r.Route("/{ownerID}", func(r chi.Router) { + r.Get("/", q.Get) + r.Put("/", q.Save) + r.Delete("/", q.Delete) + r.Get("/usage", q.Usage) + }) + return r +} + +// List returns all configured owner quotas. +// +// @Summary List owner quotas +// @Description Return all persisted owner quota overrides +// @Tags quotas +// @Produce json +// @Success 200 {array} orchestrator.OwnerQuota +// @Security ApiKeyAuth +// @Router /quotas [get] +func (q *QuotaRoutes) List(w http.ResponseWriter, r *http.Request) { + quotas, err := q.manager.ListOwnerQuotas(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + if quotas == nil { + quotas = []*orchestrator.OwnerQuota{} + } + httputil.WriteJSON(w, http.StatusOK, quotas) +} + +// Summary returns redacted quota coverage counts. +// +// @Summary Get quota summary +// @Description Return non-identifying counts for persisted owner quota overrides +// @Tags quotas +// @Produce json +// @Success 200 {object} orchestrator.QuotaSummary +// @Security ApiKeyAuth +// @Router /quotas/summary [get] +func (q *QuotaRoutes) Summary(w http.ResponseWriter, r *http.Request) { + summary, err := q.manager.QuotaSummary(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, summary) +} + +// Get returns one configured owner quota. +// +// @Summary Get owner quota +// @Description Return the persisted quota override for an owner +// @Tags quotas +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Success 200 {object} orchestrator.OwnerQuota +// @Failure 404 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /quotas/{ownerID} [get] +func (q *QuotaRoutes) Get(w http.ResponseWriter, r *http.Request) { + quota, err := q.manager.GetOwnerQuota(r.Context(), chi.URLParam(r, "ownerID")) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, quota) +} + +// Save creates or updates an owner quota. +// +// @Summary Save owner quota +// @Description Create or update quota overrides for an owner +// @Tags quotas +// @Accept json +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Param request body orchestrator.OwnerQuota true "Quota request" +// @Success 200 {object} orchestrator.OwnerQuota +// @Security ApiKeyAuth +// @Router /quotas/{ownerID} [put] +func (q *QuotaRoutes) Save(w http.ResponseWriter, r *http.Request) { + ownerID := chi.URLParam(r, "ownerID") + var quota orchestrator.OwnerQuota + if err := json.NewDecoder(r.Body).Decode("a); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + quota.OwnerID = ownerID + saved, err := q.manager.SaveOwnerQuota(r.Context(), quota) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, saved) +} + +// Delete removes an owner quota override. +// +// @Summary Delete owner quota +// @Description Delete the quota override for an owner +// @Tags quotas +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Success 200 {object} StatusResponse +// @Failure 404 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /quotas/{ownerID} [delete] +func (q *QuotaRoutes) Delete(w http.ResponseWriter, r *http.Request) { + if err := q.manager.DeleteOwnerQuota(r.Context(), chi.URLParam(r, "ownerID")); err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) +} + +// Usage returns current owner usage against quota. +// +// @Summary Get owner quota usage +// @Description Return active sandbox usage and effective quota for an owner +// @Tags quotas +// @Produce json +// @Param ownerID path string true "Owner ID" +// @Success 200 {object} orchestrator.OwnerUsage +// @Security ApiKeyAuth +// @Router /quotas/{ownerID}/usage [get] +func (q *QuotaRoutes) Usage(w http.ResponseWriter, r *http.Request) { + usage, err := q.manager.OwnerUsage(r.Context(), chi.URLParam(r, "ownerID")) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, usage) +} diff --git a/internal/api/routes/quotas_test.go b/internal/api/routes/quotas_test.go new file mode 100644 index 0000000..489cd8a --- /dev/null +++ b/internal/api/routes/quotas_test.go @@ -0,0 +1,159 @@ +package routes + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog" +) + +func setupQuotaRouter(t *testing.T) (chi.Router, *orchestrator.Manager) { + t.Helper() + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := providers.NewMockProvider() + reg.Register(mock) + reg.SetDefault("mock") + + mgr := orchestrator.NewManager(reg, st, orchestrator.NewEventBus(), zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + + r := chi.NewRouter() + r.Mount("/api/v1/quotas", NewQuotaRoutes(mgr).Routes()) + return r, mgr +} + +func TestQuotaRoutes_SaveGetUsageDelete(t *testing.T) { + r, mgr := setupQuotaRouter(t) + + body := `{"max_sandboxes":1,"max_ttl":"30m","max_exec_timeout":"10s"}` + req := httptest.NewRequest(http.MethodPut, "/api/v1/quotas/owner-a", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("save status = %d: %s", w.Code, w.Body.String()) + } + + var quota orchestrator.OwnerQuota + if err := json.NewDecoder(w.Body).Decode("a); err != nil { + t.Fatalf("decode quota: %v", err) + } + if quota.OwnerID != "owner-a" || quota.MaxSandboxes != 1 { + t.Fatalf("unexpected quota: %+v", quota) + } + + if _, err := mgr.Spawn(req.Context(), orchestrator.SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/quotas/owner-a/usage", nil) + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("usage status = %d: %s", w.Code, w.Body.String()) + } + var usage orchestrator.OwnerUsage + if err := json.NewDecoder(w.Body).Decode(&usage); err != nil { + t.Fatalf("decode usage: %v", err) + } + if !usage.QuotaConfigured || usage.ActiveSandboxes != 1 || usage.MaxSandboxes != 1 { + t.Fatalf("unexpected usage: %+v", usage) + } + + req = httptest.NewRequest(http.MethodDelete, "/api/v1/quotas/owner-a", nil) + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("delete status = %d: %s", w.Code, w.Body.String()) + } +} + +func TestQuotaRoutes_Summary(t *testing.T) { + r, _ := setupQuotaRouter(t) + + quotas := map[string]string{ + "owner-a": `{"max_sandboxes":1,"max_ttl":"30s"}`, + "owner-b": `{"max_exec_timeout":"10s"}`, + } + for ownerID, body := range quotas { + req := httptest.NewRequest(http.MethodPut, "/api/v1/quotas/"+ownerID, bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("save %s status = %d: %s", ownerID, w.Code, w.Body.String()) + } + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/quotas/summary", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("summary status = %d: %s", w.Code, w.Body.String()) + } + + var summary orchestrator.QuotaSummary + if err := json.NewDecoder(w.Body).Decode(&summary); err != nil { + t.Fatalf("decode summary: %v", err) + } + if summary.Total != 2 || summary.WithMaxSandboxes != 1 || summary.WithMaxTTL != 1 || summary.WithMaxExecTimeout != 1 { + t.Fatalf("unexpected summary: %+v", summary) + } +} + +func TestQuotaRoutes_InvalidQuotaReturnsBadRequest(t *testing.T) { + r, _ := setupQuotaRouter(t) + + tests := []struct { + name string + path string + body string + }{ + { + name: "bad duration", + path: "/api/v1/quotas/owner-a", + body: `{"max_ttl":"500ms"}`, + }, + { + name: "negative sandboxes", + path: "/api/v1/quotas/owner-a", + body: `{"max_sandboxes":-1}`, + }, + { + name: "bad owner", + path: "/api/v1/quotas/owner%20a", + body: `{"max_sandboxes":1}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, tt.path, bytes.NewBufferString(tt.body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusBadRequest, w.Body.String()) + } + }) + } +} diff --git a/internal/api/routes/sandboxes.go b/internal/api/routes/sandboxes.go index ea0ca8f..4506db6 100644 --- a/internal/api/routes/sandboxes.go +++ b/internal/api/routes/sandboxes.go @@ -1,51 +1,92 @@ package routes import ( + "context" "encoding/json" "net/http" "strconv" - "strings" "time" - "github.com/go-chi/chi/v5" + "github.com/StacyOs/stacyvm/internal/api/middleware" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" "nhooyr.io/websocket" "nhooyr.io/websocket/wsjson" ) +type sandboxPolicyStore interface { + ListPolicies(ctx context.Context, query store.PolicyQuery) ([]*store.PolicyRecord, error) +} + type SandboxRoutes struct { - manager *orchestrator.Manager + manager *orchestrator.Manager + policyStore sandboxPolicyStore } func NewSandboxRoutes(manager *orchestrator.Manager) *SandboxRoutes { return &SandboxRoutes{manager: manager} } +func NewSandboxRoutesWithPolicy(manager *orchestrator.Manager, ps sandboxPolicyStore) *SandboxRoutes { + return &SandboxRoutes{manager: manager, policyStore: ps} +} + func (s *SandboxRoutes) Routes() chi.Router { + return s.RoutesWithScopeEnforcement(false) +} + +// RoutesWithScopeEnforcement returns the router with scope checks enabled when +// enforce is true (used by the server when auth is configured). +func (s *SandboxRoutes) RoutesWithScopeEnforcement(enforce bool) chi.Router { r := chi.NewRouter() - r.Post("/", s.Create) - r.Get("/", s.List) - r.Delete("/", s.Prune) + + readScope := optionalScope(enforce, middleware.ScopeRead) + apiScope := optionalScope(enforce, middleware.ScopeAPI) + + // Read-only operations: viewer, api, operator, admin. + r.With(readScope).Get("/", s.List) + r.With(readScope).Post("/admission", s.Admission) + + // Mutating operations: api, operator, admin (not viewer). + // PolicyEnforcer runs before Create to enforce tenant image/provider/network rules. + spawnChain := []func(http.Handler) http.Handler{apiScope} + if s.policyStore != nil { + spawnChain = append(spawnChain, middleware.PolicyEnforcer(s.policyStore)) + } + r.With(spawnChain...).Post("/", s.Create) + r.With(apiScope).Delete("/", s.Prune) + r.Route("/{sandboxID}", func(r chi.Router) { - r.Get("/", s.Get) - r.Delete("/", s.Destroy) - r.Post("/extend", s.Extend) - r.Post("/exec", s.Exec) - r.Get("/exec/ws", s.ExecWebSocket) - r.Post("/files", s.WriteFile) - r.Get("/files", s.ReadFile) - r.Delete("/files", s.DeleteFile) - r.Get("/files/list", s.ListFiles) - r.Post("/files/move", s.MoveFile) - r.Post("/files/chmod", s.ChmodFile) - r.Get("/files/stat", s.StatFile) - r.Get("/files/glob", s.GlobFiles) - r.Get("/logs", s.ConsoleLog) + // Read-only. + r.With(readScope).Get("/", s.Get) + r.With(readScope).Get("/files", s.ReadFile) + r.With(readScope).Get("/files/list", s.ListFiles) + r.With(readScope).Get("/files/stat", s.StatFile) + r.With(readScope).Get("/files/glob", s.GlobFiles) + r.With(readScope).Get("/logs", s.ConsoleLog) + // Mutating. + r.With(apiScope).Delete("/", s.Destroy) + r.With(apiScope).Post("/extend", s.Extend) + r.With(apiScope).Post("/exec", s.Exec) + r.With(apiScope).Get("/exec/ws", s.ExecWebSocket) + r.With(apiScope).Post("/files", s.WriteFile) + r.With(apiScope).Delete("/files", s.DeleteFile) + r.With(apiScope).Post("/files/move", s.MoveFile) + r.With(apiScope).Post("/files/chmod", s.ChmodFile) }) return r } +// optionalScope returns RequireScope when enforce is true, otherwise a no-op. +func optionalScope(enforce bool, scope string) func(http.Handler) http.Handler { + if enforce { + return middleware.RequireScope(scope) + } + return func(next http.Handler) http.Handler { return next } +} + // Create creates a new sandbox. // // @Summary Create a sandbox @@ -56,6 +97,7 @@ func (s *SandboxRoutes) Routes() chi.Router { // @Param request body orchestrator.SpawnRequest true "Spawn request" // @Success 201 {object} orchestrator.Sandbox // @Failure 400 {object} httputil.APIError +// @Failure 429 {object} httputil.APIError // @Failure 500 {object} httputil.APIError // @Security ApiKeyAuth // @Router /sandboxes [post] @@ -66,20 +108,56 @@ func (s *SandboxRoutes) Create(w http.ResponseWriter, r *http.Request) { return } - // Extract owner from X-User-ID header if present. + identity := middleware.AuthIdentityFromContext(r.Context()) + // Extract owner from X-User-ID header, falling back to OIDC subject. if userID := r.Header.Get("X-User-ID"); userID != "" { req.OwnerID = userID + } else if identity.Subject != "" && req.OwnerID == "" { + req.OwnerID = identity.Subject } + // Stamp the tenant from the caller's identity so sandboxes are scoped. + req.TenantID = identity.TenantID sb, err := s.manager.Spawn(r.Context(), req) if err != nil { - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, sb) } +// Admission evaluates whether a spawn request would be admitted. +// +// @Summary Evaluate spawn admission +// @Description Return whether a spawn request would be allowed, queued, or denied by quota and scheduler limits +// @Tags sandboxes +// @Accept json +// @Produce json +// @Param request body orchestrator.SpawnRequest true "Spawn request" +// @Success 200 {object} orchestrator.SpawnAdmissionDecision +// @Failure 400 {object} httputil.APIError +// @Failure 500 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /sandboxes/admission [post] +func (s *SandboxRoutes) Admission(w http.ResponseWriter, r *http.Request) { + var req orchestrator.SpawnRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + if userID := r.Header.Get("X-User-ID"); userID != "" { + req.OwnerID = userID + } + + decision, err := s.manager.EvaluateSpawnRequestAdmission(r.Context(), req) + if err != nil { + writeRouteError(w, err) + return + } + httputil.WriteJSON(w, http.StatusOK, decision) +} + // List lists all active sandboxes. // // @Summary List sandboxes @@ -99,6 +177,17 @@ func (s *SandboxRoutes) List(w http.ResponseWriter, r *http.Request) { if sandboxes == nil { sandboxes = []*orchestrator.Sandbox{} } + // Enforce tenant scoping: callers with a tenant identity only see their tenant's sandboxes. + identity := middleware.AuthIdentityFromContext(r.Context()) + if identity.TenantID != "" { + filtered := sandboxes[:0] + for _, sb := range sandboxes { + if sb.TenantID == identity.TenantID { + filtered = append(filtered, sb) + } + } + sandboxes = filtered + } httputil.WriteJSON(w, http.StatusOK, sandboxes) } @@ -118,11 +207,12 @@ func (s *SandboxRoutes) Get(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") sb, err := s.manager.Get(r.Context(), id) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) + return + } + // Enforce tenant scoping. + if tenantID := middleware.AuthIdentityFromContext(r.Context()).TenantID; tenantID != "" && sb.TenantID != tenantID { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "sandbox not found") return } httputil.WriteJSON(w, http.StatusOK, sb) @@ -140,14 +230,33 @@ func (s *SandboxRoutes) Get(w http.ResponseWriter, r *http.Request) { // @Failure 500 {object} httputil.APIError // @Security ApiKeyAuth // @Router /sandboxes/{sandboxID} [delete] +// checkTenantAccess fetches the sandbox and returns false (writing 404) if the +// caller's tenant does not match the sandbox's tenant. Callers with no tenant +// (API key without X-Tenant-ID) bypass the check. +func (s *SandboxRoutes) checkTenantAccess(w http.ResponseWriter, r *http.Request, sandboxID string) bool { + tenantID := middleware.AuthIdentityFromContext(r.Context()).TenantID + if tenantID == "" { + return true + } + sb, err := s.manager.Get(r.Context(), sandboxID) + if err != nil { + writeRouteError(w, err) + return false + } + if sb.TenantID != tenantID { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "sandbox not found") + return false + } + return true +} + func (s *SandboxRoutes) Destroy(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } if err := s.manager.Destroy(r.Context(), id); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "destroyed"}) @@ -170,6 +279,9 @@ func (s *SandboxRoutes) Destroy(w http.ResponseWriter, r *http.Request) { // @Router /sandboxes/{sandboxID}/extend [post] func (s *SandboxRoutes) Extend(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } var req struct { TTL string `json:"ttl"` } @@ -196,11 +308,7 @@ func (s *SandboxRoutes) Extend(w http.ResponseWriter, r *http.Request) { sb, err := s.manager.ExtendTTL(r.Context(), id, duration) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -223,6 +331,9 @@ func (s *SandboxRoutes) Extend(w http.ResponseWriter, r *http.Request) { // @Router /sandboxes/{sandboxID}/exec [post] func (s *SandboxRoutes) Exec(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } var req orchestrator.ExecRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") @@ -236,11 +347,7 @@ func (s *SandboxRoutes) Exec(w http.ResponseWriter, r *http.Request) { result, err := s.manager.Exec(r.Context(), id, req) if err != nil { - if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "destroyed") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, result) @@ -249,7 +356,7 @@ func (s *SandboxRoutes) Exec(w http.ResponseWriter, r *http.Request) { func (s *SandboxRoutes) execStream(w http.ResponseWriter, r *http.Request, id string, req orchestrator.ExecRequest) { ch, err := s.manager.ExecStream(r.Context(), id, req) if err != nil { - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -285,6 +392,9 @@ func (s *SandboxRoutes) execStream(w http.ResponseWriter, r *http.Request, id st // @Router /sandboxes/{sandboxID}/files [post] func (s *SandboxRoutes) WriteFile(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } var req orchestrator.FileWriteRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") @@ -292,11 +402,7 @@ func (s *SandboxRoutes) WriteFile(w http.ResponseWriter, r *http.Request) { } if err := s.manager.WriteFile(r.Context(), id, req); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "written"}) @@ -318,6 +424,9 @@ func (s *SandboxRoutes) WriteFile(w http.ResponseWriter, r *http.Request) { // @Router /sandboxes/{sandboxID}/files [get] func (s *SandboxRoutes) ReadFile(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } path := r.URL.Query().Get("path") if path == "" { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "path query parameter required") @@ -326,11 +435,7 @@ func (s *SandboxRoutes) ReadFile(w http.ResponseWriter, r *http.Request) { data, err := s.manager.ReadFile(r.Context(), id, path) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -354,6 +459,9 @@ func (s *SandboxRoutes) ReadFile(w http.ResponseWriter, r *http.Request) { // @Router /sandboxes/{sandboxID}/files/list [get] func (s *SandboxRoutes) ListFiles(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } path := r.URL.Query().Get("path") if path == "" { path = "/" @@ -361,11 +469,7 @@ func (s *SandboxRoutes) ListFiles(w http.ResponseWriter, r *http.Request) { files, err := s.manager.ListFiles(r.Context(), id, path) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, files) @@ -374,6 +478,9 @@ func (s *SandboxRoutes) ListFiles(w http.ResponseWriter, r *http.Request) { // DeleteFile deletes a file from a sandbox. func (s *SandboxRoutes) DeleteFile(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } path := r.URL.Query().Get("path") if path == "" { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "path query parameter required") @@ -386,11 +493,7 @@ func (s *SandboxRoutes) DeleteFile(w http.ResponseWriter, r *http.Request) { Path: path, Recursive: recursive, }); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) @@ -399,6 +502,9 @@ func (s *SandboxRoutes) DeleteFile(w http.ResponseWriter, r *http.Request) { // MoveFile moves/renames a file in a sandbox. func (s *SandboxRoutes) MoveFile(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } var req orchestrator.FileMoveRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") @@ -406,11 +512,7 @@ func (s *SandboxRoutes) MoveFile(w http.ResponseWriter, r *http.Request) { } if err := s.manager.MoveFile(r.Context(), id, req); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "moved"}) @@ -419,6 +521,9 @@ func (s *SandboxRoutes) MoveFile(w http.ResponseWriter, r *http.Request) { // ChmodFile changes file permissions in a sandbox. func (s *SandboxRoutes) ChmodFile(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } var req orchestrator.FileChmodRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") @@ -426,11 +531,7 @@ func (s *SandboxRoutes) ChmodFile(w http.ResponseWriter, r *http.Request) { } if err := s.manager.ChmodFile(r.Context(), id, req); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "chmod applied"}) @@ -439,6 +540,9 @@ func (s *SandboxRoutes) ChmodFile(w http.ResponseWriter, r *http.Request) { // StatFile returns file info for a single file in a sandbox. func (s *SandboxRoutes) StatFile(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } path := r.URL.Query().Get("path") if path == "" { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "path query parameter required") @@ -447,11 +551,7 @@ func (s *SandboxRoutes) StatFile(w http.ResponseWriter, r *http.Request) { fi, err := s.manager.StatFile(r.Context(), id, path) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, fi) @@ -460,6 +560,9 @@ func (s *SandboxRoutes) StatFile(w http.ResponseWriter, r *http.Request) { // GlobFiles returns paths matching a glob pattern in a sandbox. func (s *SandboxRoutes) GlobFiles(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } pattern := r.URL.Query().Get("pattern") if pattern == "" { httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "pattern query parameter required") @@ -468,11 +571,7 @@ func (s *SandboxRoutes) GlobFiles(w http.ResponseWriter, r *http.Request) { matches, err := s.manager.GlobFiles(r.Context(), id, pattern) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, matches) @@ -525,6 +624,9 @@ func (s *SandboxRoutes) Prune(w http.ResponseWriter, r *http.Request) { // @Router /sandboxes/{sandboxID}/logs [get] func (s *SandboxRoutes) ConsoleLog(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } lines := 100 if q := r.URL.Query().Get("lines"); q != "" { if n, err := strconv.Atoi(q); err == nil && n > 0 { @@ -534,11 +636,7 @@ func (s *SandboxRoutes) ConsoleLog(w http.ResponseWriter, r *http.Request) { log, err := s.manager.ConsoleLog(r.Context(), id, lines) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, log) @@ -556,6 +654,9 @@ func (s *SandboxRoutes) ConsoleLog(w http.ResponseWriter, r *http.Request) { // @Router /sandboxes/{sandboxID}/exec/ws [get] func (s *SandboxRoutes) ExecWebSocket(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "sandboxID") + if !s.checkTenantAccess(w, r, id) { + return + } conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{ OriginPatterns: []string{"*"}, diff --git a/internal/api/routes/sandboxes_test.go b/internal/api/routes/sandboxes_test.go index b319158..64a9d1b 100644 --- a/internal/api/routes/sandboxes_test.go +++ b/internal/api/routes/sandboxes_test.go @@ -2,6 +2,7 @@ package routes import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -9,14 +10,24 @@ import ( "testing" "time" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/orchestrator" "github.com/StacyOs/stacyvm/internal/providers" "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" "github.com/rs/zerolog" ) func setupTestRouter(t *testing.T) (chi.Router, *orchestrator.Manager) { + t.Helper() + return setupTestRouterWithConfig(t, orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) +} + +func setupTestRouterWithConfig(t *testing.T, cfg orchestrator.ManagerConfig) (chi.Router, *orchestrator.Manager) { t.Helper() dir := t.TempDir() st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) @@ -33,12 +44,7 @@ func setupTestRouter(t *testing.T) (chi.Router, *orchestrator.Manager) { events := orchestrator.NewEventBus() logger := zerolog.Nop() - mgr := orchestrator.NewManager(reg, st, events, logger, orchestrator.ManagerConfig{ - DefaultTTL: 5 * time.Minute, - DefaultImage: "alpine:latest", - DefaultMemory: 512, - DefaultVCPUs: 1, - }) + mgr := orchestrator.NewManager(reg, st, events, logger, cfg) mgr.Start() t.Cleanup(func() { mgr.Stop() }) @@ -71,6 +77,98 @@ func TestCreateSandbox(t *testing.T) { } } +func TestSpawnAdmissionRoute(t *testing.T) { + r, _ := setupTestRouterWithConfig(t, orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: orchestrator.OperationalLimits{ + MaxSandboxes: 1, + SpawnOverflow: "queue", + SpawnQueueTimeout: time.Second, + MaxSpawnQueue: 2, + }, + }) + + body := `{"image":"alpine:latest","owner_id":"owner-a"}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusCreated { + t.Fatalf("create status = %d: %s", w.Code, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/admission", bytes.NewBufferString(`{"ttl":"1m","owner_id":"owner-b"}`)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("admission status = %d: %s", w.Code, w.Body.String()) + } + + var decision orchestrator.SpawnAdmissionDecision + if err := json.NewDecoder(w.Body).Decode(&decision); err != nil { + t.Fatalf("decode admission: %v", err) + } + if decision.Allowed || !decision.Queueable || decision.Reason != "max_sandboxes" { + t.Fatalf("unexpected admission decision: %+v", decision) + } + if decision.ActiveSandboxes != 1 || decision.MaxSandboxes != 1 { + t.Fatalf("unexpected admission counts: %+v", decision) + } +} + +func TestSpawnAdmissionRouteRejectModeNotQueueable(t *testing.T) { + r, _ := setupTestRouterWithConfig(t, orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: orchestrator.OperationalLimits{ + MaxSandboxes: 1, + }, + }) + + body := `{"image":"alpine:latest","owner_id":"owner-a"}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusCreated { + t.Fatalf("create status = %d: %s", w.Code, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/admission", bytes.NewBufferString(`{"owner_id":"owner-b"}`)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("admission status = %d: %s", w.Code, w.Body.String()) + } + + var decision orchestrator.SpawnAdmissionDecision + if err := json.NewDecoder(w.Body).Decode(&decision); err != nil { + t.Fatalf("decode admission: %v", err) + } + if decision.Allowed || decision.Queueable || decision.Reason != "max_sandboxes" { + t.Fatalf("unexpected admission decision: %+v", decision) + } +} + +func TestSpawnAdmissionRouteInvalidTTL(t *testing.T) { + r, _ := setupTestRouter(t) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/admission", bytes.NewBufferString(`{"ttl":"not-a-duration"}`)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String()) + } +} + func TestListSandboxes(t *testing.T) { r, _ := setupTestRouter(t) @@ -128,6 +226,54 @@ func TestExecInSandbox(t *testing.T) { } } +func TestExecInSandbox_Timeout(t *testing.T) { + r, _ := setupTestRouter(t) + + sbID := createTestSandbox(t, r) + execBody := `{"command":"sleep 1","timeout":"1ms"}` + req := httptest.NewRequest("POST", "/api/v1/sandboxes/"+sbID+"/exec", bytes.NewBufferString(execBody)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusRequestTimeout { + t.Fatalf("expected 408, got %d: %s", w.Code, w.Body.String()) + } +} + +func TestExecStream_MaxTimeoutLimit(t *testing.T) { + r, mgr := setupTestRouter(t) + if _, err := mgr.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "owner-a", + MaxExecTimeout: "1s", + }); err != nil { + t.Fatalf("save owner quota: %v", err) + } + + body := `{"image":"alpine:latest","owner_id":"owner-a"}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusCreated { + t.Fatalf("create status = %d: %s", w.Code, w.Body.String()) + } + var sb orchestrator.Sandbox + if err := json.NewDecoder(w.Body).Decode(&sb); err != nil { + t.Fatalf("decode sandbox: %v", err) + } + + execBody := `{"command":"echo hello","timeout":"2s","stream":true}` + req = httptest.NewRequest(http.MethodPost, "/api/v1/sandboxes/"+sb.ID+"/exec", bytes.NewBufferString(execBody)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusTooManyRequests { + t.Fatalf("expected 429, got %d: %s", w.Code, w.Body.String()) + } +} + func TestDestroyAndGet404(t *testing.T) { r, _ := setupTestRouter(t) @@ -506,6 +652,21 @@ func TestCreateSandbox_WithOwnerID(t *testing.T) { } } +func TestCreateSandbox_InvalidOwnerID(t *testing.T) { + r, _ := setupTestRouter(t) + + body := `{"image":"alpine:latest"}` + req := httptest.NewRequest("POST", "/api/v1/sandboxes", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-User-ID", "alice smith") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String()) + } +} + func TestWriteAndReadFile(t *testing.T) { r, _ := setupTestRouter(t) diff --git a/internal/api/routes/swagger_types.go b/internal/api/routes/swagger_types.go index 29fe6fb..01a292a 100644 --- a/internal/api/routes/swagger_types.go +++ b/internal/api/routes/swagger_types.go @@ -1,5 +1,11 @@ package routes +import ( + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/store" +) + // StatusResponse is a generic status response. type StatusResponse struct { Status string `json:"status" example:"destroyed"` @@ -17,6 +23,60 @@ type HealthResponse struct { Uptime string `json:"uptime" example:"2h30m15s"` } +// ProviderHealth is a provider readiness item. +type ProviderHealth struct { + Name string `json:"name" example:"docker"` + Healthy bool `json:"healthy" example:"true"` + Default bool `json:"default" example:"true"` + LatencyMS int64 `json:"latency_ms" example:"3"` + LastChecked string `json:"last_checked" example:"2026-05-08T10:30:00Z"` + Error string `json:"error,omitempty" example:"health check returned false"` + Capabilities []string `json:"capabilities" example:"spawn,exec,files"` + RuntimeCount *int `json:"runtime_count,omitempty" example:"2"` +} + +// ReadinessResponse is the response from the readiness endpoint. +type ReadinessResponse struct { + Status string `json:"status" example:"ready"` + Version string `json:"version" example:"1.0.0"` + Uptime string `json:"uptime" example:"2h30m15s"` + Providers []ProviderHealth `json:"providers"` + ReadyProviders int `json:"ready_providers" example:"1"` + TotalProviders int `json:"total_providers" example:"2"` +} + +// DiagnosticsResponse is the response from the diagnostics endpoint. +type DiagnosticsResponse struct { + GeneratedAt string `json:"generated_at" example:"2026-05-08T10:30:00Z"` + Build map[string]interface{} `json:"build"` + Process map[string]interface{} `json:"process"` + Store map[string]interface{} `json:"store"` + Limits orchestrator.OperationalLimitsInfo `json:"limits"` + Providers []ProviderHealth `json:"providers"` + Workers map[string]interface{} `json:"workers"` + Leases map[string]interface{} `json:"leases"` + Sandboxes map[string]interface{} `json:"sandboxes"` + Events orchestrator.EventBusStats `json:"events"` + Operations []orchestrator.OperationMetrics `json:"operations"` + Scheduler orchestrator.SchedulerStatus `json:"scheduler"` + Quotas orchestrator.QuotaSummary `json:"quotas"` + RateLimit middleware.RateLimitStats `json:"rate_limit"` + Remediation map[string]string `json:"remediation"` + Redactions []string `json:"redactions"` +} + +// OwnerQuotaResponse is the response for owner quota configuration. +type OwnerQuotaResponse = orchestrator.OwnerQuota + +// OwnerUsageResponse is the response for owner quota usage. +type OwnerUsageResponse = orchestrator.OwnerUsage + +// QuotaSummaryResponse is the response for redacted quota coverage counts. +type QuotaSummaryResponse = orchestrator.QuotaSummary + +// AdminAuditResponse is a redacted admin route access log record. +type AdminAuditResponse = store.AdminAuditRecord + // MetricsResponse is the response from the metrics endpoint. type MetricsResponse struct { SandboxesActive int `json:"sandboxes_active" example:"5"` diff --git a/internal/api/routes/system.go b/internal/api/routes/system.go index b266b20..352dcaf 100644 --- a/internal/api/routes/system.go +++ b/internal/api/routes/system.go @@ -1,41 +1,57 @@ package routes import ( + "bytes" + "context" "encoding/json" "fmt" "net/http" "runtime" "time" - "github.com/go-chi/chi/v5" - "github.com/google/uuid" + "github.com/StacyOs/stacyvm/internal/api/middleware" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" ) type SystemRoutes struct { registry *providers.Registry manager *orchestrator.Manager events *orchestrator.EventBus + store store.Store startTime time.Time version string + limiter *middleware.RateLimiter } -func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, version string) *SystemRoutes { +func NewSystemRoutes(registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, st store.Store, version string, limiter ...*middleware.RateLimiter) *SystemRoutes { + var rateLimiter *middleware.RateLimiter + if len(limiter) > 0 { + rateLimiter = limiter[0] + } return &SystemRoutes{ registry: registry, manager: manager, events: events, + store: st, startTime: time.Now(), version: version, + limiter: rateLimiter, } } func (s *SystemRoutes) Routes() chi.Router { r := chi.NewRouter() r.Get("/health", s.Health) + r.Get("/live", s.Live) + r.Get("/ready", s.Ready) + r.Get("/diagnostics", s.Diagnostics) r.Get("/metrics", s.Metrics) + r.Get("/metrics/prometheus", s.PrometheusMetrics) r.Get("/events", s.Events) return r } @@ -57,6 +73,145 @@ func (s *SystemRoutes) Health(w http.ResponseWriter, r *http.Request) { }) } +// Live returns process liveness. +// +// @Summary Liveness check +// @Description Return whether the StacyVM API process is alive +// @Tags system +// @Produce json +// @Success 200 {object} HealthResponse +// @Security ApiKeyAuth +// @Router /live [get] +func (s *SystemRoutes) Live(w http.ResponseWriter, r *http.Request) { + httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ + "status": "alive", + "version": s.version, + "uptime": time.Since(s.startTime).String(), + }) +} + +// Ready returns dependency readiness. +// +// @Summary Readiness check +// @Description Return whether the API is ready to serve sandbox traffic +// @Tags system +// @Produce json +// @Success 200 {object} ReadinessResponse +// @Failure 503 {object} ReadinessResponse +// @Security ApiKeyAuth +// @Router /ready [get] +func (s *SystemRoutes) Ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + providers := s.providerHealth(ctx) + readyProviders := 0 + for _, provider := range providers { + if provider.Healthy { + readyProviders++ + } + } + + statusCode := http.StatusOK + status := "ready" + if len(providers) == 0 || readyProviders == 0 { + statusCode = http.StatusServiceUnavailable + status = "not_ready" + } + + httputil.WriteJSON(w, statusCode, map[string]interface{}{ + "status": status, + "version": s.version, + "uptime": time.Since(s.startTime).String(), + "providers": providers, + "ready_providers": readyProviders, + "total_providers": len(providers), + }) +} + +// Diagnostics returns redacted operational diagnostics. +// +// @Summary Get diagnostics +// @Description Return redacted build, store, provider, sandbox, event, and operation diagnostics +// @Tags system +// @Produce json +// @Success 200 {object} DiagnosticsResponse +// @Security ApiKeyAuth +// @Router /diagnostics [get] +func (s *SystemRoutes) Diagnostics(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + + metrics, err := s.collectMetrics(ctx) + if err != nil { + writeRouteError(w, err) + return + } + + storeStatus := map[string]interface{}{ + "healthy": false, + } + if s.store != nil { + start := time.Now() + if _, err := s.store.ListSandboxes(ctx); err != nil { + storeStatus["error"] = err.Error() + } else { + storeStatus["healthy"] = true + } + storeStatus["latency_ms"] = time.Since(start).Milliseconds() + } else { + storeStatus["error"] = "store unavailable" + } + + httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ + "generated_at": time.Now().UTC().Format(time.RFC3339), + "build": map[string]interface{}{ + "version": s.version, + "goos": runtime.GOOS, + "goarch": runtime.GOARCH, + }, + "process": map[string]interface{}{ + "uptime": metrics.uptime.String(), + "goroutines": metrics.goroutines, + "memory": map[string]interface{}{ + "alloc": metrics.memoryAlloc, + "sys": metrics.memorySys, + "heap_alloc": metrics.memoryHeapAlloc, + "gc_cycles": metrics.gcCycles, + }, + }, + "store": storeStatus, + "limits": s.manager.Limits(), + "scheduler": s.manager.SchedulerStatus(), + "quotas": metrics.quotaSummary, + "rate_limit": s.rateLimitStats(), + "providers": metrics.providerHealth, + "workers": metrics.workerSummary, + "leases": metrics.leaseSummary, + "sandboxes": metrics.sandboxSummary(), + "events": metrics.eventStats, + "operations": metrics.operationMetrics, + "remediation": map[string]string{ + "admin_control_plane": "docs/admin-control-plane.md", + "deployment": "docs/deployment.md", + "production_readiness": "docs/production-readiness.md", + "public_support_matrix": "docs/public-support-matrix.md", + "release_verification": "docs/releasing.md", + "runtime_certification": "docs/runtime-certification.md", + "runtime_conformance": "docs/runtime-conformance.md", + "security_governance": "docs/security-governance.md", + "support_bundle": "docs/deployment.md#support-bundles", + "upgrade_and_rollback": "docs/deployment.md#upgrade-rehearsal-and-rollback", + }, + "redactions": []string{ + "provider secrets", + "registry credentials", + "environment secrets", + "API keys", + }, + }) +} + // Metrics returns runtime metrics. // // @Summary Get metrics @@ -67,23 +222,233 @@ func (s *SystemRoutes) Health(w http.ResponseWriter, r *http.Request) { // @Security ApiKeyAuth // @Router /metrics [get] func (s *SystemRoutes) Metrics(w http.ResponseWriter, r *http.Request) { + metrics, err := s.collectMetrics(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + + httputil.WriteJSON(w, http.StatusOK, metrics.toResponse()) +} + +// PrometheusMetrics returns Prometheus-compatible operational metrics. +// +// @Summary Get Prometheus metrics +// @Description Return runtime, provider, sandbox, event, and operation metrics in Prometheus text format +// @Tags system +// @Produce text/plain +// @Success 200 {string} string +// @Security ApiKeyAuth +// @Router /metrics/prometheus [get] +func (s *SystemRoutes) PrometheusMetrics(w http.ResponseWriter, r *http.Request) { + metrics, err := s.collectMetrics(r.Context()) + if err != nil { + writeRouteError(w, err) + return + } + + var buf bytes.Buffer + writePrometheusMetrics(&buf, metrics) + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(buf.Bytes()) +} + +type systemMetricsSnapshot struct { + uptime time.Duration + goroutines int + memoryAlloc uint64 + memorySys uint64 + memoryHeapAlloc uint64 + gcCycles uint32 + sandboxTotal int + sandboxActive int + sandboxesByState map[string]int + sandboxesByProvider map[string]int + sandboxesByWorker map[string]int + providerHealth []ProviderHealth + healthyProviders int + eventStats orchestrator.EventBusStats + operationMetrics []orchestrator.OperationMetrics + schedulerStatus orchestrator.SchedulerStatus + quotaSummary orchestrator.QuotaSummary + rateLimitStats middleware.RateLimitStats + workerSummary map[string]interface{} + leaseSummary map[string]interface{} +} + +func (s *SystemRoutes) collectMetrics(ctx context.Context) (systemMetricsSnapshot, error) { var mem runtime.MemStats runtime.ReadMemStats(&mem) - sandboxes, _ := s.manager.List(r.Context()) - active := 0 + sandboxes, err := s.manager.List(ctx) + if err != nil { + return systemMetricsSnapshot{}, err + } + + byState := make(map[string]int) + byProvider := make(map[string]int) + byWorker := make(map[string]int) for _, sb := range sandboxes { - if sb.State == orchestrator.StateRunning { - active++ + byState[string(sb.State)]++ + byProvider[sb.Provider]++ + if sb.WorkerID != "" { + byWorker[sb.WorkerID]++ } } - httputil.WriteJSON(w, http.StatusOK, map[string]interface{}{ - "goroutines": runtime.NumGoroutine(), - "memory_alloc": mem.Alloc, - "active_sandboxes": active, - "total_sandboxes": len(sandboxes), - }) + providerHealth := s.providerHealth(ctx) + healthyProviders := 0 + for _, provider := range providerHealth { + if provider.Healthy { + healthyProviders++ + } + } + eventStats := s.events.Stats() + quotaSummary, err := s.manager.QuotaSummary(ctx) + if err != nil { + return systemMetricsSnapshot{}, err + } + workerSummary := s.workerSummary(ctx) + leaseSummary := s.leaseSummary(ctx) + + return systemMetricsSnapshot{ + uptime: time.Since(s.startTime), + goroutines: runtime.NumGoroutine(), + memoryAlloc: mem.Alloc, + memorySys: mem.Sys, + memoryHeapAlloc: mem.HeapAlloc, + gcCycles: mem.NumGC, + sandboxTotal: len(sandboxes), + sandboxActive: byState[string(orchestrator.StateRunning)], + sandboxesByState: byState, + sandboxesByProvider: byProvider, + sandboxesByWorker: byWorker, + providerHealth: providerHealth, + healthyProviders: healthyProviders, + eventStats: eventStats, + operationMetrics: s.manager.OperationMetrics(), + schedulerStatus: s.manager.SchedulerStatus(), + quotaSummary: quotaSummary, + rateLimitStats: s.rateLimitStats(), + workerSummary: workerSummary, + leaseSummary: leaseSummary, + }, nil +} + +func (m systemMetricsSnapshot) toResponse() map[string]interface{} { + return map[string]interface{}{ + "uptime": m.uptime.String(), + "goroutines": m.goroutines, + "memory_alloc": m.memoryAlloc, + "memory_sys": m.memorySys, + "memory_heap_alloc": m.memoryHeapAlloc, + "gc_cycles": m.gcCycles, + "sandboxes": m.sandboxSummary(), + "providers": map[string]interface{}{ + "total": len(m.providerHealth), + "healthy": m.healthyProviders, + "items": m.providerHealth, + }, + "workers": m.workerSummary, + "leases": m.leaseSummary, + "events": m.eventStats, + "operations": m.operationMetrics, + "scheduler": m.schedulerStatus, + "quotas": m.quotaSummary, + "rate_limit": m.rateLimitStats, + } +} + +func (m systemMetricsSnapshot) sandboxSummary() map[string]interface{} { + return map[string]interface{}{ + "total": m.sandboxTotal, + "active": m.sandboxActive, + "by_state": m.sandboxesByState, + "by_provider": m.sandboxesByProvider, + "by_worker": m.sandboxesByWorker, + } +} + +func (s *SystemRoutes) providerHealth(ctx context.Context) []ProviderHealth { + return collectProviderHealth(ctx, s.registry) +} + +func (s *SystemRoutes) rateLimitStats() middleware.RateLimitStats { + if s.limiter == nil { + return middleware.RateLimitStats{} + } + return s.limiter.Stats() +} + +func (s *SystemRoutes) workerSummary(ctx context.Context) map[string]interface{} { + summary := map[string]interface{}{ + "total": 0, + "online": 0, + "stale": 0, + "unhealthy": 0, + "items": []WorkerResponse{}, + } + if s.store == nil { + return summary + } + workers, err := s.store.ListWorkers(ctx) + if err != nil { + summary["error"] = err.Error() + return summary + } + now := time.Now().UTC() + items := make([]WorkerResponse, 0, len(workers)) + for _, rec := range workers { + item := workerResponse(rec, now) + items = append(items, item) + if item.Status == "online" { + summary["online"] = summary["online"].(int) + 1 + } + if item.Stale { + summary["stale"] = summary["stale"].(int) + 1 + } + if item.Status == "unhealthy" { + summary["unhealthy"] = summary["unhealthy"].(int) + 1 + } + } + summary["total"] = len(workers) + summary["items"] = items + return summary +} + +func (s *SystemRoutes) leaseSummary(ctx context.Context) map[string]interface{} { + summary := map[string]interface{}{ + "total": 0, + "active": 0, + "expired": 0, + "by_holder": map[string]int{}, + } + if s.store == nil { + return summary + } + leases, err := s.store.ListLeases(ctx) + if err != nil { + summary["error"] = err.Error() + return summary + } + now := time.Now().UTC() + byHolder := make(map[string]int) + active := 0 + expired := 0 + for _, lease := range leases { + if lease.ExpiresAt.After(now) { + active++ + byHolder[lease.HolderID]++ + } else { + expired++ + } + } + summary["total"] = len(leases) + summary["active"] = active + summary["expired"] = expired + summary["by_holder"] = byHolder + return summary } // Events serves Server-Sent Events for real-time updates. diff --git a/internal/api/routes/system_test.go b/internal/api/routes/system_test.go new file mode 100644 index 0000000..6150e57 --- /dev/null +++ b/internal/api/routes/system_test.go @@ -0,0 +1,280 @@ +package routes + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/rs/zerolog" +) + +func setupSystemRoutes(t *testing.T, withProvider bool) (*SystemRoutes, *orchestrator.Manager) { + t.Helper() + + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + registry := providers.NewRegistry() + if withProvider { + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + } + + events := orchestrator.NewEventBus() + manager := orchestrator.NewManager(registry, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + + return NewSystemRoutes(registry, manager, events, st, "test-version"), manager +} + +func TestSystemRoutes_Live(t *testing.T) { + routes, _ := setupSystemRoutes(t, true) + req := httptest.NewRequest(http.MethodGet, "/live", nil) + w := httptest.NewRecorder() + + routes.Live(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + if body["status"] != "alive" { + t.Fatalf("status = %v, want alive", body["status"]) + } +} + +func TestSystemRoutes_Ready(t *testing.T) { + routes, _ := setupSystemRoutes(t, true) + req := httptest.NewRequest(http.MethodGet, "/ready", nil) + w := httptest.NewRecorder() + + routes.Ready(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + if body["status"] != "ready" { + t.Fatalf("status = %v, want ready", body["status"]) + } + if body["ready_providers"].(float64) != 1 { + t.Fatalf("ready providers = %v, want 1", body["ready_providers"]) + } + providersBody := body["providers"].([]interface{}) + firstProvider := providersBody[0].(map[string]interface{}) + for _, field := range []string{"latency_ms", "last_checked", "capabilities"} { + if _, ok := firstProvider[field]; !ok { + t.Fatalf("provider health missing %s: %#v", field, firstProvider) + } + } +} + +func TestSystemRoutes_ReadyNoProviders(t *testing.T) { + routes, _ := setupSystemRoutes(t, false) + req := httptest.NewRequest(http.MethodGet, "/ready", nil) + w := httptest.NewRecorder() + + routes.Ready(w, req) + + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", w.Code, http.StatusServiceUnavailable) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + if body["status"] != "not_ready" { + t.Fatalf("status = %v, want not_ready", body["status"]) + } +} + +func TestSystemRoutes_Diagnostics(t *testing.T) { + routes, manager := setupSystemRoutes(t, true) + if _, err := manager.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "team-a", + MaxSandboxes: 3, + MaxTTL: "30m", + MaxExecTimeout: "10s", + }); err != nil { + t.Fatalf("save quota: %v", err) + } + if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { + t.Fatalf("spawn: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/diagnostics", nil) + w := httptest.NewRecorder() + + routes.Diagnostics(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + for _, field := range []string{"generated_at", "build", "process", "store", "limits", "scheduler", "quotas", "rate_limit", "providers", "workers", "leases", "sandboxes", "events", "operations", "remediation", "redactions"} { + if _, ok := body[field]; !ok { + t.Fatalf("diagnostics missing %s: %#v", field, body) + } + } + workers := body["workers"].(map[string]interface{}) + if workers["total"].(float64) != 0 { + t.Fatalf("worker total = %v, want 0 before server registration", workers["total"]) + } + leases := body["leases"].(map[string]interface{}) + if leases["total"].(float64) != 1 || leases["active"].(float64) != 1 { + t.Fatalf("lease summary = %#v, want total=1 active=1", leases) + } + remediation := body["remediation"].(map[string]interface{}) + if remediation["runtime_certification"] != "docs/runtime-certification.md" { + t.Fatalf("unexpected runtime certification remediation: %#v", remediation) + } + if remediation["public_support_matrix"] != "docs/public-support-matrix.md" { + t.Fatalf("unexpected public support matrix remediation: %#v", remediation) + } + quotas := body["quotas"].(map[string]interface{}) + if quotas["total"].(float64) != 1 || quotas["with_max_sandboxes"].(float64) != 1 { + t.Fatalf("unexpected quota summary: %#v", quotas) + } + storeBody := body["store"].(map[string]interface{}) + if storeBody["healthy"] != true { + t.Fatalf("store healthy = %v, want true", storeBody["healthy"]) + } + if strings.Contains(w.Body.String(), "X-API-Key") { + t.Fatal("diagnostics response leaked API key header name") + } +} + +func TestSystemRoutes_MetricsIncludesOperationalBreakdown(t *testing.T) { + routes, manager := setupSystemRoutes(t, true) + if _, err := manager.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "team-a", + MaxSandboxes: 2, + }); err != nil { + t.Fatalf("save quota: %v", err) + } + if _, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}); err != nil { + t.Fatalf("spawn: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + w := httptest.NewRecorder() + + routes.Metrics(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + var body map[string]interface{} + decodeSystemResponse(t, w, &body) + + sandboxes := body["sandboxes"].(map[string]interface{}) + if sandboxes["total"].(float64) != 1 { + t.Fatalf("sandbox total = %v, want 1", sandboxes["total"]) + } + byWorker := sandboxes["by_worker"].(map[string]interface{}) + if byWorker["local"].(float64) != 1 { + t.Fatalf("sandbox by_worker = %#v, want local=1", byWorker) + } + providersBody := body["providers"].(map[string]interface{}) + if providersBody["healthy"].(float64) != 1 { + t.Fatalf("healthy providers = %v, want 1", providersBody["healthy"]) + } + if _, ok := body["events"].(map[string]interface{}); !ok { + t.Fatal("expected events metrics") + } + if _, ok := body["scheduler"].(map[string]interface{}); !ok { + t.Fatal("expected scheduler metrics") + } + quotas := body["quotas"].(map[string]interface{}) + if quotas["total"].(float64) != 1 { + t.Fatalf("unexpected quota metrics: %#v", quotas) + } + if _, ok := body["rate_limit"].(map[string]interface{}); !ok { + t.Fatal("expected rate limit metrics") + } + if _, ok := body["workers"].(map[string]interface{}); !ok { + t.Fatal("expected worker metrics") + } + operations := body["operations"].([]interface{}) + if len(operations) == 0 { + t.Fatal("expected operation metrics") + } +} + +func TestSystemRoutes_PrometheusMetrics(t *testing.T) { + routes, manager := setupSystemRoutes(t, true) + if _, err := manager.SaveOwnerQuota(context.Background(), orchestrator.OwnerQuota{ + OwnerID: "team-a", + MaxSandboxes: 2, + }); err != nil { + t.Fatalf("save quota: %v", err) + } + sb, err := manager.Spawn(context.Background(), orchestrator.SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if _, err := manager.Exec(context.Background(), sb.ID, orchestrator.ExecRequest{Command: "echo prometheus"}); err != nil { + t.Fatalf("exec: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/metrics/prometheus", nil) + w := httptest.NewRecorder() + + routes.PrometheusMetrics(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get("Content-Type"); !strings.Contains(got, "text/plain") { + t.Fatalf("content type = %q, want text/plain", got) + } + body := w.Body.String() + for _, want := range []string{ + "stacyvm_uptime_seconds", + "stacyvm_provider_healthy", + "stacyvm_provider_health_latency_milliseconds", + "stacyvm_spawn_queue_depth", + "stacyvm_spawn_queue_enqueued_total", + "stacyvm_spawn_queue_wait_milliseconds_count", + "stacyvm_owner_quotas_total", + `type="max_sandboxes"`, + "stacyvm_workers_total", + "stacyvm_leases_total", + "stacyvm_sandboxes_by_worker_total", + "stacyvm_rate_limit_allowed_total", + "stacyvm_operation_success_total", + `operation="spawn"`, + `operation="exec"`, + } { + if !strings.Contains(body, want) { + t.Fatalf("prometheus body missing %q:\n%s", want, body) + } + } +} + +func decodeSystemResponse(t *testing.T, w *httptest.ResponseRecorder, dst interface{}) { + t.Helper() + if err := json.Unmarshal(w.Body.Bytes(), dst); err != nil { + t.Fatalf("decode response: %v", err) + } +} diff --git a/internal/api/routes/templates.go b/internal/api/routes/templates.go index 6f23e84..2d48e70 100644 --- a/internal/api/routes/templates.go +++ b/internal/api/routes/templates.go @@ -3,11 +3,10 @@ package routes import ( "encoding/json" "net/http" - "strings" - "github.com/go-chi/chi/v5" "github.com/StacyOs/stacyvm/internal/httputil" "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/go-chi/chi/v5" ) type TemplateRoutes struct { @@ -53,11 +52,7 @@ func (t *TemplateRoutes) Create(w http.ResponseWriter, r *http.Request) { return } if err := t.registry.Create(r.Context(), &tmpl); err != nil { - if strings.Contains(err.Error(), "UNIQUE constraint") { - httputil.WriteError(w, http.StatusConflict, httputil.CodeConflict, "template already exists") - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, tmpl) @@ -101,11 +96,7 @@ func (t *TemplateRoutes) Get(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") tmpl, err := t.registry.Get(r.Context(), name) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, tmpl) @@ -135,11 +126,7 @@ func (t *TemplateRoutes) Update(w http.ResponseWriter, r *http.Request) { } tmpl.Name = name if err := t.registry.Update(r.Context(), &tmpl); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, tmpl) @@ -160,11 +147,7 @@ func (t *TemplateRoutes) Update(w http.ResponseWriter, r *http.Request) { func (t *TemplateRoutes) Delete(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") if err := t.registry.Delete(r.Context(), name); err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) @@ -188,11 +171,7 @@ func (t *TemplateRoutes) Spawn(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") tmpl, err := t.registry.Get(r.Context(), name) if err != nil { - if strings.Contains(err.Error(), "not found") { - httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, err.Error()) - return - } - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } @@ -213,7 +192,7 @@ func (t *TemplateRoutes) Spawn(w http.ResponseWriter, r *http.Request) { sb, err := t.manager.Spawn(r.Context(), req) if err != nil { - httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + writeRouteError(w, err) return } httputil.WriteJSON(w, http.StatusCreated, sb) diff --git a/internal/api/routes/templates_test.go b/internal/api/routes/templates_test.go new file mode 100644 index 0000000..b457b33 --- /dev/null +++ b/internal/api/routes/templates_test.go @@ -0,0 +1,75 @@ +package routes + +import ( + "bytes" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" + "github.com/rs/zerolog" +) + +func setupTemplateTestRouter(t *testing.T) chi.Router { + t.Helper() + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := providers.NewMockProvider() + reg.Register(mock) + if err := reg.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + events := orchestrator.NewEventBus() + mgr := orchestrator.NewManager(reg, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + mgr.Start() + t.Cleanup(func() { mgr.Stop() }) + + r := chi.NewRouter() + r.Mount("/api/v1/templates", NewTemplateRoutes(orchestrator.NewTemplateRegistry(st), mgr).Routes()) + return r +} + +func TestTemplateDuplicateReturnsConflict(t *testing.T) { + r := setupTemplateTestRouter(t) + body := `{"name":"node","image":"node:20","ttl_seconds":300}` + + for i := 0; i < 2; i++ { + req := httptest.NewRequest("POST", "/api/v1/templates", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if i == 0 && w.Code != http.StatusCreated { + t.Fatalf("first create: expected 201, got %d: %s", w.Code, w.Body.String()) + } + if i == 1 && w.Code != http.StatusConflict { + t.Fatalf("second create: expected 409, got %d: %s", w.Code, w.Body.String()) + } + } +} + +func TestTemplateMissingReturnsNotFound(t *testing.T) { + r := setupTemplateTestRouter(t) + + req := httptest.NewRequest("GET", "/api/v1/templates/missing-template", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusNotFound { + t.Fatalf("expected 404, got %d: %s", w.Code, w.Body.String()) + } +} diff --git a/internal/api/routes/tenants.go b/internal/api/routes/tenants.go new file mode 100644 index 0000000..d608192 --- /dev/null +++ b/internal/api/routes/tenants.go @@ -0,0 +1,351 @@ +package routes + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +type tenantStore interface { + CreateTenant(ctx context.Context, t *store.TenantRecord) error + GetTenant(ctx context.Context, id string) (*store.TenantRecord, error) + ListTenants(ctx context.Context) ([]*store.TenantRecord, error) + UpdateTenant(ctx context.Context, t *store.TenantRecord) error + DeleteTenant(ctx context.Context, id string) error + SaveTenantMember(ctx context.Context, m *store.TenantMemberRecord) error + GetTenantMember(ctx context.Context, tenantID, userID string) (*store.TenantMemberRecord, error) + ListTenantMembers(ctx context.Context, tenantID string) ([]*store.TenantMemberRecord, error) + DeleteTenantMember(ctx context.Context, tenantID, userID string) error + ListAdminAudit(ctx context.Context, query store.AdminAuditQuery) ([]*store.AdminAuditRecord, error) + ListOperationAudit(ctx context.Context, query store.OperationAuditQuery) ([]*store.OperationAuditRecord, error) + CreatePolicy(ctx context.Context, p *store.PolicyRecord) error + GetPolicy(ctx context.Context, id string) (*store.PolicyRecord, error) + ListPolicies(ctx context.Context, query store.PolicyQuery) ([]*store.PolicyRecord, error) + DeletePolicy(ctx context.Context, id string) error +} + +type TenantRoutes struct { + store tenantStore +} + +func NewTenantRoutes(st tenantStore) *TenantRoutes { + return &TenantRoutes{store: st} +} + +func (tr *TenantRoutes) Routes() chi.Router { + r := chi.NewRouter() + r.Get("/", tr.List) + r.Post("/", tr.Create) + r.Get("/{tenantID}", tr.Get) + r.Put("/{tenantID}", tr.Update) + r.Delete("/{tenantID}", tr.Delete) + r.Get("/{tenantID}/members", tr.ListMembers) + r.Put("/{tenantID}/members/{userID}", tr.UpsertMember) + r.Delete("/{tenantID}/members/{userID}", tr.DeleteMember) + r.Get("/{tenantID}/audit", tr.AuditExport) + r.Get("/{tenantID}/policies", tr.ListPolicies) + r.Post("/{tenantID}/policies", tr.CreatePolicy) + r.Delete("/{tenantID}/policies/{policyID}", tr.DeletePolicy) + return r +} + +type CreateTenantRequest struct { + ID string `json:"id"` + Name string `json:"name"` + OwnerID string `json:"owner_id"` + Settings any `json:"settings"` +} + +type UpsertMemberRequest struct { + Role string `json:"role"` // viewer, operator, admin +} + +type CreatePolicyRequest struct { + ResourceType string `json:"resource_type"` + Effect string `json:"effect"` + Pattern string `json:"pattern"` + Priority int `json:"priority"` +} + +func (tr *TenantRoutes) List(w http.ResponseWriter, r *http.Request) { + tenants, err := tr.store.ListTenants(r.Context()) + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + if tenants == nil { + tenants = []*store.TenantRecord{} + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{"tenants": tenants}) +} + +func (tr *TenantRoutes) Create(w http.ResponseWriter, r *http.Request) { + var req CreateTenantRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + if strings.TrimSpace(req.Name) == "" { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "name is required") + return + } + id := strings.TrimSpace(req.ID) + if id == "" { + id = "tenant-" + uuid.New().String()[:8] + } + settings := "{}" + if req.Settings != nil { + if b, err := json.Marshal(req.Settings); err == nil { + settings = string(b) + } + } + t := &store.TenantRecord{ + ID: id, + Name: req.Name, + OwnerID: strings.TrimSpace(req.OwnerID), + Settings: settings, + } + if err := tr.store.CreateTenant(r.Context(), t); err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + httputil.WriteJSON(w, http.StatusCreated, map[string]any{"tenant": t}) +} + +func (tr *TenantRoutes) Get(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "tenantID") + t, err := tr.store.GetTenant(r.Context(), id) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "tenant not found") + return + } + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{"tenant": t}) +} + +func (tr *TenantRoutes) Update(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "tenantID") + existing, err := tr.store.GetTenant(r.Context(), id) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "tenant not found") + return + } + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + var req CreateTenantRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + if req.Name != "" { + existing.Name = req.Name + } + if req.OwnerID != "" { + existing.OwnerID = req.OwnerID + } + if req.Settings != nil { + if b, err := json.Marshal(req.Settings); err == nil { + existing.Settings = string(b) + } + } + if err := tr.store.UpdateTenant(r.Context(), existing); err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{"tenant": existing}) +} + +func (tr *TenantRoutes) Delete(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "tenantID") + if err := tr.store.DeleteTenant(r.Context(), id); err != nil { + if errors.Is(err, store.ErrNotFound) { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "tenant not found") + return + } + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (tr *TenantRoutes) ListMembers(w http.ResponseWriter, r *http.Request) { + tenantID := chi.URLParam(r, "tenantID") + members, err := tr.store.ListTenantMembers(r.Context(), tenantID) + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + if members == nil { + members = []*store.TenantMemberRecord{} + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{"members": members}) +} + +func (tr *TenantRoutes) UpsertMember(w http.ResponseWriter, r *http.Request) { + tenantID := chi.URLParam(r, "tenantID") + userID := chi.URLParam(r, "userID") + var req UpsertMemberRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + role := strings.TrimSpace(req.Role) + if !isValidMemberRole(role) { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "role must be viewer, operator, or admin") + return + } + m := &store.TenantMemberRecord{TenantID: tenantID, UserID: userID, Role: role} + if err := tr.store.SaveTenantMember(r.Context(), m); err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{"member": m}) +} + +func (tr *TenantRoutes) DeleteMember(w http.ResponseWriter, r *http.Request) { + tenantID := chi.URLParam(r, "tenantID") + userID := chi.URLParam(r, "userID") + if err := tr.store.DeleteTenantMember(r.Context(), tenantID, userID); err != nil { + if errors.Is(err, store.ErrNotFound) { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "member not found") + return + } + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (tr *TenantRoutes) AuditExport(w http.ResponseWriter, r *http.Request) { + tenantID := chi.URLParam(r, "tenantID") + q := r.URL.Query() + limit := 200 + since := q.Get("since") + + adminLogs, err := tr.store.ListAdminAudit(r.Context(), store.AdminAuditQuery{ + Limit: limit, + TenantID: tenantID, + PathLike: since, // repurposed: caller can filter by path prefix + }) + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + opLogs, err := tr.store.ListOperationAudit(r.Context(), store.OperationAuditQuery{ + Limit: limit, + TenantID: tenantID, + }) + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + if adminLogs == nil { + adminLogs = []*store.AdminAuditRecord{} + } + if opLogs == nil { + opLogs = []*store.OperationAuditRecord{} + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{ + "tenant_id": tenantID, + "admin_audit": adminLogs, + "op_audit": opLogs, + "exported_at": time.Now().UTC(), + }) +} + +func (tr *TenantRoutes) ListPolicies(w http.ResponseWriter, r *http.Request) { + tenantID := chi.URLParam(r, "tenantID") + policies, err := tr.store.ListPolicies(r.Context(), store.PolicyQuery{TenantID: tenantID}) + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + if policies == nil { + policies = []*store.PolicyRecord{} + } + httputil.WriteJSON(w, http.StatusOK, map[string]any{"policies": policies}) +} + +func (tr *TenantRoutes) CreatePolicy(w http.ResponseWriter, r *http.Request) { + tenantID := chi.URLParam(r, "tenantID") + var req CreatePolicyRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + if !isValidPolicyResourceType(req.ResourceType) { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "resource_type must be image, provider, or network") + return + } + if !isValidPolicyEffect(req.Effect) { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "effect must be allow or deny") + return + } + if strings.TrimSpace(req.Pattern) == "" { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "pattern is required") + return + } + priority := req.Priority + if priority == 0 { + priority = 10 + } + p := &store.PolicyRecord{ + ID: "pol-" + uuid.New().String()[:8], + TenantID: tenantID, + ResourceType: req.ResourceType, + Effect: req.Effect, + Pattern: req.Pattern, + Priority: priority, + } + if err := tr.store.CreatePolicy(r.Context(), p); err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + httputil.WriteJSON(w, http.StatusCreated, map[string]any{"policy": p}) +} + +func (tr *TenantRoutes) DeletePolicy(w http.ResponseWriter, r *http.Request) { + policyID := chi.URLParam(r, "policyID") + if err := tr.store.DeletePolicy(r.Context(), policyID); err != nil { + if errors.Is(err, store.ErrNotFound) { + httputil.WriteError(w, http.StatusNotFound, httputil.CodeNotFound, "policy not found") + return + } + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func isValidMemberRole(role string) bool { + switch role { + case "viewer", "operator", "admin": + return true + } + return false +} + +func isValidPolicyResourceType(rt string) bool { + switch rt { + case "image", "provider", "network": + return true + } + return false +} + +func isValidPolicyEffect(effect string) bool { + return effect == "allow" || effect == "deny" +} diff --git a/internal/api/routes/token_issuer.go b/internal/api/routes/token_issuer.go new file mode 100644 index 0000000..3ec7664 --- /dev/null +++ b/internal/api/routes/token_issuer.go @@ -0,0 +1,125 @@ +package routes + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/httputil" +) + +// WorkerTokenIssuerRoutes provides a centralized endpoint that lets the +// control plane mint short-lived signed worker tokens on behalf of workers. +// Workers call this endpoint with their bootstrap credentials to receive a +// short-lived signed token without needing direct access to the signing key. +type WorkerTokenIssuerRoutes struct { + signingKey string +} + +func NewWorkerTokenIssuerRoutes(signingKey string) *WorkerTokenIssuerRoutes { + return &WorkerTokenIssuerRoutes{signingKey: signingKey} +} + +type IssueWorkerTokenRequest struct { + WorkerID string `json:"worker_id"` + TTL string `json:"ttl"` // e.g. "5m", "15m" + Scopes []string `json:"scopes"` // optional subset of worker scopes + Audience string `json:"audience"` // "worker:control-plane" or "worker:rpc" +} + +type IssueWorkerTokenResponse struct { + Token string `json:"token"` + WorkerID string `json:"worker_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +// IssueToken mints a short-lived signed worker token. +// Requires admin auth (protected by the caller's middleware chain). +func (r *WorkerTokenIssuerRoutes) IssueToken(w http.ResponseWriter, req *http.Request) { + if r.signingKey == "" { + httputil.WriteError(w, http.StatusServiceUnavailable, httputil.CodeUnavailable, + "worker signing key is not configured; set auth.worker_signing_key to enable centralized token issuance") + return + } + + var body IssueWorkerTokenRequest + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + + workerID := strings.TrimSpace(body.WorkerID) + if workerID == "" { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "worker_id is required") + return + } + + ttlStr := strings.TrimSpace(body.TTL) + if ttlStr == "" { + ttlStr = "5m" + } + ttl, err := time.ParseDuration(ttlStr) + if err != nil || ttl <= 0 { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid ttl; use a Go duration like 5m or 15m") + return + } + if ttl > middleware.MaxWorkerTokenTTL { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, + "ttl exceeds maximum allowed worker token lifetime of 15m") + return + } + + audience := strings.TrimSpace(body.Audience) + if audience == "" { + audience = middleware.WorkerTokenAudienceControlPlane + } + if audience != middleware.WorkerTokenAudienceControlPlane && audience != middleware.WorkerTokenAudienceRPC { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, + "audience must be worker:control-plane or worker:rpc") + return + } + + // Validate requested scopes up-front: only worker:* scopes may be issued. + // This prevents escalation through the issuer even if the caller holds admin credentials. + if len(body.Scopes) > 0 { + for _, s := range body.Scopes { + if !strings.HasPrefix(strings.TrimSpace(s), "worker:") { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, + "only worker:* scopes may be issued; scope "+strconv.Quote(s)+" is not permitted") + return + } + } + } + + tokenID, err := middleware.NewWorkerTokenID() + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, "failed to generate token ID") + return + } + + now := time.Now().UTC() + expiresAt := now.Add(ttl) + claims := middleware.WorkerTokenClaims{ + WorkerID: workerID, + TokenID: tokenID, + Audience: audience, + Scopes: body.Scopes, + IssuedAt: now.Unix(), + ExpiresAt: expiresAt.Unix(), + } + + token, err := middleware.SignWorkerToken(r.signingKey, claims) + if err != nil { + httputil.WriteError(w, http.StatusInternalServerError, httputil.CodeInternal, "failed to sign worker token") + return + } + + httputil.WriteJSON(w, http.StatusOK, IssueWorkerTokenResponse{ + Token: token, + WorkerID: workerID, + ExpiresAt: expiresAt, + }) +} diff --git a/internal/api/routes/workers.go b/internal/api/routes/workers.go new file mode 100644 index 0000000..9e8a355 --- /dev/null +++ b/internal/api/routes/workers.go @@ -0,0 +1,277 @@ +package routes + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/StacyOs/stacyvm/internal/workerproto" + "github.com/go-chi/chi/v5" +) + +type workerStore interface { + SaveWorker(ctx context.Context, rec *store.WorkerRecord) error + GetWorker(ctx context.Context, id string) (*store.WorkerRecord, error) + ListWorkers(ctx context.Context) ([]*store.WorkerRecord, error) + DeleteWorker(ctx context.Context, id string) error + RenewLease(ctx context.Context, resourceID, holderID string, ttl time.Duration) (*store.LeaseRecord, error) +} + +type WorkerRoutes struct { + store workerStore +} + +func NewWorkerRoutes(st workerStore) *WorkerRoutes { + return &WorkerRoutes{store: st} +} + +func (w *WorkerRoutes) Routes() chi.Router { + r := chi.NewRouter() + r.Get("/", w.List) + r.Get("/{workerID}", w.Get) + r.Post("/{workerID}/heartbeat", w.Heartbeat) + r.Post("/{workerID}/leases/{resourceID}/renew", w.RenewLease) + r.Delete("/{workerID}", w.Delete) + return r +} + +func (w *WorkerRoutes) ReadOnlyRoutes() chi.Router { + r := chi.NewRouter() + r.Get("/", w.List) + r.Get("/{workerID}", w.Get) + return r +} + +type WorkerHeartbeatRequest struct { + Hostname string `json:"hostname" example:"stacyvm-host-1"` + Status string `json:"status" example:"online"` + Providers []string `json:"providers"` + Capabilities []string `json:"capabilities"` + Capacity map[string]interface{} `json:"capacity"` +} + +type WorkerRenewLeaseRequest struct { + TTL string `json:"ttl" example:"30s"` +} + +type WorkerResponse struct { + ID string `json:"id" example:"worker-local"` + Hostname string `json:"hostname" example:"stacyvm-host-1"` + Status string `json:"status" example:"online"` + Providers []string `json:"providers"` + Capabilities []string `json:"capabilities"` + Capacity map[string]interface{} `json:"capacity"` + LastHeartbeat string `json:"last_heartbeat" example:"2026-05-09T10:30:00Z"` + CreatedAt string `json:"created_at" example:"2026-05-09T10:00:00Z"` + UpdatedAt string `json:"updated_at" example:"2026-05-09T10:30:00Z"` + Stale bool `json:"stale" example:"false"` +} + +// List returns all registered workers. +// +// @Summary List workers +// @Description Return worker registry records and heartbeat state +// @Tags workers +// @Produce json +// @Success 200 {array} WorkerResponse +// @Security ApiKeyAuth +// @Router /workers [get] +func (w *WorkerRoutes) List(rw http.ResponseWriter, r *http.Request) { + if w.store == nil { + httputil.WriteJSON(rw, http.StatusOK, []WorkerResponse{}) + return + } + records, err := w.store.ListWorkers(r.Context()) + if err != nil { + writeRouteError(rw, err) + return + } + responses := make([]WorkerResponse, 0, len(records)) + for _, rec := range records { + responses = append(responses, workerResponse(rec, time.Now().UTC())) + } + httputil.WriteJSON(rw, http.StatusOK, responses) +} + +// Get returns one registered worker. +// +// @Summary Get worker +// @Description Return one worker registry record +// @Tags workers +// @Produce json +// @Param workerID path string true "Worker ID" +// @Success 200 {object} WorkerResponse +// @Failure 404 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /workers/{workerID} [get] +func (w *WorkerRoutes) Get(rw http.ResponseWriter, r *http.Request) { + rec, err := w.store.GetWorker(r.Context(), chi.URLParam(r, "workerID")) + if err != nil { + writeRouteError(rw, err) + return + } + httputil.WriteJSON(rw, http.StatusOK, workerResponse(rec, time.Now().UTC())) +} + +// Heartbeat creates or updates a worker heartbeat. +// +// @Summary Heartbeat worker +// @Description Create or update worker registry state for a worker +// @Tags workers +// @Accept json +// @Produce json +// @Param workerID path string true "Worker ID" +// @Param request body WorkerHeartbeatRequest true "Worker heartbeat" +// @Success 200 {object} WorkerResponse +// @Security ApiKeyAuth +// @Router /workers/{workerID}/heartbeat [post] +func (w *WorkerRoutes) Heartbeat(rw http.ResponseWriter, r *http.Request) { + if w.store == nil { + httputil.WriteError(rw, http.StatusServiceUnavailable, httputil.CodeUnavailable, "worker store unavailable") + return + } + workerID := strings.TrimSpace(chi.URLParam(r, "workerID")) + if workerID == "" { + httputil.WriteError(rw, http.StatusBadRequest, httputil.CodeBadRequest, "worker id is required") + return + } + if identity := middleware.AuthIdentityFromContext(r.Context()); identity.Role == middleware.AuthRoleWorker && identity.WorkerID != workerID { + httputil.WriteError(rw, http.StatusForbidden, httputil.CodeUnauth, "worker credential does not match requested worker") + return + } + var req WorkerHeartbeatRequest + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(rw, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + } + status := strings.TrimSpace(req.Status) + if status == "" { + status = "online" + } + now := time.Now().UTC() + rec := &store.WorkerRecord{ + ID: workerID, + Hostname: strings.TrimSpace(req.Hostname), + Status: status, + Providers: mustJSON(req.Providers, []string{}), + Capabilities: mustJSON(req.Capabilities, []string{}), + Capacity: mustJSON(req.Capacity, map[string]interface{}{}), + LastHeartbeat: now, + } + if existing, err := w.store.GetWorker(r.Context(), workerID); err == nil { + rec.CreatedAt = existing.CreatedAt + } else if !errors.Is(err, store.ErrNotFound) { + writeRouteError(rw, err) + return + } + if err := w.store.SaveWorker(r.Context(), rec); err != nil { + writeRouteError(rw, err) + return + } + httputil.WriteJSON(rw, http.StatusOK, workerResponse(rec, now)) +} + +// Delete removes one worker registry record. +// +// @Summary Delete worker +// @Description Remove a worker registry record +// @Tags workers +// @Param workerID path string true "Worker ID" +// @Success 200 {object} StatusResponse +// @Failure 404 {object} httputil.APIError +// @Security ApiKeyAuth +// @Router /workers/{workerID} [delete] +func (w *WorkerRoutes) Delete(rw http.ResponseWriter, r *http.Request) { + if err := w.store.DeleteWorker(r.Context(), chi.URLParam(r, "workerID")); err != nil { + writeRouteError(rw, err) + return + } + httputil.WriteJSON(rw, http.StatusOK, map[string]string{"status": "deleted"}) +} + +func (w *WorkerRoutes) RenewLease(rw http.ResponseWriter, r *http.Request) { + if w.store == nil { + httputil.WriteError(rw, http.StatusServiceUnavailable, httputil.CodeUnavailable, "worker store unavailable") + return + } + workerID := strings.TrimSpace(chi.URLParam(r, "workerID")) + resourceID := strings.TrimSpace(chi.URLParam(r, "resourceID")) + if workerID == "" || resourceID == "" { + httputil.WriteError(rw, http.StatusBadRequest, httputil.CodeBadRequest, "worker id and resource id are required") + return + } + if identity := middleware.AuthIdentityFromContext(r.Context()); identity.Role == middleware.AuthRoleWorker && identity.WorkerID != workerID { + httputil.WriteError(rw, http.StatusForbidden, httputil.CodeUnauth, "worker credential does not match requested worker") + return + } + var req WorkerRenewLeaseRequest + if r.Body != nil { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(rw, http.StatusBadRequest, httputil.CodeBadRequest, "invalid request body") + return + } + } + ttl, err := time.ParseDuration(req.TTL) + if err != nil || ttl <= 0 { + httputil.WriteError(rw, http.StatusBadRequest, httputil.CodeBadRequest, "ttl must be a positive duration") + return + } + lease, err := w.store.RenewLease(r.Context(), resourceID, workerID, ttl) + if err != nil { + writeRouteError(rw, err) + return + } + httputil.WriteJSON(rw, http.StatusOK, workerproto.RenewLeaseResult{Lease: leaseTokenFromRecord(lease)}) +} + +func workerResponse(rec *store.WorkerRecord, now time.Time) WorkerResponse { + var providers []string + var capabilities []string + capacity := map[string]interface{}{} + _ = json.Unmarshal([]byte(rec.Providers), &providers) + _ = json.Unmarshal([]byte(rec.Capabilities), &capabilities) + _ = json.Unmarshal([]byte(rec.Capacity), &capacity) + return WorkerResponse{ + ID: rec.ID, + Hostname: rec.Hostname, + Status: rec.Status, + Providers: providers, + Capabilities: capabilities, + Capacity: capacity, + LastHeartbeat: rec.LastHeartbeat.UTC().Format(time.RFC3339), + CreatedAt: rec.CreatedAt.UTC().Format(time.RFC3339), + UpdatedAt: rec.UpdatedAt.UTC().Format(time.RFC3339), + Stale: now.Sub(rec.LastHeartbeat) > 2*time.Minute, + } +} + +func leaseTokenFromRecord(rec *store.LeaseRecord) workerproto.LeaseToken { + if rec == nil { + return workerproto.LeaseToken{} + } + return workerproto.LeaseToken{ + ResourceID: rec.ResourceID, + HolderID: rec.HolderID, + Generation: rec.Generation, + ExpiresAt: rec.ExpiresAt, + } +} + +func mustJSON(value interface{}, fallback interface{}) string { + if value == nil { + value = fallback + } + data, err := json.Marshal(value) + if err != nil { + data, _ = json.Marshal(fallback) + } + return string(data) +} diff --git a/internal/api/routes/workers_test.go b/internal/api/routes/workers_test.go new file mode 100644 index 0000000..afb0c0a --- /dev/null +++ b/internal/api/routes/workers_test.go @@ -0,0 +1,101 @@ +package routes + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/StacyOs/stacyvm/internal/store" + "github.com/go-chi/chi/v5" +) + +func setupWorkerRoutes(t *testing.T) (*WorkerRoutes, *store.SQLiteStore) { + t.Helper() + st, err := store.NewSQLiteStore(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + return NewWorkerRoutes(st), st +} + +func TestWorkerRoutes_HeartbeatListGetDelete(t *testing.T) { + routes, _ := setupWorkerRoutes(t) + router := chi.NewRouter() + router.Mount("/workers", routes.Routes()) + + body := []byte(`{ + "hostname":"host-a", + "status":"online", + "providers":["mock","docker"], + "capabilities":["spawn","exec"], + "capacity":{"max_sandboxes":10} + }`) + req := httptest.NewRequest(http.MethodPost, "/workers/worker-a/heartbeat", bytes.NewReader(body)) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("heartbeat status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var heartbeat WorkerResponse + if err := json.NewDecoder(w.Body).Decode(&heartbeat); err != nil { + t.Fatalf("decode heartbeat: %v", err) + } + if heartbeat.ID != "worker-a" || heartbeat.Hostname != "host-a" || heartbeat.Stale { + t.Fatalf("unexpected heartbeat response: %+v", heartbeat) + } + if len(heartbeat.Providers) != 2 || heartbeat.Providers[0] != "mock" { + t.Fatalf("unexpected providers: %+v", heartbeat.Providers) + } + + req = httptest.NewRequest(http.MethodGet, "/workers", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("list status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var workers []WorkerResponse + if err := json.NewDecoder(w.Body).Decode(&workers); err != nil { + t.Fatalf("decode workers: %v", err) + } + if len(workers) != 1 || workers[0].ID != "worker-a" { + t.Fatalf("unexpected workers: %+v", workers) + } + + req = httptest.NewRequest(http.MethodGet, "/workers/worker-a", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("get status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodDelete, "/workers/worker-a", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("delete status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodGet, "/workers/worker-a", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusNotFound { + t.Fatalf("get deleted status = %d, want %d", w.Code, http.StatusNotFound) + } +} + +func TestWorkerRoutes_HeartbeatRejectsInvalidJSON(t *testing.T) { + routes, _ := setupWorkerRoutes(t) + router := chi.NewRouter() + router.Mount("/workers", routes.Routes()) + + req := httptest.NewRequest(http.MethodPost, "/workers/worker-a/heartbeat", bytes.NewReader([]byte(`{`))) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index ca41bd8..3262c86 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -2,7 +2,11 @@ package api import ( "context" + "encoding/json" "net/http" + "os" + "strings" + "sync" "time" "github.com/StacyOs/stacyvm/internal/api/middleware" @@ -19,14 +23,27 @@ import ( ) type ServerConfig struct { - Addr string - APIKey string - Version string + Addr string + APIKey string + AdminAPIKey string + AdminFallbackDisabled bool + AdminAuditRetention time.Duration + CORSAllowedOrigins []string + WorkerToken string + WorkerTokens map[string]string + WorkerSigningKey string + WorkerSigningKeys []string + WorkerRevokedTokenIDs []string + Version string + RateLimit middleware.RateLimitConfig + WorkerHeartbeat time.Duration + OIDC middleware.OIDCConfig } type Server struct { - httpServer *http.Server - logger zerolog.Logger + httpServer *http.Server + logger zerolog.Logger + workerHeartbeat *localWorkerHeartbeat } // @title StacyVM API @@ -42,6 +59,12 @@ type Server struct { func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestrator.Manager, events *orchestrator.EventBus, templates *orchestrator.TemplateRegistry, pool *orchestrator.PoolManager, st store.Store, envBuild routes.BuildStarter, logger zerolog.Logger) *Server { r := chi.NewRouter() + heartbeatInterval := cfg.WorkerHeartbeat + if heartbeatInterval == 0 { + heartbeatInterval = 30 * time.Second + } + workerHeartbeat := newLocalWorkerHeartbeat(registry, manager, st, logger, heartbeatInterval) + workerHeartbeat.register(context.Background()) // Global middleware (applies to all routes including swagger) r.Use(chimw.Recoverer) @@ -54,41 +77,76 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr httpSwagger.URL("/swagger/doc.json"), )) + workerRoutes := routes.NewWorkerRoutes(st) + r.Route("/api/v1/worker", func(r chi.Router) { + r.Use(middleware.WorkerAuthWithConfig(middleware.WorkerAuthConfig{ + SharedToken: cfg.WorkerToken, + WorkerTokens: cfg.WorkerTokens, + SigningKey: cfg.WorkerSigningKey, + SigningKeys: cfg.WorkerSigningKeys, + RevokedTokenIDs: cfg.WorkerRevokedTokenIDs, + })) + r.With(middleware.RequireScope(middleware.ScopeWorkerHeartbeat)).Post("/{workerID}/heartbeat", workerRoutes.Heartbeat) + r.With(middleware.RequireScope(middleware.ScopeWorkerLease)).Post("/{workerID}/leases/{resourceID}/renew", workerRoutes.RenewLease) + }) + // API routes — with auth and CORS r.Group(func(r chi.Router) { - if cfg.APIKey != "" { - r.Use(middleware.Auth(cfg.APIKey)) + // OIDC Bearer token auth (runs before API key auth; falls through on no Bearer token). + if cfg.OIDC.Issuer != "" || cfg.OIDC.JWKSUrl != "" || cfg.OIDC.PublicKeyPEM != "" { + r.Use(middleware.OIDCAuth(cfg.OIDC)) + } + if cfg.APIKey != "" || cfg.AdminAPIKey != "" { + r.Use(middleware.AuthAny(cfg.APIKey, cfg.AdminAPIKey)) } - // CORS - r.Use(func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key, X-Request-ID, X-User-ID") - if r.Method == "OPTIONS" { - w.WriteHeader(http.StatusOK) - return - } - next.ServeHTTP(w, r) - }) - }) + r.Use(corsMiddleware(cfg.CORSAllowedOrigins)) + + var rateLimiter *middleware.RateLimiter + if cfg.RateLimit.Enabled { + rateLimiter = middleware.NewRateLimiter(cfg.RateLimit) + r.Use(rateLimiter.Middleware) + } // Routes - sandboxRoutes := routes.NewSandboxRoutes(manager) + sandboxRoutes := routes.NewSandboxRoutesWithPolicy(manager, st) providerRoutes := routes.NewProviderRoutes(registry, manager) templateRoutes := routes.NewTemplateRoutes(templates, manager) snapshotRoutes := routes.NewSnapshotRoutes(registry) - systemRoutes := routes.NewSystemRoutes(registry, manager, events, cfg.Version) + systemRoutes := routes.NewSystemRoutes(registry, manager, events, st, cfg.Version, rateLimiter) environmentRoutes := routes.NewEnvironmentRoutes(st, envBuild) - + quotaRoutes := routes.NewQuotaRoutes(manager) + adminAuditRoutes := routes.NewAdminAuditRoutes(st) + tenantRoutes := routes.NewTenantRoutes(st) + tokenIssuerRoutes := routes.NewWorkerTokenIssuerRoutes(cfg.WorkerSigningKey) + authConfigured := cfg.APIKey != "" || cfg.AdminAPIKey != "" || cfg.OIDC.Issuer != "" || cfg.OIDC.JWKSUrl != "" || cfg.OIDC.PublicKeyPEM != "" r.Route("/api/v1", func(r chi.Router) { - r.Mount("/sandboxes", sandboxRoutes.Routes()) + r.Mount("/sandboxes", sandboxRoutes.RoutesWithScopeEnforcement(authConfigured)) r.Mount("/providers", providerRoutes.Routes()) r.Mount("/templates", templateRoutes.Routes()) r.Mount("/snapshots", snapshotRoutes.Routes()) r.Mount("/environments", environmentRoutes.Routes()) + r.Mount("/quotas", quotaRoutes.Routes()) + r.Mount("/workers", workerRoutes.ReadOnlyRoutes()) r.Get("/pool/status", sandboxRoutes.VMPoolStatus) + r.Route("/admin", func(r chi.Router) { + r.Use(middleware.AdminAuth(cfg.AdminAPIKey, cfg.APIKey, !cfg.AdminFallbackDisabled)) + // Require admin scope whenever any auth mode is configured — including + // OIDC-only deployments where no API keys are set. + if authConfigured { + r.Use(middleware.RequireScope(middleware.ScopeAdmin)) + } + r.Use(middleware.AdminAudit(st, logger, cfg.AdminAuditRetention)) + r.Get("/audit", adminAuditRoutes.List) + r.Mount("/providers", providerRoutes.Routes()) + r.Mount("/quotas", quotaRoutes.Routes()) + r.Mount("/workers", workerRoutes.Routes()) + r.Mount("/tenants", tenantRoutes.Routes()) + r.Post("/worker-tokens", tokenIssuerRoutes.IssueToken) + r.Get("/diagnostics", systemRoutes.Diagnostics) + r.Get("/metrics", systemRoutes.Metrics) + r.Get("/metrics/prometheus", systemRoutes.PrometheusMetrics) + }) r.Mount("/", systemRoutes.Routes()) }) }) @@ -101,13 +159,154 @@ func NewServer(cfg ServerConfig, registry *providers.Registry, manager *orchestr WriteTimeout: 120 * time.Second, IdleTimeout: 60 * time.Second, }, - logger: logger, + logger: logger, + workerHeartbeat: workerHeartbeat, + } +} + +func corsMiddleware(allowedOrigins []string) func(http.Handler) http.Handler { + origins := normalizeCORSAllowedOrigins(allowedOrigins) + allowAll := len(origins) == 0 || origins["*"] + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + origin := strings.TrimSpace(r.Header.Get("Origin")) + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-API-Key, X-Admin-API-Key, X-Request-ID, X-User-ID, X-Tenant-ID, Authorization") + if allowAll { + w.Header().Set("Access-Control-Allow-Origin", "*") + } else if origin != "" { + w.Header().Add("Vary", "Origin") + if origins[origin] { + w.Header().Set("Access-Control-Allow-Origin", origin) + } else if r.Method == http.MethodOptions { + http.Error(w, "CORS origin is not allowed", http.StatusForbidden) + return + } + } + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusOK) + return + } + next.ServeHTTP(w, r) + }) + } +} + +func normalizeCORSAllowedOrigins(origins []string) map[string]bool { + out := make(map[string]bool, len(origins)) + for _, origin := range origins { + origin = strings.TrimSpace(origin) + if origin == "" { + continue + } + out[origin] = true + } + return out +} + +type localWorkerHeartbeat struct { + registry *providers.Registry + manager *orchestrator.Manager + store store.Store + logger zerolog.Logger + interval time.Duration + + mu sync.Mutex + cancel context.CancelFunc + done chan struct{} +} + +func newLocalWorkerHeartbeat(registry *providers.Registry, manager *orchestrator.Manager, st store.Store, logger zerolog.Logger, interval time.Duration) *localWorkerHeartbeat { + return &localWorkerHeartbeat{ + registry: registry, + manager: manager, + store: st, + logger: logger, + interval: interval, + } +} + +func (h *localWorkerHeartbeat) start() { + if h == nil || h.store == nil || h.interval <= 0 { + return + } + h.mu.Lock() + if h.cancel != nil { + h.mu.Unlock() + return + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + h.cancel = cancel + h.done = done + h.mu.Unlock() + + go func() { + defer close(done) + ticker := time.NewTicker(h.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + h.register(ctx) + } + } + }() +} + +func (h *localWorkerHeartbeat) stop() { + if h == nil { + return + } + h.mu.Lock() + cancel := h.cancel + done := h.done + h.cancel = nil + h.done = nil + h.mu.Unlock() + if cancel == nil { + return + } + cancel() + <-done +} + +func (h *localWorkerHeartbeat) register(ctx context.Context) { + if h == nil || h.store == nil { + return + } + hostname, err := os.Hostname() + if err != nil { + hostname = "local" + } + capabilities := []string{"api", "single_node", "spawn", "exec", "files"} + providerNames := h.registry.List() + providersJSON, _ := json.Marshal(providerNames) + capabilitiesJSON, _ := json.Marshal(capabilities) + capacityJSON, _ := json.Marshal(h.manager.Limits()) + if err := h.store.SaveWorker(ctx, &store.WorkerRecord{ + ID: "local", + Hostname: hostname, + Status: "online", + Providers: string(providersJSON), + Capabilities: string(capabilitiesJSON), + Capacity: string(capacityJSON), + LastHeartbeat: time.Now().UTC(), + }); err != nil { + h.logger.Warn().Err(err).Msg("failed to register local worker") } } func (s *Server) Start() error { s.logger.Info().Str("addr", s.httpServer.Addr).Msg("starting HTTP server") - return s.httpServer.ListenAndServe() + s.workerHeartbeat.start() + err := s.httpServer.ListenAndServe() + if err != nil { + s.workerHeartbeat.stop() + } + return err } func (s *Server) Handler() http.Handler { @@ -116,5 +315,6 @@ func (s *Server) Handler() http.Handler { func (s *Server) Shutdown(ctx context.Context) error { s.logger.Info().Msg("shutting down HTTP server") + s.workerHeartbeat.stop() return s.httpServer.Shutdown(ctx) } diff --git a/internal/api/server_test.go b/internal/api/server_test.go new file mode 100644 index 0000000..ad9eaf0 --- /dev/null +++ b/internal/api/server_test.go @@ -0,0 +1,537 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/orchestrator" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/store" + "github.com/rs/zerolog" +) + +type noopBuildStarter struct{} + +func (noopBuildStarter) Enqueue(buildID string) error { return nil } + +func setupTestServer(t *testing.T, cfg ServerConfig) *Server { + t.Helper() + + st, err := store.NewSQLiteStore(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + + events := orchestrator.NewEventBus() + manager := orchestrator.NewManager(registry, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + templates := orchestrator.NewTemplateRegistry(st) + pool := orchestrator.NewPoolManager(manager, templates, zerolog.Nop()) + + return NewServer(cfg, registry, manager, events, templates, pool, st, noopBuildStarter{}, zerolog.Nop()) +} + +func setupTestServerWithStore(t *testing.T, cfg ServerConfig) (*Server, *store.SQLiteStore) { + t.Helper() + + st, err := store.NewSQLiteStore(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } + + events := orchestrator.NewEventBus() + manager := orchestrator.NewManager(registry, st, events, zerolog.Nop(), orchestrator.ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + templates := orchestrator.NewTemplateRegistry(st) + pool := orchestrator.NewPoolManager(manager, templates, zerolog.Nop()) + + return NewServer(cfg, registry, manager, events, templates, pool, st, noopBuildStarter{}, zerolog.Nop()), st +} + +func TestCORSDefaultsToWildcardForLocalDevelopment(t *testing.T) { + srv := setupTestServer(t, ServerConfig{Version: "test"}) + + req := httptest.NewRequest(http.MethodOptions, "/api/v1/ready", nil) + req.Header.Set("Origin", "https://app.example.com") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "*" { + t.Fatalf("Access-Control-Allow-Origin = %q, want *", got) + } +} + +func TestCORSAllowsConfiguredOrigin(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + Version: "test", + CORSAllowedOrigins: []string{"https://console.example.com"}, + }) + + req := httptest.NewRequest(http.MethodOptions, "/api/v1/ready", nil) + req.Header.Set("Origin", "https://console.example.com") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "https://console.example.com" { + t.Fatalf("Access-Control-Allow-Origin = %q, want configured origin", got) + } + if got := w.Header().Get("Vary"); got != "Origin" { + t.Fatalf("Vary = %q, want Origin", got) + } +} + +func TestCORSRejectsDisallowedPreflightOrigin(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + Version: "test", + CORSAllowedOrigins: []string{"https://console.example.com"}, + }) + + req := httptest.NewRequest(http.MethodOptions, "/api/v1/ready", nil) + req.Header.Set("Origin", "https://evil.example.com") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusForbidden, w.Body.String()) + } + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "" { + t.Fatalf("Access-Control-Allow-Origin = %q, want empty", got) + } +} + +func TestWorkerHeartbeatUsesWorkerTokenNotAPIKey(t *testing.T) { + srv, st := setupTestServerWithStore(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + WorkerToken: "worker-secret", + Version: "test", + }) + body := []byte(`{"hostname":"worker-host","status":"online","providers":["mock"],"capabilities":["heartbeat"],"capacity":{"max_sandboxes":1}}`) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-a/heartbeat", bytes.NewReader(body)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + rec, err := st.GetWorker(context.Background(), "worker-a") + if err != nil { + t.Fatalf("get worker: %v", err) + } + if rec.Hostname != "worker-host" { + t.Fatalf("hostname = %q, want worker-host", rec.Hostname) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-b/heartbeat", bytes.NewReader(body)) + req.Header.Set("X-API-Key", "client-key") + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized { + t.Fatalf("api key status = %d, want %d", w.Code, http.StatusUnauthorized) + } +} + +func TestWorkerHeartbeatRejectsWorkerIDMismatch(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + WorkerToken: "worker-secret", + Version: "test", + }) + body := []byte(`{"hostname":"worker-host"}`) + req := httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-a/heartbeat", bytes.NewReader(body)) + req.Header.Set("X-Worker-ID", "worker-b") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusForbidden, w.Body.String()) + } +} + +func TestWorkerHeartbeatUsesPerWorkerToken(t *testing.T) { + srv, st := setupTestServerWithStore(t, ServerConfig{ + WorkerToken: "shared-worker-secret", + WorkerTokens: map[string]string{ + "worker-a": "worker-a-secret", + }, + Version: "test", + }) + body := []byte(`{"hostname":"worker-host","status":"online","providers":["mock"],"capabilities":["heartbeat"],"capacity":{"max_sandboxes":1}}`) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-a/heartbeat", bytes.NewReader(body)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-a-secret") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if _, err := st.GetWorker(context.Background(), "worker-a"); err != nil { + t.Fatalf("get worker: %v", err) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-a/heartbeat", bytes.NewReader(body)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "shared-worker-secret") + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized { + t.Fatalf("shared token status = %d, want %d for per-worker credential: %s", w.Code, http.StatusUnauthorized, w.Body.String()) + } +} + +func TestWorkerRenewLeaseUsesWorkerToken(t *testing.T) { + srv, st := setupTestServerWithStore(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + WorkerToken: "worker-secret", + Version: "test", + }) + if _, err := st.AcquireLease(context.Background(), "sb-lease", "sandbox", "worker-a", time.Minute); err != nil { + t.Fatalf("acquire lease: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-a/leases/sb-lease/renew", strings.NewReader(`{"ttl":"2m"}`)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var result struct { + Lease struct { + ResourceID string `json:"resource_id"` + HolderID string `json:"holder_id"` + Generation int64 `json:"generation"` + } `json:"lease"` + } + if err := json.NewDecoder(w.Body).Decode(&result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Lease.ResourceID != "sb-lease" || result.Lease.HolderID != "worker-a" || result.Lease.Generation != 2 { + t.Fatalf("unexpected lease result: %+v", result.Lease) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/worker/worker-a/leases/sb-lease/renew", strings.NewReader(`{"ttl":"2m"}`)) + req.Header.Set("X-API-Key", "client-key") + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized { + t.Fatalf("api key status = %d, want %d", w.Code, http.StatusUnauthorized) + } +} + +func TestAdminRoutesRequireAdminAPIKeyWhenConfigured(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + Version: "test", + }) + + tests := []struct { + name string + header string + key string + want int + }{ + {name: "client key forbidden", header: "X-API-Key", key: "client-key", want: http.StatusForbidden}, + {name: "admin api header ok", header: "X-API-Key", key: "admin-key", want: http.StatusOK}, + {name: "admin header ok", header: "X-Admin-API-Key", key: "admin-key", want: http.StatusOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set(tt.header, tt.key) + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.want, w.Body.String()) + } + }) + } +} + +func TestAdminRoutesFallbackToAPIKeyWhenAdminKeyUnset(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set("X-API-Key", "client-key") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestAdminRoutesRejectAPIKeyWhenFallbackDisabled(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + AdminFallbackDisabled: true, + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set("X-API-Key", "client-key") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusForbidden, w.Body.String()) + } +} + +func TestAdminRoutesRemainOpenWhenAuthUnset(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestAdminAPIKeyCanAuthenticateRegularRoutes(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/live", nil) + req.Header.Set("X-Admin-API-Key", "admin-key") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } +} + +func TestAdminRoutesWriteAuditLog(t *testing.T) { + srv := setupTestServer(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + Version: "test", + }) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set("X-Admin-API-Key", "admin-key") + req.Header.Set("X-User-ID", "operator-a") + req.Header.Set("User-Agent", "stacyvm-test") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("diagnostics status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/admin/audit?limit=10", nil) + req.Header.Set("X-Admin-API-Key", "admin-key") + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("audit status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + + var records []store.AdminAuditRecord + if err := json.NewDecoder(w.Body).Decode(&records); err != nil { + t.Fatalf("decode audit records: %v", err) + } + if len(records) == 0 { + t.Fatal("expected audit records") + } + var found bool + for _, rec := range records { + if rec.Path == "/api/v1/admin/diagnostics" { + found = true + if rec.Actor != "operator-a" || rec.Method != http.MethodGet || rec.Status != http.StatusOK { + t.Fatalf("unexpected diagnostics audit record: %+v", rec) + } + } + } + if !found { + t.Fatalf("diagnostics audit record not found: %+v", records) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/admin/audit?actor=operator-a&method=GET&status=200&path=diagnostics&format=csv", nil) + req.Header.Set("X-Admin-API-Key", "admin-key") + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("csv audit status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := w.Header().Get("Content-Type"); got != "text/csv; charset=utf-8" { + t.Fatalf("csv content type = %q", got) + } + if body := w.Body.String(); !strings.Contains(body, "/api/v1/admin/diagnostics") || !strings.Contains(body, "operator-a") { + t.Fatalf("csv body missing filtered audit record: %s", body) + } +} + +func TestServerRegistersLocalWorkerAndExposesWorkers(t *testing.T) { + srv := setupTestServer(t, ServerConfig{Version: "test"}) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/workers", nil) + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("workers status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var workers []map[string]interface{} + if err := json.NewDecoder(w.Body).Decode(&workers); err != nil { + t.Fatalf("decode workers: %v", err) + } + if len(workers) != 1 || workers[0]["id"] != "local" { + t.Fatalf("unexpected workers: %+v", workers) + } + + req = httptest.NewRequest(http.MethodPost, "/api/v1/admin/workers/worker-b/heartbeat", strings.NewReader(`{"hostname":"host-b","providers":["mock"],"capabilities":["spawn"],"capacity":{"max_sandboxes":5}}`)) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("heartbeat status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var heartbeat map[string]interface{} + if err := json.NewDecoder(w.Body).Decode(&heartbeat); err != nil { + t.Fatalf("decode heartbeat: %v", err) + } + if heartbeat["id"] != "worker-b" || heartbeat["status"] != "online" { + t.Fatalf("unexpected heartbeat: %+v", heartbeat) + } + + req = httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + w = httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("diagnostics status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var diagnostics map[string]interface{} + if err := json.NewDecoder(w.Body).Decode(&diagnostics); err != nil { + t.Fatalf("decode diagnostics: %v", err) + } + workerSummary := diagnostics["workers"].(map[string]interface{}) + if workerSummary["total"].(float64) != 2 { + t.Fatalf("worker total = %v, want 2", workerSummary["total"]) + } +} + +func TestServerRefreshesLocalWorkerHeartbeat(t *testing.T) { + srv, st := setupTestServerWithStore(t, ServerConfig{ + Version: "test", + WorkerHeartbeat: 10 * time.Millisecond, + }) + oldHeartbeat := time.Now().UTC().Add(-10 * time.Minute) + if err := st.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "local", + Hostname: "stale-host", + Status: "online", + Providers: `["mock"]`, + Capabilities: `["spawn"]`, + Capacity: `{}`, + LastHeartbeat: oldHeartbeat, + }); err != nil { + t.Fatalf("save stale worker: %v", err) + } + + srv.workerHeartbeat.start() + t.Cleanup(func() { srv.workerHeartbeat.stop() }) + + deadline := time.Now().Add(250 * time.Millisecond) + for time.Now().Before(deadline) { + worker, err := st.GetWorker(context.Background(), "local") + if err != nil { + t.Fatalf("get local worker: %v", err) + } + if worker.LastHeartbeat.After(oldHeartbeat) && worker.Hostname != "stale-host" { + return + } + time.Sleep(5 * time.Millisecond) + } + worker, _ := st.GetWorker(context.Background(), "local") + t.Fatalf("local worker heartbeat was not refreshed: %+v", worker) +} + +func TestAdminRoutesPruneAuditLogWithRetention(t *testing.T) { + srv, st := setupTestServerWithStore(t, ServerConfig{ + APIKey: "client-key", + AdminAPIKey: "admin-key", + AdminAuditRetention: time.Hour, + Version: "test", + }) + ctx := t.Context() + if err := st.CreateAdminAudit(ctx, &store.AdminAuditRecord{ + Actor: "old-operator", + Method: http.MethodGet, + Path: "/api/v1/admin/old", + Status: http.StatusOK, + CreatedAt: time.Now().Add(-2 * time.Hour), + }); err != nil { + t.Fatalf("create old audit: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics", nil) + req.Header.Set("X-Admin-API-Key", "admin-key") + w := httptest.NewRecorder() + srv.Handler().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("diagnostics status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + + records, err := st.ListAdminAudit(ctx, store.AdminAuditQuery{Limit: 10}) + if err != nil { + t.Fatalf("list audit: %v", err) + } + for _, rec := range records { + if rec.Actor == "old-operator" { + t.Fatalf("old audit record was not pruned: %+v", records) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index bc033b4..a2dc54b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,19 +2,23 @@ package config import ( "fmt" + "net/url" "os" "os/exec" "path/filepath" "strings" + "time" "github.com/spf13/viper" ) type Config struct { Server ServerConfig `mapstructure:"server"` + Worker WorkerConfig `mapstructure:"worker"` Providers ProvidersConfig `mapstructure:"providers"` Defaults DefaultsConfig `mapstructure:"defaults"` Auth AuthConfig `mapstructure:"auth"` + RateLimit RateLimitConfig `mapstructure:"rate_limit"` Database DatabaseConfig `mapstructure:"database"` Logging LoggingConfig `mapstructure:"logging"` Pool PoolConfig `mapstructure:"pool"` @@ -31,15 +35,38 @@ type PoolConfig struct { } type ServerConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - PreviewDomain string `mapstructure:"preview_domain"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + PreviewDomain string `mapstructure:"preview_domain"` + CORSAllowedOrigins []string `mapstructure:"cors_allowed_origins"` } func (s ServerConfig) Addr() string { return fmt.Sprintf("%s:%d", s.Host, s.Port) } +type WorkerConfig struct { + ID string `mapstructure:"id"` + ControlPlaneURL string `mapstructure:"control_plane_url"` + ListenAddr string `mapstructure:"listen_addr"` + PreviewDomain string `mapstructure:"preview_domain"` + HeartbeatInterval string `mapstructure:"heartbeat_interval"` + ShutdownTimeout string `mapstructure:"shutdown_timeout"` + RPCTLS WorkerRPCTLSConfig `mapstructure:"rpc_tls"` +} + +type WorkerRPCTLSConfig struct { + Enabled bool `mapstructure:"enabled"` + ServerCertFile string `mapstructure:"server_cert_file"` + ServerKeyFile string `mapstructure:"server_key_file"` + ClientCAFile string `mapstructure:"client_ca_file"` + CAFile string `mapstructure:"ca_file"` + ClientCertFile string `mapstructure:"client_cert_file"` + ClientKeyFile string `mapstructure:"client_key_file"` + ServerName string `mapstructure:"server_name"` + InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` +} + type ProvidersConfig struct { Default string `mapstructure:"default"` Mock MockConfig `mapstructure:"mock"` @@ -63,20 +90,20 @@ type PRootConfig struct { } type DockerConfig struct { - Enabled bool `mapstructure:"enabled"` - Socket string `mapstructure:"socket"` - Runtime string `mapstructure:"runtime"` - DefaultImage string `mapstructure:"default_image"` - NetworkMode string `mapstructure:"network_mode"` - SeccompProfile string `mapstructure:"seccomp_profile"` - ReadOnlyRootfs bool `mapstructure:"read_only_rootfs"` - Memory string `mapstructure:"memory"` - CPUs string `mapstructure:"cpus"` - PidsLimit int64 `mapstructure:"pids_limit"` - User string `mapstructure:"user"` - DroppedCaps []string `mapstructure:"dropped_caps"` - AddedCaps []string `mapstructure:"added_caps"` - Tmpfs map[string]string `mapstructure:"tmpfs"` + Enabled bool `mapstructure:"enabled"` + Socket string `mapstructure:"socket"` + Runtime string `mapstructure:"runtime"` + DefaultImage string `mapstructure:"default_image"` + NetworkMode string `mapstructure:"network_mode"` + SeccompProfile string `mapstructure:"seccomp_profile"` + ReadOnlyRootfs bool `mapstructure:"read_only_rootfs"` + Memory string `mapstructure:"memory"` + CPUs string `mapstructure:"cpus"` + PidsLimit int64 `mapstructure:"pids_limit"` + User string `mapstructure:"user"` + DroppedCaps []string `mapstructure:"dropped_caps"` + AddedCaps []string `mapstructure:"added_caps"` + Tmpfs map[string]string `mapstructure:"tmpfs"` PoolSecurity PoolSecurityConfig `mapstructure:"pool_security"` } @@ -115,22 +142,64 @@ type FirecrackerConfig struct { } type DefaultsConfig struct { - TTL string `mapstructure:"ttl"` - Image string `mapstructure:"image"` - MemoryMB int `mapstructure:"memory_mb"` - VCPUs int `mapstructure:"vcpus"` - DiskSizeMB int `mapstructure:"disk_size_mb"` - PoolSize int `mapstructure:"pool_size"` - PoolTemplate string `mapstructure:"pool_template"` + TTL string `mapstructure:"ttl"` + Image string `mapstructure:"image"` + MemoryMB int `mapstructure:"memory_mb"` + VCPUs int `mapstructure:"vcpus"` + DiskSizeMB int `mapstructure:"disk_size_mb"` + PoolSize int `mapstructure:"pool_size"` + PoolTemplate string `mapstructure:"pool_template"` + MaxTTL string `mapstructure:"max_ttl"` + DefaultExecTimeout string `mapstructure:"default_exec_timeout"` + MaxExecTimeout string `mapstructure:"max_exec_timeout"` + MaxSandboxes int `mapstructure:"max_sandboxes"` + MaxSandboxesPerOwner int `mapstructure:"max_sandboxes_per_owner"` + SpawnOverflow string `mapstructure:"spawn_overflow"` + SpawnQueueTimeout string `mapstructure:"spawn_queue_timeout"` + MaxSpawnQueue int `mapstructure:"max_spawn_queue"` } type AuthConfig struct { - Enabled bool `mapstructure:"enabled"` - APIKey string `mapstructure:"api_key"` + Enabled bool `mapstructure:"enabled"` + APIKey string `mapstructure:"api_key"` + AdminAPIKey string `mapstructure:"admin_api_key"` + WorkerToken string `mapstructure:"worker_token"` + WorkerTokenFile string `mapstructure:"worker_token_file"` + WorkerTokens map[string]string `mapstructure:"worker_tokens"` + WorkerSigningKey string `mapstructure:"worker_signing_key"` + WorkerSigningKeyFile string `mapstructure:"worker_signing_key_file"` + WorkerSigningKeys []string `mapstructure:"worker_signing_keys"` + WorkerRevokedTokenIDs []string `mapstructure:"worker_revoked_token_ids"` + AdminFallbackEnabled bool `mapstructure:"admin_fallback_enabled"` + AdminAuditRetention string `mapstructure:"admin_audit_retention"` + + // OIDC/JWT configuration for enterprise SSO + OIDCEnabled bool `mapstructure:"oidc_enabled"` + OIDCIssuer string `mapstructure:"oidc_issuer"` + OIDCAudience string `mapstructure:"oidc_audience"` + OIDCJWKSUrl string `mapstructure:"oidc_jwks_url"` + OIDCPublicKey string `mapstructure:"oidc_public_key"` + OIDCPublicKeyFile string `mapstructure:"oidc_public_key_file"` + OIDCGroupsClaim string `mapstructure:"oidc_groups_claim"` + OIDCTenantClaim string `mapstructure:"oidc_tenant_claim"` + OIDCAdminGroups []string `mapstructure:"oidc_admin_groups"` + OIDCOperatorGroups []string `mapstructure:"oidc_operator_groups"` + OIDCViewerGroups []string `mapstructure:"oidc_viewer_groups"` +} + +type RateLimitConfig struct { + Enabled bool `mapstructure:"enabled"` + RequestsPerMinute int `mapstructure:"requests_per_minute"` + Burst int `mapstructure:"burst"` + KeyBy string `mapstructure:"key_by"` + BucketTTL string `mapstructure:"bucket_ttl"` + CleanupInterval string `mapstructure:"cleanup_interval"` } type DatabaseConfig struct { - Path string `mapstructure:"path"` + Driver string `mapstructure:"driver"` + Path string `mapstructure:"path"` + DSN string `mapstructure:"dsn"` } type LoggingConfig struct { @@ -142,6 +211,22 @@ func setDefaults(v *viper.Viper) { v.SetDefault("server.host", "0.0.0.0") v.SetDefault("server.port", 7423) v.SetDefault("server.preview_domain", "localhost") + v.SetDefault("server.cors_allowed_origins", []string{"*"}) + v.SetDefault("worker.id", "") + v.SetDefault("worker.control_plane_url", "http://localhost:7423") + v.SetDefault("worker.listen_addr", "") + v.SetDefault("worker.preview_domain", "") + v.SetDefault("worker.heartbeat_interval", "30s") + v.SetDefault("worker.shutdown_timeout", "10s") + v.SetDefault("worker.rpc_tls.enabled", false) + v.SetDefault("worker.rpc_tls.server_cert_file", "") + v.SetDefault("worker.rpc_tls.server_key_file", "") + v.SetDefault("worker.rpc_tls.client_ca_file", "") + v.SetDefault("worker.rpc_tls.ca_file", "") + v.SetDefault("worker.rpc_tls.client_cert_file", "") + v.SetDefault("worker.rpc_tls.client_key_file", "") + v.SetDefault("worker.rpc_tls.server_name", "") + v.SetDefault("worker.rpc_tls.insecure_skip_verify", false) v.SetDefault("providers.default", "docker") v.SetDefault("providers.mock.enabled", false) @@ -195,11 +280,49 @@ func setDefaults(v *viper.Viper) { v.SetDefault("defaults.disk_size_mb", 1024) v.SetDefault("defaults.pool_size", 0) v.SetDefault("defaults.pool_template", "") + v.SetDefault("defaults.max_ttl", "24h") + v.SetDefault("defaults.default_exec_timeout", "0s") + v.SetDefault("defaults.max_exec_timeout", "10m") + v.SetDefault("defaults.max_sandboxes", 0) + v.SetDefault("defaults.max_sandboxes_per_owner", 0) + v.SetDefault("defaults.spawn_overflow", "reject") + v.SetDefault("defaults.spawn_queue_timeout", "30s") + v.SetDefault("defaults.max_spawn_queue", 100) v.SetDefault("auth.enabled", false) v.SetDefault("auth.api_key", "") + v.SetDefault("auth.admin_api_key", "") + v.SetDefault("auth.worker_token", "") + v.SetDefault("auth.worker_token_file", "") + v.SetDefault("auth.worker_tokens", map[string]string{}) + v.SetDefault("auth.worker_signing_key", "") + v.SetDefault("auth.worker_signing_key_file", "") + v.SetDefault("auth.worker_signing_keys", []string{}) + v.SetDefault("auth.worker_revoked_token_ids", []string{}) + v.SetDefault("auth.admin_fallback_enabled", true) + v.SetDefault("auth.admin_audit_retention", "0s") + v.SetDefault("auth.oidc_enabled", false) + v.SetDefault("auth.oidc_issuer", "") + v.SetDefault("auth.oidc_audience", "") + v.SetDefault("auth.oidc_jwks_url", "") + v.SetDefault("auth.oidc_public_key", "") + v.SetDefault("auth.oidc_public_key_file", "") + v.SetDefault("auth.oidc_groups_claim", "groups") + v.SetDefault("auth.oidc_tenant_claim", "tenant_id") + v.SetDefault("auth.oidc_admin_groups", []string{}) + v.SetDefault("auth.oidc_operator_groups", []string{}) + v.SetDefault("auth.oidc_viewer_groups", []string{}) + + v.SetDefault("rate_limit.enabled", false) + v.SetDefault("rate_limit.requests_per_minute", 120) + v.SetDefault("rate_limit.burst", 60) + v.SetDefault("rate_limit.key_by", "owner") + v.SetDefault("rate_limit.bucket_ttl", "15m") + v.SetDefault("rate_limit.cleanup_interval", "1m") v.SetDefault("database.path", "stacyvm.db") + v.SetDefault("database.driver", "sqlite") + v.SetDefault("database.dsn", "") v.SetDefault("logging.level", "info") v.SetDefault("logging.format", "json") @@ -246,6 +369,21 @@ func (c *Config) ResolveAgentPath() string { } func Load() (*Config, error) { + configPaths := []string{"stacyvm.yaml"} + if home, err := os.UserHomeDir(); err == nil { + configPaths = append(configPaths, filepath.Join(home, ".stacyvm", "config.yaml")) + } + return load(configPaths, false) +} + +func LoadFile(path string) (*Config, error) { + if strings.TrimSpace(path) == "" { + return nil, fmt.Errorf("config path is required") + } + return load([]string{path}, true) +} + +func load(configPaths []string, requireConfig bool) (*Config, error) { v := viper.New() setDefaults(v) @@ -255,12 +393,6 @@ func Load() (*Config, error) { v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.AutomaticEnv() - // Try explicit config files in order of priority - configPaths := []string{"stacyvm.yaml"} - if home, err := os.UserHomeDir(); err == nil { - configPaths = append(configPaths, filepath.Join(home, ".stacyvm", "config.yaml")) - } - loaded := false for _, p := range configPaths { if _, err := os.Stat(p); err == nil { @@ -270,6 +402,8 @@ func Load() (*Config, error) { } loaded = true break + } else if requireConfig { + return nil, fmt.Errorf("config file %s: %w", p, err) } } _ = loaded // defaults are fine if no config file found @@ -278,6 +412,152 @@ func Load() (*Config, error) { if err := v.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("unmarshaling config: %w", err) } + if err := cfg.resolveAuthSecretFiles(); err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, err + } return &cfg, nil } + +func (c *Config) resolveAuthSecretFiles() error { + if err := resolveSecretFile("auth.worker_token", &c.Auth.WorkerToken, "auth.worker_token_file", c.Auth.WorkerTokenFile); err != nil { + return err + } + if err := resolveSecretFile("auth.worker_signing_key", &c.Auth.WorkerSigningKey, "auth.worker_signing_key_file", c.Auth.WorkerSigningKeyFile); err != nil { + return err + } + if err := resolveSecretFile("auth.oidc_public_key", &c.Auth.OIDCPublicKey, "auth.oidc_public_key_file", c.Auth.OIDCPublicKeyFile); err != nil { + return err + } + return nil +} + +func resolveSecretFile(valueName string, value *string, fileName, path string) error { + path = strings.TrimSpace(path) + if path == "" { + return nil + } + if strings.TrimSpace(*value) != "" { + return fmt.Errorf("%s and %s cannot both be set", valueName, fileName) + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("%s: %w", fileName, err) + } + secret := strings.TrimSpace(string(data)) + if secret == "" { + return fmt.Errorf("%s is empty", fileName) + } + *value = secret + return nil +} + +func (c *Config) Validate() error { + switch driver := strings.ToLower(strings.TrimSpace(c.Database.Driver)); driver { + case "", "sqlite", "sqlite3": + if strings.TrimSpace(c.Database.Path) == "" { + return fmt.Errorf("database.path is required for sqlite") + } + case "postgres", "postgresql": + if strings.TrimSpace(c.Database.DSN) == "" { + return fmt.Errorf("database.dsn is required for postgres") + } + default: + return fmt.Errorf("unsupported database.driver %q", c.Database.Driver) + } + + durationFields := map[string]string{ + "defaults.ttl": c.Defaults.TTL, + "defaults.max_ttl": c.Defaults.MaxTTL, + "defaults.default_exec_timeout": c.Defaults.DefaultExecTimeout, + "defaults.max_exec_timeout": c.Defaults.MaxExecTimeout, + "defaults.spawn_queue_timeout": c.Defaults.SpawnQueueTimeout, + "rate_limit.bucket_ttl": c.RateLimit.BucketTTL, + "rate_limit.cleanup_interval": c.RateLimit.CleanupInterval, + "auth.admin_audit_retention": c.Auth.AdminAuditRetention, + "worker.heartbeat_interval": c.Worker.HeartbeatInterval, + "worker.shutdown_timeout": c.Worker.ShutdownTimeout, + "providers.custom.timeout": c.Providers.Custom.Timeout, + "providers.proot.default_timeout": c.Providers.PRoot.DefaultTimeout, + } + for name, value := range durationFields { + if err := validateDuration(name, value); err != nil { + return err + } + } + + if c.Defaults.MaxSandboxes < 0 { + return fmt.Errorf("defaults.max_sandboxes cannot be negative") + } + if c.Defaults.MaxSandboxesPerOwner < 0 { + return fmt.Errorf("defaults.max_sandboxes_per_owner cannot be negative") + } + if c.Defaults.MaxSpawnQueue < 0 { + return fmt.Errorf("defaults.max_spawn_queue cannot be negative") + } + if c.RateLimit.RequestsPerMinute < 0 { + return fmt.Errorf("rate_limit.requests_per_minute cannot be negative") + } + if c.RateLimit.Burst < 0 { + return fmt.Errorf("rate_limit.burst cannot be negative") + } + if !isOneOf(c.Defaults.SpawnOverflow, "", "reject", "queue") { + return fmt.Errorf("defaults.spawn_overflow must be reject or queue") + } + if !isOneOf(c.RateLimit.KeyBy, "", "owner", "api_key", "ip") { + return fmt.Errorf("rate_limit.key_by must be owner, api_key, or ip") + } + if !isOneOf(c.Pool.Overflow, "", "reject", "queue") { + return fmt.Errorf("pool.overflow must be reject or queue") + } + if err := validateCORSAllowedOrigins(c.Server.CORSAllowedOrigins); err != nil { + return err + } + return nil +} + +func validateCORSAllowedOrigins(origins []string) error { + for _, origin := range origins { + origin = strings.TrimSpace(origin) + if origin == "" || origin == "*" { + continue + } + u, err := url.Parse(origin) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("server.cors_allowed_origins contains invalid origin %q", origin) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("server.cors_allowed_origins origin %q must use http or https", origin) + } + if u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("server.cors_allowed_origins origin %q must not include path, query, or fragment", origin) + } + } + return nil +} + +func validateDuration(name, value string) error { + if value == "" { + return nil + } + d, err := time.ParseDuration(value) + if err != nil { + return fmt.Errorf("%s must be a valid duration: %w", name, err) + } + if d < 0 { + return fmt.Errorf("%s cannot be negative", name) + } + return nil +} + +func isOneOf(value string, allowed ...string) bool { + for _, candidate := range allowed { + if value == candidate { + return true + } + } + return false +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..db15d5d --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,302 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadRejectsInvalidDuration(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + spawn_queue_timeout: "soon" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected invalid duration error") + } + if !strings.Contains(err.Error(), "defaults.spawn_queue_timeout") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadRejectsInvalidEnums(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + spawn_overflow: "stall" +rate_limit: + key_by: "cookie" +pool: + overflow: "stall" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected invalid enum error") + } + if !strings.Contains(err.Error(), "defaults.spawn_overflow") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadRejectsNegativeLimits(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + max_spawn_queue: -1 +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected negative limit error") + } + if !strings.Contains(err.Error(), "defaults.max_spawn_queue") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadRejectsUnsupportedDatabaseDriver(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +database: + driver: "mysql" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected unsupported database driver error") + } + if !strings.Contains(err.Error(), "database.driver") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadRejectsInvalidCORSOrigin(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +server: + cors_allowed_origins: + - "https://console.example.com/path" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected invalid CORS origin error") + } + if !strings.Contains(err.Error(), "server.cors_allowed_origins") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadAcceptsExplicitCORSOrigins(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +server: + cors_allowed_origins: + - "https://console.example.com" + - "http://localhost:5173" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load() + if err != nil { + t.Fatalf("load config: %v", err) + } + if len(cfg.Server.CORSAllowedOrigins) != 2 { + t.Fatalf("CORS origins = %#v, want two", cfg.Server.CORSAllowedOrigins) + } +} + +func TestLoadRequiresPostgresDSN(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +database: + driver: "postgres" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + _, err := Load() + if err == nil { + t.Fatal("expected postgres dsn error") + } + if !strings.Contains(err.Error(), "database.dsn") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestLoadAcceptsPhaseThreeConfig(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +defaults: + spawn_overflow: "queue" + spawn_queue_timeout: "45s" + max_spawn_queue: 25 +rate_limit: + enabled: true + requests_per_minute: 240 + burst: 80 + key_by: "api_key" + bucket_ttl: "30m" + cleanup_interval: "2m" +auth: + admin_api_key: "admin-secret" + admin_fallback_enabled: false + admin_audit_retention: "2160h" +pool: + overflow: "queue" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load() + if err != nil { + t.Fatalf("load config: %v", err) + } + if cfg.Defaults.SpawnOverflow != "queue" || cfg.Defaults.SpawnQueueTimeout != "45s" || cfg.Defaults.MaxSpawnQueue != 25 { + t.Fatalf("unexpected defaults config: %+v", cfg.Defaults) + } + if !cfg.RateLimit.Enabled || cfg.RateLimit.KeyBy != "api_key" || cfg.RateLimit.BucketTTL != "30m" { + t.Fatalf("unexpected rate limit config: %+v", cfg.RateLimit) + } + if cfg.Auth.AdminAPIKey != "admin-secret" { + t.Fatalf("admin api key = %q, want admin-secret", cfg.Auth.AdminAPIKey) + } + if cfg.Auth.AdminFallbackEnabled { + t.Fatal("admin fallback enabled = true, want false") + } + if cfg.Auth.AdminAuditRetention != "2160h" { + t.Fatalf("admin audit retention = %q, want 2160h", cfg.Auth.AdminAuditRetention) + } +} + +func TestLoadAcceptsWorkerRuntimeConfig(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.WriteFile("stacyvm.yaml", []byte(` +worker: + id: "worker-a" + control_plane_url: "http://control-plane:7423" + listen_addr: "127.0.0.1:7430" + heartbeat_interval: "5s" + shutdown_timeout: "15s" + rpc_tls: + enabled: true + server_cert_file: "/etc/stacyvm/tls/worker.crt" + server_key_file: "/etc/stacyvm/tls/worker.key" + client_ca_file: "/etc/stacyvm/tls/control-plane-ca.crt" + ca_file: "/etc/stacyvm/tls/worker-ca.crt" + client_cert_file: "/etc/stacyvm/tls/control-plane.crt" + client_key_file: "/etc/stacyvm/tls/control-plane.key" + server_name: "worker-a.internal" +auth: + worker_token: "worker-secret" + worker_signing_key: "0123456789abcdef0123456789abcdef" + worker_signing_keys: + - "old-worker-signing-key-with-at-least-32-bytes" + worker_revoked_token_ids: + - "revoked-token-id" + worker_tokens: + worker-a: "worker-a-secret" +`), 0644); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := Load() + if err != nil { + t.Fatalf("load config: %v", err) + } + if cfg.Worker.ID != "worker-a" { + t.Fatalf("worker id = %q, want worker-a", cfg.Worker.ID) + } + if cfg.Worker.ControlPlaneURL != "http://control-plane:7423" { + t.Fatalf("control plane URL = %q", cfg.Worker.ControlPlaneURL) + } + if cfg.Worker.ListenAddr != "127.0.0.1:7430" { + t.Fatalf("listen addr = %q", cfg.Worker.ListenAddr) + } + if !cfg.Worker.RPCTLS.Enabled || cfg.Worker.RPCTLS.ServerName != "worker-a.internal" { + t.Fatalf("worker rpc tls = %+v, want enabled worker-a.internal config", cfg.Worker.RPCTLS) + } + if cfg.Auth.WorkerToken != "worker-secret" { + t.Fatalf("worker token = %q, want worker-secret", cfg.Auth.WorkerToken) + } + if cfg.Auth.WorkerSigningKey != "0123456789abcdef0123456789abcdef" { + t.Fatalf("worker signing key = %q, want configured key", cfg.Auth.WorkerSigningKey) + } + if len(cfg.Auth.WorkerSigningKeys) != 1 || cfg.Auth.WorkerSigningKeys[0] != "old-worker-signing-key-with-at-least-32-bytes" { + t.Fatalf("worker signing keys = %#v, want old rotation key", cfg.Auth.WorkerSigningKeys) + } + if len(cfg.Auth.WorkerRevokedTokenIDs) != 1 || cfg.Auth.WorkerRevokedTokenIDs[0] != "revoked-token-id" { + t.Fatalf("worker revoked token ids = %#v, want revoked-token-id", cfg.Auth.WorkerRevokedTokenIDs) + } + if cfg.Auth.WorkerTokens["worker-a"] != "worker-a-secret" { + t.Fatalf("worker-a token = %q, want worker-a-secret", cfg.Auth.WorkerTokens["worker-a"]) + } +} + +func TestLoadResolvesWorkerSecretFiles(t *testing.T) { + dir := t.TempDir() + tokenPath := filepath.Join(dir, "worker-token") + signingKeyPath := filepath.Join(dir, "worker-signing-key") + if err := os.WriteFile(tokenPath, []byte("worker-token-from-file\n"), 0o600); err != nil { + t.Fatalf("write token file: %v", err) + } + if err := os.WriteFile(signingKeyPath, []byte("worker-signing-key-from-file-with-32-bytes\n"), 0o600); err != nil { + t.Fatalf("write signing key file: %v", err) + } + configPath := filepath.Join(dir, "stacyvm.yaml") + if err := os.WriteFile(configPath, []byte(fmt.Sprintf(` +auth: + worker_token_file: %q + worker_signing_key_file: %q +`, tokenPath, signingKeyPath)), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := LoadFile(configPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + if cfg.Auth.WorkerToken != "worker-token-from-file" { + t.Fatalf("worker token = %q, want token from file", cfg.Auth.WorkerToken) + } + if cfg.Auth.WorkerSigningKey != "worker-signing-key-from-file-with-32-bytes" { + t.Fatalf("worker signing key = %q, want key from file", cfg.Auth.WorkerSigningKey) + } +} + +func TestLoadRejectsAmbiguousWorkerSecretFile(t *testing.T) { + dir := t.TempDir() + tokenPath := filepath.Join(dir, "worker-token") + if err := os.WriteFile(tokenPath, []byte("worker-token-from-file\n"), 0o600); err != nil { + t.Fatalf("write token file: %v", err) + } + configPath := filepath.Join(dir, "stacyvm.yaml") + if err := os.WriteFile(configPath, []byte(fmt.Sprintf(` +auth: + worker_token: "worker-token-inline" + worker_token_file: %q +`, tokenPath)), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + if _, err := LoadFile(configPath); err == nil { + t.Fatal("expected ambiguous worker token config to fail") + } +} diff --git a/internal/httputil/response.go b/internal/httputil/response.go index b78691d..2c9cc74 100644 --- a/internal/httputil/response.go +++ b/internal/httputil/response.go @@ -8,12 +8,14 @@ import ( type ErrorCode string const ( - CodeNotFound ErrorCode = "NOT_FOUND" - CodeBadRequest ErrorCode = "BAD_REQUEST" - CodeInternal ErrorCode = "INTERNAL_ERROR" - CodeUnauth ErrorCode = "UNAUTHORIZED" - CodeConflict ErrorCode = "CONFLICT" - CodeUnavailable ErrorCode = "UNAVAILABLE" + CodeNotFound ErrorCode = "NOT_FOUND" + CodeBadRequest ErrorCode = "BAD_REQUEST" + CodeInternal ErrorCode = "INTERNAL_ERROR" + CodeUnauth ErrorCode = "UNAUTHORIZED" + CodeConflict ErrorCode = "CONFLICT" + CodeUnavailable ErrorCode = "UNAVAILABLE" + CodeTimeout ErrorCode = "TIMEOUT" + CodeResourceLimit ErrorCode = "RESOURCE_LIMIT" ) type APIError struct { diff --git a/internal/orchestrator/errors.go b/internal/orchestrator/errors.go new file mode 100644 index 0000000..6562a78 --- /dev/null +++ b/internal/orchestrator/errors.go @@ -0,0 +1,22 @@ +package orchestrator + +import ( + "errors" + "fmt" + + "github.com/StacyOs/stacyvm/internal/providers" +) + +var ( + ErrInvalidInput = errors.New("invalid input") + ErrSandboxNotFound = providers.ErrSandboxNotFound + ErrSandboxDestroyed = providers.ErrSandboxDestroyed + ErrProviderNotFound = providers.ErrProviderNotFound + ErrProviderUnavailable = providers.ErrProviderUnavailable + ErrExecTimeout = providers.ErrExecTimeout + ErrResourceLimit = providers.ErrResourceLimit +) + +func InvalidInputError(message string) error { + return fmt.Errorf("%w: %s", ErrInvalidInput, message) +} diff --git a/internal/orchestrator/events.go b/internal/orchestrator/events.go index fbcce10..666df58 100644 --- a/internal/orchestrator/events.go +++ b/internal/orchestrator/events.go @@ -2,6 +2,7 @@ package orchestrator import ( "encoding/json" + "strconv" "sync" "time" ) @@ -9,14 +10,25 @@ import ( type EventType string const ( - EventSandboxCreated EventType = "sandbox.created" - EventSandboxRunning EventType = "sandbox.running" - EventSandboxDestroyed EventType = "sandbox.destroyed" - EventSandboxError EventType = "sandbox.error" - EventExecStarted EventType = "exec.started" - EventExecCompleted EventType = "exec.completed" - EventFileWritten EventType = "file.written" - EventFileRead EventType = "file.read" + EventSandboxCreated EventType = "sandbox.created" + EventSandboxRunning EventType = "sandbox.running" + EventSandboxDestroyed EventType = "sandbox.destroyed" + EventSandboxError EventType = "sandbox.error" + EventExecStarted EventType = "exec.started" + EventExecCompleted EventType = "exec.completed" + EventExecFailed EventType = "exec.failed" + EventExecTimeout EventType = "exec.timeout" + EventFileWritten EventType = "file.written" + EventFileRead EventType = "file.read" + EventOperationFailed EventType = "operation.failed" + EventResourceLimit EventType = "resource.limit" + EventProviderFailed EventType = "provider.failed" + EventReconcileAction EventType = "reconcile.action" + EventSpawnQueued EventType = "spawn.queued" + EventSpawnDequeued EventType = "spawn.dequeued" + EventSpawnQueueTimeout EventType = "spawn.queue_timeout" + EventQuotaSaved EventType = "quota.saved" + EventQuotaDeleted EventType = "quota.deleted" ) type Event struct { @@ -33,12 +45,26 @@ const ( ) // EventBus is an in-process pub/sub system with ring buffer history. +// For single-node deployments this is sufficient. +// For HA deployments with multiple control-plane replicas, attach a Postgres +// LISTEN/NOTIFY bridge via AttachDurableBridge so cross-replica events are +// delivered to local subscribers. type EventBus struct { mu sync.RWMutex subscribers map[string]chan Event history []Event historySize int nextID int + + // onPublish is called after every local Publish. Used by the durable bridge + // to forward events to Postgres NOTIFY. Must be set before any Publish call. + onPublish func(Event) +} + +type EventBusStats struct { + Subscribers int `json:"subscribers"` + HistorySize int `json:"history_size"` + EventsTotal int `json:"events_total"` } func NewEventBus() *EventBus { @@ -58,6 +84,9 @@ func (eb *EventBus) Publish(evt Event) { evt.Timestamp = time.Now() } eb.nextID++ + if evt.ID == "" { + evt.ID = stringID(eb.nextID) + } // Ring buffer: append or overwrite oldest if len(eb.history) < eb.historySize { @@ -73,6 +102,52 @@ func (eb *EventBus) Publish(evt Event) { // Drop if subscriber is slow — non-blocking } } + + // Fire the durable bridge hook outside the lock to avoid deadlock. + hook := eb.onPublish + if hook != nil { + go hook(evt) + } +} + +// publishLocal delivers an event to local subscribers without firing the +// durable bridge hook. Used by the LISTEN goroutine to inject remote events +// received from Postgres NOTIFY. +func (eb *EventBus) publishLocal(evt Event) { + eb.mu.Lock() + defer eb.mu.Unlock() + + if evt.Timestamp.IsZero() { + evt.Timestamp = time.Now() + } + eb.nextID++ + if evt.ID == "" { + evt.ID = stringID(eb.nextID) + } + if len(eb.history) < eb.historySize { + eb.history = append(eb.history, evt) + } else { + eb.history[eb.nextID%eb.historySize] = evt + } + for _, ch := range eb.subscribers { + select { + case ch <- evt: + default: + } + } +} + +// AttachDurableBridge wires an onPublish hook into this bus. Call once before +// the bus is used. The hook runs in a goroutine per publish so it must be +// safe to call concurrently. +func (eb *EventBus) AttachDurableBridge(hook func(Event)) { + eb.mu.Lock() + defer eb.mu.Unlock() + eb.onPublish = hook +} + +func stringID(id int) string { + return "evt-" + strconv.Itoa(id) } // Subscribe creates a new subscription and returns a channel + unsubscribe key. @@ -113,3 +188,14 @@ func (eb *EventBus) History(limit int) []Event { copy(result, eb.history[start:]) return result } + +func (eb *EventBus) Stats() EventBusStats { + eb.mu.RLock() + defer eb.mu.RUnlock() + + return EventBusStats{ + Subscribers: len(eb.subscribers), + HistorySize: len(eb.history), + EventsTotal: eb.nextID, + } +} diff --git a/internal/orchestrator/events_durable.go b/internal/orchestrator/events_durable.go new file mode 100644 index 0000000..72977cd --- /dev/null +++ b/internal/orchestrator/events_durable.go @@ -0,0 +1,151 @@ +package orchestrator + +import ( + "context" + "encoding/json" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/rs/zerolog" +) + +const pgChannel = "stacyvm_events" + +// bridgeEnvelope wraps an Event with the source instance ID so the publishing +// instance can ignore its own reflected NOTIFY and avoid double-delivery. +type bridgeEnvelope struct { + SourceID string `json:"_src"` + Event +} + +// DurableBridge connects an EventBus to Postgres LISTEN/NOTIFY so that events +// published on one control-plane replica reach subscribers on all other replicas. +// +// Each bridge instance stamps published events with a unique instanceID. When +// its own NOTIFY is reflected back by Postgres, the listener skips it — +// preventing double-delivery to local subscribers. +// +// Usage: +// +// bus := orchestrator.NewEventBus() +// bridge, err := orchestrator.NewDurableBridge(ctx, dsn, bus, logger) +// if err != nil { ... } +// defer bridge.Stop() +// +// The bridge is a no-op for SQLite deployments — callers should only attach it +// when the store driver is Postgres. +type DurableBridge struct { + bus *EventBus + instanceID string + conn *pgx.Conn + cancel context.CancelFunc + done chan struct{} + logger zerolog.Logger +} + +// NewDurableBridge opens a dedicated Postgres connection for LISTEN/NOTIFY, +// attaches the onPublish hook to the given EventBus, and starts the listener +// goroutine. The returned bridge must be stopped with Stop() on shutdown. +func NewDurableBridge(ctx context.Context, dsn string, bus *EventBus, logger zerolog.Logger) (*DurableBridge, error) { + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + return nil, err + } + if _, err := conn.Exec(ctx, "LISTEN "+pgChannel); err != nil { + conn.Close(ctx) + return nil, err + } + + listenCtx, cancel := context.WithCancel(context.Background()) + b := &DurableBridge{ + bus: bus, + instanceID: uuid.New().String(), + conn: conn, + cancel: cancel, + done: make(chan struct{}), + logger: logger, + } + + // Wire the publish hook: NOTIFY Postgres when a local event is published. + bus.AttachDurableBridge(func(evt Event) { + b.notify(evt) + }) + + go b.listen(listenCtx) + return b, nil +} + +// Stop cancels the LISTEN goroutine and closes the connection. +func (b *DurableBridge) Stop() { + b.cancel() + <-b.done + b.conn.Close(context.Background()) +} + +func (b *DurableBridge) notify(evt Event) { + env := bridgeEnvelope{SourceID: b.instanceID, Event: evt} + payload, err := json.Marshal(env) + if err != nil { + b.logger.Warn().Err(err).Msg("durable bridge: failed to marshal event for NOTIFY") + return + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // pg_notify payload limit is 8000 bytes; strip Event.Data for large payloads. + if len(payload) > 7900 { + env.Data = nil + payload, _ = json.Marshal(env) + } + if _, err := b.conn.Exec(ctx, "SELECT pg_notify($1, $2)", pgChannel, string(payload)); err != nil { + b.logger.Warn().Err(err).Msg("durable bridge: pg_notify failed") + } +} + +func (b *DurableBridge) listen(ctx context.Context) { + defer close(b.done) + for { + notification, err := b.conn.WaitForNotification(ctx) + if err != nil { + if ctx.Err() != nil { + return // clean shutdown + } + b.logger.Warn().Err(err).Msg("durable bridge: lost LISTEN connection; events may be missed until reconnect") + // Attempt reconnect with exponential backoff (up to 20 s per attempt). + for attempt := 1; attempt <= 10; attempt++ { + select { + case <-ctx.Done(): + return + case <-time.After(time.Duration(attempt) * 2 * time.Second): + } + newConn, connErr := pgx.Connect(ctx, b.conn.Config().ConnString()) + if connErr != nil { + b.logger.Warn().Err(connErr).Int("attempt", attempt).Msg("durable bridge: reconnect failed") + continue + } + if _, execErr := newConn.Exec(ctx, "LISTEN "+pgChannel); execErr != nil { + newConn.Close(ctx) + b.logger.Warn().Err(execErr).Int("attempt", attempt).Msg("durable bridge: LISTEN after reconnect failed") + continue + } + b.conn = newConn + b.logger.Info().Int("attempt", attempt).Msg("durable bridge: reconnected") + break + } + continue + } + + var env bridgeEnvelope + if err := json.Unmarshal([]byte(notification.Payload), &env); err != nil { + b.logger.Warn().Err(err).Msg("durable bridge: failed to unmarshal notification payload") + continue + } + // Skip our own reflected NOTIFY — local subscribers already received + // this event via the in-process bus (double-delivery prevention). + if env.SourceID == b.instanceID { + continue + } + // publishLocal avoids re-notifying Postgres (prevents infinite loop). + b.bus.publishLocal(env.Event) + } +} diff --git a/internal/orchestrator/events_test.go b/internal/orchestrator/events_test.go index fe45104..0e5571b 100644 --- a/internal/orchestrator/events_test.go +++ b/internal/orchestrator/events_test.go @@ -22,6 +22,9 @@ func TestEventBus_SubscribePublish(t *testing.T) { if evt.SandboxID != "sb-001" { t.Fatalf("expected sb-001, got %s", evt.SandboxID) } + if evt.ID == "" { + t.Fatal("expected event ID") + } case <-time.After(time.Second): t.Fatal("timeout waiting for event") } diff --git a/internal/orchestrator/manager.go b/internal/orchestrator/manager.go index 171fb4a..474cb15 100644 --- a/internal/orchestrator/manager.go +++ b/internal/orchestrator/manager.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/json" + "errors" "fmt" "io" "path/filepath" @@ -11,9 +12,12 @@ import ( "sync" "time" + "github.com/StacyOs/stacyvm/internal/api/middleware" "github.com/StacyOs/stacyvm/internal/config" "github.com/StacyOs/stacyvm/internal/providers" "github.com/StacyOs/stacyvm/internal/store" + "github.com/StacyOs/stacyvm/internal/worker" + "github.com/StacyOs/stacyvm/internal/workerproto" "github.com/rs/zerolog" ) @@ -22,14 +26,26 @@ type Manager struct { store store.Store events *EventBus logger zerolog.Logger - - mu sync.RWMutex - sandboxes map[string]*Sandbox - - defaultTTL time.Duration - defaultImage string - defaultMemory int - defaultVCPUs int + metrics *MetricsRecorder + + mu sync.RWMutex + sandboxes map[string]*Sandbox + admissionMu sync.Mutex + queueMu sync.Mutex + queueWaiters int + capacityCh chan struct{} + queueStats spawnQueueStats + + defaultTTL time.Duration + defaultImage string + defaultMemory int + defaultVCPUs int + limits OperationalLimits + workerID string + workerToken string + workerSigningKey string + workerRevokedTokenIDs []string + workerRPCTLS worker.TLSConfig vmPoolMgr *VMPoolManager poolConfig config.PoolConfig @@ -40,31 +56,56 @@ type Manager struct { cancel context.CancelFunc } +type spawnQueueStats struct { + queuedTotal uint64 + dequeuedTotal uint64 + timeoutTotal uint64 + waitCount uint64 + waitTotal time.Duration + waitMax time.Duration +} + +const sandboxLeaseGrace = 5 * time.Minute + type ManagerConfig struct { - DefaultTTL time.Duration - DefaultImage string - DefaultMemory int - DefaultVCPUs int - Pool config.PoolConfig - PreviewDomain string + DefaultTTL time.Duration + DefaultImage string + DefaultMemory int + DefaultVCPUs int + Pool config.PoolConfig + PreviewDomain string + Limits OperationalLimits + WorkerID string + WorkerToken string + WorkerSigningKey string + WorkerRevokedTokenIDs []string + WorkerRPCTLS worker.TLSConfig } func NewManager(registry *providers.Registry, st store.Store, events *EventBus, logger zerolog.Logger, cfg ManagerConfig) *Manager { ctx, cancel := context.WithCancel(context.Background()) m := &Manager{ - registry: registry, - store: st, - events: events, - logger: logger.With().Str("component", "manager").Logger(), - sandboxes: make(map[string]*Sandbox), - defaultTTL: cfg.DefaultTTL, - defaultImage: cfg.DefaultImage, - defaultMemory: cfg.DefaultMemory, - defaultVCPUs: cfg.DefaultVCPUs, - poolConfig: cfg.Pool, - previewDomain: cfg.PreviewDomain, - ctx: ctx, - cancel: cancel, + registry: registry, + store: st, + events: events, + logger: logger.With().Str("component", "manager").Logger(), + metrics: NewMetricsRecorder(), + sandboxes: make(map[string]*Sandbox), + capacityCh: make(chan struct{}), + defaultTTL: cfg.DefaultTTL, + defaultImage: cfg.DefaultImage, + defaultMemory: cfg.DefaultMemory, + defaultVCPUs: cfg.DefaultVCPUs, + limits: cfg.Limits, + workerID: strings.TrimSpace(cfg.WorkerID), + workerToken: strings.TrimSpace(cfg.WorkerToken), + workerSigningKey: strings.TrimSpace(cfg.WorkerSigningKey), + workerRevokedTokenIDs: cfg.WorkerRevokedTokenIDs, + workerRPCTLS: cfg.WorkerRPCTLS, + poolConfig: cfg.Pool, + previewDomain: cfg.PreviewDomain, + ctx: ctx, + cancel: cancel, } if m.defaultTTL == 0 { m.defaultTTL = 30 * time.Minute @@ -78,6 +119,18 @@ func NewManager(registry *providers.Registry, st store.Store, events *EventBus, if m.defaultVCPUs == 0 { m.defaultVCPUs = 1 } + if m.limits.SpawnOverflow == "" { + m.limits.SpawnOverflow = "reject" + } + if m.limits.SpawnQueueTimeout == 0 { + m.limits.SpawnQueueTimeout = 30 * time.Second + } + if m.limits.MaxSpawnQueue == 0 { + m.limits.MaxSpawnQueue = 100 + } + if m.workerID == "" { + m.workerID = "local" + } return m } @@ -118,12 +171,282 @@ func (m *Manager) pruneExpired() { } } +// Reconcile refreshes persisted sandbox state against provider runtime state. +// It is intended for startup recovery after the server process restarts. +func (m *Manager) Reconcile(ctx context.Context) error { + records, err := m.store.ListSandboxes(ctx) + if err != nil { + return fmt.Errorf("listing sandboxes for reconciliation: %w", err) + } + + known := make(map[string]struct{}, len(records)) + for _, rec := range records { + known[rec.ID] = struct{}{} + if SandboxState(rec.State) == StateDestroyed { + continue + } + if handled, err := m.reconcileRemoteOwnedSandbox(ctx, rec); handled || err != nil { + if err != nil { + return err + } + continue + } + + prov, err := m.registry.Get(rec.Provider) + if err != nil { + m.logger.Warn().Err(err).Str("sandbox", rec.ID).Str("provider", rec.Provider).Msg("reconcile: provider unavailable") + m.publishOperationalEvent(EventProviderFailed, rec.ID, map[string]interface{}{ + "operation": "reconcile.status", + "provider": rec.Provider, + "error": err.Error(), + }) + if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateError)); updateErr != nil { + return fmt.Errorf("marking sandbox %s error: %w", rec.ID, updateErr) + } + continue + } + + status, err := prov.Status(ctx, rec.ID) + if err != nil { + if errors.Is(err, providers.ErrSandboxNotFound) || errors.Is(err, providers.ErrSandboxDestroyed) { + if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateDestroyed)); updateErr != nil { + return fmt.Errorf("marking stale sandbox %s destroyed: %w", rec.ID, updateErr) + } + m.mu.Lock() + delete(m.sandboxes, rec.ID) + m.mu.Unlock() + m.logger.Info().Str("sandbox", rec.ID).Msg("reconcile: stale sandbox marked destroyed") + m.publishOperationalEvent(EventReconcileAction, rec.ID, map[string]interface{}{ + "action": "marked_destroyed", + "provider": rec.Provider, + "reason": err.Error(), + }) + continue + } + m.logger.Warn().Err(err).Str("sandbox", rec.ID).Msg("reconcile: provider status failed") + m.publishOperationalEvent(EventProviderFailed, rec.ID, map[string]interface{}{ + "operation": "reconcile.status", + "provider": rec.Provider, + "error": err.Error(), + }) + if updateErr := m.store.UpdateSandboxState(ctx, rec.ID, string(StateError)); updateErr != nil { + return fmt.Errorf("marking sandbox %s error: %w", rec.ID, updateErr) + } + continue + } + + state := SandboxState(status.State) + if state == "" { + state = StateRunning + } + if state == StateDestroyed { + if err := m.store.UpdateSandboxState(ctx, rec.ID, string(StateDestroyed)); err != nil { + return fmt.Errorf("marking sandbox %s destroyed: %w", rec.ID, err) + } + continue + } + if state != SandboxState(rec.State) { + if err := m.store.UpdateSandboxState(ctx, rec.ID, string(state)); err != nil { + return fmt.Errorf("updating reconciled sandbox %s state: %w", rec.ID, err) + } + } + + sb := recordToSandbox(rec) + sb.State = state + m.applyPreviewDomain(ctx, sb) + m.mu.Lock() + m.sandboxes[rec.ID] = sb + m.mu.Unlock() + } + + if err := m.reconcileProviderRuntimes(ctx, known); err != nil { + return err + } + + return nil +} + +func (m *Manager) reconcileRemoteOwnedSandbox(ctx context.Context, rec *store.SandboxRecord) (bool, error) { + if rec == nil || strings.TrimSpace(rec.WorkerID) == "" || rec.WorkerID == m.workerID { + return false, nil + } + workerRec, err := m.store.GetWorker(ctx, rec.WorkerID) + reason := "" + if err != nil { + reason = "worker_missing" + } else if time.Since(workerRec.LastHeartbeat) > workerHeartbeatStaleAfter { + reason = "worker_stale" + } else if strings.EqualFold(workerRec.Status, "draining") { + m.publishRemoteOwnershipEvent(rec, "worker_draining", "worker is draining; keeping existing ownership") + sb := recordToSandbox(rec) + if refreshed, refreshErr := m.refreshRemoteSandboxStatus(ctx, sb); refreshErr == nil { + m.mu.Lock() + m.sandboxes[rec.ID] = refreshed + m.mu.Unlock() + } + return true, nil + } else if !strings.EqualFold(workerRec.Status, "online") { + reason = "worker_" + strings.TrimSpace(workerRec.Status) + } + if reason == "" { + sb := recordToSandbox(rec) + refreshed, refreshErr := m.refreshRemoteSandboxStatus(ctx, sb) + if refreshErr != nil { + reason = "worker_rpc_unavailable" + } else { + m.mu.Lock() + m.sandboxes[rec.ID] = refreshed + m.mu.Unlock() + return true, nil + } + } + + state := StateUnhealthy + action := "marked_unhealthy" + if !rec.ExpiresAt.IsZero() && time.Now().UTC().After(rec.ExpiresAt) { + state = StateExpired + action = "marked_expired" + _ = m.store.ReleaseLease(ctx, rec.ID, rec.WorkerID) + } + if SandboxState(rec.State) != state { + if err := m.store.UpdateSandboxState(ctx, rec.ID, string(state)); err != nil { + return true, fmt.Errorf("marking remote sandbox %s %s: %w", rec.ID, state, err) + } + } + sb := recordToSandbox(rec) + sb.State = state + m.applyPreviewDomain(ctx, sb) + m.mu.Lock() + m.sandboxes[rec.ID] = sb + m.mu.Unlock() + m.publishRemoteOwnershipEvent(rec, action, reason) + return true, nil +} + +func (m *Manager) publishRemoteOwnershipEvent(rec *store.SandboxRecord, action, reason string) { + m.logger.Info(). + Str("sandbox", rec.ID). + Str("worker", rec.WorkerID). + Str("action", action). + Str("reason", reason). + Msg("reconcile: remote sandbox ownership policy applied") + m.publishOperationalEvent(EventReconcileAction, rec.ID, map[string]interface{}{ + "action": action, + "reason": reason, + "worker": rec.WorkerID, + }) +} + +func (m *Manager) reconcileProviderRuntimes(ctx context.Context, known map[string]struct{}) error { + for _, name := range m.registry.List() { + prov, err := m.registry.Get(name) + if err != nil { + continue + } + lister, ok := prov.(providers.RuntimeSandboxLister) + if !ok { + continue + } + + runtimes, err := lister.ListRuntimeSandboxes(ctx) + if err != nil { + m.logger.Warn().Err(err).Str("provider", name).Msg("reconcile: runtime inventory failed") + m.publishOperationalEvent(EventProviderFailed, "", map[string]interface{}{ + "operation": "reconcile.runtime_inventory", + "provider": name, + "error": err.Error(), + }) + continue + } + for _, runtime := range runtimes { + if _, ok := known[runtime.ID]; ok { + continue + } + if runtime.State == "" { + runtime.State = string(StateRunning) + } + if SandboxState(runtime.State) == StateDestroyed { + continue + } + if runtime.Provider == "" { + runtime.Provider = prov.Name() + } + if runtime.Image == "" { + runtime.Image = m.defaultImage + } + if runtime.CreatedAt.IsZero() { + runtime.CreatedAt = time.Now().UTC() + } + expiresAt := runtime.CreatedAt.Add(m.defaultTTL) + if expiresAt.Before(time.Now()) { + expiresAt = time.Now().Add(m.defaultTTL) + } + if _, err := m.acquireSandboxLease(ctx, runtime.ID, expiresAt); err != nil { + m.logger.Warn().Err(err).Str("sandbox", runtime.ID).Msg("reconcile: skipping runtime adoption because lease is unavailable") + continue + } + + metaJSON, _ := json.Marshal(runtime.Metadata) + rec := &store.SandboxRecord{ + ID: runtime.ID, + State: runtime.State, + Provider: runtime.Provider, + Image: runtime.Image, + MemoryMB: m.defaultMemory, + VCPUs: m.defaultVCPUs, + Metadata: string(metaJSON), + WorkerID: m.workerID, + CreatedAt: runtime.CreatedAt, + ExpiresAt: expiresAt, + UpdatedAt: time.Now().UTC(), + } + if err := m.store.CreateSandbox(ctx, rec); err != nil { + if errors.Is(err, store.ErrConflict) { + continue + } + return fmt.Errorf("adopting runtime sandbox %s: %w", runtime.ID, err) + } + + sb := recordToSandbox(rec) + sb.PreviewDomain = m.previewDomain + m.mu.Lock() + m.sandboxes[runtime.ID] = sb + m.mu.Unlock() + known[runtime.ID] = struct{}{} + + m.logger.Info(). + Str("sandbox", runtime.ID). + Str("provider", runtime.Provider). + Msg("reconcile: adopted provider runtime") + m.publishOperationalEvent(EventReconcileAction, runtime.ID, map[string]interface{}{ + "action": "adopted_runtime", + "provider": runtime.Provider, + "image": runtime.Image, + }) + } + } + return nil +} + func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) { + start := time.Now() + metricsProvider := req.Provider + if metricsProvider == "" { + metricsProvider = m.registry.Default() + } + var metricsErr error + defer func() { + m.recordOperation(OperationSpawn, metricsProvider, time.Since(start), metricsErr) + }() + providerName := req.Provider prov, err := m.registry.Get(providerName) if err != nil { + metricsErr = err + m.publishOperationFailure(EventProviderFailed, "", OperationSpawn, metricsProvider, err) return nil, fmt.Errorf("getting provider: %w", err) } + metricsProvider = prov.Name() image := req.Image if image == "" { @@ -145,12 +468,40 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) } ttl = parsed } + ownerID, err := normalizeOptionalOwnerID(req.OwnerID) + if err != nil { + metricsErr = err + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + return nil, err + } + req.OwnerID = ownerID + if err := m.acquireSpawnAdmission(ctx, req.OwnerID, ttl, metricsProvider); err != nil { + metricsErr = err + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + return nil, err + } + defer m.admissionMu.Unlock() now := time.Now() // Pool mode: acquire a VM slot instead of spawning a new VM. if m.vmPoolMgr != nil { - return m.spawnPooled(ctx, prov, req, image, memMB, vcpus, ttl, now) + sb, err := m.spawnPooled(ctx, prov, req, image, memMB, vcpus, ttl, now) + metricsErr = err + if err != nil { + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + } + return sb, err + } + + placement := m.currentSpawnPlacement(ctx, metricsProvider) + if placement.SelectedID != "" && placement.SelectedID != m.workerID { + sb, err := m.spawnRemote(ctx, placement.SelectedID, req, metricsProvider, image, memMB, vcpus, ttl, now) + metricsErr = err + if err != nil { + m.publishFailureForError("", OperationSpawn, metricsProvider, err) + } + return sb, err } sb := &Sandbox{ @@ -160,6 +511,8 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) MemoryMB: memMB, VCPUs: vcpus, OwnerID: req.OwnerID, + TenantID: req.TenantID, + WorkerID: m.workerID, CreatedAt: now, ExpiresAt: now.Add(ttl), Metadata: req.Metadata, @@ -174,10 +527,18 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) Metadata: req.Metadata, }) if err != nil { + metricsErr = err + m.publishFailureForError("", OperationSpawn, metricsProvider, err) return nil, fmt.Errorf("spawning sandbox: %w", err) } sb.ID = id sb.State = StateRunning + if _, err := m.acquireSandboxLease(ctx, sb.ID, sb.ExpiresAt); err != nil { + _ = prov.Destroy(ctx, id) + metricsErr = err + m.publishOperationFailure(EventOperationFailed, id, OperationSpawn, metricsProvider, err) + return nil, fmt.Errorf("acquiring sandbox lease: %w", err) + } // Persist metaJSON, _ := json.Marshal(sb.Metadata) @@ -190,13 +551,18 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) VCPUs: sb.VCPUs, Metadata: string(metaJSON), OwnerID: sb.OwnerID, + TenantID: sb.TenantID, VMID: sb.VMID, + WorkerID: sb.WorkerID, CreatedAt: sb.CreatedAt, ExpiresAt: sb.ExpiresAt, UpdatedAt: now, }); err != nil { // Best effort: destroy the sandbox if DB write fails prov.Destroy(ctx, id) + _ = m.releaseSandboxLease(ctx, id) + metricsErr = err + m.publishOperationFailure(EventOperationFailed, id, OperationSpawn, metricsProvider, err) return nil, fmt.Errorf("persisting sandbox: %w", err) } @@ -212,11 +578,84 @@ func (m *Manager) Spawn(ctx context.Context, req SpawnRequest) (*Sandbox, error) Type: EventSandboxRunning, SandboxID: id, }) + m.auditOperation(ctx, "sandbox.spawn", sb, sb.ID, sb.Image, "success", "state=running") m.logger.Info().Str("sandbox", id).Str("provider", prov.Name()).Str("image", image).Msg("sandbox spawned") return sb, nil } +func (m *Manager) spawnRemote(ctx context.Context, workerID string, req SpawnRequest, providerName, image string, memMB, vcpus int, ttl time.Duration, now time.Time) (*Sandbox, error) { + client, err := m.remoteWorkerRPCClient(ctx, workerID) + if err != nil { + return nil, err + } + sandboxID := generateSandboxID() + expiresAt := now.Add(ttl) + lease, err := m.acquireSandboxLeaseFor(ctx, sandboxID, workerID, expiresAt) + if err != nil { + return nil, fmt.Errorf("acquiring remote sandbox lease: %w", err) + } + result, err := client.Spawn(ctx, "spawn-"+sandboxID, leaseTokenFromStore(lease), workerproto.SpawnParams{ + SandboxID: sandboxID, + Image: image, + Provider: providerName, + MemoryMB: memMB, + VCPUs: vcpus, + OwnerID: req.OwnerID, + TTL: ttl.String(), + Metadata: req.Metadata, + }) + if err != nil { + _ = m.store.ReleaseLease(ctx, sandboxID, workerID) + return nil, fmt.Errorf("remote worker spawn: %w", err) + } + sb := &Sandbox{ + ID: sandboxID, + State: StateRunning, + Provider: result.Provider, + Image: image, + MemoryMB: memMB, + VCPUs: vcpus, + OwnerID: req.OwnerID, + VMID: result.RuntimeID, + WorkerID: workerID, + CreatedAt: now, + ExpiresAt: expiresAt, + Metadata: req.Metadata, + PreviewDomain: m.previewDomainForWorker(ctx, workerID), + } + if sb.Provider == "" { + sb.Provider = providerName + } + metaJSON, _ := json.Marshal(sb.Metadata) + if err := m.store.CreateSandbox(ctx, &store.SandboxRecord{ + ID: sb.ID, + State: string(sb.State), + Provider: sb.Provider, + Image: sb.Image, + MemoryMB: sb.MemoryMB, + VCPUs: sb.VCPUs, + Metadata: string(metaJSON), + OwnerID: sb.OwnerID, + VMID: sb.VMID, + WorkerID: sb.WorkerID, + CreatedAt: sb.CreatedAt, + ExpiresAt: sb.ExpiresAt, + UpdatedAt: now, + }); err != nil { + _ = m.store.ReleaseLease(ctx, sandboxID, workerID) + return nil, fmt.Errorf("persisting remote sandbox: %w", err) + } + m.mu.Lock() + m.sandboxes[sandboxID] = sb + m.mu.Unlock() + m.events.Publish(Event{Type: EventSandboxCreated, SandboxID: sandboxID}) + m.events.Publish(Event{Type: EventSandboxRunning, SandboxID: sandboxID}) + m.auditOperation(ctx, "sandbox.spawn", sb, sb.ID, sb.Image, "success", "state=running remote_worker="+workerID) + m.logger.Info().Str("sandbox", sandboxID).Str("runtime", sb.VMID).Str("worker", workerID).Str("provider", sb.Provider).Str("image", image).Msg("remote sandbox spawned") + return sb, nil +} + // spawnPooled creates a logical sandbox on a shared pool VM. func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req SpawnRequest, image string, memMB, vcpus int, ttl time.Duration, now time.Time) (*Sandbox, error) { // Generate a logical sandbox ID (not tied to a VM). @@ -227,6 +666,9 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req // Acquire a VM slot (may spawn a new VM if needed). vmID, err := m.vmPoolMgr.Acquire(ctx, sandboxID) if err != nil { + if errors.Is(err, ErrVMPoolFull) { + return nil, providers.ResourceLimitError(err.Error()) + } return nil, fmt.Errorf("pool acquire: %w", err) } @@ -239,11 +681,17 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req VCPUs: vcpus, OwnerID: req.OwnerID, VMID: vmID, + WorkerID: m.workerID, CreatedAt: now, ExpiresAt: now.Add(ttl), Metadata: req.Metadata, PreviewDomain: m.previewDomain, } + if _, err := m.acquireSandboxLease(ctx, sb.ID, sb.ExpiresAt); err != nil { + m.vmPoolMgr.Release(vmID, sandboxID) + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationSpawn, prov.Name(), err) + return nil, fmt.Errorf("acquiring sandbox lease: %w", err) + } // Create the sandbox's isolated workspace on the VM. workspaceDir := "/workspace/" + sandboxID @@ -252,6 +700,8 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req }) if err != nil { m.vmPoolMgr.Release(vmID, sandboxID) + _ = m.releaseSandboxLease(ctx, sandboxID) + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationSpawn, prov.Name(), err) return nil, fmt.Errorf("creating workspace: %w", err) } @@ -267,11 +717,14 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req Metadata: string(metaJSON), OwnerID: sb.OwnerID, VMID: sb.VMID, + WorkerID: sb.WorkerID, CreatedAt: sb.CreatedAt, ExpiresAt: sb.ExpiresAt, UpdatedAt: now, }); err != nil { m.vmPoolMgr.Release(vmID, sandboxID) + _ = m.releaseSandboxLease(ctx, sandboxID) + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationSpawn, prov.Name(), err) return nil, fmt.Errorf("persisting sandbox: %w", err) } @@ -281,6 +734,7 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req m.events.Publish(Event{Type: EventSandboxCreated, SandboxID: sandboxID}) m.events.Publish(Event{Type: EventSandboxRunning, SandboxID: sandboxID}) + m.auditOperation(ctx, "sandbox.spawn", sb, sb.ID, sb.Image, "success", "state=running pooled=true") m.logger.Info(). Str("sandbox", sandboxID). @@ -291,10 +745,39 @@ func (m *Manager) spawnPooled(ctx context.Context, prov providers.Provider, req } func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) (*ExecResult, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationExec, metricsProvider, time.Since(start), metricsErr) + }() + + if err := validateExecRequest(req); err != nil { + metricsErr = err + m.publishFailureForError(sandboxID, OperationExec, metricsProvider, err) + return nil, err + } + sb, prov, err := m.getSandboxAndProvider(sandboxID) if err != nil { + metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExec, metricsProvider, err) return nil, err } + metricsProvider = sb.Provider + + execCtx := ctx + var cancel context.CancelFunc + timeout, err := m.resolveExecTimeout(req.Timeout, sb.OwnerID) + if err != nil { + metricsErr = err + m.publishFailureForError(sandboxID, OperationExec, metricsProvider, err) + return nil, err + } + if timeout > 0 { + execCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } m.events.Publish(Event{ Type: EventExecStarted, @@ -303,21 +786,36 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( // In pool mode, default workdir to the sandbox's workspace. workDir := req.WorkDir - if workDir == "" && sb.VMID != "" { + if workDir == "" && sb.VMID != "" && !m.isRemoteOwnedSandbox(sb) { workDir = "/workspace/" + sandboxID } - start := time.Now() - result, err := prov.Exec(ctx, m.resolveVMID(sb), providers.ExecOptions{ - Command: req.Command, - Args: req.Args, - Env: req.Env, - WorkDir: workDir, - }) + execStart := time.Now() + var result *providers.ExecResult + if m.isRemoteOwnedSandbox(sb) { + result, err = m.execRemote(execCtx, sb, req, workDir) + } else { + result, err = prov.Exec(execCtx, m.resolveVMID(sb), providers.ExecOptions{ + Command: req.Command, + Args: req.Args, + Mode: req.Mode, + Env: req.Env, + WorkDir: workDir, + }) + } if err != nil { + if execCtx.Err() == context.DeadlineExceeded { + metricsErr = providers.ExecTimeoutError(sandboxID) + m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExec, metricsProvider, metricsErr) + m.auditOperation(ctx, "exec", sb, sandboxID, req.Command, "failure", metricsErr.Error()) + return nil, metricsErr + } + metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExec, metricsProvider, err) + m.auditOperation(ctx, "exec", sb, sandboxID, req.Command, "failure", err.Error()) return nil, fmt.Errorf("exec: %w", err) } - duration := time.Since(start) + duration := time.Since(execStart) execResult := &ExecResult{ ExitCode: result.ExitCode, @@ -341,190 +839,1473 @@ func (m *Manager) Exec(ctx context.Context, sandboxID string, req ExecRequest) ( Type: EventExecCompleted, SandboxID: sandboxID, }) + m.auditOperation(ctx, "exec", sb, sandboxID, req.Command, "success", fmt.Sprintf("exit=%d duration=%s mode=%s", result.ExitCode, duration.String(), req.Mode)) + + return execResult, nil +} + +func (m *Manager) execRemote(ctx context.Context, sb *Sandbox, req ExecRequest, workDir string) (*providers.ExecResult, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return nil, err + } + runtimeID := strings.TrimSpace(sb.VMID) + if runtimeID == "" { + runtimeID = sb.ID + } + result, err := client.Exec(ctx, "exec-"+sb.ID, workerproto.ExecParams{ + SandboxID: sb.ID, + Provider: sb.Provider, + RuntimeID: runtimeID, + Command: req.Command, + Args: req.Args, + Mode: req.Mode, + Env: req.Env, + WorkDir: workDir, + Timeout: req.Timeout, + }) + if err != nil { + return nil, fmt.Errorf("remote worker exec: %w", err) + } + return &providers.ExecResult{ + ExitCode: result.ExitCode, + Stdout: result.Stdout, + Stderr: result.Stderr, + }, nil +} + +func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequest) (<-chan providers.StreamChunk, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + if metricsErr != nil { + m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), metricsErr) + } + }() + + if err := validateExecRequest(req); err != nil { + metricsErr = err + m.publishFailureForError(sandboxID, OperationExecStream, metricsProvider, err) + return nil, err + } + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, err) + return nil, err + } + metricsProvider = sb.Provider + + execCtx := ctx + var cancel context.CancelFunc + timeout, err := m.resolveExecTimeout(req.Timeout, sb.OwnerID) + if err != nil { + metricsErr = err + m.publishFailureForError(sandboxID, OperationExecStream, metricsProvider, err) + return nil, err + } + if timeout > 0 { + execCtx, cancel = context.WithTimeout(ctx, timeout) + } + + workDir := req.WorkDir + if workDir == "" && sb.VMID != "" && !m.isRemoteOwnedSandbox(sb) { + workDir = "/workspace/" + sandboxID + } + + var ch <-chan providers.StreamChunk + if m.isRemoteOwnedSandbox(sb) { + ch, err = m.execStreamRemote(execCtx, sb, req, workDir) + } else { + ch, err = prov.ExecStream(execCtx, m.resolveVMID(sb), providers.ExecOptions{ + Command: req.Command, + Args: req.Args, + Mode: req.Mode, + Env: req.Env, + WorkDir: workDir, + }) + } + if err != nil { + if cancel != nil { + cancel() + } + if execCtx.Err() == context.DeadlineExceeded { + metricsErr = providers.ExecTimeoutError(sandboxID) + m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExecStream, metricsProvider, metricsErr) + return nil, metricsErr + } + metricsErr = err + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, err) + return nil, err + } + if cancel == nil { + out := make(chan providers.StreamChunk, 64) + go func() { + defer close(out) + for chunk := range ch { + out <- chunk + } + m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), nil) + }() + return out, nil + } + + out := make(chan providers.StreamChunk, 64) + // metricsErr is read by the defer above; use a goroutine-local variable to + // avoid a data race between the defer (which runs at ExecStream return) and + // this goroutine (which may still be running). + go func() { + defer close(out) + defer cancel() + var goroutineErr error + interrupted := false + for chunk := range ch { + select { + case out <- chunk: + case <-execCtx.Done(): + interrupted = true + } + } + if errors.Is(execCtx.Err(), context.DeadlineExceeded) { + goroutineErr = providers.ExecTimeoutError(sandboxID) + m.publishOperationFailure(EventExecTimeout, sandboxID, OperationExecStream, metricsProvider, goroutineErr) + select { + case out <- providers.StreamChunk{Stream: "stderr", Data: goroutineErr.Error()}: + case <-ctx.Done(): + } + } else if interrupted && execCtx.Err() != nil { + goroutineErr = execCtx.Err() + m.publishOperationFailure(EventExecFailed, sandboxID, OperationExecStream, metricsProvider, goroutineErr) + } + m.recordOperation(OperationExecStream, metricsProvider, time.Since(start), goroutineErr) + }() + return out, nil +} + +func (m *Manager) execStreamRemote(ctx context.Context, sb *Sandbox, req ExecRequest, workDir string) (<-chan providers.StreamChunk, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return nil, err + } + runtimeID := strings.TrimSpace(sb.VMID) + if runtimeID == "" { + runtimeID = sb.ID + } + chunks, errs, err := client.ExecStreamLive(ctx, "exec-stream-"+sb.ID, workerproto.ExecParams{ + SandboxID: sb.ID, + Provider: sb.Provider, + RuntimeID: runtimeID, + Command: req.Command, + Args: req.Args, + Mode: req.Mode, + Env: req.Env, + WorkDir: workDir, + Timeout: req.Timeout, + }) + if err != nil { + return nil, fmt.Errorf("remote worker exec stream: %w", err) + } + out := make(chan providers.StreamChunk, 64) + go func() { + defer close(out) + for chunk := range chunks { + select { + case out <- providers.StreamChunk{Stream: chunk.Stream, Data: chunk.Data}: + case <-ctx.Done(): + return + } + } + for err := range errs { + if err != nil { + select { + case out <- providers.StreamChunk{Stream: "stderr", Data: err.Error()}: + case <-ctx.Done(): + } + } + } + }() + return out, nil +} + +func validateExecRequest(req ExecRequest) error { + switch strings.TrimSpace(req.Mode) { + case "", providers.ExecModeShell: + return nil + case providers.ExecModeArgv: + if strings.TrimSpace(req.Command) == "" { + return InvalidInputError("argv exec mode requires command") + } + return nil + default: + return InvalidInputError(fmt.Sprintf("unsupported exec mode %q", req.Mode)) + } +} + +func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWriteRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileWrite, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileWrite, metricsProvider, err) + return err + } + metricsProvider = sb.Provider + + mode := req.Mode + if mode == "" { + mode = "0644" + } + + path, err := m.scopedPathForOperation(sb, req.Path) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileWrite, metricsProvider, err) + m.auditOperation(ctx, "file.write", sb, sandboxID, req.Path, "failure", err.Error()) + return err + } + if m.isRemoteOwnedSandbox(sb) { + err = m.remoteFileWrite(ctx, sb, path, []byte(req.Content), mode) + } else { + err = prov.WriteFile(ctx, m.resolveVMID(sb), path, strings.NewReader(req.Content), mode) + } + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileWrite, metricsProvider, err) + m.auditOperation(ctx, "file.write", sb, sandboxID, req.Path, "failure", err.Error()) + return fmt.Errorf("writing file: %w", err) + } + + m.events.Publish(Event{ + Type: EventFileWritten, + SandboxID: sandboxID, + }) + m.auditOperation(ctx, "file.write", sb, sandboxID, req.Path, "success", "mode="+mode) + return nil +} + +func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ([]byte, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileRead, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) + return nil, err + } + metricsProvider = sb.Provider + + scopedPath, err := m.scopedPathForOperation(sb, path) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) + m.auditOperation(ctx, "file.read", sb, sandboxID, path, "failure", err.Error()) + return nil, err + } + var buf []byte + if m.isRemoteOwnedSandbox(sb) { + buf, err = m.remoteFileRead(ctx, sb, scopedPath) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) + m.auditOperation(ctx, "file.read", sb, sandboxID, path, "failure", err.Error()) + return nil, fmt.Errorf("reading file: %w", err) + } + } else { + rc, err := prov.ReadFile(ctx, m.resolveVMID(sb), scopedPath) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) + m.auditOperation(ctx, "file.read", sb, sandboxID, path, "failure", err.Error()) + return nil, fmt.Errorf("reading file: %w", err) + } + defer rc.Close() + + buf, err = io.ReadAll(rc) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileRead, metricsProvider, err) + return nil, fmt.Errorf("reading file content: %w", err) + } + } + + m.events.Publish(Event{ + Type: EventFileRead, + SandboxID: sandboxID, + }) + m.auditOperation(ctx, "file.read", sb, sandboxID, path, "success", fmt.Sprintf("bytes=%d", len(buf))) + return buf, nil +} + +func (m *Manager) ListFiles(ctx context.Context, sandboxID string, path string) ([]FileInfo, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileList, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileList, metricsProvider, err) + return nil, err + } + metricsProvider = sb.Provider + + scopedPath, err := m.scopedPathForOperation(sb, path) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileList, metricsProvider, err) + m.auditOperation(ctx, "file.list", sb, sandboxID, path, "failure", err.Error()) + return nil, err + } + var pFiles []providers.FileInfo + if m.isRemoteOwnedSandbox(sb) { + pFiles, err = m.remoteFileList(ctx, sb, scopedPath) + } else { + pFiles, err = prov.ListFiles(ctx, m.resolveVMID(sb), scopedPath) + } + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileList, metricsProvider, err) + m.auditOperation(ctx, "file.list", sb, sandboxID, path, "failure", err.Error()) + return nil, err + } + + files := make([]FileInfo, len(pFiles)) + for i, f := range pFiles { + files[i] = FileInfo{ + Path: f.Path, + Size: f.Size, + Mode: f.Mode, + IsDir: f.IsDir, + ModTime: f.ModTime, + } + } + m.auditOperation(ctx, "file.list", sb, sandboxID, path, "success", fmt.Sprintf("count=%d", len(files))) + return files, nil +} + +func (m *Manager) DeleteFile(ctx context.Context, sandboxID string, req FileDeleteRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileDelete, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileDelete, metricsProvider, err) + return err + } + metricsProvider = sb.Provider + + path, err := m.scopedPathForOperation(sb, req.Path) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileDelete, metricsProvider, err) + m.auditOperation(ctx, "file.delete", sb, sandboxID, req.Path, "failure", err.Error()) + return err + } + if m.isRemoteOwnedSandbox(sb) { + metricsErr = m.remoteFileDelete(ctx, sb, path, req.Recursive) + } else { + metricsErr = prov.DeleteFile(ctx, m.resolveVMID(sb), path, req.Recursive) + } + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileDelete, metricsProvider, metricsErr) + m.auditOperation(ctx, "file.delete", sb, sandboxID, req.Path, "failure", metricsErr.Error()) + return metricsErr + } + m.auditOperation(ctx, "file.delete", sb, sandboxID, req.Path, "success", fmt.Sprintf("recursive=%t", req.Recursive)) + return nil +} + +func (m *Manager) MoveFile(ctx context.Context, sandboxID string, req FileMoveRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileMove, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileMove, metricsProvider, err) + return err + } + metricsProvider = sb.Provider + + oldPath, err := m.scopedPathForOperation(sb, req.OldPath) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileMove, metricsProvider, err) + m.auditOperation(ctx, "file.move", sb, sandboxID, req.OldPath, "failure", err.Error()) + return err + } + newPath, err := m.scopedPathForOperation(sb, req.NewPath) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileMove, metricsProvider, err) + m.auditOperation(ctx, "file.move", sb, sandboxID, req.NewPath, "failure", err.Error()) + return err + } + if m.isRemoteOwnedSandbox(sb) { + metricsErr = m.remoteFileMove(ctx, sb, oldPath, newPath) + } else { + metricsErr = prov.MoveFile(ctx, m.resolveVMID(sb), oldPath, newPath) + } + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileMove, metricsProvider, metricsErr) + m.auditOperation(ctx, "file.move", sb, sandboxID, req.OldPath+" -> "+req.NewPath, "failure", metricsErr.Error()) + return metricsErr + } + m.auditOperation(ctx, "file.move", sb, sandboxID, req.OldPath+" -> "+req.NewPath, "success", "") + return nil +} + +func (m *Manager) ChmodFile(ctx context.Context, sandboxID string, req FileChmodRequest) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileChmod, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileChmod, metricsProvider, err) + return err + } + metricsProvider = sb.Provider + + path, err := m.scopedPathForOperation(sb, req.Path) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileChmod, metricsProvider, err) + m.auditOperation(ctx, "file.chmod", sb, sandboxID, req.Path, "failure", err.Error()) + return err + } + if m.isRemoteOwnedSandbox(sb) { + metricsErr = m.remoteFileChmod(ctx, sb, path, req.Mode) + } else { + metricsErr = prov.ChmodFile(ctx, m.resolveVMID(sb), path, req.Mode) + } + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileChmod, metricsProvider, metricsErr) + m.auditOperation(ctx, "file.chmod", sb, sandboxID, req.Path, "failure", metricsErr.Error()) + return metricsErr + } + m.auditOperation(ctx, "file.chmod", sb, sandboxID, req.Path, "success", "mode="+req.Mode) + return nil +} + +func (m *Manager) StatFile(ctx context.Context, sandboxID string, path string) (*FileInfo, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileStat, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileStat, metricsProvider, err) + return nil, err + } + metricsProvider = sb.Provider + + scopedPath, err := m.scopedPathForOperation(sb, path) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileStat, metricsProvider, err) + m.auditOperation(ctx, "file.stat", sb, sandboxID, path, "failure", err.Error()) + return nil, err + } + var fi *providers.FileInfo + if m.isRemoteOwnedSandbox(sb) { + fi, err = m.remoteFileStat(ctx, sb, scopedPath) + } else { + fi, err = prov.StatFile(ctx, m.resolveVMID(sb), scopedPath) + } + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileStat, metricsProvider, err) + m.auditOperation(ctx, "file.stat", sb, sandboxID, path, "failure", err.Error()) + return nil, err + } + m.auditOperation(ctx, "file.stat", sb, sandboxID, path, "success", "") + return &FileInfo{ + Path: fi.Path, + Size: fi.Size, + Mode: fi.Mode, + IsDir: fi.IsDir, + ModTime: fi.ModTime, + }, nil +} + +func (m *Manager) GlobFiles(ctx context.Context, sandboxID string, pattern string) ([]string, error) { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationFileGlob, metricsProvider, time.Since(start), metricsErr) + }() + + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileGlob, metricsProvider, err) + return nil, err + } + metricsProvider = sb.Provider + + scopedPattern, err := m.scopedPathForOperation(sb, pattern) + if err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileGlob, metricsProvider, err) + m.auditOperation(ctx, "file.glob", sb, sandboxID, pattern, "failure", err.Error()) + return nil, err + } + var matches []string + if m.isRemoteOwnedSandbox(sb) { + matches, err = m.remoteFileGlob(ctx, sb, scopedPattern) + } else { + matches, err = prov.GlobFiles(ctx, m.resolveVMID(sb), scopedPattern) + } + metricsErr = err + if err != nil { + m.publishOperationFailure(EventOperationFailed, sandboxID, OperationFileGlob, metricsProvider, err) + m.auditOperation(ctx, "file.glob", sb, sandboxID, pattern, "failure", err.Error()) + return matches, err + } + m.auditOperation(ctx, "file.glob", sb, sandboxID, pattern, "success", fmt.Sprintf("count=%d", len(matches))) + return matches, err +} + +func (m *Manager) remoteFileParams(sb *Sandbox) workerproto.FileParams { + runtimeID := strings.TrimSpace(sb.VMID) + if runtimeID == "" { + runtimeID = sb.ID + } + return workerproto.FileParams{ + SandboxID: sb.ID, + Provider: sb.Provider, + RuntimeID: runtimeID, + } +} + +func (m *Manager) remoteFileWrite(ctx context.Context, sb *Sandbox, path string, content []byte, mode string) error { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return err + } + params := m.remoteFileParams(sb) + params.Path = path + params.Content = content + params.Mode = mode + return client.FileWrite(ctx, "file-write-"+sb.ID, params) +} + +func (m *Manager) remoteFileRead(ctx context.Context, sb *Sandbox, path string) ([]byte, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return nil, err + } + params := m.remoteFileParams(sb) + params.Path = path + result, err := client.FileRead(ctx, "file-read-"+sb.ID, params) + if err != nil { + return nil, err + } + return result.Content, nil +} + +func (m *Manager) remoteFileList(ctx context.Context, sb *Sandbox, path string) ([]providers.FileInfo, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return nil, err + } + params := m.remoteFileParams(sb) + params.Path = path + result, err := client.FileList(ctx, "file-list-"+sb.ID, params) + if err != nil { + return nil, err + } + return fromWorkerFileInfo(result.Files), nil +} + +func (m *Manager) remoteFileDelete(ctx context.Context, sb *Sandbox, path string, recursive bool) error { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return err + } + params := m.remoteFileParams(sb) + params.Path = path + params.Recursive = recursive + return client.FileDelete(ctx, "file-delete-"+sb.ID, params) +} + +func (m *Manager) remoteFileMove(ctx context.Context, sb *Sandbox, oldPath, newPath string) error { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return err + } + params := m.remoteFileParams(sb) + params.OldPath = oldPath + params.NewPath = newPath + return client.FileMove(ctx, "file-move-"+sb.ID, params) +} + +func (m *Manager) remoteFileChmod(ctx context.Context, sb *Sandbox, path, mode string) error { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return err + } + params := m.remoteFileParams(sb) + params.Path = path + params.Mode = mode + return client.FileChmod(ctx, "file-chmod-"+sb.ID, params) +} + +func (m *Manager) remoteFileStat(ctx context.Context, sb *Sandbox, path string) (*providers.FileInfo, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return nil, err + } + params := m.remoteFileParams(sb) + params.Path = path + result, err := client.FileStat(ctx, "file-stat-"+sb.ID, params) + if err != nil { + return nil, err + } + file := providers.FileInfo{ + Path: result.File.Path, + Size: result.File.Size, + Mode: result.File.Mode, + IsDir: result.File.IsDir, + ModTime: result.File.ModTime, + } + return &file, nil +} + +func (m *Manager) remoteFileGlob(ctx context.Context, sb *Sandbox, pattern string) ([]string, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return nil, err + } + params := m.remoteFileParams(sb) + params.Pattern = pattern + result, err := client.FileGlob(ctx, "file-glob-"+sb.ID, params) + if err != nil { + return nil, err + } + return result.Matches, nil +} + +func fromWorkerFileInfo(files []workerproto.FileInfo) []providers.FileInfo { + out := make([]providers.FileInfo, len(files)) + for i, file := range files { + out[i] = providers.FileInfo{ + Path: file.Path, + Size: file.Size, + Mode: file.Mode, + IsDir: file.IsDir, + ModTime: file.ModTime, + } + } + return out +} + +// scopedPath prefixes a path with the sandbox workspace when running in pool mode. +func (m *Manager) scopedPath(sb *Sandbox, path string) string { + if sb.VMID == "" || m.isRemoteOwnedSandbox(sb) { + return path // dedicated VM, no scoping + } + base := "/workspace/" + sb.ID + cleaned := filepath.Clean(filepath.Join(base, path)) + if !strings.HasPrefix(cleaned, base+"/") && cleaned != base { + return base + } + return cleaned +} + +func (m *Manager) scopedPathForOperation(sb *Sandbox, path string) (string, error) { + if strings.TrimSpace(path) == "" { + return "", InvalidInputError("path is required") + } + if sb.VMID == "" || m.isRemoteOwnedSandbox(sb) { + return path, nil + } + base := "/workspace/" + sb.ID + cleaned := filepath.Clean(filepath.Join(base, path)) + if !strings.HasPrefix(cleaned, base+"/") && cleaned != base { + return "", InvalidInputError(fmt.Sprintf("path %q escapes sandbox workspace", path)) + } + return cleaned, nil +} + +func (m *Manager) auditOperation(ctx context.Context, action string, sb *Sandbox, sandboxID, resource, status, detail string) { + rec := &store.OperationAuditRecord{ + Action: action, + SandboxID: sandboxID, + Resource: resource, + Status: status, + Detail: truncateAuditDetail(detail), + CreatedAt: time.Now().UTC(), + } + if sb != nil { + rec.Actor = sb.OwnerID + rec.Provider = sb.Provider + if rec.SandboxID == "" { + rec.SandboxID = sb.ID + } + } + if err := m.store.CreateOperationAudit(ctx, rec); err != nil { + m.logger.Debug().Err(err).Str("action", action).Str("sandbox", sandboxID).Msg("operation audit write failed") + } +} + +func truncateAuditDetail(detail string) string { + const maxAuditDetailLen = 2048 + if len(detail) <= maxAuditDetailLen { + return detail + } + return detail[:maxAuditDetailLen] +} + +// resolveVMID returns the VM sandbox ID for provider calls. +// In pool mode, operations go to the VM (sb.VMID); in 1:1 mode, to the sandbox itself. +func (m *Manager) resolveVMID(sb *Sandbox) string { + if sb.VMID != "" { + return sb.VMID + } + return sb.ID +} + +func (m *Manager) isRemoteOwnedSandbox(sb *Sandbox) bool { + return sb != nil && strings.TrimSpace(sb.WorkerID) != "" && sb.WorkerID != m.workerID +} + +// VMPoolStatus returns the current VM pool status. Returns nil if pool is disabled. +func (m *Manager) VMPoolStatus() *VMPoolStatus { + if m.vmPoolMgr == nil { + return nil + } + status := m.vmPoolMgr.Status() + return &status +} + +func (m *Manager) OperationMetrics() []OperationMetrics { + return m.metrics.Snapshot() +} + +func (m *Manager) Limits() OperationalLimitsInfo { + return OperationalLimitsInfo{ + MaxSandboxes: m.limits.MaxSandboxes, + MaxSandboxesPerOwner: m.limits.MaxSandboxesPerOwner, + DefaultExecTimeout: m.limits.DefaultExecTimeout.String(), + MaxExecTimeout: m.limits.MaxExecTimeout.String(), + MaxTTL: m.limits.MaxTTL.String(), + SpawnOverflow: m.limits.SpawnOverflow, + SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), + MaxSpawnQueue: m.limits.MaxSpawnQueue, + } +} + +func (m *Manager) SchedulerStatus() SchedulerStatus { + m.queueMu.Lock() + waitAvg := time.Duration(0) + if m.queueStats.waitCount > 0 { + waitAvg = time.Duration(int64(m.queueStats.waitTotal) / int64(m.queueStats.waitCount)) + } + queueWaiters := m.queueWaiters + queuedTotal := m.queueStats.queuedTotal + dequeuedTotal := m.queueStats.dequeuedTotal + timeoutTotal := m.queueStats.timeoutTotal + waitCount := m.queueStats.waitCount + waitTotal := m.queueStats.waitTotal + waitMax := m.queueStats.waitMax + m.queueMu.Unlock() + + placement := workerPlacement{SelectedID: m.workerID, Eligible: 1} + if records, err := m.store.ListSandboxes(context.Background()); err == nil { + placement = m.evaluateWorkerPlacement(context.Background(), m.registry.Default(), records) + } + return SchedulerStatus{ + SpawnOverflow: m.limits.SpawnOverflow, + SpawnQueueDepth: queueWaiters, + MaxSpawnQueue: m.limits.MaxSpawnQueue, + SpawnQueueTimeout: m.limits.SpawnQueueTimeout.String(), + AdmissionControl: "worker_aware_local", + WorkerID: m.workerID, + SelectedWorkerID: placement.SelectedID, + EligibleWorkers: placement.Eligible, + SpawnQueuedTotal: queuedTotal, + SpawnDequeuedTotal: dequeuedTotal, + SpawnQueueTimeouts: timeoutTotal, + SpawnQueueWaitCount: waitCount, + SpawnQueueWaitTotal: waitTotal.String(), + SpawnQueueWaitMax: waitMax.String(), + SpawnQueueWaitAvg: waitAvg.String(), + SpawnQueueWaitTotalMS: waitTotal.Milliseconds(), + SpawnQueueWaitMaxMS: waitMax.Milliseconds(), + SpawnQueueWaitAvgMS: waitAvg.Milliseconds(), + } +} + +func (m *Manager) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuota, error) { + ownerID, err := normalizeOwnerID(ownerID) + if err != nil { + return nil, err + } + rec, err := m.store.GetOwnerQuota(ctx, ownerID) + if err != nil { + return nil, err + } + return ownerQuotaFromRecord(rec), nil +} + +func (m *Manager) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuota, error) { + records, err := m.store.ListOwnerQuotas(ctx) + if err != nil { + return nil, err + } + quotas := make([]*OwnerQuota, 0, len(records)) + for _, rec := range records { + quotas = append(quotas, ownerQuotaFromRecord(rec)) + } + return quotas, nil +} + +func (m *Manager) QuotaSummary(ctx context.Context) (QuotaSummary, error) { + records, err := m.store.ListOwnerQuotas(ctx) + if err != nil { + return QuotaSummary{}, err + } + summary := QuotaSummary{Total: len(records)} + for _, quota := range records { + if quota.MaxSandboxes > 0 { + summary.WithMaxSandboxes++ + } + if quota.MaxTTLSeconds > 0 { + summary.WithMaxTTL++ + } + if quota.MaxExecTimeoutSeconds > 0 { + summary.WithMaxExecTimeout++ + } + } + return summary, nil +} + +func (m *Manager) SaveOwnerQuota(ctx context.Context, quota OwnerQuota) (*OwnerQuota, error) { + ownerID, err := normalizeOwnerID(quota.OwnerID) + if err != nil { + return nil, err + } + quota.OwnerID = ownerID + maxTTL, err := parseOptionalDurationSeconds(quota.MaxTTL) + if err != nil { + return nil, InvalidInputError(fmt.Sprintf("parsing max_ttl: %v", err)) + } + maxExecTimeout, err := parseOptionalDurationSeconds(quota.MaxExecTimeout) + if err != nil { + return nil, InvalidInputError(fmt.Sprintf("parsing max_exec_timeout: %v", err)) + } + if quota.MaxSandboxes < 0 { + return nil, InvalidInputError("max_sandboxes cannot be negative") + } + rec := &store.OwnerQuotaRecord{ + OwnerID: quota.OwnerID, + MaxSandboxes: quota.MaxSandboxes, + MaxTTLSeconds: maxTTL, + MaxExecTimeoutSeconds: maxExecTimeout, + } + if err := m.store.SaveOwnerQuota(ctx, rec); err != nil { + return nil, err + } + saved, err := m.GetOwnerQuota(ctx, quota.OwnerID) + if err != nil { + return nil, err + } + m.publishOperationalEvent(EventQuotaSaved, "", map[string]interface{}{ + "owner_id": saved.OwnerID, + "max_sandboxes": saved.MaxSandboxes, + "max_ttl": saved.MaxTTL, + "max_exec_timeout": saved.MaxExecTimeout, + }) + m.notifySpawnCapacity() + return saved, nil +} + +func (m *Manager) DeleteOwnerQuota(ctx context.Context, ownerID string) error { + ownerID, err := normalizeOwnerID(ownerID) + if err != nil { + return err + } + if err := m.store.DeleteOwnerQuota(ctx, ownerID); err != nil { + return err + } + m.publishOperationalEvent(EventQuotaDeleted, "", map[string]interface{}{ + "owner_id": ownerID, + }) + m.notifySpawnCapacity() + return nil +} + +func (m *Manager) OwnerUsage(ctx context.Context, ownerID string) (*OwnerUsage, error) { + ownerID, err := normalizeOwnerID(ownerID) + if err != nil { + return nil, err + } + records, err := m.store.ListSandboxesByOwner(ctx, ownerID) + if err != nil { + return nil, err + } + usage := &OwnerUsage{ + OwnerID: ownerID, + ActiveSandboxes: len(records), + MaxSandboxes: m.limits.MaxSandboxesPerOwner, + MaxTTL: m.limits.MaxTTL.String(), + MaxExecTimeout: m.limits.MaxExecTimeout.String(), + } + if quota, err := m.store.GetOwnerQuota(ctx, ownerID); err == nil { + usage.QuotaConfigured = true + if quota.MaxSandboxes > 0 { + usage.MaxSandboxes = quota.MaxSandboxes + } + if quota.MaxTTLSeconds > 0 { + usage.MaxTTL = (time.Duration(quota.MaxTTLSeconds) * time.Second).String() + } + if quota.MaxExecTimeoutSeconds > 0 { + usage.MaxExecTimeout = (time.Duration(quota.MaxExecTimeoutSeconds) * time.Second).String() + } + } + return usage, nil +} + +func (m *Manager) recordOperation(operation, provider string, duration time.Duration, err error) { + if m.metrics == nil { + return + } + m.metrics.RecordOperation(operation, provider, duration, err) +} + +func (m *Manager) publishFailureForError(sandboxID, operation, provider string, err error) { + if errors.Is(err, providers.ErrResourceLimit) { + m.publishOperationFailure(EventResourceLimit, sandboxID, operation, provider, err) + return + } + if errors.Is(err, providers.ErrProviderUnavailable) || errors.Is(err, providers.ErrProviderNotFound) { + m.publishOperationFailure(EventProviderFailed, sandboxID, operation, provider, err) + return + } + m.publishOperationFailure(EventOperationFailed, sandboxID, operation, provider, err) +} + +func (m *Manager) publishOperationFailure(eventType EventType, sandboxID, operation, provider string, err error) { + if err == nil { + return + } + m.publishOperationalEvent(eventType, sandboxID, map[string]interface{}{ + "operation": operation, + "provider": provider, + "error": err.Error(), + }) +} + +func (m *Manager) publishOperationalEvent(eventType EventType, sandboxID string, data map[string]interface{}) { + if m.events == nil { + return + } + payload, _ := json.Marshal(data) + m.events.Publish(Event{ + Type: eventType, + SandboxID: sandboxID, + Data: payload, + }) +} + +func (m *Manager) acquireSpawnAdmission(ctx context.Context, ownerID string, ttl time.Duration, provider string) error { + for { + if err := m.waitForSpawnCapacity(ctx, ownerID, ttl, provider); err != nil { + return err + } - return execResult, nil + m.admissionMu.Lock() + decision, err := m.evaluateSpawnAdmission(ctx, ownerID, ttl, provider) + if err != nil { + m.admissionMu.Unlock() + return err + } + if decision.Allowed { + return nil + } + m.admissionMu.Unlock() + + if !decision.Queueable || !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + return spawnAdmissionError(decision) + } + } } -func (m *Manager) ExecStream(ctx context.Context, sandboxID string, req ExecRequest) (<-chan providers.StreamChunk, error) { - sb, prov, err := m.getSandboxAndProvider(sandboxID) +func (m *Manager) waitForSpawnCapacity(ctx context.Context, ownerID string, ttl time.Duration, provider string) error { + decision, err := m.evaluateSpawnAdmission(ctx, ownerID, ttl, provider) if err != nil { - return nil, err + return err + } + if decision.Allowed { + return nil + } + if !decision.Queueable || !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + return spawnAdmissionError(decision) } - workDir := req.WorkDir - if workDir == "" && sb.VMID != "" { - workDir = "/workspace/" + sandboxID + m.queueMu.Lock() + if m.queueWaiters >= m.limits.MaxSpawnQueue { + m.queueMu.Unlock() + return providers.ResourceLimitError(fmt.Sprintf("spawn queue full (%d)", m.limits.MaxSpawnQueue)) } + m.queueWaiters++ + m.queueStats.queuedTotal++ + depth := m.queueWaiters + capacityCh := m.capacityCh + m.queueMu.Unlock() + queuedAt := time.Now() - return prov.ExecStream(ctx, m.resolveVMID(sb), providers.ExecOptions{ - Command: req.Command, - Args: req.Args, - Env: req.Env, - WorkDir: workDir, + m.publishOperationalEvent(EventSpawnQueued, "", map[string]interface{}{ + "operation": OperationSpawn, + "provider": provider, + "owner_id": ownerID, + "depth": depth, }) -} + defer func() { + m.queueMu.Lock() + m.queueWaiters-- + m.queueMu.Unlock() + }() -func (m *Manager) WriteFile(ctx context.Context, sandboxID string, req FileWriteRequest) error { - sb, prov, err := m.getSandboxAndProvider(sandboxID) - if err != nil { - return err + waitCtx := ctx + cancel := func() {} + if m.limits.SpawnQueueTimeout > 0 { + waitCtx, cancel = context.WithTimeout(ctx, m.limits.SpawnQueueTimeout) } + defer cancel() - mode := req.Mode - if mode == "" { - mode = "0644" + for { + select { + case <-waitCtx.Done(): + if errors.Is(waitCtx.Err(), context.DeadlineExceeded) { + err := providers.ResourceLimitError(fmt.Sprintf("spawn queue timeout after %s", m.limits.SpawnQueueTimeout)) + waitDuration := time.Since(queuedAt) + m.recordSpawnQueueTimeout(waitDuration) + m.publishOperationalEvent(EventSpawnQueueTimeout, "", map[string]interface{}{ + "operation": OperationSpawn, + "provider": provider, + "owner_id": ownerID, + "error": err.Error(), + "wait_ms": waitDuration.Milliseconds(), + }) + return err + } + return waitCtx.Err() + case <-capacityCh: + decision, err = m.evaluateSpawnAdmission(ctx, ownerID, ttl, provider) + if err != nil { + return err + } + if decision.Allowed { + waitDuration := time.Since(queuedAt) + m.recordSpawnDequeued(waitDuration) + m.publishOperationalEvent(EventSpawnDequeued, "", map[string]interface{}{ + "operation": OperationSpawn, + "provider": provider, + "owner_id": ownerID, + "wait_ms": waitDuration.Milliseconds(), + }) + return nil + } + if !decision.Queueable { + return spawnAdmissionError(decision) + } + m.queueMu.Lock() + capacityCh = m.capacityCh + m.queueMu.Unlock() + } } +} - path := m.scopedPath(sb, req.Path) - if err := prov.WriteFile(ctx, m.resolveVMID(sb), path, strings.NewReader(req.Content), mode); err != nil { - return fmt.Errorf("writing file: %w", err) +func (m *Manager) recordSpawnDequeued(waitDuration time.Duration) { + m.queueMu.Lock() + defer m.queueMu.Unlock() + m.queueStats.dequeuedTotal++ + m.queueStats.waitCount++ + m.queueStats.waitTotal += waitDuration + if waitDuration > m.queueStats.waitMax { + m.queueStats.waitMax = waitDuration } - - m.events.Publish(Event{ - Type: EventFileWritten, - SandboxID: sandboxID, - }) - return nil } -func (m *Manager) ReadFile(ctx context.Context, sandboxID string, path string) ([]byte, error) { - sb, prov, err := m.getSandboxAndProvider(sandboxID) - if err != nil { - return nil, err +func (m *Manager) recordSpawnQueueTimeout(waitDuration time.Duration) { + m.queueMu.Lock() + defer m.queueMu.Unlock() + m.queueStats.timeoutTotal++ + m.queueStats.waitCount++ + m.queueStats.waitTotal += waitDuration + if waitDuration > m.queueStats.waitMax { + m.queueStats.waitMax = waitDuration } +} - rc, err := prov.ReadFile(ctx, m.resolveVMID(sb), m.scopedPath(sb, path)) - if err != nil { - return nil, fmt.Errorf("reading file: %w", err) +func (m *Manager) notifySpawnCapacity() { + m.queueMu.Lock() + close(m.capacityCh) + m.capacityCh = make(chan struct{}) + m.queueMu.Unlock() +} + +func (m *Manager) acquireSandboxLease(ctx context.Context, sandboxID string, expiresAt time.Time) (*store.LeaseRecord, error) { + return m.acquireSandboxLeaseFor(ctx, sandboxID, m.workerID, expiresAt) +} + +func (m *Manager) acquireSandboxLeaseFor(ctx context.Context, sandboxID, workerID string, expiresAt time.Time) (*store.LeaseRecord, error) { + ttl := time.Until(expiresAt) + sandboxLeaseGrace + if ttl <= 0 { + ttl = sandboxLeaseGrace } - defer rc.Close() + return m.store.AcquireLease(ctx, sandboxID, "sandbox", workerID, ttl) +} - buf, err := io.ReadAll(rc) - if err != nil { - return nil, fmt.Errorf("reading file content: %w", err) +func (m *Manager) releaseSandboxLease(ctx context.Context, sandboxID string) error { + err := m.store.ReleaseLease(ctx, sandboxID, m.workerID) + if errors.Is(err, store.ErrNotFound) { + return nil } + return err +} - m.events.Publish(Event{ - Type: EventFileRead, - SandboxID: sandboxID, - }) - return buf, nil +func (m *Manager) EvaluateSpawnAdmission(ctx context.Context, ownerID string, ttl time.Duration) (SpawnAdmissionDecision, error) { + return m.evaluateSpawnAdmission(ctx, ownerID, ttl, "") } -func (m *Manager) ListFiles(ctx context.Context, sandboxID string, path string) ([]FileInfo, error) { - sb, prov, err := m.getSandboxAndProvider(sandboxID) - if err != nil { - return nil, err +func (m *Manager) evaluateSpawnAdmission(ctx context.Context, ownerID string, ttl time.Duration, provider string) (SpawnAdmissionDecision, error) { + maxTTL, maxPerOwner := m.ownerLimitOverrides(ctx, ownerID) + decision := SpawnAdmissionDecision{ + Allowed: true, + MaxSandboxes: m.limits.MaxSandboxes, + MaxOwnerSandboxes: maxPerOwner, + } + if maxTTL > 0 { + decision.MaxTTL = maxTTL.String() + } + if maxTTL > 0 && ttl > maxTTL { + decision.Allowed = false + decision.Reason = "max_ttl" + return decision, nil } - pFiles, err := prov.ListFiles(ctx, m.resolveVMID(sb), m.scopedPath(sb, path)) + records, err := m.store.ListSandboxes(ctx) if err != nil { - return nil, err + return SpawnAdmissionDecision{}, fmt.Errorf("checking sandbox limits: %w", err) + } + placement := m.evaluateWorkerPlacement(ctx, provider, records) + decision.SelectedWorkerID = placement.SelectedID + decision.EligibleWorkers = placement.Eligible + decision.WorkerReason = placement.Reason + if placement.SelectedID == "" { + decision.Allowed = false + decision.Reason = "worker_unavailable" + return decision, nil + } + if placement.SelectedID != m.workerID { + decision.WorkerReason = "remote_worker_selected" + if !m.canUseRemoteWorker(ctx, placement.SelectedID) { + decision.Allowed = false + decision.Reason = "remote_worker_rpc_unavailable" + return decision, nil + } } - files := make([]FileInfo, len(pFiles)) - for i, f := range pFiles { - files[i] = FileInfo{ - Path: f.Path, - Size: f.Size, - Mode: f.Mode, - IsDir: f.IsDir, - ModTime: f.ModTime, + total := 0 + ownerTotal := 0 + for _, rec := range records { + if SandboxState(rec.State) == StateDestroyed { + continue + } + total++ + if ownerID != "" && rec.OwnerID == ownerID { + ownerTotal++ } } - return files, nil + decision.ActiveSandboxes = total + decision.ActiveOwnerSandboxes = ownerTotal + if m.limits.MaxSandboxes > 0 && total >= m.limits.MaxSandboxes { + decision.Allowed = false + decision.Queueable = true + decision.Reason = "max_sandboxes" + return decision, nil + } + if ownerID != "" && maxPerOwner > 0 && ownerTotal >= maxPerOwner { + decision.Allowed = false + decision.Queueable = true + decision.Reason = "max_sandboxes_per_owner" + return decision, nil + } + return decision, nil } -func (m *Manager) DeleteFile(ctx context.Context, sandboxID string, req FileDeleteRequest) error { - sb, prov, err := m.getSandboxAndProvider(sandboxID) +func (m *Manager) currentSpawnPlacement(ctx context.Context, provider string) workerPlacement { + records, err := m.store.ListSandboxes(ctx) if err != nil { - return err + return workerPlacement{SelectedID: m.workerID, Eligible: 1, Reason: "local_fallback"} } + return m.evaluateWorkerPlacement(ctx, provider, records) +} - path := m.scopedPath(sb, req.Path) - return prov.DeleteFile(ctx, m.resolveVMID(sb), path, req.Recursive) +func (m *Manager) canUseRemoteWorker(ctx context.Context, workerID string) bool { + _, err := m.remoteWorkerRPCClient(ctx, workerID) + return err == nil } -func (m *Manager) MoveFile(ctx context.Context, sandboxID string, req FileMoveRequest) error { - sb, prov, err := m.getSandboxAndProvider(sandboxID) +func (m *Manager) remoteWorkerRPCClient(ctx context.Context, workerID string) (worker.RPCClient, error) { + var zero worker.RPCClient + if strings.TrimSpace(m.workerToken) == "" && strings.TrimSpace(m.workerSigningKey) == "" { + return zero, fmt.Errorf("worker token is required for remote worker RPC") + } + rec, err := m.store.GetWorker(ctx, workerID) if err != nil { - return err + return zero, err + } + rpcURL := workerRPCURL(rec) + if rpcURL == "" { + return zero, fmt.Errorf("worker %s has no rpc_url", workerID) + } + client := worker.RPCClient{ + BaseURL: rpcURL, + WorkerID: workerID, + Token: m.workerToken, + RPCTLS: m.workerRPCTLS, + } + if strings.TrimSpace(client.Token) == "" && strings.TrimSpace(m.workerSigningKey) != "" { + client.TokenFunc = func() (string, error) { + now := time.Now().UTC() + tokenID, err := middleware.NewWorkerTokenID() + if err != nil { + return "", err + } + return middleware.SignWorkerToken(m.workerSigningKey, middleware.WorkerTokenClaims{ + WorkerID: workerID, + TokenID: tokenID, + Audience: middleware.WorkerTokenAudienceRPC, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(5 * time.Minute).Unix(), + }) + } } + return client, nil +} - oldPath := m.scopedPath(sb, req.OldPath) - newPath := m.scopedPath(sb, req.NewPath) - return prov.MoveFile(ctx, m.resolveVMID(sb), oldPath, newPath) +func leaseTokenFromStore(rec *store.LeaseRecord) workerproto.LeaseToken { + if rec == nil { + return workerproto.LeaseToken{} + } + return workerproto.LeaseToken{ + ResourceID: rec.ResourceID, + HolderID: rec.HolderID, + Generation: rec.Generation, + ExpiresAt: rec.ExpiresAt, + } } -func (m *Manager) ChmodFile(ctx context.Context, sandboxID string, req FileChmodRequest) error { - sb, prov, err := m.getSandboxAndProvider(sandboxID) +func generateSandboxID() string { + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + return fmt.Sprintf("sb-%d", time.Now().UnixNano()) + } + return fmt.Sprintf("sb-%08x", b) +} + +// EvaluateSpawnRequestAdmission evaluates a spawn request against the current +// quota and scheduler limits without creating provider resources. +func (m *Manager) EvaluateSpawnRequestAdmission(ctx context.Context, req SpawnRequest) (SpawnAdmissionDecision, error) { + ttl := m.defaultTTL + if req.TTL != "" { + parsed, err := time.ParseDuration(req.TTL) + if err != nil { + return SpawnAdmissionDecision{}, fmt.Errorf("%w: parsing TTL: %v", ErrInvalidInput, err) + } + ttl = parsed + } + ownerID, err := normalizeOptionalOwnerID(req.OwnerID) if err != nil { - return err + return SpawnAdmissionDecision{}, err + } + providerName := req.Provider + if providerName == "" { + providerName = m.registry.Default() } + decision, err := m.evaluateSpawnAdmission(ctx, ownerID, ttl, providerName) + if err != nil { + return SpawnAdmissionDecision{}, err + } + if decision.Queueable && !strings.EqualFold(m.limits.SpawnOverflow, "queue") { + decision.Queueable = false + } + return decision, nil +} - path := m.scopedPath(sb, req.Path) - return prov.ChmodFile(ctx, m.resolveVMID(sb), path, req.Mode) +func spawnAdmissionError(decision SpawnAdmissionDecision) error { + switch decision.Reason { + case "max_ttl": + return providers.ResourceLimitError(fmt.Sprintf("ttl exceeds max ttl %s", decision.MaxTTL)) + case "max_sandboxes": + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", decision.MaxSandboxes)) + case "max_sandboxes_per_owner": + return providers.ResourceLimitError(fmt.Sprintf("max sandboxes per owner reached (%d)", decision.MaxOwnerSandboxes)) + case "worker_unavailable": + return providers.ResourceLimitError("no eligible worker available") + case "remote_worker_rpc_unavailable": + return providers.ResourceLimitError("selected worker requires remote worker RPC") + default: + return providers.ResourceLimitError("spawn admission denied") + } } -func (m *Manager) StatFile(ctx context.Context, sandboxID string, path string) (*FileInfo, error) { - sb, prov, err := m.getSandboxAndProvider(sandboxID) - if err != nil { - return nil, err +func (m *Manager) resolveExecTimeout(raw string, ownerID string) (time.Duration, error) { + timeout := m.limits.DefaultExecTimeout + if raw != "" { + parsed, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("parsing exec timeout: %w", err) + } + timeout = parsed + } + if timeout < 0 { + return 0, providers.ResourceLimitError("exec timeout cannot be negative") } + maxExecTimeout := m.ownerMaxExecTimeout(context.Background(), ownerID) + if maxExecTimeout > 0 && timeout > maxExecTimeout { + return 0, providers.ResourceLimitError(fmt.Sprintf("exec timeout %s exceeds max exec timeout %s", timeout, maxExecTimeout)) + } + return timeout, nil +} - scopedPath := m.scopedPath(sb, path) - fi, err := prov.StatFile(ctx, m.resolveVMID(sb), scopedPath) +func (m *Manager) ownerLimitOverrides(ctx context.Context, ownerID string) (time.Duration, int) { + maxTTL := m.limits.MaxTTL + maxPerOwner := m.limits.MaxSandboxesPerOwner + if ownerID == "" { + return maxTTL, maxPerOwner + } + quota, err := m.store.GetOwnerQuota(ctx, ownerID) if err != nil { - return nil, err + return maxTTL, maxPerOwner } - return &FileInfo{ - Path: fi.Path, - Size: fi.Size, - Mode: fi.Mode, - IsDir: fi.IsDir, - ModTime: fi.ModTime, - }, nil + if quota.MaxTTLSeconds > 0 { + maxTTL = time.Duration(quota.MaxTTLSeconds) * time.Second + } + if quota.MaxSandboxes > 0 { + maxPerOwner = quota.MaxSandboxes + } + return maxTTL, maxPerOwner } -func (m *Manager) GlobFiles(ctx context.Context, sandboxID string, pattern string) ([]string, error) { - sb, prov, err := m.getSandboxAndProvider(sandboxID) +func (m *Manager) ownerMaxExecTimeout(ctx context.Context, ownerID string) time.Duration { + maxExecTimeout := m.limits.MaxExecTimeout + if ownerID == "" { + return maxExecTimeout + } + quota, err := m.store.GetOwnerQuota(ctx, ownerID) if err != nil { - return nil, err + return maxExecTimeout + } + if quota.MaxExecTimeoutSeconds > 0 { + return time.Duration(quota.MaxExecTimeoutSeconds) * time.Second } + return maxExecTimeout +} - scopedPattern := m.scopedPath(sb, pattern) - return prov.GlobFiles(ctx, m.resolveVMID(sb), scopedPattern) +func ownerQuotaFromRecord(rec *store.OwnerQuotaRecord) *OwnerQuota { + return &OwnerQuota{ + OwnerID: rec.OwnerID, + MaxSandboxes: rec.MaxSandboxes, + MaxTTL: optionalSecondsString(rec.MaxTTLSeconds), + MaxExecTimeout: optionalSecondsString(rec.MaxExecTimeoutSeconds), + CreatedAt: rec.CreatedAt, + UpdatedAt: rec.UpdatedAt, + } } -// scopedPath prefixes a path with the sandbox workspace when running in pool mode. -func (m *Manager) scopedPath(sb *Sandbox, path string) string { - if sb.VMID == "" { - return path // dedicated VM, no scoping +func parseOptionalDurationSeconds(raw string) (int64, error) { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "0" || raw == "0s" { + return 0, nil } - base := "/workspace/" + sb.ID - cleaned := filepath.Clean(filepath.Join(base, path)) - if !strings.HasPrefix(cleaned, base+"/") && cleaned != base { - return base + d, err := time.ParseDuration(raw) + if err != nil { + return 0, err } - return cleaned + if d < 0 { + return 0, fmt.Errorf("duration cannot be negative") + } + if d > 0 && d < time.Second { + return 0, fmt.Errorf("duration must be at least 1s") + } + if d%time.Second != 0 { + return 0, fmt.Errorf("duration must use whole seconds") + } + return int64(d.Seconds()), nil } -// resolveVMID returns the VM sandbox ID for provider calls. -// In pool mode, operations go to the VM (sb.VMID); in 1:1 mode, to the sandbox itself. -func (m *Manager) resolveVMID(sb *Sandbox) string { - if sb.VMID != "" { - return sb.VMID +func normalizeOwnerID(ownerID string) (string, error) { + ownerID = strings.TrimSpace(ownerID) + if ownerID == "" { + return "", InvalidInputError("owner_id is required") } - return sb.ID + if len(ownerID) > 128 { + return "", InvalidInputError("owner_id must be 128 characters or fewer") + } + if strings.ContainsAny(ownerID, `/\`) { + return "", InvalidInputError("owner_id cannot contain path separators") + } + for _, r := range ownerID { + if r <= 31 || r == 127 || r == ' ' || r == '\t' || r == '\n' || r == '\r' { + return "", InvalidInputError("owner_id cannot contain whitespace or control characters") + } + } + return ownerID, nil } -// VMPoolStatus returns the current VM pool status. Returns nil if pool is disabled. -func (m *Manager) VMPoolStatus() *VMPoolStatus { - if m.vmPoolMgr == nil { - return nil +func normalizeOptionalOwnerID(ownerID string) (string, error) { + ownerID = strings.TrimSpace(ownerID) + if ownerID == "" { + return "", nil } - status := m.vmPoolMgr.Status() - return &status + return normalizeOwnerID(ownerID) +} + +func optionalSecondsString(seconds int64) string { + if seconds <= 0 { + return "0s" + } + return (time.Duration(seconds) * time.Second).String() } // InitVMPool initializes the VM pool manager if pool mode is enabled. @@ -578,8 +2359,13 @@ func (m *Manager) spawnDirect(ctx context.Context, req SpawnRequest) (*Sandbox, VCPUs: vcpus, CreatedAt: now, ExpiresAt: now.Add(m.defaultTTL), + WorkerID: m.workerID, PreviewDomain: m.previewDomain, } + if _, err := m.acquireSandboxLease(ctx, sb.ID, sb.ExpiresAt); err != nil { + _ = prov.Destroy(ctx, id) + return nil, fmt.Errorf("acquiring pool VM lease: %w", err) + } metaJSON, _ := json.Marshal(sb.Metadata) if err := m.store.CreateSandbox(ctx, &store.SandboxRecord{ @@ -590,11 +2376,13 @@ func (m *Manager) spawnDirect(ctx context.Context, req SpawnRequest) (*Sandbox, MemoryMB: sb.MemoryMB, VCPUs: sb.VCPUs, Metadata: string(metaJSON), + WorkerID: sb.WorkerID, CreatedAt: sb.CreatedAt, ExpiresAt: sb.ExpiresAt, UpdatedAt: now, }); err != nil { prov.Destroy(ctx, id) + _ = m.releaseSandboxLease(ctx, id) return nil, fmt.Errorf("persisting pool VM: %w", err) } @@ -607,11 +2395,56 @@ func (m *Manager) spawnDirect(ctx context.Context, req SpawnRequest) (*Sandbox, } func (m *Manager) ConsoleLog(ctx context.Context, sandboxID string, lines int) ([]string, error) { - _, prov, err := m.getSandboxAndProvider(sandboxID) + sb, prov, err := m.getSandboxAndProvider(sandboxID) + if err != nil { + return nil, err + } + if m.isRemoteOwnedSandbox(sb) { + return m.remoteConsoleLog(ctx, sb, lines) + } + return prov.ConsoleLog(ctx, m.resolveVMID(sb), lines) +} + +func (m *Manager) remoteConsoleLog(ctx context.Context, sb *Sandbox, lines int) ([]string, error) { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) if err != nil { return nil, err } - return prov.ConsoleLog(ctx, sandboxID, lines) + runtimeID := strings.TrimSpace(sb.VMID) + if runtimeID == "" { + runtimeID = sb.ID + } + result, err := client.Logs(ctx, "logs-"+sb.ID, workerproto.LogsParams{ + SandboxID: sb.ID, + Provider: sb.Provider, + RuntimeID: runtimeID, + Lines: lines, + }) + if err != nil { + return nil, fmt.Errorf("remote worker logs: %w", err) + } + return result.Lines, nil +} + +func (m *Manager) applyPreviewDomain(ctx context.Context, sb *Sandbox) { + if sb == nil { + return + } + sb.PreviewDomain = m.previewDomainForWorker(ctx, sb.WorkerID) +} + +func (m *Manager) previewDomainForWorker(ctx context.Context, workerID string) string { + if strings.TrimSpace(workerID) == "" || workerID == m.workerID { + return m.previewDomain + } + rec, err := m.store.GetWorker(ctx, workerID) + if err != nil { + return m.previewDomain + } + if domain := workerPreviewDomain(rec); domain != "" { + return domain + } + return m.previewDomain } func (m *Manager) Get(ctx context.Context, id string) (*Sandbox, error) { @@ -620,17 +2453,66 @@ func (m *Manager) Get(ctx context.Context, id string) (*Sandbox, error) { m.mu.RUnlock() if ok { + m.applyPreviewDomain(ctx, sb) + if refreshed, err := m.refreshRemoteSandboxStatus(ctx, sb); err == nil { + return refreshed, nil + } return sb, nil } // Fall back to store rec, err := m.store.GetSandbox(ctx, id) if err != nil { - return nil, err + return nil, providers.SandboxNotFoundError(id) } sb = recordToSandbox(rec) + m.applyPreviewDomain(ctx, sb) if sb.State == StateDestroyed { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, providers.SandboxDestroyedError(id) + } + if refreshed, err := m.refreshRemoteSandboxStatus(ctx, sb); err == nil { + return refreshed, nil + } + return sb, nil +} + +func (m *Manager) refreshRemoteSandboxStatus(ctx context.Context, sb *Sandbox) (*Sandbox, error) { + m.applyPreviewDomain(ctx, sb) + if sb == nil || strings.TrimSpace(sb.WorkerID) == "" || sb.WorkerID == m.workerID { + return sb, nil + } + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + m.logger.Debug().Err(err).Str("sandbox", sb.ID).Str("worker", sb.WorkerID).Msg("remote status unavailable") + return nil, err + } + runtimeID := strings.TrimSpace(sb.VMID) + if runtimeID == "" { + runtimeID = sb.ID + } + status, err := client.Status(ctx, "status-"+sb.ID, workerproto.StatusParams{ + SandboxID: sb.ID, + Provider: sb.Provider, + RuntimeID: runtimeID, + }) + if err != nil { + m.logger.Debug().Err(err).Str("sandbox", sb.ID).Str("worker", sb.WorkerID).Msg("remote status failed") + return nil, err + } + state := SandboxState(status.State) + if state == "" { + return sb, nil + } + if state != sb.State { + sb.State = state + _ = m.store.UpdateSandboxState(ctx, sb.ID, string(state)) + m.mu.Lock() + if current, ok := m.sandboxes[sb.ID]; ok { + current.State = state + } else { + m.sandboxes[sb.ID] = sb + } + m.mu.Unlock() } return sb, nil } @@ -655,36 +2537,87 @@ func (m *Manager) List(ctx context.Context) ([]*Sandbox, error) { sandboxes := make([]*Sandbox, len(records)) for i, r := range records { sandboxes[i] = recordToSandbox(r) + m.applyPreviewDomain(ctx, sandboxes[i]) } return sandboxes, nil } func (m *Manager) Destroy(ctx context.Context, id string) error { + start := time.Now() + metricsProvider := "unknown" + var metricsErr error + defer func() { + m.recordOperation(OperationDestroy, metricsProvider, time.Since(start), metricsErr) + }() + // Check if this is a pooled sandbox. m.mu.RLock() sb := m.sandboxes[id] m.mu.RUnlock() + if sb != nil { + metricsProvider = sb.Provider + } + + if sb == nil { + if rec, err := m.store.GetSandbox(ctx, id); err == nil { + sb = recordToSandbox(rec) + metricsProvider = sb.Provider + } + } + if sb != nil { + if strings.TrimSpace(sb.WorkerID) != "" && sb.WorkerID != m.workerID { + metricsErr = m.destroyRemote(ctx, id, sb) + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, id, OperationDestroy, metricsProvider, metricsErr) + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "failure", metricsErr.Error()) + return metricsErr + } + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "success", "remote_worker="+sb.WorkerID) + return nil + } + if _, err := m.acquireSandboxLease(ctx, id, sb.ExpiresAt); err != nil { + metricsErr = err + m.publishOperationFailure(EventOperationFailed, id, OperationDestroy, metricsProvider, err) + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "failure", "lease_unavailable: "+err.Error()) + return err + } + } if sb != nil && sb.VMID != "" && m.vmPoolMgr != nil { - return m.destroyPooled(ctx, id, sb) + metricsErr = m.destroyPooled(ctx, id, sb) + if metricsErr != nil { + m.publishOperationFailure(EventOperationFailed, id, OperationDestroy, metricsProvider, metricsErr) + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "failure", metricsErr.Error()) + return metricsErr + } + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "success", "pooled=true") + return nil } prov, err := m.getProvider(id) if err != nil { // If we can't find the provider, just update the store m.store.DeleteSandbox(ctx, id) + _ = m.releaseSandboxLease(ctx, id) m.mu.Lock() delete(m.sandboxes, id) m.mu.Unlock() + m.notifySpawnCapacity() + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "success", "provider_unavailable=true") return nil } + if sb != nil { + metricsProvider = sb.Provider + } if err := prov.Destroy(ctx, id); err != nil { // Debug-level: this is expected when VMs were killed externally (e.g. process restart). m.logger.Debug().Err(err).Str("sandbox", id).Msg("provider destroy failed (VM may already be gone)") + m.publishOperationFailure(EventProviderFailed, id, OperationDestroy, metricsProvider, err) } m.store.UpdateSandboxState(ctx, id, string(StateDestroyed)) + _ = m.releaseSandboxLease(ctx, id) m.mu.Lock() if sb, ok := m.sandboxes[id]; ok { sb.State = StateDestroyed @@ -696,11 +2629,55 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { Type: EventSandboxDestroyed, SandboxID: id, }) + m.notifySpawnCapacity() + m.auditOperation(ctx, "sandbox.destroy", sb, id, "", "success", "") m.logger.Info().Str("sandbox", id).Msg("sandbox destroyed") return nil } +func (m *Manager) destroyRemote(ctx context.Context, id string, sb *Sandbox) error { + client, err := m.remoteWorkerRPCClient(ctx, sb.WorkerID) + if err != nil { + return err + } + lease, err := m.store.GetLease(ctx, id) + if err != nil { + return fmt.Errorf("getting remote sandbox lease: %w", err) + } + if lease.HolderID != sb.WorkerID { + return fmt.Errorf("remote sandbox lease holder %q does not match worker %q", lease.HolderID, sb.WorkerID) + } + runtimeID := strings.TrimSpace(sb.VMID) + if runtimeID == "" { + runtimeID = id + } + if err := client.Destroy(ctx, "destroy-"+id, leaseTokenFromStore(lease), workerproto.DestroyParams{ + SandboxID: id, + Provider: sb.Provider, + RuntimeID: runtimeID, + }); err != nil { + return fmt.Errorf("remote worker destroy: %w", err) + } + if err := m.store.UpdateSandboxState(ctx, id, string(StateDestroyed)); err != nil { + return err + } + _ = m.store.ReleaseLease(ctx, id, sb.WorkerID) + m.mu.Lock() + if current, ok := m.sandboxes[id]; ok { + current.State = StateDestroyed + } + delete(m.sandboxes, id) + m.mu.Unlock() + m.events.Publish(Event{ + Type: EventSandboxDestroyed, + SandboxID: id, + }) + m.notifySpawnCapacity() + m.logger.Info().Str("sandbox", id).Str("worker", sb.WorkerID).Str("runtime", runtimeID).Msg("remote sandbox destroyed") + return nil +} + // destroyPooled cleans up a pooled sandbox's workspace and releases the VM slot. func (m *Manager) destroyPooled(ctx context.Context, id string, sb *Sandbox) error { vmID := sb.VMID @@ -722,6 +2699,7 @@ func (m *Manager) destroyPooled(ctx context.Context, id string, sb *Sandbox) err } // Clean up the VM's sandbox record too. m.store.DeleteSandbox(ctx, vmID) + _ = m.releaseSandboxLease(ctx, vmID) m.mu.Lock() delete(m.sandboxes, vmID) m.mu.Unlock() @@ -730,6 +2708,7 @@ func (m *Manager) destroyPooled(ctx context.Context, id string, sb *Sandbox) err // Clean up the logical sandbox record. m.store.UpdateSandboxState(ctx, id, string(StateDestroyed)) + _ = m.releaseSandboxLease(ctx, id) m.mu.Lock() if cached, ok := m.sandboxes[id]; ok { cached.State = StateDestroyed @@ -738,6 +2717,7 @@ func (m *Manager) destroyPooled(ctx context.Context, id string, sb *Sandbox) err m.mu.Unlock() m.events.Publish(Event{Type: EventSandboxDestroyed, SandboxID: id}) + m.notifySpawnCapacity() m.logger.Info().Str("sandbox", id).Str("vm_id", vmID).Msg("pooled sandbox destroyed") return nil } @@ -793,13 +2773,13 @@ func (m *Manager) getSandboxAndProvider(id string) (*Sandbox, providers.Provider if !ok { rec, err := m.store.GetSandbox(context.Background(), id) if err != nil { - return nil, nil, fmt.Errorf("sandbox %q not found", id) + return nil, nil, providers.SandboxNotFoundError(id) } sb = recordToSandbox(rec) } if sb.State == StateDestroyed { - return nil, nil, fmt.Errorf("sandbox %q is destroyed", id) + return nil, nil, providers.SandboxDestroyedError(id) } prov, err := m.registry.Get(sb.Provider) @@ -818,7 +2798,7 @@ func (m *Manager) getProvider(id string) (providers.Provider, error) { if !ok { rec, err := m.store.GetSandbox(context.Background(), id) if err != nil { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, providers.SandboxNotFoundError(id) } sb = recordToSandbox(rec) } @@ -837,10 +2817,11 @@ func recordToSandbox(r *store.SandboxRecord) *Sandbox { MemoryMB: r.MemoryMB, VCPUs: r.VCPUs, OwnerID: r.OwnerID, + TenantID: r.TenantID, VMID: r.VMID, + WorkerID: r.WorkerID, CreatedAt: r.CreatedAt, ExpiresAt: r.ExpiresAt, Metadata: metadata, } } - diff --git a/internal/orchestrator/manager_test.go b/internal/orchestrator/manager_test.go index 8578776..a48583a 100644 --- a/internal/orchestrator/manager_test.go +++ b/internal/orchestrator/manager_test.go @@ -2,17 +2,1429 @@ package orchestrator import ( "context" + "encoding/json" + "errors" + "net/http/httptest" "path/filepath" + "strings" + "sync" "testing" "time" "github.com/StacyOs/stacyvm/internal/providers" "github.com/StacyOs/stacyvm/internal/store" + "github.com/StacyOs/stacyvm/internal/worker" "github.com/rs/zerolog" ) func setupManager(t *testing.T) *Manager { t.Helper() + return setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) +} + +func setupManagerWithConfig(t *testing.T, cfg ManagerConfig) *Manager { + t.Helper() + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := providers.NewMockProvider() + reg.Register(mock) + reg.SetDefault("mock") + + events := NewEventBus() + logger := zerolog.Nop() + + m := NewManager(reg, st, events, logger, cfg) + m.Start() + t.Cleanup(func() { m.Stop() }) + return m +} + +type slowSpawnProvider struct { + providers.Provider + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (p *slowSpawnProvider) Spawn(ctx context.Context, opts providers.SpawnOptions) (string, error) { + p.once.Do(func() { close(p.entered) }) + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-p.release: + } + return p.Provider.Spawn(ctx, opts) +} + +type cancellableStreamProvider struct { + providers.Provider + started chan struct{} + filled chan struct{} + once sync.Once + fill sync.Once +} + +func (p *cancellableStreamProvider) ExecStream(ctx context.Context, sandboxID string, opts providers.ExecOptions) (<-chan providers.StreamChunk, error) { + ch := make(chan providers.StreamChunk, 64) + go func() { + defer close(ch) + p.once.Do(func() { close(p.started) }) + sent := 0 + for { + select { + case <-ctx.Done(): + return + case ch <- providers.StreamChunk{Stream: "stdout", Data: "streaming\n"}: + sent++ + if sent >= 128 { + p.fill.Do(func() { close(p.filled) }) + } + } + } + }() + return ch, nil +} + +type remoteLiveStreamProvider struct { + providers.Provider + release chan struct{} +} + +func (p *remoteLiveStreamProvider) ExecStream(ctx context.Context, sandboxID string, opts providers.ExecOptions) (<-chan providers.StreamChunk, error) { + ch := make(chan providers.StreamChunk, 2) + go func() { + defer close(ch) + ch <- providers.StreamChunk{Stream: "stdout", Data: "remote first\n"} + select { + case <-ctx.Done(): + case <-p.release: + ch <- providers.StreamChunk{Stream: "stdout", Data: "remote second\n"} + } + }() + return ch, nil +} + +func TestManager_SpawnAndGet(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, err := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if sb.State != StateRunning { + t.Fatalf("expected running, got %s", sb.State) + } + + got, err := m.Get(ctx, sb.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.ID != sb.ID { + t.Fatalf("ID mismatch") + } +} + +func TestManager_RemoteSpawnUsesWorkerRPC(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + "preview_domain": "worker-preview.localhost", + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if sb.WorkerID != "worker-remote" { + t.Fatalf("worker id = %q, want worker-remote", sb.WorkerID) + } + if sb.PreviewDomain != "worker-preview.localhost" { + t.Fatalf("preview domain = %q, want worker-preview.localhost", sb.PreviewDomain) + } + if sb.VMID == "" { + t.Fatal("expected remote runtime id in VMID") + } + if _, err := remoteMock.Status(context.Background(), sb.VMID); err != nil { + t.Fatalf("remote runtime status: %v", err) + } + rec, err := m.store.GetSandbox(context.Background(), sb.ID) + if err != nil { + t.Fatalf("get sandbox record: %v", err) + } + if rec.WorkerID != "worker-remote" || rec.VMID != sb.VMID { + t.Fatalf("unexpected record ownership: %+v", rec) + } + got, err := m.Get(context.Background(), sb.ID) + if err != nil { + t.Fatalf("get sandbox: %v", err) + } + if got.PreviewDomain != "worker-preview.localhost" { + t.Fatalf("get preview domain = %q, want worker-preview.localhost", got.PreviewDomain) + } + listed, err := m.List(context.Background()) + if err != nil { + t.Fatalf("list sandboxes: %v", err) + } + if len(listed) != 1 || listed[0].PreviewDomain != "worker-preview.localhost" { + t.Fatalf("listed preview domain = %+v, want worker-preview.localhost", listed) + } + lease, err := m.store.GetLease(context.Background(), sb.ID) + if err != nil { + t.Fatalf("get lease: %v", err) + } + if lease.HolderID != "worker-remote" { + t.Fatalf("lease holder = %q, want worker-remote", lease.HolderID) + } +} + +func TestManager_RemoteSpawnUsesSignedWorkerRPC(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerSigningKey: "worker-signing-key-with-at-least-32-bytes", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if sb.WorkerID != "worker-remote" { + t.Fatalf("worker id = %q, want worker-remote", sb.WorkerID) + } + if sb.VMID == "" { + t.Fatal("expected remote runtime id in VMID") + } +} + +func TestManager_GetRefreshesRemoteStatus(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if err := remoteMock.Destroy(context.Background(), sb.VMID); err != nil { + t.Fatalf("destroy remote runtime: %v", err) + } + + got, err := m.Get(context.Background(), sb.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.State != StateDestroyed { + t.Fatalf("state = %s, want destroyed", got.State) + } + rec, err := m.store.GetSandbox(context.Background(), sb.ID) + if err != nil { + t.Fatalf("get record: %v", err) + } + if rec.State != string(StateDestroyed) { + t.Fatalf("stored state = %s, want destroyed", rec.State) + } +} + +func TestManager_DestroyRoutesToRemoteWorker(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status","destroy"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + if err := m.Destroy(context.Background(), sb.ID); err != nil { + t.Fatalf("destroy: %v", err) + } + status, err := remoteMock.Status(context.Background(), sb.VMID) + if err != nil { + t.Fatalf("remote runtime status: %v", err) + } + if status.State != "destroyed" { + t.Fatalf("remote runtime state = %s, want destroyed", status.State) + } + rec, err := m.store.GetSandbox(context.Background(), sb.ID) + if err != nil { + t.Fatalf("get record: %v", err) + } + if rec.State != string(StateDestroyed) { + t.Fatalf("stored state = %s, want destroyed", rec.State) + } + if _, err := m.store.GetLease(context.Background(), sb.ID); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("lease err = %v, want not found", err) + } +} + +func TestManager_ExecRoutesToRemoteWorker(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status","exec"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + result, err := m.Exec(context.Background(), sb.ID, ExecRequest{Command: "echo remote exec"}) + if err != nil { + t.Fatalf("exec: %v", err) + } + if result.ExitCode != 0 || result.Stdout != "remote exec\n" { + t.Fatalf("unexpected exec result: %+v", result) + } + logs, err := m.store.ListExecLogs(context.Background(), sb.ID) + if err != nil { + t.Fatalf("list exec logs: %v", err) + } + if len(logs) != 1 || logs[0].Stdout != "remote exec\n" { + t.Fatalf("unexpected exec logs: %+v", logs) + } +} + +func TestManager_ExecStreamRoutesToRemoteWorker(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status","exec","exec_stream"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + ch, err := m.ExecStream(context.Background(), sb.ID, ExecRequest{Command: "echo remote stream"}) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + var stdout strings.Builder + for chunk := range ch { + if chunk.Stream == "stdout" { + stdout.WriteString(chunk.Data) + } + } + if stdout.String() != "remote stream\n" { + t.Fatalf("stdout = %q, want remote stream", stdout.String()) + } +} + +func TestManager_ExecStreamRoutesLiveRemoteChunks(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + release := make(chan struct{}) + remoteRegistry.Register(&remoteLiveStreamProvider{Provider: remoteMock, release: release}) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status","exec","exec_stream"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + ch, err := m.ExecStream(context.Background(), sb.ID, ExecRequest{Command: "ignored"}) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + select { + case chunk := <-ch: + if chunk.Data != "remote first\n" { + t.Fatalf("first remote chunk = %+v, want remote first", chunk) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for first remote chunk") + } + close(release) + var rest strings.Builder + for chunk := range ch { + if chunk.Stream == "stdout" { + rest.WriteString(chunk.Data) + } + } + if !strings.Contains(rest.String(), "remote second") { + t.Fatalf("remaining stdout = %q, want remote second", rest.String()) + } +} + +func TestManager_FileOperationsRouteToRemoteWorker(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status","files"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + if err := m.WriteFile(context.Background(), sb.ID, FileWriteRequest{Path: "/workspace/remote.txt", Content: "remote file", Mode: "0644"}); err != nil { + t.Fatalf("write: %v", err) + } + content, err := m.ReadFile(context.Background(), sb.ID, "/workspace/remote.txt") + if err != nil { + t.Fatalf("read: %v", err) + } + if string(content) != "remote file" { + t.Fatalf("content = %q, want remote file", string(content)) + } + files, err := m.ListFiles(context.Background(), sb.ID, "/workspace") + if err != nil { + t.Fatalf("list: %v", err) + } + if len(files) == 0 { + t.Fatal("expected listed files") + } + if err := m.ChmodFile(context.Background(), sb.ID, FileChmodRequest{Path: "/workspace/remote.txt", Mode: "0755"}); err != nil { + t.Fatalf("chmod: %v", err) + } + stat, err := m.StatFile(context.Background(), sb.ID, "/workspace/remote.txt") + if err != nil { + t.Fatalf("stat: %v", err) + } + if stat.Size != int64(len("remote file")) { + t.Fatalf("stat size = %d, want %d", stat.Size, len("remote file")) + } + matches, err := m.GlobFiles(context.Background(), sb.ID, "/workspace/*.txt") + if err != nil { + t.Fatalf("glob: %v", err) + } + if len(matches) != 1 { + t.Fatalf("matches = %+v, want one match", matches) + } + if err := m.MoveFile(context.Background(), sb.ID, FileMoveRequest{OldPath: "/workspace/remote.txt", NewPath: "/workspace/moved.txt"}); err != nil { + t.Fatalf("move: %v", err) + } + content, err = m.ReadFile(context.Background(), sb.ID, "/workspace/moved.txt") + if err != nil { + t.Fatalf("read moved: %v", err) + } + if string(content) != "remote file" { + t.Fatalf("moved content = %q, want remote file", string(content)) + } + if err := m.DeleteFile(context.Background(), sb.ID, FileDeleteRequest{Path: "/workspace/moved.txt"}); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := m.ReadFile(context.Background(), sb.ID, "/workspace/moved.txt"); err == nil { + t.Fatal("expected read deleted file to fail") + } +} + +func TestManager_ConsoleLogRoutesToRemoteWorker(t *testing.T) { + remoteRegistry := providers.NewRegistry() + remoteMock := providers.NewMockProvider() + remoteRegistry.Register(remoteMock) + if err := remoteRegistry.SetDefault("mock"); err != nil { + t.Fatalf("set remote default: %v", err) + } + server := httptest.NewServer((&worker.RPCServer{ + WorkerID: "worker-remote", + Token: "worker-secret", + Registry: remoteRegistry, + }).Handler()) + defer server.Close() + + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + WorkerToken: "worker-secret", + }) + providersJSON, _ := json.Marshal([]string{"mock"}) + capacityJSON, _ := json.Marshal(map[string]interface{}{ + "max_sandboxes": 10, + "rpc_url": server.URL, + }) + now := time.Now().UTC() + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: string(providersJSON), + Capabilities: `["remote_worker","spawn","status","logs"]`, + Capacity: string(capacityJSON), + LastHeartbeat: now, + }); err != nil { + t.Fatalf("save worker: %v", err) + } + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + lines, err := m.ConsoleLog(context.Background(), sb.ID, 2) + if err != nil { + t.Fatalf("console log: %v", err) + } + if len(lines) != 2 { + t.Fatalf("log line count = %d, want 2: %+v", len(lines), lines) + } + for _, line := range lines { + if strings.Contains(line, sb.ID) { + t.Fatalf("expected worker runtime id in logs, got control-plane id in %q", line) + } + } + if !strings.Contains(lines[0], "workspace initialized") { + t.Fatalf("unexpected logs: %+v", lines) + } +} + +func TestManager_List(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + m.Spawn(ctx, SpawnRequest{Image: "ubuntu:latest"}) + + list, err := m.List(ctx) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(list) != 2 { + t.Fatalf("expected 2, got %d", len(list)) + } +} + +func TestManager_ReconcileMarksMissingRuntimeDestroyed(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + now := time.Now().UTC() + + if err := m.store.CreateSandbox(ctx, &store.SandboxRecord{ + ID: "sb-missing-runtime", + State: string(StateRunning), + Provider: "mock", + Image: "alpine:latest", + MemoryMB: 512, + VCPUs: 1, + Metadata: "{}", + CreatedAt: now, + ExpiresAt: now.Add(time.Hour), + UpdatedAt: now, + }); err != nil { + t.Fatalf("create stale sandbox record: %v", err) + } + + if err := m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + + rec, err := m.store.GetSandbox(ctx, "sb-missing-runtime") + if err != nil { + t.Fatalf("get reconciled sandbox: %v", err) + } + if rec.State != string(StateDestroyed) { + t.Fatalf("expected destroyed after reconcile, got %s", rec.State) + } + assertEventType(t, m.events.History(10), EventReconcileAction) +} + +func TestManager_ReconcileMarksStaleRemoteWorkerUnhealthy(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + now := time.Now().UTC() + if err := m.store.SaveWorker(ctx, &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "online", + Providers: `["mock"]`, + Capabilities: `["remote_worker","spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now.Add(-10 * time.Minute), + }); err != nil { + t.Fatalf("save worker: %v", err) + } + if err := m.store.CreateSandbox(ctx, &store.SandboxRecord{ + ID: "sb-remote-stale", + State: string(StateRunning), + Provider: "mock", + Image: "alpine:latest", + MemoryMB: 512, + VCPUs: 1, + Metadata: "{}", + WorkerID: "worker-remote", + VMID: "runtime-remote", + CreatedAt: now, + ExpiresAt: now.Add(time.Hour), + UpdatedAt: now, + }); err != nil { + t.Fatalf("create sandbox: %v", err) + } + + if err := m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + rec, err := m.store.GetSandbox(ctx, "sb-remote-stale") + if err != nil { + t.Fatalf("get sandbox: %v", err) + } + if rec.State != string(StateUnhealthy) { + t.Fatalf("state = %s, want unhealthy", rec.State) + } + assertEventType(t, m.events.History(10), EventReconcileAction) +} + +func TestManager_ReconcileMarksExpiredRemoteWorkerSandboxExpiredAndReleasesLease(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + now := time.Now().UTC() + if err := m.store.SaveWorker(ctx, &store.WorkerRecord{ + ID: "worker-remote", + Hostname: "remote-host", + Status: "offline", + Providers: `["mock"]`, + Capabilities: `["remote_worker","spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now.Add(-10 * time.Minute), + }); err != nil { + t.Fatalf("save worker: %v", err) + } + if err := m.store.CreateSandbox(ctx, &store.SandboxRecord{ + ID: "sb-remote-expired", + State: string(StateRunning), + Provider: "mock", + Image: "alpine:latest", + MemoryMB: 512, + VCPUs: 1, + Metadata: "{}", + WorkerID: "worker-remote", + VMID: "runtime-remote", + CreatedAt: now.Add(-2 * time.Hour), + ExpiresAt: now.Add(-time.Hour), + UpdatedAt: now, + }); err != nil { + t.Fatalf("create sandbox: %v", err) + } + if _, err := m.store.AcquireLease(ctx, "sb-remote-expired", "sandbox", "worker-remote", time.Hour); err != nil { + t.Fatalf("acquire lease: %v", err) + } + + if err := m.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + rec, err := m.store.GetSandbox(ctx, "sb-remote-expired") + if err != nil { + t.Fatalf("get sandbox: %v", err) + } + if rec.State != string(StateExpired) { + t.Fatalf("state = %s, want expired", rec.State) + } + if _, err := m.store.GetLease(ctx, "sb-remote-expired"); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("lease err = %v, want not found", err) + } + assertEventType(t, m.events.History(10), EventReconcileAction) +} + +type runtimeListerProvider struct { + providers.Provider + runtimes []providers.RuntimeSandbox +} + +func (p *runtimeListerProvider) ListRuntimeSandboxes(ctx context.Context) ([]providers.RuntimeSandbox, error) { + return p.runtimes, nil +} + +func TestManager_ReconcileAdoptsProviderRuntime(t *testing.T) { + dir := t.TempDir() + st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("new store: %v", err) + } + t.Cleanup(func() { st.Close() }) + + reg := providers.NewRegistry() + mock := &runtimeListerProvider{ + Provider: providers.NewMockProvider(), + runtimes: []providers.RuntimeSandbox{{ + ID: "sb-adopted-runtime", + State: string(StateRunning), + Provider: "mock", + Image: "alpine:latest", + CreatedAt: time.Now().UTC(), + Metadata: map[string]string{"source": "runtime"}, + }}, + } + reg.Register(mock) + reg.SetDefault("mock") + + m := NewManager(reg, st, NewEventBus(), zerolog.Nop(), ManagerConfig{ + DefaultTTL: time.Hour, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + + if err := m.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + rec, err := st.GetSandbox(context.Background(), "sb-adopted-runtime") + if err != nil { + t.Fatalf("get adopted sandbox: %v", err) + } + if rec.State != string(StateRunning) { + t.Fatalf("expected running adopted state, got %s", rec.State) + } + if rec.Provider != "mock" { + t.Fatalf("expected mock provider, got %s", rec.Provider) + } + assertEventType(t, m.events.History(10), EventReconcileAction) +} + +func TestManager_Exec(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + + result, err := m.Exec(ctx, sb.ID, ExecRequest{Command: "echo hello from manager"}) + if err != nil { + t.Fatalf("exec: %v", err) + } + if result.ExitCode != 0 { + t.Fatalf("expected exit 0, got %d", result.ExitCode) + } + if result.Stdout == "" { + t.Fatal("expected stdout") + } +} + +func TestManager_ExecArgvMode(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + + result, err := m.Exec(ctx, sb.ID, ExecRequest{ + Mode: providers.ExecModeArgv, + Command: "printf", + Args: []string{"%s", "$HOME && echo injected"}, + }) + if err != nil { + t.Fatalf("exec: %v", err) + } + if result.Stdout != "$HOME && echo injected" { + t.Fatalf("stdout = %q, want literal argv payload", result.Stdout) + } +} + +func TestManager_ExecRejectsUnsupportedMode(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + + _, err := m.Exec(ctx, sb.ID, ExecRequest{ + Mode: "raw", + Command: "echo nope", + }) + if !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected ErrInvalidInput, got %v", err) + } +} + +func TestManager_OperationMetrics(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, err := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if _, err := m.Exec(ctx, sb.ID, ExecRequest{Command: "echo metrics"}); err != nil { + t.Fatalf("exec: %v", err) + } + if err := m.WriteFile(ctx, sb.ID, FileWriteRequest{Path: "/workspace/metrics.txt", Content: "ok"}); err != nil { + t.Fatalf("write: %v", err) + } + if err := m.Destroy(ctx, sb.ID); err != nil { + t.Fatalf("destroy: %v", err) + } + + metrics := m.OperationMetrics() + assertOperationMetric(t, metrics, OperationSpawn, "mock") + assertOperationMetric(t, metrics, OperationExec, "mock") + assertOperationMetric(t, metrics, OperationFileWrite, "mock") + assertOperationMetric(t, metrics, OperationDestroy, "mock") +} + +func assertOperationMetric(t *testing.T, metrics []OperationMetrics, operation, provider string) { + t.Helper() + for _, metric := range metrics { + if metric.Operation == operation && metric.Provider == provider { + if metric.SuccessTotal == 0 { + t.Fatalf("%s/%s success total = 0", operation, provider) + } + return + } + } + t.Fatalf("operation metric %s/%s not found in %+v", operation, provider, metrics) +} + +func TestManager_ExecTimeout(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + + _, err := m.Exec(ctx, sb.ID, ExecRequest{ + Command: "sleep 1", + Timeout: "1ms", + }) + if !errors.Is(err, ErrExecTimeout) { + t.Fatalf("expected ErrExecTimeout, got %v", err) + } + assertEventType(t, m.events.History(10), EventExecTimeout) +} + +func TestManager_MaxExecTimeoutLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxExecTimeout: 50 * time.Millisecond, + }, + }) + sb, _ := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + + _, err := m.Exec(context.Background(), sb.ID, ExecRequest{ + Command: "echo nope", + Timeout: "1s", + }) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected resource limit, got %v", err) + } + assertEventType(t, m.events.History(10), EventResourceLimit) +} + +func TestManager_SpawnLimits(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + MaxSandboxesPerOwner: 1, + MaxTTL: time.Hour, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-b"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected total resource limit, got %v", err) + } + assertEventType(t, m.events.History(10), EventResourceLimit) +} + +func TestManager_SpawnOwnerLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxesPerOwner: 1, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected owner resource limit, got %v", err) + } +} + +func TestManager_SpawnOwnerIDValidation(t *testing.T) { + m := setupManager(t) + + sb, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: " owner-trimmed "}) + if err != nil { + t.Fatalf("spawn trimmed owner: %v", err) + } + if sb.OwnerID != "owner-trimmed" { + t.Fatalf("owner_id = %q, want owner-trimmed", sb.OwnerID) + } + + for _, ownerID := range []string{"owner/a", "owner a", strings.Repeat("a", 129)} { + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: ownerID}); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected invalid owner for %q, got %v", ownerID, err) + } + } +} + +func TestManager_PersistentOwnerQuotaLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + }) + _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: "owner-quota", + MaxSandboxes: 1, + MaxTTL: "30m", + MaxExecTimeout: "2s", + }) + if err != nil { + t.Fatalf("save owner quota: %v", err) + } + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-quota", TTL: "10m"}); err != nil { + t.Fatalf("spawn: %v", err) + } + _, err = m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-quota", TTL: "10m"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected owner quota resource limit, got %v", err) + } + + usage, err := m.OwnerUsage(context.Background(), "owner-quota") + if err != nil { + t.Fatalf("owner usage: %v", err) + } + if !usage.QuotaConfigured || usage.ActiveSandboxes != 1 || usage.MaxSandboxes != 1 { + t.Fatalf("unexpected owner usage: %+v", usage) + } + assertEventType(t, m.events.History(20), EventQuotaSaved) +} + +func TestManager_OwnerQuotaDeletePublishesEvent(t *testing.T) { + m := setupManager(t) + + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{OwnerID: "owner-delete", MaxSandboxes: 1}); err != nil { + t.Fatalf("save quota: %v", err) + } + if err := m.DeleteOwnerQuota(context.Background(), "owner-delete"); err != nil { + t.Fatalf("delete quota: %v", err) + } + assertEventType(t, m.events.History(20), EventQuotaDeleted) +} + +func TestManager_QuotaSummary(t *testing.T) { + m := setupManager(t) + + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: "owner-a", + MaxSandboxes: 2, + MaxTTL: "30s", + }); err != nil { + t.Fatalf("save owner-a quota: %v", err) + } + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: "owner-b", + MaxExecTimeout: "5s", + }); err != nil { + t.Fatalf("save owner-b quota: %v", err) + } + + summary, err := m.QuotaSummary(context.Background()) + if err != nil { + t.Fatalf("quota summary: %v", err) + } + if summary.Total != 2 || summary.WithMaxSandboxes != 1 || summary.WithMaxTTL != 1 || summary.WithMaxExecTimeout != 1 { + t.Fatalf("unexpected quota summary: %+v", summary) + } +} + +func TestManager_PersistentOwnerQuotaTTLLimit(t *testing.T) { + m := setupManager(t) + if _, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{OwnerID: "owner-ttl", MaxTTL: "5m"}); err != nil { + t.Fatalf("save quota: %v", err) + } + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-ttl", TTL: "10m"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected ttl quota resource limit, got %v", err) + } +} + +func TestManager_OwnerQuotaValidation(t *testing.T) { + m := setupManager(t) + + tests := []OwnerQuota{ + {OwnerID: " ", MaxSandboxes: 1}, + {OwnerID: "owner/a", MaxSandboxes: 1}, + {OwnerID: "owner a", MaxSandboxes: 1}, + {OwnerID: "owner-a", MaxSandboxes: -1}, + {OwnerID: "owner-a", MaxTTL: "500ms"}, + {OwnerID: "owner-a", MaxExecTimeout: "1.5s"}, + } + for _, quota := range tests { + if _, err := m.SaveOwnerQuota(context.Background(), quota); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected invalid input for %+v, got %v", quota, err) + } + } + + saved, err := m.SaveOwnerQuota(context.Background(), OwnerQuota{ + OwnerID: " owner-trimmed ", + MaxSandboxes: 2, + MaxTTL: "10s", + }) + if err != nil { + t.Fatalf("save trimmed owner quota: %v", err) + } + if saved.OwnerID != "owner-trimmed" || saved.MaxTTL != "10s" { + t.Fatalf("unexpected saved quota: %+v", saved) + } +} + +func TestManager_SpawnMaxTTLLimit(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxTTL: time.Hour, + }, + }) + + _, err := m.Spawn(context.Background(), SpawnRequest{TTL: "2h"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected ttl resource limit, got %v", err) + } +} + +func TestManager_EvaluateSpawnAdmissionAllowsWhenUnderLimits(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 2, + MaxSandboxesPerOwner: 2, + MaxTTL: time.Hour, + }, + }) + + decision, err := m.EvaluateSpawnAdmission(context.Background(), "owner-a", 30*time.Minute) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if !decision.Allowed || decision.Queueable || decision.Reason != "" { + t.Fatalf("unexpected admission decision: %+v", decision) + } + if decision.MaxSandboxes != 2 || decision.MaxOwnerSandboxes != 2 || decision.MaxTTL != "1h0m0s" { + t.Fatalf("unexpected admission limits: %+v", decision) + } +} + +func TestManager_EvaluateSpawnAdmissionDeniesQueueableCapacity(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("spawn: %v", err) + } + decision, err := m.EvaluateSpawnAdmission(context.Background(), "owner-b", 5*time.Minute) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if decision.Allowed || !decision.Queueable || decision.Reason != "max_sandboxes" { + t.Fatalf("unexpected admission decision: %+v", decision) + } + if decision.ActiveSandboxes != 1 || decision.MaxSandboxes != 1 { + t.Fatalf("unexpected admission counts: %+v", decision) + } +} + +func TestManager_EvaluateSpawnAdmissionDeniesNonQueueableTTL(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxTTL: time.Hour, + }, + }) + + decision, err := m.EvaluateSpawnAdmission(context.Background(), "owner-a", 2*time.Hour) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if decision.Allowed || decision.Queueable || decision.Reason != "max_ttl" { + t.Fatalf("unexpected admission decision: %+v", decision) + } +} + +func TestManager_EvaluateSpawnRequestAdmissionSelectsLocalWorker(t *testing.T) { + m := setupManager(t) + now := time.Now().UTC() + for _, worker := range []*store.WorkerRecord{ + { + ID: "local", + Status: "online", + Providers: `["mock"]`, + Capabilities: `["spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now, + }, + { + ID: "worker-b", + Status: "online", + Providers: `["mock"]`, + Capabilities: `["spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now, + }, + } { + if err := m.store.SaveWorker(context.Background(), worker); err != nil { + t.Fatalf("save worker: %v", err) + } + } + + decision, err := m.EvaluateSpawnRequestAdmission(context.Background(), SpawnRequest{Provider: "mock"}) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if !decision.Allowed || decision.SelectedWorkerID != "local" || decision.EligibleWorkers != 2 { + t.Fatalf("unexpected worker placement: %+v", decision) + } +} + +func TestManager_EvaluateSpawnRequestAdmissionRejectsRemoteWorkerUntilRPC(t *testing.T) { + m := setupManager(t) + now := time.Now().UTC() + for _, worker := range []*store.WorkerRecord{ + { + ID: "local", + Status: "draining", + Providers: `["mock"]`, + Capabilities: `["spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now, + }, + { + ID: "worker-b", + Status: "online", + Providers: `["mock"]`, + Capabilities: `["spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now, + }, + } { + if err := m.store.SaveWorker(context.Background(), worker); err != nil { + t.Fatalf("save worker: %v", err) + } + } + + decision, err := m.EvaluateSpawnRequestAdmission(context.Background(), SpawnRequest{Provider: "mock"}) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if decision.Allowed || decision.Queueable || decision.SelectedWorkerID != "worker-b" || decision.Reason != "remote_worker_rpc_unavailable" { + t.Fatalf("unexpected remote worker decision: %+v", decision) + } +} + +func TestManager_EvaluateSpawnRequestAdmissionRejectsStaleLocalWorker(t *testing.T) { + m := setupManager(t) + if err := m.store.SaveWorker(context.Background(), &store.WorkerRecord{ + ID: "local", + Status: "online", + Providers: `["mock"]`, + Capabilities: `["spawn"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: time.Now().UTC().Add(-10 * time.Minute), + }); err != nil { + t.Fatalf("save worker: %v", err) + } + + decision, err := m.EvaluateSpawnRequestAdmission(context.Background(), SpawnRequest{Provider: "mock"}) + if err != nil { + t.Fatalf("evaluate admission: %v", err) + } + if decision.Allowed || decision.Reason != "worker_unavailable" || decision.EligibleWorkers != 0 { + t.Fatalf("unexpected stale worker decision: %+v", decision) + } +} + +func TestManager_SpawnAdmissionSerializesConcurrentCreates(t *testing.T) { dir := t.TempDir() st, err := store.NewSQLiteStore(filepath.Join(dir, "test.db")) if err != nil { @@ -20,78 +1432,360 @@ func setupManager(t *testing.T) *Manager { } t.Cleanup(func() { st.Close() }) + base := providers.NewMockProvider() + slow := &slowSpawnProvider{ + Provider: base, + entered: make(chan struct{}), + release: make(chan struct{}), + } reg := providers.NewRegistry() - mock := providers.NewMockProvider() - reg.Register(mock) - reg.SetDefault("mock") - - events := NewEventBus() - logger := zerolog.Nop() + reg.Register(slow) + if err := reg.SetDefault("mock"); err != nil { + t.Fatalf("set default provider: %v", err) + } - m := NewManager(reg, st, events, logger, ManagerConfig{ + m := NewManager(reg, st, NewEventBus(), zerolog.Nop(), ManagerConfig{ DefaultTTL: 5 * time.Minute, DefaultImage: "alpine:latest", DefaultMemory: 512, DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + }, }) m.Start() t.Cleanup(func() { m.Stop() }) - return m + + firstCh := make(chan error, 1) + go func() { + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}) + firstCh <- err + }() + + select { + case <-slow.entered: + case <-time.After(time.Second): + t.Fatal("first spawn did not enter provider") + } + + secondCh := make(chan error, 1) + go func() { + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-b"}) + secondCh <- err + }() + + select { + case err := <-secondCh: + t.Fatalf("second spawn completed before first persisted: %v", err) + case <-time.After(25 * time.Millisecond): + } + + close(slow.release) + if err := <-firstCh; err != nil { + t.Fatalf("first spawn: %v", err) + } + + err = <-secondCh + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected second spawn resource limit, got %v", err) + } + + list, err := m.List(context.Background()) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(list) != 1 { + t.Fatalf("expected one persisted sandbox, got %d", len(list)) + } } -func TestManager_SpawnAndGet(t *testing.T) { - m := setupManager(t) +func TestManager_SpawnQueueWaitsForCapacity(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + SpawnOverflow: "queue", + SpawnQueueTimeout: 500 * time.Millisecond, + MaxSpawnQueue: 2, + }, + }) ctx := context.Background() - sb, err := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + first, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-a"}) if err != nil { - t.Fatalf("spawn: %v", err) + t.Fatalf("first spawn: %v", err) } - if sb.State != StateRunning { - t.Fatalf("expected running, got %s", sb.State) + + type spawnResult struct { + sb *Sandbox + err error } + resultCh := make(chan spawnResult, 1) + go func() { + sb, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-b"}) + resultCh <- spawnResult{sb: sb, err: err} + }() - got, err := m.Get(ctx, sb.ID) - if err != nil { - t.Fatalf("get: %v", err) + select { + case result := <-resultCh: + t.Fatalf("second spawn returned before capacity opened: sb=%v err=%v", result.sb, result.err) + case <-time.After(25 * time.Millisecond): } - if got.ID != sb.ID { - t.Fatalf("ID mismatch") + + if err := m.Destroy(ctx, first.ID); err != nil { + t.Fatalf("destroy first: %v", err) + } + + select { + case result := <-resultCh: + if result.err != nil { + t.Fatalf("second spawn: %v", result.err) + } + if result.sb == nil || result.sb.OwnerID != "owner-b" { + t.Fatalf("unexpected second spawn: %+v", result.sb) + } + case <-time.After(time.Second): + t.Fatal("second spawn did not resume after capacity opened") + } + + events := m.events.History(20) + assertEventType(t, events, EventSpawnQueued) + assertEventType(t, events, EventSpawnDequeued) + status := m.SchedulerStatus() + if status.SpawnQueuedTotal != 1 || status.SpawnDequeuedTotal != 1 || status.SpawnQueueWaitCount != 1 { + t.Fatalf("unexpected queue status: %+v", status) + } + if status.SpawnQueueWaitTotalMS <= 0 || status.SpawnQueueWaitMaxMS <= 0 || status.SpawnQueueWaitAvgMS <= 0 { + t.Fatalf("expected positive queue wait metrics: %+v", status) } } -func TestManager_List(t *testing.T) { - m := setupManager(t) +func TestManager_SpawnQueueResumesWhenQuotaChanges(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + SpawnOverflow: "queue", + SpawnQueueTimeout: 500 * time.Millisecond, + MaxSpawnQueue: 2, + }, + }) ctx := context.Background() + if _, err := m.SaveOwnerQuota(ctx, OwnerQuota{OwnerID: "owner-a", MaxSandboxes: 1}); err != nil { + t.Fatalf("save initial quota: %v", err) + } + if _, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("first spawn: %v", err) + } - m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) - m.Spawn(ctx, SpawnRequest{Image: "ubuntu:latest"}) + type spawnResult struct { + sb *Sandbox + err error + } + resultCh := make(chan spawnResult, 1) + go func() { + sb, err := m.Spawn(ctx, SpawnRequest{OwnerID: "owner-a"}) + resultCh <- spawnResult{sb: sb, err: err} + }() - list, err := m.List(ctx) - if err != nil { - t.Fatalf("list: %v", err) + select { + case result := <-resultCh: + t.Fatalf("second spawn returned before quota changed: sb=%v err=%v", result.sb, result.err) + case <-time.After(25 * time.Millisecond): } - if len(list) != 2 { - t.Fatalf("expected 2, got %d", len(list)) + + if _, err := m.SaveOwnerQuota(ctx, OwnerQuota{OwnerID: "owner-a", MaxSandboxes: 2}); err != nil { + t.Fatalf("increase quota: %v", err) + } + + select { + case result := <-resultCh: + if result.err != nil { + t.Fatalf("second spawn: %v", result.err) + } + if result.sb == nil || result.sb.OwnerID != "owner-a" { + t.Fatalf("unexpected second spawn: %+v", result.sb) + } + case <-time.After(time.Second): + t.Fatal("second spawn did not resume after quota changed") + } + + status := m.SchedulerStatus() + if status.SpawnQueuedTotal != 1 || status.SpawnDequeuedTotal != 1 { + t.Fatalf("unexpected queue status: %+v", status) } } -func TestManager_Exec(t *testing.T) { +func TestManager_SpawnQueueTimesOut(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + MaxSandboxes: 1, + SpawnOverflow: "queue", + SpawnQueueTimeout: 20 * time.Millisecond, + MaxSpawnQueue: 2, + }, + }) + + if _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-a"}); err != nil { + t.Fatalf("first spawn: %v", err) + } + + _, err := m.Spawn(context.Background(), SpawnRequest{OwnerID: "owner-b"}) + if !errors.Is(err, providers.ErrResourceLimit) { + t.Fatalf("expected queue timeout resource limit, got %v", err) + } + assertEventType(t, m.events.History(20), EventSpawnQueueTimeout) + status := m.SchedulerStatus() + if status.SpawnQueuedTotal != 1 || status.SpawnQueueTimeouts != 1 || status.SpawnQueueWaitCount != 1 { + t.Fatalf("unexpected timeout queue status: %+v", status) + } +} + +func TestManager_SchedulerStatus(t *testing.T) { + m := setupManagerWithConfig(t, ManagerConfig{ + DefaultTTL: 5 * time.Minute, + DefaultImage: "alpine:latest", + DefaultMemory: 512, + DefaultVCPUs: 1, + Limits: OperationalLimits{ + SpawnOverflow: "queue", + SpawnQueueTimeout: 10 * time.Second, + MaxSpawnQueue: 7, + }, + }) + + status := m.SchedulerStatus() + if status.SpawnOverflow != "queue" || status.MaxSpawnQueue != 7 || status.SpawnQueueTimeout != "10s" || status.AdmissionControl != "worker_aware_local" || status.WorkerID != "local" || status.SelectedWorkerID != "local" || status.EligibleWorkers != 1 { + t.Fatalf("unexpected scheduler status: %+v", status) + } + if status.SpawnQueueDepth != 0 { + t.Fatalf("queue depth = %d, want 0", status.SpawnQueueDepth) + } + if status.SpawnQueuedTotal != 0 || status.SpawnQueueWaitTotal != "0s" || status.SpawnQueueWaitAvg != "0s" { + t.Fatalf("unexpected empty queue metrics: %+v", status) + } +} + +func TestManager_ExecStreamTimeoutEmitsErrorChunk(t *testing.T) { m := setupManager(t) ctx := context.Background() sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) - result, err := m.Exec(ctx, sb.ID, ExecRequest{Command: "echo hello from manager"}) + ch, err := m.ExecStream(ctx, sb.ID, ExecRequest{ + Command: "sleep 1", + Timeout: "1ms", + }) if err != nil { - t.Fatalf("exec: %v", err) + t.Fatalf("exec stream: %v", err) } - if result.ExitCode != 0 { - t.Fatalf("expected exit 0, got %d", result.ExitCode) + + var sawTimeout bool + for chunk := range ch { + if chunk.Stream == "stderr" && strings.Contains(chunk.Data, ErrExecTimeout.Error()) { + sawTimeout = true + } } - if result.Stdout == "" { - t.Fatal("expected stdout") + if !sawTimeout { + t.Fatal("expected timeout error chunk") + } + assertEventType(t, m.events.History(10), EventExecTimeout) +} + +func TestManager_ExecStreamCancellationDoesNotEmitTimeout(t *testing.T) { + m := setupManager(t) + base, err := m.registry.Get("mock") + if err != nil { + t.Fatalf("get mock provider: %v", err) + } + streamProvider := &cancellableStreamProvider{ + Provider: base, + started: make(chan struct{}), + filled: make(chan struct{}), + } + m.registry.Register(streamProvider) + + sb, err := m.Spawn(context.Background(), SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + ch, err := m.ExecStream(ctx, sb.ID, ExecRequest{Command: "stream forever"}) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + + select { + case <-streamProvider.filled: + case <-time.After(time.Second): + t.Fatal("stream provider did not fill the stream buffers") + } + deadline := time.After(time.Second) + for len(ch) < cap(ch) { + select { + case <-deadline: + t.Fatalf("stream output buffer length = %d, want %d", len(ch), cap(ch)) + default: + time.Sleep(time.Millisecond) + } + } + cancel() + time.Sleep(100 * time.Millisecond) + + drained := make(chan struct{}) + go func() { + defer close(drained) + for range ch { + } + }() + select { + case <-drained: + case <-time.After(time.Second): + t.Fatal("stream did not close after cancellation") + } + + for _, event := range m.events.History(20) { + if event.Type == EventExecTimeout { + t.Fatalf("unexpected timeout event after cancellation: %+v", event) + } + } +} + +func TestManager_PublishesOperationFailureEvent(t *testing.T) { + m := setupManager(t) + + if _, err := m.Exec(context.Background(), "sb-does-not-exist", ExecRequest{Command: "echo nope"}); err == nil { + t.Fatal("expected exec error") + } + + assertEventType(t, m.events.History(10), EventExecFailed) +} + +func assertEventType(t *testing.T, events []Event, eventType EventType) Event { + t.Helper() + for _, event := range events { + if event.Type == eventType { + if event.ID == "" { + t.Fatalf("event %s has empty ID", eventType) + } + if len(event.Data) == 0 { + t.Fatalf("event %s has empty data", eventType) + } + return event + } } + t.Fatalf("event %s not found in %+v", eventType, events) + return Event{} } func TestManager_WriteAndReadFile(t *testing.T) { @@ -117,6 +1811,88 @@ func TestManager_WriteAndReadFile(t *testing.T) { } } +func TestManager_PooledScopedPathRejectsTraversal(t *testing.T) { + m := setupManager(t) + sb := &Sandbox{ID: "sb-pool", VMID: "vm-shared"} + + for _, path := range []string{ + "../../etc/passwd", + "../sb-other/secret.txt", + "/../../../root/.ssh/id_rsa", + "/workspace/../../../etc/shadow", + } { + if got, err := m.scopedPathForOperation(sb, path); err == nil { + t.Fatalf("scopedPathForOperation(%q) = %q, want traversal error", path, got) + } + } +} + +func TestManager_FileOperationsRejectPooledTraversal(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + sb := &Sandbox{ID: "sb-pool", VMID: "vm-shared", Provider: "mock", OwnerID: "owner-a"} + + m.mu.Lock() + m.sandboxes[sb.ID] = sb + m.mu.Unlock() + + cases := []struct { + name string + run func() error + }{ + {"write", func() error { return m.WriteFile(ctx, sb.ID, FileWriteRequest{Path: "../../escape.txt", Content: "x"}) }}, + {"read", func() error { _, err := m.ReadFile(ctx, sb.ID, "../../escape.txt"); return err }}, + {"list", func() error { _, err := m.ListFiles(ctx, sb.ID, "../../"); return err }}, + {"delete", func() error { return m.DeleteFile(ctx, sb.ID, FileDeleteRequest{Path: "../../escape.txt"}) }}, + {"move_old", func() error { + return m.MoveFile(ctx, sb.ID, FileMoveRequest{OldPath: "../../escape.txt", NewPath: "ok.txt"}) + }}, + {"move_new", func() error { + return m.MoveFile(ctx, sb.ID, FileMoveRequest{OldPath: "ok.txt", NewPath: "../../escape.txt"}) + }}, + {"chmod", func() error { return m.ChmodFile(ctx, sb.ID, FileChmodRequest{Path: "../../escape.txt", Mode: "0644"}) }}, + {"stat", func() error { _, err := m.StatFile(ctx, sb.ID, "../../escape.txt"); return err }}, + {"glob", func() error { _, err := m.GlobFiles(ctx, sb.ID, "../../*"); return err }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if err := tc.run(); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected ErrInvalidInput, got %v", err) + } + }) + } +} + +func TestManager_OperationAuditForExecAndFile(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, _ := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest", OwnerID: "owner-a"}) + if _, err := m.Exec(ctx, sb.ID, ExecRequest{Command: "echo audit"}); err != nil { + t.Fatalf("exec: %v", err) + } + if err := m.WriteFile(ctx, sb.ID, FileWriteRequest{Path: "/workspace/audit.txt", Content: "ok"}); err != nil { + t.Fatalf("write: %v", err) + } + + records, err := m.store.ListOperationAudit(ctx, store.OperationAuditQuery{SandboxID: sb.ID, Limit: 20}) + if err != nil { + t.Fatalf("list operation audit: %v", err) + } + seen := map[string]bool{} + for _, rec := range records { + seen[rec.Action] = true + if rec.Actor != "owner-a" { + t.Fatalf("actor = %q, want owner-a in record %+v", rec.Actor, rec) + } + } + for _, action := range []string{"sandbox.spawn", "exec", "file.write"} { + if !seen[action] { + t.Fatalf("missing audit action %s in %+v", action, records) + } + } +} + func TestManager_Destroy(t *testing.T) { m := setupManager(t) ctx := context.Background() @@ -135,6 +1911,58 @@ func TestManager_Destroy(t *testing.T) { } } +func TestManager_SpawnCreatesAndDestroyReleasesLease(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, err := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + lease, err := m.store.GetLease(ctx, sb.ID) + if err != nil { + t.Fatalf("get lease: %v", err) + } + if lease.HolderID != "local" || lease.ResourceType != "sandbox" { + t.Fatalf("unexpected lease: %+v", lease) + } + + if err := m.Destroy(ctx, sb.ID); err != nil { + t.Fatalf("destroy: %v", err) + } + if _, err := m.store.GetLease(ctx, sb.ID); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("get released lease err = %v, want ErrNotFound", err) + } +} + +func TestManager_DestroyRequiresLease(t *testing.T) { + m := setupManager(t) + ctx := context.Background() + + sb, err := m.Spawn(ctx, SpawnRequest{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if err := m.store.ReleaseLease(ctx, sb.ID, "local"); err != nil { + t.Fatalf("release local lease: %v", err) + } + if _, err := m.store.AcquireLease(ctx, sb.ID, "sandbox", "worker-b", time.Hour); err != nil { + t.Fatalf("acquire competing lease: %v", err) + } + + err = m.Destroy(ctx, sb.ID) + if !errors.Is(err, store.ErrConflict) { + t.Fatalf("destroy err = %v, want ErrConflict", err) + } + got, err := m.Get(ctx, sb.ID) + if err != nil { + t.Fatalf("get sandbox after failed destroy: %v", err) + } + if got.State != StateRunning { + t.Fatalf("sandbox state = %s, want running", got.State) + } +} + func TestManager_TTLExpiry(t *testing.T) { m := setupManager(t) ctx := context.Background() diff --git a/internal/orchestrator/metrics.go b/internal/orchestrator/metrics.go new file mode 100644 index 0000000..3f74506 --- /dev/null +++ b/internal/orchestrator/metrics.go @@ -0,0 +1,134 @@ +package orchestrator + +import ( + "sort" + "sync" + "time" +) + +const ( + OperationSpawn = "spawn" + OperationExec = "exec" + OperationExecStream = "exec_stream" + OperationDestroy = "destroy" + OperationFileWrite = "file_write" + OperationFileRead = "file_read" + OperationFileList = "file_list" + OperationFileDelete = "file_delete" + OperationFileMove = "file_move" + OperationFileChmod = "file_chmod" + OperationFileStat = "file_stat" + OperationFileGlob = "file_glob" +) + +type OperationMetrics struct { + Operation string `json:"operation"` + Provider string `json:"provider"` + SuccessTotal uint64 `json:"success_total"` + FailureTotal uint64 `json:"failure_total"` + LatencyCount uint64 `json:"latency_count"` + LatencyTotalMS uint64 `json:"latency_total_ms"` + LatencyMinMS uint64 `json:"latency_min_ms"` + LatencyMaxMS uint64 `json:"latency_max_ms"` + LatencyAvgMS uint64 `json:"latency_avg_ms"` + LastError string `json:"last_error,omitempty"` + LastObservedUnix int64 `json:"last_observed_unix,omitempty"` +} + +type operationMetricKey struct { + operation string + provider string +} + +type operationMetricBucket struct { + successTotal uint64 + failureTotal uint64 + latencyCount uint64 + latencyTotalMS uint64 + latencyMinMS uint64 + latencyMaxMS uint64 + lastError string + lastObservedUnix int64 +} + +type MetricsRecorder struct { + mu sync.RWMutex + operations map[operationMetricKey]*operationMetricBucket +} + +func NewMetricsRecorder() *MetricsRecorder { + return &MetricsRecorder{operations: make(map[operationMetricKey]*operationMetricBucket)} +} + +func (r *MetricsRecorder) RecordOperation(operation, provider string, duration time.Duration, err error) { + if r == nil { + return + } + if provider == "" { + provider = "unknown" + } + latencyMS := uint64(duration.Milliseconds()) + key := operationMetricKey{operation: operation, provider: provider} + + r.mu.Lock() + defer r.mu.Unlock() + + bucket := r.operations[key] + if bucket == nil { + bucket = &operationMetricBucket{latencyMinMS: latencyMS} + r.operations[key] = bucket + } + if err != nil { + bucket.failureTotal++ + bucket.lastError = err.Error() + } else { + bucket.successTotal++ + } + bucket.latencyCount++ + bucket.latencyTotalMS += latencyMS + if latencyMS < bucket.latencyMinMS { + bucket.latencyMinMS = latencyMS + } + if latencyMS > bucket.latencyMaxMS { + bucket.latencyMaxMS = latencyMS + } + bucket.lastObservedUnix = time.Now().Unix() +} + +func (r *MetricsRecorder) Snapshot() []OperationMetrics { + if r == nil { + return nil + } + + r.mu.RLock() + defer r.mu.RUnlock() + + out := make([]OperationMetrics, 0, len(r.operations)) + for key, bucket := range r.operations { + avg := uint64(0) + if bucket.latencyCount > 0 { + avg = bucket.latencyTotalMS / bucket.latencyCount + } + out = append(out, OperationMetrics{ + Operation: key.operation, + Provider: key.provider, + SuccessTotal: bucket.successTotal, + FailureTotal: bucket.failureTotal, + LatencyCount: bucket.latencyCount, + LatencyTotalMS: bucket.latencyTotalMS, + LatencyMinMS: bucket.latencyMinMS, + LatencyMaxMS: bucket.latencyMaxMS, + LatencyAvgMS: avg, + LastError: bucket.lastError, + LastObservedUnix: bucket.lastObservedUnix, + }) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].Operation == out[j].Operation { + return out[i].Provider < out[j].Provider + } + return out[i].Operation < out[j].Operation + }) + return out +} diff --git a/internal/orchestrator/scheduler.go b/internal/orchestrator/scheduler.go new file mode 100644 index 0000000..07b7642 --- /dev/null +++ b/internal/orchestrator/scheduler.go @@ -0,0 +1,117 @@ +package orchestrator + +import ( + "context" + "encoding/json" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/store" +) + +const workerHeartbeatStaleAfter = 2 * time.Minute + +type workerPlacement struct { + SelectedID string + Eligible int + Reason string +} + +type workerCapacity struct { + MaxSandboxes int `json:"max_sandboxes"` + RPCURL string `json:"rpc_url"` + PreviewDomain string `json:"preview_domain"` +} + +func (m *Manager) evaluateWorkerPlacement(ctx context.Context, provider string, sandboxes []*store.SandboxRecord) workerPlacement { + workers, err := m.store.ListWorkers(ctx) + if err != nil || len(workers) == 0 { + return workerPlacement{SelectedID: m.workerID, Eligible: 1, Reason: "local_fallback"} + } + + activeByWorker := make(map[string]int) + for _, sb := range sandboxes { + state := SandboxState(sb.State) + if state == StateDestroyed || state == StateExpired { + continue + } + workerID := strings.TrimSpace(sb.WorkerID) + if workerID == "" { + workerID = m.workerID + } + activeByWorker[workerID]++ + } + + now := time.Now().UTC() + bestID := "" + bestCount := 0 + eligible := 0 + for _, worker := range workers { + workerID := strings.TrimSpace(worker.ID) + if workerID == "" || !strings.EqualFold(worker.Status, "online") { + continue + } + if now.Sub(worker.LastHeartbeat) > workerHeartbeatStaleAfter { + continue + } + if provider != "" && !workerSupportsProvider(worker, provider) { + continue + } + count := activeByWorker[workerID] + if cap := workerMaxSandboxes(worker); cap > 0 && count >= cap { + continue + } + eligible++ + if workerID == m.workerID { + bestID = workerID + bestCount = count + continue + } + if bestID == "" || count < bestCount { + bestID = workerID + bestCount = count + } + } + + if bestID == "" { + return workerPlacement{Reason: "no_eligible_worker"} + } + return workerPlacement{SelectedID: bestID, Eligible: eligible} +} + +func workerSupportsProvider(worker *store.WorkerRecord, provider string) bool { + var providers []string + if err := json.Unmarshal([]byte(worker.Providers), &providers); err != nil { + return false + } + if len(providers) == 0 { + return true + } + for _, name := range providers { + if strings.EqualFold(strings.TrimSpace(name), provider) { + return true + } + } + return false +} + +func workerMaxSandboxes(worker *store.WorkerRecord) int { + return workerCapacityFromRecord(worker).MaxSandboxes +} + +func workerRPCURL(worker *store.WorkerRecord) string { + return strings.TrimSpace(workerCapacityFromRecord(worker).RPCURL) +} + +func workerPreviewDomain(worker *store.WorkerRecord) string { + return strings.TrimSpace(workerCapacityFromRecord(worker).PreviewDomain) +} + +func workerCapacityFromRecord(worker *store.WorkerRecord) workerCapacity { + var capacity workerCapacity + if worker == nil { + return capacity + } + _ = json.Unmarshal([]byte(worker.Capacity), &capacity) + return capacity +} diff --git a/internal/orchestrator/types.go b/internal/orchestrator/types.go index 13a4cb3..637a478 100644 --- a/internal/orchestrator/types.go +++ b/internal/orchestrator/types.go @@ -8,6 +8,8 @@ const ( StateCreating SandboxState = "creating" StateRunning SandboxState = "running" StateIdle SandboxState = "idle" + StateUnhealthy SandboxState = "unhealthy" + StateExpired SandboxState = "expired" StateDestroyed SandboxState = "destroyed" StateError SandboxState = "error" ) @@ -20,7 +22,9 @@ type Sandbox struct { MemoryMB int `json:"memory_mb"` VCPUs int `json:"vcpus"` OwnerID string `json:"owner_id,omitempty"` + TenantID string `json:"tenant_id,omitempty"` VMID string `json:"vm_id,omitempty"` + WorkerID string `json:"worker_id,omitempty"` CreatedAt time.Time `json:"created_at"` ExpiresAt time.Time `json:"expires_at"` Metadata map[string]string `json:"metadata,omitempty"` @@ -35,12 +39,14 @@ type SpawnRequest struct { TTL string `json:"ttl,omitempty"` Template string `json:"template,omitempty"` OwnerID string `json:"owner_id,omitempty"` + TenantID string `json:"tenant_id,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` } type ExecRequest struct { Command string `json:"command"` Args []string `json:"args,omitempty"` + Mode string `json:"mode,omitempty"` Env map[string]string `json:"env,omitempty"` WorkDir string `json:"workdir,omitempty"` Timeout string `json:"timeout,omitempty"` @@ -97,3 +103,85 @@ type SandboxInfo struct { FileCount int `json:"file_count"` PreviewDomain string `json:"preview_domain,omitempty"` } + +type OperationalLimits struct { + MaxSandboxes int `json:"max_sandboxes"` + MaxSandboxesPerOwner int `json:"max_sandboxes_per_owner"` + DefaultExecTimeout time.Duration `json:"default_exec_timeout"` + MaxExecTimeout time.Duration `json:"max_exec_timeout"` + MaxTTL time.Duration `json:"max_ttl"` + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueTimeout time.Duration `json:"spawn_queue_timeout"` + MaxSpawnQueue int `json:"max_spawn_queue"` +} + +type OperationalLimitsInfo struct { + MaxSandboxes int `json:"max_sandboxes"` + MaxSandboxesPerOwner int `json:"max_sandboxes_per_owner"` + DefaultExecTimeout string `json:"default_exec_timeout"` + MaxExecTimeout string `json:"max_exec_timeout"` + MaxTTL string `json:"max_ttl"` + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueTimeout string `json:"spawn_queue_timeout"` + MaxSpawnQueue int `json:"max_spawn_queue"` +} + +type SchedulerStatus struct { + SpawnOverflow string `json:"spawn_overflow"` + SpawnQueueDepth int `json:"spawn_queue_depth"` + MaxSpawnQueue int `json:"max_spawn_queue"` + SpawnQueueTimeout string `json:"spawn_queue_timeout"` + AdmissionControl string `json:"admission_control"` + WorkerID string `json:"worker_id"` + SelectedWorkerID string `json:"selected_worker_id,omitempty"` + EligibleWorkers int `json:"eligible_workers"` + SpawnQueuedTotal uint64 `json:"spawn_queued_total"` + SpawnDequeuedTotal uint64 `json:"spawn_dequeued_total"` + SpawnQueueTimeouts uint64 `json:"spawn_queue_timeouts"` + SpawnQueueWaitCount uint64 `json:"spawn_queue_wait_count"` + SpawnQueueWaitTotal string `json:"spawn_queue_wait_total"` + SpawnQueueWaitMax string `json:"spawn_queue_wait_max"` + SpawnQueueWaitAvg string `json:"spawn_queue_wait_avg"` + SpawnQueueWaitTotalMS int64 `json:"spawn_queue_wait_total_ms"` + SpawnQueueWaitMaxMS int64 `json:"spawn_queue_wait_max_ms"` + SpawnQueueWaitAvgMS int64 `json:"spawn_queue_wait_avg_ms"` +} + +type SpawnAdmissionDecision struct { + Allowed bool `json:"allowed"` + Queueable bool `json:"queueable"` + Reason string `json:"reason,omitempty"` + ActiveSandboxes int `json:"active_sandboxes"` + MaxSandboxes int `json:"max_sandboxes"` + ActiveOwnerSandboxes int `json:"active_owner_sandboxes,omitempty"` + MaxOwnerSandboxes int `json:"max_owner_sandboxes,omitempty"` + MaxTTL string `json:"max_ttl,omitempty"` + SelectedWorkerID string `json:"selected_worker_id,omitempty"` + EligibleWorkers int `json:"eligible_workers,omitempty"` + WorkerReason string `json:"worker_reason,omitempty"` +} + +type OwnerQuota struct { + OwnerID string `json:"owner_id"` + MaxSandboxes int `json:"max_sandboxes"` + MaxTTL string `json:"max_ttl"` + MaxExecTimeout string `json:"max_exec_timeout"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type OwnerUsage struct { + OwnerID string `json:"owner_id"` + ActiveSandboxes int `json:"active_sandboxes"` + MaxSandboxes int `json:"max_sandboxes"` + MaxTTL string `json:"max_ttl"` + MaxExecTimeout string `json:"max_exec_timeout"` + QuotaConfigured bool `json:"quota_configured"` +} + +type QuotaSummary struct { + Total int `json:"total"` + WithMaxSandboxes int `json:"with_max_sandboxes"` + WithMaxTTL int `json:"with_max_ttl"` + WithMaxExecTimeout int `json:"with_max_exec_timeout"` +} diff --git a/internal/providers/custom.go b/internal/providers/custom.go index 1b451a7..7b67d15 100644 --- a/internal/providers/custom.go +++ b/internal/providers/custom.go @@ -63,6 +63,23 @@ func NewCustomProvider(cfg CustomProviderConfig) *CustomProvider { func (p *CustomProvider) Name() string { return p.name } +func customHTTPError(operation string, code int, data []byte, sandboxID string) error { + switch code { + case http.StatusNotFound: + return SandboxNotFoundError(sandboxID) + case http.StatusGone: + return SandboxDestroyedError(sandboxID) + case http.StatusRequestTimeout: + return ExecTimeoutError(sandboxID) + case http.StatusTooManyRequests: + return ResourceLimitError(operation) + case http.StatusServiceUnavailable: + return ProviderUnavailableError("custom", fmt.Errorf("%s", string(data))) + default: + return fmt.Errorf("%s failed (HTTP %d): %s", operation, code, string(data)) + } +} + // doRequest is a shared helper that builds, signs, and executes an HTTP // request against the remote custom endpoint. func (p *CustomProvider) doRequest(ctx context.Context, method, path string, body interface{}) ([]byte, int, error) { @@ -143,7 +160,7 @@ func (p *CustomProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, return "", fmt.Errorf("custom spawn: %w", err) } if code >= 400 { - return "", fmt.Errorf("custom spawn failed (HTTP %d): %s", code, string(data)) + return "", customHTTPError("custom spawn", code, data, "") } var result struct { @@ -159,7 +176,7 @@ func (p *CustomProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, } // Exec runs a command in the sandbox. -// POST /exec { sandbox_id, command, args, env, workdir } +// POST /exec { sandbox_id, command, args, mode, env, workdir } // Expects response: { "exit_code": 0, "stdout": "...", "stderr": "..." } func (p *CustomProvider) Exec(ctx context.Context, sandboxID string, opts ExecOptions) (*ExecResult, error) { body := map[string]interface{}{ @@ -169,6 +186,9 @@ func (p *CustomProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp if len(opts.Args) > 0 { body["args"] = opts.Args } + if opts.Mode != "" { + body["mode"] = opts.Mode + } if opts.Env != nil { body["env"] = opts.Env } @@ -181,7 +201,7 @@ func (p *CustomProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp return nil, fmt.Errorf("custom exec: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom exec failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom exec", code, data, sandboxID) } var result struct { @@ -200,7 +220,7 @@ func (p *CustomProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp } // ExecStream runs a command and streams NDJSON output chunks. -// POST /exec { sandbox_id, command, args, env, workdir, stream: true } +// POST /exec { sandbox_id, command, args, mode, env, workdir, stream: true } // Expects NDJSON response: { "stream": "stdout"|"stderr", "data": "..." } func (p *CustomProvider) ExecStream(ctx context.Context, sandboxID string, opts ExecOptions) (<-chan StreamChunk, error) { body := map[string]interface{}{ @@ -211,6 +231,9 @@ func (p *CustomProvider) ExecStream(ctx context.Context, sandboxID string, opts if len(opts.Args) > 0 { body["args"] = opts.Args } + if opts.Mode != "" { + body["mode"] = opts.Mode + } if opts.Env != nil { body["env"] = opts.Env } @@ -225,7 +248,7 @@ func (p *CustomProvider) ExecStream(ctx context.Context, sandboxID string, opts if resp.StatusCode >= 400 { data, _ := io.ReadAll(resp.Body) resp.Body.Close() - return nil, fmt.Errorf("custom exec stream failed (HTTP %d): %s", resp.StatusCode, string(data)) + return nil, customHTTPError("custom exec stream", resp.StatusCode, data, sandboxID) } ch := make(chan StreamChunk, 64) @@ -244,6 +267,12 @@ func (p *CustomProvider) ExecStream(ctx context.Context, sandboxID string, opts ch <- chunk } } + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } }() return ch, nil @@ -271,7 +300,7 @@ func (p *CustomProvider) WriteFile(ctx context.Context, sandboxID string, path s return fmt.Errorf("custom write: %w", err) } if code >= 400 { - return fmt.Errorf("custom write failed (HTTP %d): %s", code, string(respData)) + return customHTTPError("custom write", code, respData, sandboxID) } return nil } @@ -288,7 +317,7 @@ func (p *CustomProvider) ReadFile(ctx context.Context, sandboxID string, path st return nil, fmt.Errorf("custom read: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom read failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom read", code, data, sandboxID) } return io.NopCloser(bytes.NewReader(data)), nil } @@ -305,7 +334,7 @@ func (p *CustomProvider) ListFiles(ctx context.Context, sandboxID string, path s return nil, fmt.Errorf("custom list: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom list failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom list", code, data, sandboxID) } var files []FileInfo @@ -326,7 +355,7 @@ func (p *CustomProvider) DeleteFile(ctx context.Context, sandboxID string, path return fmt.Errorf("custom delete: %w", err) } if code >= 400 { - return fmt.Errorf("custom delete failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom delete", code, data, sandboxID) } return nil } @@ -342,7 +371,7 @@ func (p *CustomProvider) MoveFile(ctx context.Context, sandboxID string, oldPath return fmt.Errorf("custom move: %w", err) } if code >= 400 { - return fmt.Errorf("custom move failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom move", code, data, sandboxID) } return nil } @@ -358,7 +387,7 @@ func (p *CustomProvider) ChmodFile(ctx context.Context, sandboxID string, path s return fmt.Errorf("custom chmod: %w", err) } if code >= 400 { - return fmt.Errorf("custom chmod failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom chmod", code, data, sandboxID) } return nil } @@ -373,7 +402,7 @@ func (p *CustomProvider) StatFile(ctx context.Context, sandboxID string, path st return nil, fmt.Errorf("custom stat: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom stat failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom stat", code, data, sandboxID) } var fi FileInfo @@ -393,7 +422,7 @@ func (p *CustomProvider) GlobFiles(ctx context.Context, sandboxID string, patter return nil, fmt.Errorf("custom glob: %w", err) } if code >= 400 { - return nil, fmt.Errorf("custom glob failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom glob", code, data, sandboxID) } var matches []string @@ -411,10 +440,10 @@ func (p *CustomProvider) Status(ctx context.Context, sandboxID string) (*Sandbox return nil, fmt.Errorf("custom status: %w", err) } if code == 404 { - return &SandboxStatus{ID: sandboxID, State: "destroyed"}, nil + return nil, SandboxNotFoundError(sandboxID) } if code >= 400 { - return nil, fmt.Errorf("custom status failed (HTTP %d): %s", code, string(data)) + return nil, customHTTPError("custom status", code, data, sandboxID) } var result struct { @@ -445,7 +474,7 @@ func (p *CustomProvider) Destroy(ctx context.Context, sandboxID string) error { return fmt.Errorf("custom destroy: %w", err) } if code >= 400 && code != 404 { - return fmt.Errorf("custom destroy failed (HTTP %d): %s", code, string(data)) + return customHTTPError("custom destroy", code, data, sandboxID) } return nil } diff --git a/internal/providers/custom_conformance_test.go b/internal/providers/custom_conformance_test.go new file mode 100644 index 0000000..30c8ef2 --- /dev/null +++ b/internal/providers/custom_conformance_test.go @@ -0,0 +1,242 @@ +package providers + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "path" + "strconv" + "strings" + "testing" + "time" +) + +func TestCustomProvider_Conformance(t *testing.T) { + server := newFakeCustomProviderServer(t) + defer server.Close() + + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + return NewCustomProvider(CustomProviderConfig{ + BaseURL: server.URL, + Timeout: 5 * time.Second, + }) + }) +} + +type fakeCustomBackend struct { + mock *MockProvider +} + +func newFakeCustomProviderServer(t *testing.T) *httptest.Server { + t.Helper() + backend := &fakeCustomBackend{mock: NewMockProvider()} + mux := http.NewServeMux() + mux.HandleFunc("/health", backend.health) + mux.HandleFunc("/spawn", backend.spawn) + mux.HandleFunc("/exec", backend.exec) + mux.HandleFunc("/files", backend.files) + mux.HandleFunc("/files/list", backend.listFiles) + mux.HandleFunc("/files/move", backend.moveFile) + mux.HandleFunc("/files/chmod", backend.chmodFile) + mux.HandleFunc("/files/stat", backend.statFile) + mux.HandleFunc("/files/glob", backend.globFiles) + mux.HandleFunc("/status/", backend.status) + mux.HandleFunc("/sandboxes/", backend.destroy) + return httptest.NewServer(mux) +} + +func (b *fakeCustomBackend) health(w http.ResponseWriter, r *http.Request) { + writeFakeJSON(w, http.StatusOK, map[string]bool{"ok": true}) +} + +func (b *fakeCustomBackend) spawn(w http.ResponseWriter, r *http.Request) { + id, err := b.mock.Spawn(r.Context(), SpawnOptions{}) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, map[string]string{"id": id}) +} + +func (b *fakeCustomBackend) exec(w http.ResponseWriter, r *http.Request) { + var req struct { + SandboxID string `json:"sandbox_id"` + Command string `json:"command"` + Args []string `json:"args"` + Mode string `json:"mode"` + Stream bool `json:"stream"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if req.Stream { + ch, err := b.mock.ExecStream(r.Context(), req.SandboxID, ExecOptions{Command: req.Command, Args: req.Args, Mode: req.Mode}) + if err != nil { + writeFakeError(w, err) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + enc := json.NewEncoder(w) + for chunk := range ch { + _ = enc.Encode(chunk) + } + return + } + result, err := b.mock.Exec(r.Context(), req.SandboxID, ExecOptions{Command: req.Command, Args: req.Args, Mode: req.Mode}) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, map[string]any{ + "exit_code": result.ExitCode, + "stdout": result.Stdout, + "stderr": result.Stderr, + }) +} + +func (b *fakeCustomBackend) files(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + rc, err := b.mock.ReadFile(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("path")) + if err != nil { + writeFakeError(w, err) + return + } + defer rc.Close() + _, _ = io.Copy(w, rc) + case http.MethodPost: + var req struct { + SandboxID string `json:"sandbox_id"` + Path string `json:"path"` + Content string `json:"content"` + Mode string `json:"mode"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.WriteFile(r.Context(), req.SandboxID, req.Path, strings.NewReader(req.Content), req.Mode); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) + case http.MethodDelete: + var req struct { + SandboxID string `json:"sandbox_id"` + Path string `json:"path"` + Recursive bool `json:"recursive"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.DeleteFile(r.Context(), req.SandboxID, req.Path, req.Recursive); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func (b *fakeCustomBackend) listFiles(w http.ResponseWriter, r *http.Request) { + files, err := b.mock.ListFiles(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("path")) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, files) +} + +func (b *fakeCustomBackend) moveFile(w http.ResponseWriter, r *http.Request) { + var req struct { + SandboxID string `json:"sandbox_id"` + OldPath string `json:"old_path"` + NewPath string `json:"new_path"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.MoveFile(r.Context(), req.SandboxID, req.OldPath, req.NewPath); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (b *fakeCustomBackend) chmodFile(w http.ResponseWriter, r *http.Request) { + var req struct { + SandboxID string `json:"sandbox_id"` + Path string `json:"path"` + Mode string `json:"mode"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err := b.mock.ChmodFile(r.Context(), req.SandboxID, req.Path, req.Mode); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (b *fakeCustomBackend) statFile(w http.ResponseWriter, r *http.Request) { + fi, err := b.mock.StatFile(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("path")) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, fi) +} + +func (b *fakeCustomBackend) globFiles(w http.ResponseWriter, r *http.Request) { + matches, err := b.mock.GlobFiles(r.Context(), r.URL.Query().Get("sandbox_id"), r.URL.Query().Get("pattern")) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, matches) +} + +func (b *fakeCustomBackend) status(w http.ResponseWriter, r *http.Request) { + id := path.Base(r.URL.Path) + status, err := b.mock.Status(r.Context(), id) + if err != nil { + writeFakeError(w, err) + return + } + writeFakeJSON(w, http.StatusOK, status) +} + +func (b *fakeCustomBackend) destroy(w http.ResponseWriter, r *http.Request) { + id := path.Base(r.URL.Path) + if err := b.mock.Destroy(r.Context(), id); err != nil { + writeFakeError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func writeFakeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeFakeError(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + if errors.Is(err, ErrSandboxNotFound) { + status = http.StatusNotFound + } + if errors.Is(err, ErrSandboxDestroyed) { + status = http.StatusGone + } + http.Error(w, strconv.Quote(err.Error()), status) +} diff --git a/internal/providers/docker.go b/internal/providers/docker.go index 7d50942..33a06c9 100644 --- a/internal/providers/docker.go +++ b/internal/providers/docker.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "crypto/rand" + "encoding/json" "fmt" "io" "path" @@ -15,6 +16,7 @@ import ( "time" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" dockerimage "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" @@ -150,7 +152,15 @@ func (d *DockerProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, sandboxID := fmt.Sprintf("sb-%x", b) labels := map[string]string{ - "stacyvm": "true", + "stacyvm": "true", + "stacyvm.provider": "docker", + "stacyvm.sandbox": sandboxID, + "stacyvm.image": image, + } + if len(opts.Metadata) > 0 { + if data, err := json.Marshal(opts.Metadata); err == nil { + labels["stacyvm.metadata"] = string(data) + } } if d.config.PreviewDomain != "" { @@ -226,13 +236,13 @@ func (d *DockerProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp workDir = "/workspace" } - shellCmd := opts.Command - for _, a := range opts.Args { - shellCmd += " " + shellQuoteDocker(a) + cmd, err := buildExecCommand(opts) + if err != nil { + return nil, err } execCfg := container.ExecOptions{ - Cmd: []string{"sh", "-c", shellCmd}, + Cmd: cmd, WorkingDir: workDir, AttachStdout: true, AttachStderr: true, @@ -241,22 +251,34 @@ func (d *DockerProvider) Exec(ctx context.Context, sandboxID string, opts ExecOp execID, err := d.cli.ContainerExecCreate(ctx, sandboxID, execCfg) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec create: %w", err) } resp, err := d.cli.ContainerExecAttach(ctx, execID.ID, container.ExecStartOptions{}) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec attach: %w", err) } defer resp.Close() var stdout, stderr bytes.Buffer if _, err := stdcopy.StdCopy(&stdout, &stderr, resp.Reader); err != nil && err != io.EOF { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } d.logger.Debug().Err(err).Msg("stdcopy exec error") } inspect, err := d.cli.ContainerExecInspect(ctx, execID.ID) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } return nil, fmt.Errorf("exec inspect: %w", err) } @@ -277,13 +299,13 @@ func (d *DockerProvider) ExecStream(ctx context.Context, sandboxID string, opts workDir = "/workspace" } - shellCmd := opts.Command - for _, a := range opts.Args { - shellCmd += " " + shellQuoteDocker(a) + cmd, err := buildExecCommand(opts) + if err != nil { + return nil, err } execCfg := container.ExecOptions{ - Cmd: []string{"sh", "-c", shellCmd}, + Cmd: cmd, WorkingDir: workDir, AttachStdout: true, AttachStderr: true, @@ -309,6 +331,12 @@ func (d *DockerProvider) ExecStream(ctx context.Context, sandboxID string, opts if _, err := stdcopy.StdCopy(stdoutW, stderrW, resp.Reader); err != nil && err != io.EOF { d.logger.Debug().Err(err).Msg("stdcopy stream error") } + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } }() return ch, nil @@ -529,6 +557,9 @@ func (d *DockerProvider) GlobFiles(ctx context.Context, sandboxID string, patter func (d *DockerProvider) Status(ctx context.Context, sandboxID string) (*SandboxStatus, error) { info, err := d.cli.ContainerInspect(ctx, sandboxID) if err != nil { + if client.IsErrNotFound(err) { + return nil, SandboxNotFoundError(sandboxID) + } return nil, fmt.Errorf("inspect container %q: %w", sandboxID, err) } @@ -560,6 +591,9 @@ func (d *DockerProvider) Destroy(ctx context.Context, sandboxID string) error { d.logger.Debug().Err(err).Msg("container stop (continuing with remove)") } if err := d.cli.ContainerRemove(ctx, sandboxID, container.RemoveOptions{Force: true}); err != nil { + if client.IsErrNotFound(err) { + return SandboxNotFoundError(sandboxID) + } return fmt.Errorf("removing container %q: %w", sandboxID, err) } @@ -605,20 +639,111 @@ func (d *DockerProvider) ConsoleLog(ctx context.Context, sandboxID string, lines return result, nil } +func (d *DockerProvider) ListRuntimeSandboxes(ctx context.Context) ([]RuntimeSandbox, error) { + args := filters.NewArgs(filters.Arg("label", "stacyvm=true")) + containers, err := d.cli.ContainerList(ctx, container.ListOptions{All: true, Filters: args}) + if err != nil { + return nil, fmt.Errorf("list stacyvm containers: %w", err) + } + + out := make([]RuntimeSandbox, 0, len(containers)) + for _, c := range containers { + id := c.Labels["stacyvm.sandbox"] + if id == "" && len(c.Names) > 0 { + id = strings.TrimPrefix(c.Names[0], "/") + } + if id == "" { + id = c.ID + } + + image := c.Labels["stacyvm.image"] + if image == "" { + image = c.Image + } + + metadata := map[string]string{} + if raw := c.Labels["stacyvm.metadata"]; raw != "" { + _ = json.Unmarshal([]byte(raw), &metadata) + } + + out = append(out, RuntimeSandbox{ + ID: id, + State: dockerContainerState(c.State), + Provider: d.Name(), + Image: image, + CreatedAt: time.Unix(c.Created, 0).UTC(), + Metadata: metadata, + }) + d.rememberSandbox(id, image, dockerContainerState(c.State)) + } + return out, nil +} + // --------------------------------------------------------------------------- // Private helpers // --------------------------------------------------------------------------- func (d *DockerProvider) getSandbox(id string) (*dockerSandbox, error) { d.mu.RLock() - defer d.mu.RUnlock() sb, ok := d.sandboxes[id] + d.mu.RUnlock() if !ok { - return nil, fmt.Errorf("sandbox %q not found (may have been destroyed)", id) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + info, err := d.cli.ContainerInspect(ctx, id) + if err != nil { + if client.IsErrNotFound(err) { + return nil, SandboxNotFoundError(id) + } + return nil, fmt.Errorf("inspect container %q: %w", id, err) + } + if info.Config == nil || info.Config.Labels["stacyvm"] != "true" { + return nil, SandboxNotFoundError(id) + } + image := info.Config.Labels["stacyvm.image"] + if image == "" { + image = info.Config.Image + } + state := "unknown" + if info.State != nil { + state = dockerContainerState(info.State.Status) + } + return d.rememberSandbox(id, image, state), nil } return sb, nil } +func (d *DockerProvider) rememberSandbox(id, image, state string) *dockerSandbox { + if state == "" { + state = "running" + } + d.mu.Lock() + defer d.mu.Unlock() + sb := &dockerSandbox{id: id, image: image, state: state} + d.sandboxes[id] = sb + return sb +} + +func dockerContainerState(state string) string { + switch state { + case "running": + return "running" + case "paused": + return "paused" + case "restarting": + return "restarting" + case "created": + return "creating" + case "exited", "dead", "removing": + return "stopped" + default: + if state == "" { + return "unknown" + } + return state + } +} + // containerExec runs a one-shot command inside the container and returns the result. func (d *DockerProvider) containerExec(ctx context.Context, containerID string, cmd string) (*ExecResult, error) { execCfg := container.ExecOptions{ @@ -801,4 +926,3 @@ func parseStatOutput(output string) []FileInfo { } return files } - diff --git a/internal/providers/docker_test.go b/internal/providers/docker_test.go index 55852b5..195bb66 100644 --- a/internal/providers/docker_test.go +++ b/internal/providers/docker_test.go @@ -2,6 +2,7 @@ package providers import ( "context" + "errors" "io" "os" "os/exec" @@ -127,6 +128,9 @@ func TestDockerHealthy_NoDocker(t *testing.T) { func skipIfNoDocker(t *testing.T) { t.Helper() + if os.Getenv("STACYVM_DOCKER_INTEGRATION") != "1" { + t.Skip("set STACYVM_DOCKER_INTEGRATION=1 to run Docker integration tests") + } if _, err := exec.LookPath("docker"); err != nil { t.Skip("docker not available in PATH") } @@ -181,6 +185,44 @@ func TestDockerIntegration_Healthy(t *testing.T) { } } +func TestDockerIntegration_Conformance(t *testing.T) { + skipIfNoDocker(t) + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + p, err := newTestDockerProvider(t) + if err != nil { + t.Fatalf("provider: %v", err) + } + return p + }) +} + +func TestDockerIntegration_ListRuntimeSandboxes(t *testing.T) { + skipIfNoDocker(t) + p, err := newTestDockerProvider(t) + if err != nil { + t.Fatalf("provider: %v", err) + } + id := spawnTestSandbox(t, p) + + runtimes, err := p.ListRuntimeSandboxes(context.Background()) + if err != nil { + t.Fatalf("list runtime sandboxes: %v", err) + } + for _, runtime := range runtimes { + if runtime.ID == id { + if runtime.Provider != "docker" { + t.Fatalf("provider = %q, want docker", runtime.Provider) + } + if runtime.Image == "" { + t.Fatal("runtime image is empty") + } + return + } + } + t.Fatalf("spawned sandbox %s not found in runtime inventory", id) +} + func TestDockerIntegration_SpawnAndDestroy(t *testing.T) { skipIfNoDocker(t) p, err := newTestDockerProvider(t) @@ -536,6 +578,9 @@ func TestDockerIntegration_StatusNotFound(t *testing.T) { if err == nil { t.Error("expected error for nonexistent container") } + if !errors.Is(err, ErrSandboxNotFound) { + t.Fatalf("expected ErrSandboxNotFound, got %v", err) + } } func TestDockerIntegration_DestroyNotFound(t *testing.T) { @@ -549,6 +594,9 @@ func TestDockerIntegration_DestroyNotFound(t *testing.T) { if err == nil { t.Error("expected error for nonexistent container") } + if !errors.Is(err, ErrSandboxNotFound) { + t.Fatalf("expected ErrSandboxNotFound, got %v", err) + } } func TestDockerIntegration_PoolWorkspaceIsolation(t *testing.T) { @@ -563,8 +611,8 @@ func TestDockerIntegration_PoolWorkspaceIsolation(t *testing.T) { userA, userB := "sb-aaaa0001", "sb-bbbb0002" // Create workspace dirs - p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userA}) //nolint:errcheck - p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userB}) //nolint:errcheck + p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userA}) //nolint:errcheck + p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + userB}) //nolint:errcheck // User A writes secret p.Exec(ctx, vmID, ExecOptions{Command: "echo TOP_SECRET > /workspace/" + userA + "/secret.txt"}) //nolint:errcheck @@ -593,7 +641,7 @@ func TestDockerIntegration_ConcurrentUsers(t *testing.T) { users := []string{"sb-u001", "sb-u002", "sb-u003"} for _, u := range users { - p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + u}) //nolint:errcheck + p.Exec(ctx, vmID, ExecOptions{Command: "mkdir -p /workspace/" + u}) //nolint:errcheck p.Exec(ctx, vmID, ExecOptions{Command: "echo " + u + " > /workspace/" + u + "/id.txt"}) //nolint:errcheck } diff --git a/internal/providers/e2b.go b/internal/providers/e2b.go index 66d1a0d..ff9f186 100644 --- a/internal/providers/e2b.go +++ b/internal/providers/e2b.go @@ -85,8 +85,20 @@ func (p *E2BProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, err } func (p *E2BProvider) Exec(ctx context.Context, sandboxID string, opts ExecOptions) (*ExecResult, error) { + mode, err := normalizeExecMode(opts.Mode) + if err != nil { + return nil, err + } + if mode == ExecModeArgv { + return nil, fmt.Errorf("e2b provider does not support argv exec mode") + } + + command := opts.Command + for _, arg := range opts.Args { + command += " " + shellQuoteArg(arg) + } body := map[string]interface{}{ - "cmd": opts.Command, + "cmd": command, } if opts.WorkDir != "" { body["cwd"] = opts.WorkDir diff --git a/internal/providers/errors.go b/internal/providers/errors.go new file mode 100644 index 0000000..dc8fc9c --- /dev/null +++ b/internal/providers/errors.go @@ -0,0 +1,42 @@ +package providers + +import ( + "errors" + "fmt" +) + +var ( + ErrSandboxNotFound = errors.New("sandbox not found") + ErrSandboxDestroyed = errors.New("sandbox destroyed") + ErrProviderNotFound = errors.New("provider not found") + ErrProviderUnavailable = errors.New("provider unavailable") + ErrExecTimeout = errors.New("exec timeout") + ErrResourceLimit = errors.New("resource limit exceeded") +) + +func SandboxNotFoundError(id string) error { + return fmt.Errorf("%w: %s", ErrSandboxNotFound, id) +} + +func SandboxDestroyedError(id string) error { + return fmt.Errorf("%w: %s", ErrSandboxDestroyed, id) +} + +func ProviderNotFoundError(name string) error { + return fmt.Errorf("%w: %s", ErrProviderNotFound, name) +} + +func ProviderUnavailableError(name string, err error) error { + if err == nil { + return fmt.Errorf("%w: %s", ErrProviderUnavailable, name) + } + return fmt.Errorf("%w: %s: %v", ErrProviderUnavailable, name, err) +} + +func ExecTimeoutError(sandboxID string) error { + return fmt.Errorf("%w: %s", ErrExecTimeout, sandboxID) +} + +func ResourceLimitError(resource string) error { + return fmt.Errorf("%w: %s", ErrResourceLimit, resource) +} diff --git a/internal/providers/exec_options_test.go b/internal/providers/exec_options_test.go new file mode 100644 index 0000000..83add82 --- /dev/null +++ b/internal/providers/exec_options_test.go @@ -0,0 +1,43 @@ +package providers + +import ( + "reflect" + "testing" +) + +func TestBuildExecCommandShellMode(t *testing.T) { + got, err := buildExecCommand(ExecOptions{ + Command: "echo", + Args: []string{"hello world", "it's ok"}, + }) + if err != nil { + t.Fatalf("build exec command: %v", err) + } + + want := []string{"sh", "-c", "echo 'hello world' 'it'\"'\"'s ok'"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("command = %#v, want %#v", got, want) + } +} + +func TestBuildExecCommandArgvMode(t *testing.T) { + got, err := buildExecCommand(ExecOptions{ + Mode: ExecModeArgv, + Command: "printf", + Args: []string{"%s", "$HOME && echo injected"}, + }) + if err != nil { + t.Fatalf("build exec command: %v", err) + } + + want := []string{"printf", "%s", "$HOME && echo injected"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("command = %#v, want %#v", got, want) + } +} + +func TestBuildExecCommandRejectsUnsupportedMode(t *testing.T) { + if _, err := buildExecCommand(ExecOptions{Mode: "raw", Command: "echo"}); err == nil { + t.Fatal("expected unsupported mode error") + } +} diff --git a/internal/providers/firecracker.go b/internal/providers/firecracker.go index f10fcb3..5e8463e 100644 --- a/internal/providers/firecracker.go +++ b/internal/providers/firecracker.go @@ -37,7 +37,7 @@ type FirecrackerProvider struct { // snapshotInfo holds paths to a base snapshot's files. type snapshotInfo struct { - dir string // snapshot directory + dir string // snapshot directory vmstatePath string // CPU/device state memoryPath string // full RAM snapshot rootfsPath string // clean baseline rootfs @@ -489,7 +489,7 @@ func (p *FirecrackerProvider) Spawn(ctx context.Context, opts SpawnOptions) (str // Machine config. if err := api.put(ctx, "/machine-config", map[string]any{ - "vcpu_count": vcpus, + "vcpu_count": vcpus, "mem_size_mib": memMB, }); err != nil { cmd.Process.Kill() @@ -595,7 +595,7 @@ func (p *FirecrackerProvider) getVM(sandboxID string) (*vmInstance, error) { defer p.mu.RUnlock() vm, ok := p.vms[sandboxID] if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return vm, nil } @@ -609,6 +609,7 @@ func (p *FirecrackerProvider) Exec(ctx context.Context, sandboxID string, opts E params, _ := agentproto.MarshalParams(&agentproto.ExecParams{ Command: opts.Command, Args: opts.Args, + Mode: opts.Mode, WorkDir: opts.WorkDir, Env: opts.Env, }) @@ -646,6 +647,7 @@ func (p *FirecrackerProvider) ExecStream(ctx context.Context, sandboxID string, params, _ := agentproto.MarshalParams(&agentproto.ExecParams{ Command: opts.Command, Args: opts.Args, + Mode: opts.Mode, WorkDir: opts.WorkDir, Env: opts.Env, }) @@ -665,13 +667,33 @@ func (p *FirecrackerProvider) ExecStream(ctx context.Context, sandboxID string, go func() { defer vm.connMu.Unlock() defer close(ch) + defer vm.conn.SetReadDeadline(time.Time{}) //nolint:errcheck for { + if deadline, ok := ctx.Deadline(); ok { + _ = vm.conn.SetReadDeadline(deadline) + } sresp, err := agentproto.ReadStreamResponse(vm.conn) if err != nil { + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } return } if sresp.Data != "" { - ch <- StreamChunk{Stream: sresp.Stream, Data: sresp.Data} + select { + case ch <- StreamChunk{Stream: sresp.Stream, Data: sresp.Data}: + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + default: + } + } + return + } } if sresp.Done { return @@ -928,7 +950,7 @@ func (p *FirecrackerProvider) Destroy(ctx context.Context, sandboxID string) err vm, ok := p.vms[sandboxID] if !ok { p.mu.Unlock() - return fmt.Errorf("sandbox %q not found", sandboxID) + return SandboxNotFoundError(sandboxID) } delete(p.vms, sandboxID) p.mu.Unlock() @@ -962,7 +984,7 @@ func (p *FirecrackerProvider) ConsoleLog(ctx context.Context, sandboxID string, vm, ok := p.vms[sandboxID] p.mu.RUnlock() if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return vm.consoleBuf.Lines(lines), nil } @@ -1067,7 +1089,7 @@ var syscall0 = os.Signal(signalZero(0)) type signalZero int -func (signalZero) Signal() {} +func (signalZero) Signal() {} func (signalZero) String() string { return "signal 0" } func generateRequestID() string { diff --git a/internal/providers/firecracker_test.go b/internal/providers/firecracker_test.go index 5fecdac..232e3c9 100644 --- a/internal/providers/firecracker_test.go +++ b/internal/providers/firecracker_test.go @@ -1,6 +1,9 @@ package providers import ( + "os" + "os/exec" + "runtime" "strings" "testing" @@ -109,3 +112,54 @@ func TestStatusNotFound(t *testing.T) { t.Error("Status should return error for nonexistent sandbox") } } + +func TestFirecrackerProvider_Integration_Conformance(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("firecracker conformance requires Linux") + } + if _, err := os.Stat("/dev/kvm"); err != nil { + t.Skip("firecracker conformance requires /dev/kvm") + } + + firecrackerPath := os.Getenv("STACYVM_FIRECRACKER_PATH") + if firecrackerPath == "" { + var err error + firecrackerPath, err = exec.LookPath("firecracker") + if err != nil { + t.Skip("firecracker binary not found; set STACYVM_FIRECRACKER_PATH") + } + } + kernelPath := os.Getenv("STACYVM_KERNEL_PATH") + if kernelPath == "" { + t.Skip("set STACYVM_KERNEL_PATH to run firecracker conformance") + } + rootfsPath := os.Getenv("STACYVM_ROOTFS_PATH") + if rootfsPath == "" { + t.Skip("set STACYVM_ROOTFS_PATH to run firecracker conformance") + } + agentPath := os.Getenv("STACYVM_AGENT_PATH") + if agentPath == "" { + agentPath = "./bin/stacyvm-agent" + } + for name, path := range map[string]string{ + "kernel": kernelPath, + "rootfs": rootfsPath, + "agent": agentPath, + } { + if _, err := os.Stat(path); err != nil { + t.Skipf("%s path %q is unavailable: %v", name, path, err) + } + } + + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + return NewFirecrackerProvider(FirecrackerProviderConfig{ + FirecrackerPath: firecrackerPath, + KernelPath: kernelPath, + DefaultRootfs: rootfsPath, + AgentPath: agentPath, + DataDir: t.TempDir(), + DefaultMemoryMB: 256, + }, zerolog.Nop()) + }) +} diff --git a/internal/providers/mock.go b/internal/providers/mock.go index 341b4a2..a38d076 100644 --- a/internal/providers/mock.go +++ b/internal/providers/mock.go @@ -73,10 +73,10 @@ func (m *MockProvider) getSandbox(id string) (*mockSandbox, error) { defer m.mu.RUnlock() sb, ok := m.sandboxes[id] if !ok { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, SandboxNotFoundError(id) } if sb.state == "destroyed" { - return nil, fmt.Errorf("sandbox %q is destroyed", id) + return nil, SandboxDestroyedError(id) } return sb, nil } @@ -87,9 +87,12 @@ func (m *MockProvider) Exec(ctx context.Context, sandboxID string, opts ExecOpti return nil, err } - // Use sandbox root as working dir and set up env so /workspace resolves - args := append([]string{"-c", opts.Command}, opts.Args...) - cmd := exec.CommandContext(ctx, "sh", args...) + // Use sandbox root as working dir and set up env so /workspace resolves. + args, err := buildExecCommand(opts) + if err != nil { + return nil, err + } + cmd := exec.CommandContext(ctx, args[0], args[1:]...) cmd.Dir = filepath.Join(sb.root, "workspace") if opts.WorkDir != "" { cmd.Dir = filepath.Join(sb.root, opts.WorkDir) @@ -111,6 +114,9 @@ func (m *MockProvider) Exec(ctx context.Context, sandboxID string, opts ExecOpti err = cmd.Run() exitCode := 0 if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } if exitErr, ok := err.(*exec.ExitError); ok { exitCode = exitErr.ExitCode() } else { @@ -131,8 +137,11 @@ func (m *MockProvider) ExecStream(ctx context.Context, sandboxID string, opts Ex return nil, err } - args := append([]string{"-c", opts.Command}, opts.Args...) - cmd := exec.CommandContext(ctx, "sh", args...) + args, err := buildExecCommand(opts) + if err != nil { + return nil, err + } + cmd := exec.CommandContext(ctx, args[0], args[1:]...) cmd.Dir = filepath.Join(sb.root, "workspace") if opts.WorkDir != "" { cmd.Dir = filepath.Join(sb.root, opts.WorkDir) @@ -354,7 +363,7 @@ func (m *MockProvider) Status(ctx context.Context, sandboxID string) (*SandboxSt defer m.mu.RUnlock() sb, ok := m.sandboxes[sandboxID] if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return &SandboxStatus{ ID: sb.id, @@ -367,7 +376,7 @@ func (m *MockProvider) Destroy(ctx context.Context, sandboxID string) error { defer m.mu.Unlock() sb, ok := m.sandboxes[sandboxID] if !ok { - return fmt.Errorf("sandbox %q not found", sandboxID) + return SandboxNotFoundError(sandboxID) } sb.state = "destroyed" return os.RemoveAll(sb.root) diff --git a/internal/providers/mock_test.go b/internal/providers/mock_test.go index 4421b20..7b27a9c 100644 --- a/internal/providers/mock_test.go +++ b/internal/providers/mock_test.go @@ -22,6 +22,13 @@ func TestMockProvider_Healthy(t *testing.T) { } } +func TestMockProvider_Conformance(t *testing.T) { + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + return NewMockProvider() + }) +} + func TestMockProvider_Spawn(t *testing.T) { p := NewMockProvider() ctx := context.Background() @@ -68,6 +75,48 @@ func TestMockProvider_Exec(t *testing.T) { } } +func TestMockProvider_ExecArgvModeDoesNotUseShell(t *testing.T) { + p := NewMockProvider() + ctx := context.Background() + + id, err := p.Spawn(ctx, SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + defer p.Destroy(ctx, id) + + result, err := p.Exec(ctx, id, ExecOptions{ + Mode: ExecModeArgv, + Command: "printf", + Args: []string{"%s", "$HOME && echo injected"}, + }) + if err != nil { + t.Fatalf("exec: %v", err) + } + if got := result.Stdout; got != "$HOME && echo injected" { + t.Fatalf("stdout = %q, want literal argv payload", got) + } +} + +func TestMockProvider_ExecRejectsUnsupportedMode(t *testing.T) { + p := NewMockProvider() + ctx := context.Background() + + id, err := p.Spawn(ctx, SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + defer p.Destroy(ctx, id) + + _, err = p.Exec(ctx, id, ExecOptions{ + Mode: "raw", + Command: "echo nope", + }) + if err == nil { + t.Fatal("expected unsupported mode error") + } +} + func TestMockProvider_ExecNonZero(t *testing.T) { p := NewMockProvider() ctx := context.Background() diff --git a/internal/providers/proot.go b/internal/providers/proot.go index 3879b44..4276ddc 100644 --- a/internal/providers/proot.go +++ b/internal/providers/proot.go @@ -110,7 +110,7 @@ func (p *PRootProvider) Spawn(ctx context.Context, opts SpawnOptions) (string, e } } if activeCount >= p.config.MaxSandboxes { - return "", fmt.Errorf("max sandboxes reached (%d)", p.config.MaxSandboxes) + return "", ResourceLimitError(fmt.Sprintf("max sandboxes reached (%d)", p.config.MaxSandboxes)) } id := generatePRootSandboxID() @@ -139,10 +139,10 @@ func (p *PRootProvider) getSandbox(id string) (*prootSandbox, error) { defer p.mu.RUnlock() sb, ok := p.sandboxes[id] if !ok { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, SandboxNotFoundError(id) } if sb.state == "destroyed" { - return nil, fmt.Errorf("sandbox %q is destroyed", id) + return nil, SandboxDestroyedError(id) } return sb, nil } @@ -169,7 +169,7 @@ func (p *PRootProvider) safePath(sb *prootSandbox, path string) (string, error) // BuildCommand constructs the proot exec.Cmd for a sandbox and exec options. // Exported for testing. -func (p *PRootProvider) BuildCommand(ctx context.Context, sb *prootSandbox, opts ExecOptions) *exec.Cmd { +func (p *PRootProvider) BuildCommand(ctx context.Context, sb *prootSandbox, opts ExecOptions) (*exec.Cmd, error) { args := []string{ "-0", "-r", p.config.RootfsPath, @@ -184,8 +184,14 @@ func (p *PRootProvider) BuildCommand(ctx context.Context, sb *prootSandbox, opts args[len(args)-1] = opts.WorkDir } - // Build the command: sh -c - args = append(args, "/bin/sh", "-c", opts.Command) + execArgs, err := buildExecCommand(opts) + if err != nil { + return nil, err + } + if mode, _ := normalizeExecMode(opts.Mode); mode == ExecModeShell { + execArgs[0] = "/bin/sh" + } + args = append(args, execArgs...) cmd := exec.CommandContext(ctx, p.config.PRootBinary, args...) @@ -201,7 +207,7 @@ func (p *PRootProvider) BuildCommand(ctx context.Context, sb *prootSandbox, opts } cmd.Env = env - return cmd + return cmd, nil } func (p *PRootProvider) Exec(ctx context.Context, sandboxID string, opts ExecOptions) (*ExecResult, error) { @@ -214,7 +220,11 @@ func (p *PRootProvider) Exec(ctx context.Context, sandboxID string, opts ExecOpt execCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - cmd := p.BuildCommand(execCtx, sb, opts) + cmd, err := p.BuildCommand(execCtx, sb, opts) + if err != nil { + cancel() + return nil, err + } var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -223,6 +233,9 @@ func (p *PRootProvider) Exec(ctx context.Context, sandboxID string, opts ExecOpt err = cmd.Run() exitCode := 0 if err != nil { + if execCtx.Err() == context.DeadlineExceeded { + return nil, ExecTimeoutError(sandboxID) + } if exitErr, ok := err.(*exec.ExitError); ok { exitCode = exitErr.ExitCode() } else { @@ -246,7 +259,11 @@ func (p *PRootProvider) ExecStream(ctx context.Context, sandboxID string, opts E timeout := p.config.DefaultTimeout execCtx, cancel := context.WithTimeout(ctx, timeout) - cmd := p.BuildCommand(execCtx, sb, opts) + cmd, err := p.BuildCommand(execCtx, sb, opts) + if err != nil { + cancel() + return nil, err + } stdoutPipe, err := cmd.StdoutPipe() if err != nil { @@ -295,7 +312,12 @@ func (p *PRootProvider) ExecStream(ctx context.Context, sandboxID string, opts E go readStream("stderr", stderrPipe) wg.Wait() - cmd.Wait() + if err := cmd.Wait(); err != nil && execCtx.Err() == context.DeadlineExceeded { + select { + case ch <- StreamChunk{Stream: "stderr", Data: ExecTimeoutError(sandboxID).Error()}: + case <-ctx.Done(): + } + } }() return ch, nil @@ -499,7 +521,7 @@ func (p *PRootProvider) Status(ctx context.Context, sandboxID string) (*SandboxS defer p.mu.RUnlock() sb, ok := p.sandboxes[sandboxID] if !ok { - return nil, fmt.Errorf("sandbox %q not found", sandboxID) + return nil, SandboxNotFoundError(sandboxID) } return &SandboxStatus{ ID: sb.id, @@ -512,7 +534,7 @@ func (p *PRootProvider) Destroy(ctx context.Context, sandboxID string) error { sb, ok := p.sandboxes[sandboxID] if !ok { p.mu.Unlock() - return fmt.Errorf("sandbox %q not found", sandboxID) + return SandboxNotFoundError(sandboxID) } sb.state = "destroyed" delete(p.sandboxes, sandboxID) diff --git a/internal/providers/proot_test.go b/internal/providers/proot_test.go index 8ef590d..b6e8bb6 100644 --- a/internal/providers/proot_test.go +++ b/internal/providers/proot_test.go @@ -67,10 +67,13 @@ func TestPRootProvider_BuildCommand(t *testing.T) { workspace: "/tmp/test-workspace", } - cmd := p.BuildCommand(context.Background(), sb, ExecOptions{ + cmd, err := p.BuildCommand(context.Background(), sb, ExecOptions{ Command: "echo hello", Env: map[string]string{"FOO": "bar"}, }) + if err != nil { + t.Fatalf("build command: %v", err) + } // Verify binary if cmd.Path != "/usr/bin/proot" && !strings.HasSuffix(cmd.Path, "proot") { @@ -113,10 +116,13 @@ func TestPRootProvider_BuildCommand_WorkDir(t *testing.T) { p := testPRootProvider(t) sb := &prootSandbox{id: "sb-test", workspace: "/tmp/ws"} - cmd := p.BuildCommand(context.Background(), sb, ExecOptions{ + cmd, err := p.BuildCommand(context.Background(), sb, ExecOptions{ Command: "ls", WorkDir: "/app", }) + if err != nil { + t.Fatalf("build command: %v", err) + } args := strings.Join(cmd.Args, " ") if !strings.Contains(args, "-w /app") { @@ -124,6 +130,28 @@ func TestPRootProvider_BuildCommand_WorkDir(t *testing.T) { } } +func TestPRootProvider_BuildCommandArgvMode(t *testing.T) { + p := testPRootProvider(t) + sb := &prootSandbox{id: "sb-test", workspace: "/tmp/ws"} + + cmd, err := p.BuildCommand(context.Background(), sb, ExecOptions{ + Mode: ExecModeArgv, + Command: "printf", + Args: []string{"%s", "$HOME"}, + }) + if err != nil { + t.Fatalf("build command: %v", err) + } + + args := strings.Join(cmd.Args, " ") + if strings.Contains(args, "/bin/sh -c") { + t.Fatalf("argv mode should not use shell, got args: %s", args) + } + if !strings.Contains(args, "printf %s $HOME") { + t.Fatalf("expected direct argv payload, got args: %s", args) + } +} + func TestPRootProvider_MaxSandboxes(t *testing.T) { p := testPRootProvider(t) ctx := context.Background() @@ -538,6 +566,21 @@ func TestPRootProvider_Integration_Exec(t *testing.T) { } } +func TestPRootProvider_Integration_Conformance(t *testing.T) { + prootPath := skipIfNoPRoot(t) + runProviderConformance(t, func(t *testing.T) Provider { + t.Helper() + tmpDir := t.TempDir() + return NewPRootProvider(PRootProviderConfig{ + RootfsPath: "/", + PRootBinary: prootPath, + WorkspaceBase: filepath.Join(tmpDir, "workspaces"), + DefaultTimeout: 30 * time.Second, + MaxSandboxes: 5, + }, testPRootLogger()) + }) +} + func TestPRootProvider_Integration_ExecStream(t *testing.T) { prootPath := skipIfNoPRoot(t) tmpDir := t.TempDir() diff --git a/internal/providers/provider.go b/internal/providers/provider.go index dd6aef6..5c60f1b 100644 --- a/internal/providers/provider.go +++ b/internal/providers/provider.go @@ -2,10 +2,17 @@ package providers import ( "context" + "fmt" "io" + "strings" "time" ) +const ( + ExecModeShell = "shell" + ExecModeArgv = "argv" +) + type SpawnOptions struct { Image string MemoryMB int @@ -17,6 +24,7 @@ type SpawnOptions struct { type ExecOptions struct { Command string Args []string + Mode string Env map[string]string WorkDir string } @@ -45,7 +53,21 @@ type SandboxStatus struct { State string } +// RuntimeSandbox describes a provider-owned runtime discovered outside the +// orchestrator's in-memory state, usually during startup reconciliation. +type RuntimeSandbox struct { + ID string + State string + Provider string + Image string + CreatedAt time.Time + Metadata map[string]string +} + // Provider defines the interface for sandbox execution backends. +// +// Implementations must satisfy the behavior documented in +// docs/provider-contract.md and exercised by provider_conformance_test.go. type Provider interface { // Name returns the unique provider identifier. Name() string @@ -108,3 +130,43 @@ type SnapshotSummary struct { type SnapshotLister interface { ListSnapshots() []SnapshotSummary } + +// RuntimeSandboxLister is implemented by providers that can enumerate +// already-running runtimes for startup reconciliation. +type RuntimeSandboxLister interface { + ListRuntimeSandboxes(ctx context.Context) ([]RuntimeSandbox, error) +} + +func normalizeExecMode(mode string) (string, error) { + switch strings.TrimSpace(mode) { + case "", ExecModeShell: + return ExecModeShell, nil + case ExecModeArgv: + return ExecModeArgv, nil + default: + return "", fmt.Errorf("unsupported exec mode %q", mode) + } +} + +func shellQuoteArg(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" +} + +func buildExecCommand(opts ExecOptions) ([]string, error) { + mode, err := normalizeExecMode(opts.Mode) + if err != nil { + return nil, err + } + if mode == ExecModeArgv { + if strings.TrimSpace(opts.Command) == "" { + return nil, fmt.Errorf("argv exec mode requires command") + } + return append([]string{opts.Command}, opts.Args...), nil + } + + shellCmd := opts.Command + for _, arg := range opts.Args { + shellCmd += " " + shellQuoteArg(arg) + } + return []string{"sh", "-c", shellCmd}, nil +} diff --git a/internal/providers/provider_conformance_test.go b/internal/providers/provider_conformance_test.go new file mode 100644 index 0000000..7b0f6c4 --- /dev/null +++ b/internal/providers/provider_conformance_test.go @@ -0,0 +1,157 @@ +package providers + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" +) + +type providerFactory func(t *testing.T) Provider + +func runProviderConformance(t *testing.T, factory providerFactory) { + t.Helper() + + t.Run("spawn status and destroy", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + + id, err := p.Spawn(ctx, SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if id == "" { + t.Fatal("spawn returned an empty sandbox ID") + } + + status, err := p.Status(ctx, id) + if err != nil { + t.Fatalf("status: %v", err) + } + if status.ID != id { + t.Fatalf("status ID = %q, want %q", status.ID, id) + } + if status.State != "running" { + t.Fatalf("status state = %q, want running", status.State) + } + + if err := p.Destroy(ctx, id); err != nil { + t.Fatalf("destroy: %v", err) + } + + _, err = p.Exec(ctx, id, ExecOptions{Command: "echo after destroy"}) + if !errors.Is(err, ErrSandboxDestroyed) && !errors.Is(err, ErrSandboxNotFound) { + t.Fatalf("exec after destroy error = %v, want sandbox lifecycle error", err) + } + }) + + t.Run("exec captures success and nonzero exit", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + id := spawnConformanceSandbox(t, p) + t.Cleanup(func() { _ = p.Destroy(context.Background(), id) }) + + result, err := p.Exec(ctx, id, ExecOptions{Command: "echo conformance-ok"}) + if err != nil { + t.Fatalf("exec success: %v", err) + } + if result.ExitCode != 0 { + t.Fatalf("exit code = %d, want 0", result.ExitCode) + } + if !strings.Contains(result.Stdout, "conformance-ok") { + t.Fatalf("stdout = %q, want conformance-ok", result.Stdout) + } + + result, err = p.Exec(ctx, id, ExecOptions{Command: "exit 7"}) + if err != nil { + t.Fatalf("exec nonzero: %v", err) + } + if result.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", result.ExitCode) + } + }) + + t.Run("exec stream emits output", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + id := spawnConformanceSandbox(t, p) + t.Cleanup(func() { _ = p.Destroy(context.Background(), id) }) + + ch, err := p.ExecStream(ctx, id, ExecOptions{Command: "printf stream-ok"}) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + var out strings.Builder + for chunk := range ch { + out.WriteString(chunk.Data) + } + if !strings.Contains(out.String(), "stream-ok") { + t.Fatalf("stream output = %q, want stream-ok", out.String()) + } + }) + + t.Run("file operations round trip", func(t *testing.T) { + p := factory(t) + ctx := context.Background() + id := spawnConformanceSandbox(t, p) + t.Cleanup(func() { _ = p.Destroy(context.Background(), id) }) + + const originalPath = "/workspace/contract.txt" + const movedPath = "/workspace/contract-moved.txt" + const content = "provider contract file" + + if err := p.WriteFile(ctx, id, originalPath, bytes.NewReader([]byte(content)), "0644"); err != nil { + t.Fatalf("write file: %v", err) + } + + rc, err := p.ReadFile(ctx, id, originalPath) + if err != nil { + t.Fatalf("read file: %v", err) + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatalf("read content: %v", err) + } + if string(data) != content { + t.Fatalf("content = %q, want %q", string(data), content) + } + + if _, err := p.StatFile(ctx, id, originalPath); err != nil { + t.Fatalf("stat file: %v", err) + } + + matches, err := p.GlobFiles(ctx, id, "/workspace/contract*.txt") + if err != nil { + t.Fatalf("glob files: %v", err) + } + if len(matches) == 0 { + t.Fatal("glob files returned no matches") + } + + if err := p.MoveFile(ctx, id, originalPath, movedPath); err != nil { + t.Fatalf("move file: %v", err) + } + if err := p.ChmodFile(ctx, id, movedPath, "0755"); err != nil { + t.Fatalf("chmod file: %v", err) + } + if err := p.DeleteFile(ctx, id, movedPath, false); err != nil { + t.Fatalf("delete file: %v", err) + } + }) +} + +func spawnConformanceSandbox(t *testing.T, p Provider) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + id, err := p.Spawn(ctx, SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + return id +} diff --git a/internal/providers/registry.go b/internal/providers/registry.go index 5ab7261..02071a3 100644 --- a/internal/providers/registry.go +++ b/internal/providers/registry.go @@ -50,7 +50,7 @@ func (r *Registry) Get(name string) (Provider, error) { p, ok := r.providers[name] if !ok { - return nil, fmt.Errorf("provider %q not found", name) + return nil, ProviderNotFoundError(name) } return p, nil } diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 0000000..85eda47 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,34 @@ +package store + +import ( + "errors" + "fmt" + "strings" +) + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") +) + +func NotFoundError(resource, id string) error { + if id == "" { + return fmt.Errorf("%w: %s", ErrNotFound, resource) + } + return fmt.Errorf("%w: %s %q", ErrNotFound, resource, id) +} + +func ConflictError(msg string) error { + return fmt.Errorf("%w: %s", ErrConflict, msg) +} + +func IsConstraintError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "UNIQUE constraint") || + strings.Contains(msg, "constraint failed") || + strings.Contains(msg, "constraint violation") || + strings.Contains(msg, "duplicate key value violates unique constraint") +} diff --git a/internal/store/factory.go b/internal/store/factory.go new file mode 100644 index 0000000..c5517b8 --- /dev/null +++ b/internal/store/factory.go @@ -0,0 +1,41 @@ +package store + +import ( + "errors" + "fmt" + "strings" +) + +const ( + DriverSQLite = "sqlite" + DriverPostgres = "postgres" +) + +var ErrUnsupportedDriver = errors.New("unsupported store driver") + +type Config struct { + Driver string + Path string + DSN string +} + +func Open(cfg Config) (Store, error) { + driver := strings.ToLower(strings.TrimSpace(cfg.Driver)) + if driver == "" { + driver = DriverSQLite + } + switch driver { + case DriverSQLite, "sqlite3": + if strings.TrimSpace(cfg.Path) == "" { + return nil, fmt.Errorf("sqlite database path is required") + } + return NewSQLiteStore(cfg.Path) + case DriverPostgres, "postgresql": + if strings.TrimSpace(cfg.DSN) == "" { + return nil, fmt.Errorf("postgres database dsn is required") + } + return NewPostgresStore(cfg.DSN) + default: + return nil, fmt.Errorf("%w: %s", ErrUnsupportedDriver, driver) + } +} diff --git a/internal/store/factory_test.go b/internal/store/factory_test.go new file mode 100644 index 0000000..ebec220 --- /dev/null +++ b/internal/store/factory_test.go @@ -0,0 +1,34 @@ +package store + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestOpenDefaultsToSQLite(t *testing.T) { + dir := t.TempDir() + st, err := Open(Config{Path: filepath.Join(dir, "test.db")}) + if err != nil { + t.Fatalf("open: %v", err) + } + if closer, ok := st.(*SQLiteStore); ok { + closer.Close() + } else { + t.Fatalf("store type = %T, want *SQLiteStore", st) + } +} + +func TestOpenPostgresRejectsMissingDSN(t *testing.T) { + _, err := Open(Config{Driver: DriverPostgres}) + if err == nil || !strings.Contains(err.Error(), "postgres database dsn is required") { + t.Fatalf("err = %v, want missing postgres dsn error", err) + } +} + +func TestOpenRejectsMissingSQLitePath(t *testing.T) { + _, err := Open(Config{Driver: DriverSQLite}) + if err == nil { + t.Fatal("expected missing sqlite path error") + } +} diff --git a/internal/store/migrations.go b/internal/store/migrations.go index 031ef50..a34f869 100644 --- a/internal/store/migrations.go +++ b/internal/store/migrations.go @@ -1,9 +1,11 @@ package store -var migrations = []struct { +type migration struct { version int sql string -}{ +} + +var migrations = []migration{ { version: 1, sql: ` @@ -143,6 +145,148 @@ ALTER TABLE sandboxes ADD COLUMN owner_id TEXT NOT NULL DEFAULT ''; ALTER TABLE sandboxes ADD COLUMN vm_id TEXT NOT NULL DEFAULT ''; CREATE INDEX IF NOT EXISTS idx_sandboxes_owner ON sandboxes(owner_id); CREATE INDEX IF NOT EXISTS idx_sandboxes_vm ON sandboxes(vm_id); +`, + }, + { + version: 5, + sql: ` +CREATE TABLE IF NOT EXISTS owner_quotas ( + owner_id TEXT PRIMARY KEY, + max_sandboxes INTEGER NOT NULL DEFAULT 0, + max_ttl_seconds INTEGER NOT NULL DEFAULT 0, + max_exec_timeout_seconds INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) +); +`, + }, + { + version: 6, + sql: ` +CREATE TABLE IF NOT EXISTS admin_audit_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + actor TEXT NOT NULL DEFAULT '', + method TEXT NOT NULL, + path TEXT NOT NULL, + status INTEGER NOT NULL DEFAULT 0, + duration_ms INTEGER NOT NULL DEFAULT 0, + request_id TEXT NOT NULL DEFAULT '', + remote_addr TEXT NOT NULL DEFAULT '', + user_agent TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_admin_audit_logs_created_at ON admin_audit_logs(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_admin_audit_logs_actor ON admin_audit_logs(actor); +`, + }, + { + version: 7, + sql: ` +CREATE TABLE IF NOT EXISTS operation_audit_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + actor TEXT NOT NULL DEFAULT '', + action TEXT NOT NULL, + sandbox_id TEXT NOT NULL DEFAULT '', + resource TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT '', + detail TEXT NOT NULL DEFAULT '', + created_at DATETIME NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_created_at ON operation_audit_logs(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_actor ON operation_audit_logs(actor); +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_sandbox ON operation_audit_logs(sandbox_id); +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_action ON operation_audit_logs(action); +`, + }, + { + version: 8, + sql: ` +CREATE TABLE IF NOT EXISTS workers ( + id TEXT PRIMARY KEY, + hostname TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'online', + providers TEXT NOT NULL DEFAULT '[]', + capabilities TEXT NOT NULL DEFAULT '[]', + capacity TEXT NOT NULL DEFAULT '{}', + last_heartbeat DATETIME NOT NULL DEFAULT (datetime('now')), + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_workers_status ON workers(status); +CREATE INDEX IF NOT EXISTS idx_workers_last_heartbeat ON workers(last_heartbeat DESC); +`, + }, + { + version: 9, + sql: ` +ALTER TABLE sandboxes ADD COLUMN worker_id TEXT NOT NULL DEFAULT 'local'; +CREATE INDEX IF NOT EXISTS idx_sandboxes_worker ON sandboxes(worker_id); +`, + }, + { + version: 10, + sql: ` +CREATE TABLE IF NOT EXISTS leases ( + resource_id TEXT PRIMARY KEY, + resource_type TEXT NOT NULL DEFAULT '', + holder_id TEXT NOT NULL, + generation INTEGER NOT NULL DEFAULT 1, + expires_at DATETIME NOT NULL, + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_leases_holder ON leases(holder_id); +CREATE INDEX IF NOT EXISTS idx_leases_expires_at ON leases(expires_at); +`, + }, + { + version: 11, + sql: ` +CREATE TABLE IF NOT EXISTS tenants ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + owner_id TEXT NOT NULL DEFAULT '', + settings TEXT NOT NULL DEFAULT '{}', + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_tenants_owner ON tenants(owner_id); + +CREATE TABLE IF NOT EXISTS tenant_members ( + tenant_id TEXT NOT NULL, + user_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'viewer', + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (tenant_id, user_id) +); + +CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id); + +CREATE TABLE IF NOT EXISTS policies ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + resource_type TEXT NOT NULL, + effect TEXT NOT NULL DEFAULT 'allow', + pattern TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 10, + created_at DATETIME NOT NULL DEFAULT (datetime('now')), + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_policies_tenant ON policies(tenant_id); +CREATE INDEX IF NOT EXISTS idx_policies_resource ON policies(resource_type); + +ALTER TABLE admin_audit_logs ADD COLUMN tenant_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE operation_audit_logs ADD COLUMN tenant_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE sandboxes ADD COLUMN tenant_id TEXT NOT NULL DEFAULT ''; +CREATE INDEX IF NOT EXISTS idx_sandboxes_tenant ON sandboxes(tenant_id); `, }, } diff --git a/internal/store/postgres.go b/internal/store/postgres.go new file mode 100644 index 0000000..4d14104 --- /dev/null +++ b/internal/store/postgres.go @@ -0,0 +1,1285 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "strconv" + "strings" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" +) + +type PostgresStore struct { + db *sql.DB +} + +func NewPostgresStore(dsn string) (*PostgresStore, error) { + db, err := sql.Open("pgx", dsn) + if err != nil { + return nil, fmt.Errorf("opening database: %w", err) + } + + db.SetMaxOpenConns(10) + db.SetMaxIdleConns(5) + + s := &PostgresStore{db: db} + if err := s.migrate(); err != nil { + db.Close() + return nil, fmt.Errorf("running migrations: %w", err) + } + + return s, nil +} + +func (s *PostgresStore) migrate() error { + // Ensure schema_migrations table exists + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`) + if err != nil { + return err + } + + for _, m := range postgresMigrations { + var count int + err := s.db.QueryRow("SELECT COUNT(*) FROM schema_migrations WHERE version = $1", m.version).Scan(&count) + if err != nil { + return err + } + if count > 0 { + continue + } + + tx, err := s.db.Begin() + if err != nil { + return err + } + + if _, err := tx.Exec(m.sql); err != nil { + tx.Rollback() + return fmt.Errorf("migration v%d: %w", m.version, err) + } + if _, err := tx.Exec("INSERT INTO schema_migrations (version) VALUES ($1)", m.version); err != nil { + tx.Rollback() + return err + } + + if err := tx.Commit(); err != nil { + return err + } + } + + return nil +} + +func (s *PostgresStore) execContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + return s.db.ExecContext(ctx, rebindPostgres(query), args...) +} + +func (s *PostgresStore) queryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + return s.db.QueryContext(ctx, rebindPostgres(query), args...) +} + +func (s *PostgresStore) queryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + return s.db.QueryRowContext(ctx, rebindPostgres(query), args...) +} + +func rebindPostgres(query string) string { + var b strings.Builder + b.Grow(len(query) + 8) + arg := 1 + for _, r := range query { + if r == '?' { + b.WriteByte('$') + b.WriteString(strconv.Itoa(arg)) + arg++ + continue + } + b.WriteRune(r) + } + return b.String() +} + +// --- Sandbox CRUD --- + +func (s *PostgresStore) CreateSandbox(ctx context.Context, sb *SandboxRecord) error { + if strings.TrimSpace(sb.WorkerID) == "" { + sb.WorkerID = "local" + } + _, err := s.execContext(ctx, ` + INSERT INTO sandboxes (id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + sb.ID, sb.State, sb.Provider, sb.Image, sb.MemoryMB, sb.VCPUs, sb.Metadata, + sb.OwnerID, sb.TenantID, sb.VMID, sb.WorkerID, + sb.CreatedAt.UTC(), sb.ExpiresAt.UTC(), sb.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetSandbox(ctx context.Context, id string) (*SandboxRecord, error) { + sb := &SandboxRecord{} + err := s.queryRowContext(ctx, ` + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at + FROM sandboxes WHERE id = ?`, id, + ).Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("sandbox", id) + } + return sb, err +} + +func (s *PostgresStore) ListSandboxes(ctx context.Context) ([]*SandboxRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at + FROM sandboxes WHERE state != 'destroyed' ORDER BY created_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var sandboxes []*SandboxRecord + for rows.Next() { + sb := &SandboxRecord{} + if err := rows.Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { + return nil, err + } + sandboxes = append(sandboxes, sb) + } + return sandboxes, rows.Err() +} + +func (s *PostgresStore) UpdateSandboxState(ctx context.Context, id string, state string) error { + res, err := s.execContext(ctx, ` + UPDATE sandboxes SET state = ?, updated_at = ? WHERE id = ?`, + state, time.Now().UTC(), id, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("sandbox", id) + } + return nil +} + +func (s *PostgresStore) UpdateSandboxExpiresAt(ctx context.Context, id string, expiresAt time.Time) error { + res, err := s.execContext(ctx, ` + UPDATE sandboxes SET expires_at = ?, updated_at = ? WHERE id = ? AND state != 'destroyed'`, + expiresAt.UTC(), time.Now().UTC(), id, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("sandbox", id) + } + return nil +} + +func (s *PostgresStore) DeleteSandbox(ctx context.Context, id string) error { + return s.UpdateSandboxState(ctx, id, "destroyed") +} + +func (s *PostgresStore) ListExpiredSandboxes(ctx context.Context, before time.Time) ([]*SandboxRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at + FROM sandboxes WHERE state NOT IN ('destroyed') AND expires_at < ?`, + before.UTC(), + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var sandboxes []*SandboxRecord + for rows.Next() { + sb := &SandboxRecord{} + if err := rows.Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { + return nil, err + } + sandboxes = append(sandboxes, sb) + } + return sandboxes, rows.Err() +} + +func (s *PostgresStore) ListSandboxesByOwner(ctx context.Context, ownerID string) ([]*SandboxRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at + FROM sandboxes WHERE state != 'destroyed' AND owner_id = ? ORDER BY created_at DESC`, ownerID) + if err != nil { + return nil, err + } + defer rows.Close() + + var sandboxes []*SandboxRecord + for rows.Next() { + sb := &SandboxRecord{} + if err := rows.Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { + return nil, err + } + sandboxes = append(sandboxes, sb) + } + return sandboxes, rows.Err() +} + +func (s *PostgresStore) CountSandboxesByVM(ctx context.Context, vmID string) (int, error) { + var count int + err := s.queryRowContext(ctx, ` + SELECT COUNT(*) FROM sandboxes WHERE state != 'destroyed' AND vm_id = ?`, vmID).Scan(&count) + return count, err +} + +func (s *PostgresStore) SaveOwnerQuota(ctx context.Context, quota *OwnerQuotaRecord) error { + now := time.Now().UTC() + if quota.CreatedAt.IsZero() { + quota.CreatedAt = now + } + quota.UpdatedAt = now + _, err := s.execContext(ctx, ` + INSERT INTO owner_quotas (owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(owner_id) DO UPDATE SET + max_sandboxes = excluded.max_sandboxes, + max_ttl_seconds = excluded.max_ttl_seconds, + max_exec_timeout_seconds = excluded.max_exec_timeout_seconds, + updated_at = excluded.updated_at`, + quota.OwnerID, quota.MaxSandboxes, quota.MaxTTLSeconds, quota.MaxExecTimeoutSeconds, + quota.CreatedAt.UTC(), quota.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuotaRecord, error) { + quota := &OwnerQuotaRecord{} + err := s.queryRowContext(ctx, ` + SELECT owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at + FROM owner_quotas WHERE owner_id = ?`, ownerID, + ).Scan("a.OwnerID, "a.MaxSandboxes, "a.MaxTTLSeconds, "a.MaxExecTimeoutSeconds, "a.CreatedAt, "a.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("owner_quota", ownerID) + } + return quota, err +} + +func (s *PostgresStore) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuotaRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at + FROM owner_quotas ORDER BY owner_id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var quotas []*OwnerQuotaRecord + for rows.Next() { + quota := &OwnerQuotaRecord{} + if err := rows.Scan("a.OwnerID, "a.MaxSandboxes, "a.MaxTTLSeconds, + "a.MaxExecTimeoutSeconds, "a.CreatedAt, "a.UpdatedAt); err != nil { + return nil, err + } + quotas = append(quotas, quota) + } + return quotas, rows.Err() +} + +func (s *PostgresStore) DeleteOwnerQuota(ctx context.Context, ownerID string) error { + res, err := s.execContext(ctx, `DELETE FROM owner_quotas WHERE owner_id = ?`, ownerID) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("owner_quota", ownerID) + } + return nil +} + +func (s *PostgresStore) CreateAdminAudit(ctx context.Context, rec *AdminAuditRecord) error { + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now().UTC() + } + res, err := s.execContext(ctx, ` + INSERT INTO admin_audit_logs (actor, method, path, status, duration_ms, request_id, remote_addr, user_agent, tenant_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + rec.Actor, rec.Method, rec.Path, rec.Status, rec.DurationMS, rec.RequestID, + rec.RemoteAddr, rec.UserAgent, rec.TenantID, rec.CreatedAt.UTC(), + ) + if err != nil { + return err + } + rec.ID, _ = res.LastInsertId() + return nil +} + +func (s *PostgresStore) ListAdminAudit(ctx context.Context, query AdminAuditQuery) ([]*AdminAuditRecord, error) { + if query.Limit <= 0 { + query.Limit = 100 + } + if query.Limit > 500 { + query.Limit = 500 + } + + clauses := []string{"1=1"} + args := make([]interface{}, 0, 6) + if query.Actor != "" { + clauses = append(clauses, "actor = ?") + args = append(args, query.Actor) + } + if query.Method != "" { + clauses = append(clauses, "method = ?") + args = append(args, query.Method) + } + if query.Status > 0 { + clauses = append(clauses, "status = ?") + args = append(args, query.Status) + } + if query.PathLike != "" { + clauses = append(clauses, "path LIKE ?") + args = append(args, "%"+query.PathLike+"%") + } + if query.TenantID != "" { + clauses = append(clauses, "tenant_id = ?") + args = append(args, query.TenantID) + } + args = append(args, query.Limit) + + rows, err := s.queryContext(ctx, ` + SELECT id, actor, method, path, status, duration_ms, request_id, remote_addr, user_agent, tenant_id, created_at + FROM admin_audit_logs WHERE `+strings.Join(clauses, " AND ")+` + ORDER BY created_at DESC, id DESC LIMIT ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*AdminAuditRecord + for rows.Next() { + rec := &AdminAuditRecord{} + if err := rows.Scan(&rec.ID, &rec.Actor, &rec.Method, &rec.Path, &rec.Status, + &rec.DurationMS, &rec.RequestID, &rec.RemoteAddr, &rec.UserAgent, &rec.TenantID, &rec.CreatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +func (s *PostgresStore) DeleteAdminAuditBefore(ctx context.Context, before time.Time) (int64, error) { + res, err := s.execContext(ctx, `DELETE FROM admin_audit_logs WHERE created_at < ?`, before.UTC()) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +func (s *PostgresStore) CreateOperationAudit(ctx context.Context, rec *OperationAuditRecord) error { + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now().UTC() + } + res, err := s.execContext(ctx, ` + INSERT INTO operation_audit_logs (actor, action, sandbox_id, resource, provider, status, detail, tenant_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + rec.Actor, rec.Action, rec.SandboxID, rec.Resource, rec.Provider, rec.Status, rec.Detail, rec.TenantID, rec.CreatedAt.UTC(), + ) + if err != nil { + return err + } + rec.ID, _ = res.LastInsertId() + return nil +} + +func (s *PostgresStore) ListOperationAudit(ctx context.Context, query OperationAuditQuery) ([]*OperationAuditRecord, error) { + if query.Limit <= 0 { + query.Limit = 100 + } + if query.Limit > 500 { + query.Limit = 500 + } + + clauses := []string{"1=1"} + args := make([]interface{}, 0, 8) + if query.Actor != "" { + clauses = append(clauses, "actor = ?") + args = append(args, query.Actor) + } + if query.Action != "" { + clauses = append(clauses, "action = ?") + args = append(args, query.Action) + } + if query.SandboxID != "" { + clauses = append(clauses, "sandbox_id = ?") + args = append(args, query.SandboxID) + } + if query.Resource != "" { + clauses = append(clauses, "resource = ?") + args = append(args, query.Resource) + } + if query.Status != "" { + clauses = append(clauses, "status = ?") + args = append(args, query.Status) + } + if query.TenantID != "" { + clauses = append(clauses, "tenant_id = ?") + args = append(args, query.TenantID) + } + args = append(args, query.Limit) + + rows, err := s.queryContext(ctx, ` + SELECT id, actor, action, sandbox_id, resource, provider, status, detail, tenant_id, created_at + FROM operation_audit_logs WHERE `+strings.Join(clauses, " AND ")+` + ORDER BY created_at DESC, id DESC LIMIT ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*OperationAuditRecord + for rows.Next() { + rec := &OperationAuditRecord{} + if err := rows.Scan(&rec.ID, &rec.Actor, &rec.Action, &rec.SandboxID, &rec.Resource, + &rec.Provider, &rec.Status, &rec.Detail, &rec.TenantID, &rec.CreatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +// --- Workers --- + +func (s *PostgresStore) SaveWorker(ctx context.Context, rec *WorkerRecord) error { + now := time.Now().UTC() + if rec.CreatedAt.IsZero() { + rec.CreatedAt = now + } + if rec.LastHeartbeat.IsZero() { + rec.LastHeartbeat = now + } + rec.UpdatedAt = now + _, err := s.execContext(ctx, ` + INSERT INTO workers (id, hostname, status, providers, capabilities, capacity, last_heartbeat, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + hostname = excluded.hostname, + status = excluded.status, + providers = excluded.providers, + capabilities = excluded.capabilities, + capacity = excluded.capacity, + last_heartbeat = excluded.last_heartbeat, + updated_at = excluded.updated_at`, + rec.ID, rec.Hostname, rec.Status, rec.Providers, rec.Capabilities, rec.Capacity, + rec.LastHeartbeat.UTC(), rec.CreatedAt.UTC(), rec.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetWorker(ctx context.Context, id string) (*WorkerRecord, error) { + rec := &WorkerRecord{} + err := s.queryRowContext(ctx, ` + SELECT id, hostname, status, providers, capabilities, capacity, last_heartbeat, created_at, updated_at + FROM workers WHERE id = ?`, id, + ).Scan(&rec.ID, &rec.Hostname, &rec.Status, &rec.Providers, &rec.Capabilities, &rec.Capacity, + &rec.LastHeartbeat, &rec.CreatedAt, &rec.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("worker", id) + } + return rec, err +} + +func (s *PostgresStore) ListWorkers(ctx context.Context) ([]*WorkerRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, hostname, status, providers, capabilities, capacity, last_heartbeat, created_at, updated_at + FROM workers ORDER BY id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*WorkerRecord + for rows.Next() { + rec := &WorkerRecord{} + if err := rows.Scan(&rec.ID, &rec.Hostname, &rec.Status, &rec.Providers, &rec.Capabilities, + &rec.Capacity, &rec.LastHeartbeat, &rec.CreatedAt, &rec.UpdatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +func (s *PostgresStore) DeleteWorker(ctx context.Context, id string) error { + res, err := s.execContext(ctx, `DELETE FROM workers WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("worker", id) + } + return nil +} + +// --- Leases --- + +func (s *PostgresStore) AcquireLease(ctx context.Context, resourceID, resourceType, holderID string, ttl time.Duration) (*LeaseRecord, error) { + if strings.TrimSpace(resourceID) == "" { + return nil, ConflictError("lease resource id is required") + } + if strings.TrimSpace(holderID) == "" { + return nil, ConflictError("lease holder id is required") + } + if ttl <= 0 { + return nil, ConflictError("lease ttl must be positive") + } + now := time.Now().UTC() + expiresAt := now.Add(ttl) + res, err := s.execContext(ctx, ` + INSERT INTO leases (resource_id, resource_type, holder_id, generation, expires_at, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?, ?) + ON CONFLICT(resource_id) DO UPDATE SET + resource_type = excluded.resource_type, + holder_id = excluded.holder_id, + generation = leases.generation + 1, + expires_at = excluded.expires_at, + updated_at = excluded.updated_at + WHERE leases.expires_at <= ? OR leases.holder_id = excluded.holder_id`, + resourceID, resourceType, holderID, expiresAt, now, now, now, + ) + if err != nil { + return nil, err + } + rows, _ := res.RowsAffected() + if rows == 0 { + return nil, ConflictError("lease is held by another worker") + } + return s.GetLease(ctx, resourceID) +} + +func (s *PostgresStore) RenewLease(ctx context.Context, resourceID, holderID string, ttl time.Duration) (*LeaseRecord, error) { + if ttl <= 0 { + return nil, ConflictError("lease ttl must be positive") + } + now := time.Now().UTC() + res, err := s.execContext(ctx, ` + UPDATE leases + SET generation = generation + 1, expires_at = ?, updated_at = ? + WHERE resource_id = ? AND holder_id = ? AND expires_at > ?`, + now.Add(ttl), now, resourceID, holderID, now, + ) + if err != nil { + return nil, err + } + rows, _ := res.RowsAffected() + if rows == 0 { + return nil, ConflictError("lease is not held by worker or has expired") + } + return s.GetLease(ctx, resourceID) +} + +func (s *PostgresStore) GetLease(ctx context.Context, resourceID string) (*LeaseRecord, error) { + rec := &LeaseRecord{} + err := s.queryRowContext(ctx, ` + SELECT resource_id, resource_type, holder_id, generation, expires_at, created_at, updated_at + FROM leases WHERE resource_id = ?`, resourceID, + ).Scan(&rec.ResourceID, &rec.ResourceType, &rec.HolderID, &rec.Generation, &rec.ExpiresAt, &rec.CreatedAt, &rec.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("lease", resourceID) + } + return rec, err +} + +func (s *PostgresStore) ListLeases(ctx context.Context) ([]*LeaseRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT resource_id, resource_type, holder_id, generation, expires_at, created_at, updated_at + FROM leases ORDER BY resource_id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*LeaseRecord + for rows.Next() { + rec := &LeaseRecord{} + if err := rows.Scan(&rec.ResourceID, &rec.ResourceType, &rec.HolderID, &rec.Generation, &rec.ExpiresAt, &rec.CreatedAt, &rec.UpdatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +func (s *PostgresStore) ReleaseLease(ctx context.Context, resourceID, holderID string) error { + res, err := s.execContext(ctx, `DELETE FROM leases WHERE resource_id = ? AND holder_id = ?`, resourceID, holderID) + if err != nil { + return err + } + rows, _ := res.RowsAffected() + if rows == 0 { + return NotFoundError("lease", resourceID) + } + return nil +} + +// --- Exec Logs --- + +func (s *PostgresStore) CreateExecLog(ctx context.Context, log *ExecLogRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO exec_logs (sandbox_id, command, exit_code, stdout, stderr, duration, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + log.SandboxID, log.Command, log.ExitCode, log.Stdout, log.Stderr, log.Duration, log.CreatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) ListExecLogs(ctx context.Context, sandboxID string) ([]*ExecLogRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, sandbox_id, command, exit_code, stdout, stderr, duration, created_at + FROM exec_logs WHERE sandbox_id = ? ORDER BY created_at DESC`, sandboxID) + if err != nil { + return nil, err + } + defer rows.Close() + + var logs []*ExecLogRecord + for rows.Next() { + l := &ExecLogRecord{} + if err := rows.Scan(&l.ID, &l.SandboxID, &l.Command, &l.ExitCode, &l.Stdout, &l.Stderr, + &l.Duration, &l.CreatedAt); err != nil { + return nil, err + } + logs = append(logs, l) + } + return logs, rows.Err() +} + +// --- Provider Configs --- + +func (s *PostgresStore) GetProviderConfig(ctx context.Context, name string) (*ProviderConfigRecord, error) { + cfg := &ProviderConfigRecord{} + err := s.queryRowContext(ctx, ` + SELECT name, config, enabled, updated_at FROM provider_configs WHERE name = ?`, name, + ).Scan(&cfg.Name, &cfg.Config, &cfg.Enabled, &cfg.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("provider config", name) + } + return cfg, err +} + +func (s *PostgresStore) SaveProviderConfig(ctx context.Context, cfg *ProviderConfigRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO provider_configs (name, config, enabled, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET config = excluded.config, enabled = excluded.enabled, updated_at = excluded.updated_at`, + cfg.Name, cfg.Config, cfg.Enabled, time.Now().UTC(), + ) + return err +} + +func (s *PostgresStore) ListProviderConfigs(ctx context.Context) ([]*ProviderConfigRecord, error) { + rows, err := s.queryContext(ctx, `SELECT name, config, enabled, updated_at FROM provider_configs`) + if err != nil { + return nil, err + } + defer rows.Close() + + var configs []*ProviderConfigRecord + for rows.Next() { + cfg := &ProviderConfigRecord{} + if err := rows.Scan(&cfg.Name, &cfg.Config, &cfg.Enabled, &cfg.UpdatedAt); err != nil { + return nil, err + } + configs = append(configs, cfg) + } + return configs, rows.Err() +} + +// --- Templates --- + +func (s *PostgresStore) CreateTemplate(ctx context.Context, t *TemplateRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO templates (name, version, image, description, setup, allowed_hosts, memory_mb, cpu_cores, ttl_seconds, env, secrets, pool_size, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + t.Name, t.Version, t.Image, t.Description, t.Setup, t.AllowedHosts, + t.MemoryMB, t.CPUCores, t.TTLSeconds, t.Env, t.Secrets, t.PoolSize, + t.CreatedAt.UTC(), t.UpdatedAt.UTC(), + ) + if IsConstraintError(err) { + return ConflictError("template already exists") + } + return err +} + +func (s *PostgresStore) GetTemplate(ctx context.Context, name string) (*TemplateRecord, error) { + t := &TemplateRecord{} + err := s.queryRowContext(ctx, ` + SELECT name, version, image, description, setup, allowed_hosts, memory_mb, cpu_cores, ttl_seconds, env, secrets, pool_size, created_at, updated_at + FROM templates WHERE name = ?`, name, + ).Scan(&t.Name, &t.Version, &t.Image, &t.Description, &t.Setup, &t.AllowedHosts, + &t.MemoryMB, &t.CPUCores, &t.TTLSeconds, &t.Env, &t.Secrets, &t.PoolSize, + &t.CreatedAt, &t.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("template", name) + } + return t, err +} + +func (s *PostgresStore) ListTemplates(ctx context.Context) ([]*TemplateRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT name, version, image, description, setup, allowed_hosts, memory_mb, cpu_cores, ttl_seconds, env, secrets, pool_size, created_at, updated_at + FROM templates ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + + var templates []*TemplateRecord + for rows.Next() { + t := &TemplateRecord{} + if err := rows.Scan(&t.Name, &t.Version, &t.Image, &t.Description, &t.Setup, &t.AllowedHosts, + &t.MemoryMB, &t.CPUCores, &t.TTLSeconds, &t.Env, &t.Secrets, &t.PoolSize, + &t.CreatedAt, &t.UpdatedAt); err != nil { + return nil, err + } + templates = append(templates, t) + } + return templates, rows.Err() +} + +func (s *PostgresStore) UpdateTemplate(ctx context.Context, t *TemplateRecord) error { + res, err := s.execContext(ctx, ` + UPDATE templates SET version = ?, image = ?, description = ?, setup = ?, allowed_hosts = ?, + memory_mb = ?, cpu_cores = ?, ttl_seconds = ?, env = ?, secrets = ?, pool_size = ?, updated_at = ? + WHERE name = ?`, + t.Version, t.Image, t.Description, t.Setup, t.AllowedHosts, + t.MemoryMB, t.CPUCores, t.TTLSeconds, t.Env, t.Secrets, t.PoolSize, + time.Now().UTC(), t.Name, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("template", t.Name) + } + return nil +} + +func (s *PostgresStore) DeleteTemplate(ctx context.Context, name string) error { + res, err := s.execContext(ctx, `DELETE FROM templates WHERE name = ?`, name) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("template", name) + } + return nil +} + +// --- Environment Specs --- + +func (s *PostgresStore) CreateEnvironmentSpec(ctx context.Context, spec *EnvironmentSpecRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO environment_specs (id, owner_id, name, base_image, python_packages, apt_packages, python_version, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + spec.ID, spec.OwnerID, spec.Name, spec.BaseImage, spec.PythonPackages, spec.AptPackages, spec.PythonVersion, + spec.CreatedAt.UTC(), spec.UpdatedAt.UTC(), + ) + if IsConstraintError(err) { + return ConflictError("spec name already exists for this owner") + } + return err +} + +func (s *PostgresStore) GetEnvironmentSpec(ctx context.Context, id string) (*EnvironmentSpecRecord, error) { + spec := &EnvironmentSpecRecord{} + err := s.queryRowContext(ctx, ` + SELECT id, owner_id, name, base_image, python_packages, apt_packages, python_version, created_at, updated_at + FROM environment_specs WHERE id = ?`, id, + ).Scan( + &spec.ID, &spec.OwnerID, &spec.Name, &spec.BaseImage, &spec.PythonPackages, &spec.AptPackages, + &spec.PythonVersion, &spec.CreatedAt, &spec.UpdatedAt, + ) + if err == sql.ErrNoRows { + return nil, NotFoundError("environment spec", id) + } + return spec, err +} + +func (s *PostgresStore) ListEnvironmentSpecs(ctx context.Context, ownerID string) ([]*EnvironmentSpecRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, owner_id, name, base_image, python_packages, apt_packages, python_version, created_at, updated_at + FROM environment_specs + WHERE owner_id = ? + ORDER BY created_at DESC`, ownerID) + if err != nil { + return nil, err + } + defer rows.Close() + + var specs []*EnvironmentSpecRecord + for rows.Next() { + spec := &EnvironmentSpecRecord{} + if err := rows.Scan( + &spec.ID, &spec.OwnerID, &spec.Name, &spec.BaseImage, &spec.PythonPackages, &spec.AptPackages, + &spec.PythonVersion, &spec.CreatedAt, &spec.UpdatedAt, + ); err != nil { + return nil, err + } + specs = append(specs, spec) + } + return specs, rows.Err() +} + +func (s *PostgresStore) UpdateEnvironmentSpec(ctx context.Context, spec *EnvironmentSpecRecord) error { + res, err := s.execContext(ctx, ` + UPDATE environment_specs + SET name = ?, base_image = ?, python_packages = ?, apt_packages = ?, python_version = ?, updated_at = ? + WHERE id = ?`, + spec.Name, spec.BaseImage, spec.PythonPackages, spec.AptPackages, spec.PythonVersion, + time.Now().UTC(), spec.ID, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("environment spec", spec.ID) + } + return nil +} + +func (s *PostgresStore) DeleteEnvironmentSpec(ctx context.Context, id string) error { + res, err := s.execContext(ctx, `DELETE FROM environment_specs WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("environment spec", id) + } + return nil +} + +// --- Environment Builds --- + +func (s *PostgresStore) CreateEnvironmentBuild(ctx context.Context, build *EnvironmentBuildRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO environment_builds ( + id, spec_id, status, current_step, log_blob, image_size_bytes, digest_local, error, created_at, finished_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + build.ID, build.SpecID, build.Status, build.CurrentStep, build.LogBlob, build.ImageSizeBytes, + build.DigestLocal, build.Error, build.CreatedAt.UTC(), postgresNullableTime(build.FinishedAt), build.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetEnvironmentBuild(ctx context.Context, id string) (*EnvironmentBuildRecord, error) { + build := &EnvironmentBuildRecord{} + var finishedAt sql.NullTime + + err := s.queryRowContext(ctx, ` + SELECT id, spec_id, status, current_step, log_blob, image_size_bytes, digest_local, error, created_at, finished_at, updated_at + FROM environment_builds WHERE id = ?`, id, + ).Scan( + &build.ID, &build.SpecID, &build.Status, &build.CurrentStep, &build.LogBlob, &build.ImageSizeBytes, + &build.DigestLocal, &build.Error, &build.CreatedAt, &finishedAt, &build.UpdatedAt, + ) + if err == sql.ErrNoRows { + return nil, NotFoundError("environment build", id) + } + if err != nil { + return nil, err + } + if finishedAt.Valid { + t := finishedAt.Time + build.FinishedAt = &t + } + return build, nil +} + +func (s *PostgresStore) ListEnvironmentBuilds(ctx context.Context, specID string) ([]*EnvironmentBuildRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, spec_id, status, current_step, log_blob, image_size_bytes, digest_local, error, created_at, finished_at, updated_at + FROM environment_builds + WHERE spec_id = ? + ORDER BY created_at DESC`, specID) + if err != nil { + return nil, err + } + defer rows.Close() + + var builds []*EnvironmentBuildRecord + for rows.Next() { + build := &EnvironmentBuildRecord{} + var finishedAt sql.NullTime + if err := rows.Scan( + &build.ID, &build.SpecID, &build.Status, &build.CurrentStep, &build.LogBlob, &build.ImageSizeBytes, + &build.DigestLocal, &build.Error, &build.CreatedAt, &finishedAt, &build.UpdatedAt, + ); err != nil { + return nil, err + } + if finishedAt.Valid { + t := finishedAt.Time + build.FinishedAt = &t + } + builds = append(builds, build) + } + return builds, rows.Err() +} + +func (s *PostgresStore) UpdateEnvironmentBuild(ctx context.Context, build *EnvironmentBuildRecord) error { + res, err := s.execContext(ctx, ` + UPDATE environment_builds + SET status = ?, current_step = ?, log_blob = ?, image_size_bytes = ?, digest_local = ?, error = ?, finished_at = ?, updated_at = ? + WHERE id = ?`, + build.Status, build.CurrentStep, build.LogBlob, build.ImageSizeBytes, build.DigestLocal, build.Error, + postgresNullableTime(build.FinishedAt), time.Now().UTC(), build.ID, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("environment build", build.ID) + } + return nil +} + +// --- Environment Artifacts --- + +func (s *PostgresStore) SaveEnvironmentArtifact(ctx context.Context, artifact *EnvironmentArtifactRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO environment_artifacts (build_id, target, image_ref, digest, status, error, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(build_id, target) DO UPDATE SET + image_ref = excluded.image_ref, + digest = excluded.digest, + status = excluded.status, + error = excluded.error, + updated_at = excluded.updated_at`, + artifact.BuildID, artifact.Target, artifact.ImageRef, artifact.Digest, artifact.Status, artifact.Error, + time.Now().UTC(), time.Now().UTC(), + ) + return err +} + +func (s *PostgresStore) ListEnvironmentArtifacts(ctx context.Context, buildID string) ([]*EnvironmentArtifactRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, build_id, target, image_ref, digest, status, error, created_at, updated_at + FROM environment_artifacts + WHERE build_id = ? + ORDER BY id ASC`, buildID) + if err != nil { + return nil, err + } + defer rows.Close() + + var artifacts []*EnvironmentArtifactRecord + for rows.Next() { + artifact := &EnvironmentArtifactRecord{} + if err := rows.Scan( + &artifact.ID, &artifact.BuildID, &artifact.Target, &artifact.ImageRef, &artifact.Digest, + &artifact.Status, &artifact.Error, &artifact.CreatedAt, &artifact.UpdatedAt, + ); err != nil { + return nil, err + } + artifacts = append(artifacts, artifact) + } + return artifacts, rows.Err() +} + +// --- Registry Connections --- + +func (s *PostgresStore) SaveRegistryConnection(ctx context.Context, conn *RegistryConnectionRecord) error { + _, err := s.execContext(ctx, ` + INSERT INTO registry_connections (id, owner_id, provider, username, secret_ref, is_default, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + owner_id = excluded.owner_id, + provider = excluded.provider, + username = excluded.username, + secret_ref = excluded.secret_ref, + is_default = excluded.is_default, + updated_at = excluded.updated_at`, + conn.ID, conn.OwnerID, conn.Provider, conn.Username, conn.SecretRef, conn.IsDefault, + time.Now().UTC(), time.Now().UTC(), + ) + if IsConstraintError(err) { + return ConflictError("registry connection already exists") + } + return err +} + +func (s *PostgresStore) GetRegistryConnection(ctx context.Context, id string) (*RegistryConnectionRecord, error) { + conn := &RegistryConnectionRecord{} + err := s.queryRowContext(ctx, ` + SELECT id, owner_id, provider, username, secret_ref, is_default, created_at, updated_at + FROM registry_connections + WHERE id = ?`, id, + ).Scan(&conn.ID, &conn.OwnerID, &conn.Provider, &conn.Username, &conn.SecretRef, &conn.IsDefault, &conn.CreatedAt, &conn.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("registry connection", id) + } + return conn, err +} + +func (s *PostgresStore) ListRegistryConnections(ctx context.Context, ownerID string) ([]*RegistryConnectionRecord, error) { + rows, err := s.queryContext(ctx, ` + SELECT id, owner_id, provider, username, secret_ref, is_default, created_at, updated_at + FROM registry_connections + WHERE owner_id = ? + ORDER BY created_at DESC`, ownerID) + if err != nil { + return nil, err + } + defer rows.Close() + + var conns []*RegistryConnectionRecord + for rows.Next() { + conn := &RegistryConnectionRecord{} + if err := rows.Scan( + &conn.ID, &conn.OwnerID, &conn.Provider, &conn.Username, &conn.SecretRef, &conn.IsDefault, &conn.CreatedAt, &conn.UpdatedAt, + ); err != nil { + return nil, err + } + conns = append(conns, conn) + } + return conns, rows.Err() +} + +func (s *PostgresStore) DeleteRegistryConnection(ctx context.Context, id string) error { + res, err := s.execContext(ctx, `DELETE FROM registry_connections WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("registry connection", id) + } + return nil +} + +func postgresNullableTime(t *time.Time) any { + if t == nil { + return nil + } + return t.UTC() +} + +// --- Tenants --- + +func (s *PostgresStore) CreateTenant(ctx context.Context, t *TenantRecord) error { + now := time.Now().UTC() + if t.CreatedAt.IsZero() { + t.CreatedAt = now + } + t.UpdatedAt = now + if t.Settings == "" { + t.Settings = "{}" + } + _, err := s.execContext(ctx, + `INSERT INTO tenants (id, name, owner_id, settings, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`, + t.ID, t.Name, t.OwnerID, t.Settings, t.CreatedAt.UTC(), t.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetTenant(ctx context.Context, id string) (*TenantRecord, error) { + t := &TenantRecord{} + err := s.queryRowContext(ctx, + `SELECT id, name, owner_id, settings, created_at, updated_at FROM tenants WHERE id = ?`, id, + ).Scan(&t.ID, &t.Name, &t.OwnerID, &t.Settings, &t.CreatedAt, &t.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("tenant", id) + } + return t, err +} + +func (s *PostgresStore) ListTenants(ctx context.Context) ([]*TenantRecord, error) { + rows, err := s.queryContext(ctx, + `SELECT id, name, owner_id, settings, created_at, updated_at FROM tenants ORDER BY created_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + var tenants []*TenantRecord + for rows.Next() { + t := &TenantRecord{} + if err := rows.Scan(&t.ID, &t.Name, &t.OwnerID, &t.Settings, &t.CreatedAt, &t.UpdatedAt); err != nil { + return nil, err + } + tenants = append(tenants, t) + } + return tenants, rows.Err() +} + +func (s *PostgresStore) UpdateTenant(ctx context.Context, t *TenantRecord) error { + t.UpdatedAt = time.Now().UTC() + res, err := s.execContext(ctx, + `UPDATE tenants SET name = ?, owner_id = ?, settings = ?, updated_at = ? WHERE id = ?`, + t.Name, t.OwnerID, t.Settings, t.UpdatedAt.UTC(), t.ID, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("tenant", t.ID) + } + return nil +} + +func (s *PostgresStore) DeleteTenant(ctx context.Context, id string) error { + res, err := s.execContext(ctx, `DELETE FROM tenants WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("tenant", id) + } + return nil +} + +// --- Tenant members --- + +func (s *PostgresStore) SaveTenantMember(ctx context.Context, m *TenantMemberRecord) error { + now := time.Now().UTC() + if m.CreatedAt.IsZero() { + m.CreatedAt = now + } + m.UpdatedAt = now + _, err := s.execContext(ctx, ` + INSERT INTO tenant_members (tenant_id, user_id, role, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (tenant_id, user_id) DO UPDATE SET role = EXCLUDED.role, updated_at = EXCLUDED.updated_at`, + m.TenantID, m.UserID, m.Role, m.CreatedAt.UTC(), m.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetTenantMember(ctx context.Context, tenantID, userID string) (*TenantMemberRecord, error) { + m := &TenantMemberRecord{} + err := s.queryRowContext(ctx, + `SELECT tenant_id, user_id, role, created_at, updated_at FROM tenant_members WHERE tenant_id = ? AND user_id = ?`, + tenantID, userID, + ).Scan(&m.TenantID, &m.UserID, &m.Role, &m.CreatedAt, &m.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("tenant_member", tenantID+"/"+userID) + } + return m, err +} + +func (s *PostgresStore) ListTenantMembers(ctx context.Context, tenantID string) ([]*TenantMemberRecord, error) { + rows, err := s.queryContext(ctx, + `SELECT tenant_id, user_id, role, created_at, updated_at FROM tenant_members WHERE tenant_id = ? ORDER BY created_at ASC`, + tenantID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var members []*TenantMemberRecord + for rows.Next() { + m := &TenantMemberRecord{} + if err := rows.Scan(&m.TenantID, &m.UserID, &m.Role, &m.CreatedAt, &m.UpdatedAt); err != nil { + return nil, err + } + members = append(members, m) + } + return members, rows.Err() +} + +func (s *PostgresStore) DeleteTenantMember(ctx context.Context, tenantID, userID string) error { + res, err := s.execContext(ctx, `DELETE FROM tenant_members WHERE tenant_id = ? AND user_id = ?`, tenantID, userID) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("tenant_member", tenantID+"/"+userID) + } + return nil +} + +// --- Policies --- + +func (s *PostgresStore) CreatePolicy(ctx context.Context, p *PolicyRecord) error { + now := time.Now().UTC() + if p.CreatedAt.IsZero() { + p.CreatedAt = now + } + p.UpdatedAt = now + _, err := s.execContext(ctx, + `INSERT INTO policies (id, tenant_id, resource_type, effect, pattern, priority, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + p.ID, p.TenantID, p.ResourceType, p.Effect, p.Pattern, p.Priority, p.CreatedAt.UTC(), p.UpdatedAt.UTC(), + ) + return err +} + +func (s *PostgresStore) GetPolicy(ctx context.Context, id string) (*PolicyRecord, error) { + p := &PolicyRecord{} + err := s.queryRowContext(ctx, + `SELECT id, tenant_id, resource_type, effect, pattern, priority, created_at, updated_at FROM policies WHERE id = ?`, id, + ).Scan(&p.ID, &p.TenantID, &p.ResourceType, &p.Effect, &p.Pattern, &p.Priority, &p.CreatedAt, &p.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("policy", id) + } + return p, err +} + +func (s *PostgresStore) ListPolicies(ctx context.Context, query PolicyQuery) ([]*PolicyRecord, error) { + q := `SELECT id, tenant_id, resource_type, effect, pattern, priority, created_at, updated_at FROM policies WHERE 1=1` + var args []any + if query.TenantID != "" { + q += " AND (tenant_id = ? OR tenant_id = '')" + args = append(args, query.TenantID) + } else { + q += " AND tenant_id = ''" + } + if query.ResourceType != "" { + q += " AND resource_type = ?" + args = append(args, query.ResourceType) + } + q += " ORDER BY priority ASC, created_at ASC" + rows, err := s.queryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var policies []*PolicyRecord + for rows.Next() { + p := &PolicyRecord{} + if err := rows.Scan(&p.ID, &p.TenantID, &p.ResourceType, &p.Effect, &p.Pattern, &p.Priority, &p.CreatedAt, &p.UpdatedAt); err != nil { + return nil, err + } + policies = append(policies, p) + } + return policies, rows.Err() +} + +func (s *PostgresStore) DeletePolicy(ctx context.Context, id string) error { + res, err := s.execContext(ctx, `DELETE FROM policies WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("policy", id) + } + return nil +} + +func (s *PostgresStore) Close() error { + return s.db.Close() +} diff --git a/internal/store/postgres_lease_test.go b/internal/store/postgres_lease_test.go new file mode 100644 index 0000000..06eb445 --- /dev/null +++ b/internal/store/postgres_lease_test.go @@ -0,0 +1,122 @@ +package store + +import ( + "context" + "errors" + "os" + "strconv" + "sync" + "testing" + "time" +) + +func TestPostgresLeaseConcurrency(t *testing.T) { + dsn := os.Getenv("STACYVM_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("set STACYVM_POSTGRES_TEST_DSN to run Postgres lease concurrency test") + } + + const contenders = 16 + ctx := context.Background() + stores := make([]*PostgresStore, 0, contenders) + for i := 0; i < contenders; i++ { + st, err := NewPostgresStore(dsn) + if err != nil { + t.Fatalf("open postgres store %d: %v", i, err) + } + stores = append(stores, st) + t.Cleanup(func() { _ = st.Close() }) + } + resetPostgresContractStore(t, stores[0]) + + var wg sync.WaitGroup + winners := make(chan string, contenders) + failures := make(chan error, contenders) + for i := 0; i < contenders; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + holderID := "worker-" + strconv.Itoa(i) + lease, err := stores[i].AcquireLease(ctx, "lease-race", "sandbox", holderID, time.Minute) + if err == nil { + winners <- lease.HolderID + return + } + if !errors.Is(err, ErrConflict) { + failures <- err + } + }() + } + wg.Wait() + close(winners) + close(failures) + + if len(failures) > 0 { + t.Fatalf("unexpected acquire error: %v", <-failures) + } + if len(winners) != 1 { + t.Fatalf("lease winners = %d, want 1", len(winners)) + } + winner := <-winners + lease, err := stores[0].GetLease(ctx, "lease-race") + if err != nil { + t.Fatalf("get lease: %v", err) + } + if lease.HolderID != winner { + t.Fatalf("stored holder = %q, want winning holder %q", lease.HolderID, winner) + } + + renewed, err := stores[0].RenewLease(ctx, "lease-race", winner, time.Nanosecond) + if err != nil { + t.Fatalf("renew lease: %v", err) + } + if renewed.Generation != lease.Generation+1 { + t.Fatalf("renewed generation = %d, want %d", renewed.Generation, lease.Generation+1) + } + time.Sleep(2 * time.Millisecond) + + var takeoverWG sync.WaitGroup + takeovers := make(chan string, contenders) + takeoverFailures := make(chan error, contenders) + for i := 0; i < contenders; i++ { + i := i + takeoverWG.Add(1) + go func() { + defer takeoverWG.Done() + holderID := "takeover-worker-" + strconv.Itoa(i) + lease, err := stores[i].AcquireLease(ctx, "lease-race", "sandbox", holderID, time.Minute) + if err == nil { + takeovers <- lease.HolderID + return + } + if !errors.Is(err, ErrConflict) { + takeoverFailures <- err + } + }() + } + takeoverWG.Wait() + close(takeovers) + close(takeoverFailures) + + if len(takeoverFailures) > 0 { + t.Fatalf("unexpected takeover error: %v", <-takeoverFailures) + } + if len(takeovers) != 1 { + t.Fatalf("takeover winners = %d, want 1", len(takeovers)) + } + takeoverWinner := <-takeovers + if takeoverWinner == winner { + t.Fatalf("takeover holder = original holder %q, want a new holder", takeoverWinner) + } + lease, err = stores[0].GetLease(ctx, "lease-race") + if err != nil { + t.Fatalf("get takeover lease: %v", err) + } + if lease.HolderID != takeoverWinner { + t.Fatalf("stored takeover holder = %q, want %q", lease.HolderID, takeoverWinner) + } + if lease.Generation <= renewed.Generation { + t.Fatalf("takeover generation = %d, want > %d", lease.Generation, renewed.Generation) + } +} diff --git a/internal/store/postgres_migration_rehearsal_test.go b/internal/store/postgres_migration_rehearsal_test.go new file mode 100644 index 0000000..269f848 --- /dev/null +++ b/internal/store/postgres_migration_rehearsal_test.go @@ -0,0 +1,45 @@ +package store + +import ( + "os" + "testing" +) + +func TestPostgresMigrationRehearsal(t *testing.T) { + dsn := os.Getenv("STACYVM_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("set STACYVM_POSTGRES_TEST_DSN to run Postgres migration rehearsal") + } + + first, err := NewPostgresStore(dsn) + if err != nil { + t.Fatalf("first open postgres store: %v", err) + } + if err := first.Close(); err != nil { + t.Fatalf("close first postgres store: %v", err) + } + + second, err := NewPostgresStore(dsn) + if err != nil { + t.Fatalf("second open postgres store: %v", err) + } + defer second.Close() + + var applied int + if err := second.db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&applied); err != nil { + t.Fatalf("count postgres migrations: %v", err) + } + if applied != len(postgresMigrations) { + t.Fatalf("applied postgres migrations = %d, want %d", applied, len(postgresMigrations)) + } + + for _, migration := range postgresMigrations { + var exists int + if err := second.db.QueryRow("SELECT COUNT(*) FROM schema_migrations WHERE version = $1", migration.version).Scan(&exists); err != nil { + t.Fatalf("check postgres migration %d: %v", migration.version, err) + } + if exists != 1 { + t.Fatalf("postgres migration %d recorded %d times, want once", migration.version, exists) + } + } +} diff --git a/internal/store/postgres_migrations.go b/internal/store/postgres_migrations.go new file mode 100644 index 0000000..5c0876b --- /dev/null +++ b/internal/store/postgres_migrations.go @@ -0,0 +1,287 @@ +package store + +var postgresMigrations = []migration{ + { + version: 1, + sql: ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS sandboxes ( + id TEXT PRIMARY KEY, + state TEXT NOT NULL DEFAULT 'creating', + provider TEXT NOT NULL, + image TEXT NOT NULL DEFAULT '', + memory_mb INTEGER NOT NULL DEFAULT 512, + vcpus INTEGER NOT NULL DEFAULT 1, + metadata TEXT NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS exec_logs ( + id BIGSERIAL PRIMARY KEY, + sandbox_id TEXT NOT NULL REFERENCES sandboxes(id), + command TEXT NOT NULL, + exit_code INTEGER NOT NULL DEFAULT 0, + stdout TEXT NOT NULL DEFAULT '', + stderr TEXT NOT NULL DEFAULT '', + duration TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_exec_logs_sandbox ON exec_logs(sandbox_id); + +CREATE TABLE IF NOT EXISTS provider_configs ( + name TEXT PRIMARY KEY, + config TEXT NOT NULL DEFAULT '{}', + enabled BOOLEAN NOT NULL DEFAULT false, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +`, + }, + { + version: 2, + sql: ` +CREATE TABLE IF NOT EXISTS templates ( + name TEXT PRIMARY KEY, + version INTEGER NOT NULL DEFAULT 1, + image TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + setup TEXT NOT NULL DEFAULT '[]', + allowed_hosts TEXT NOT NULL DEFAULT '[]', + memory_mb INTEGER NOT NULL DEFAULT 512, + cpu_cores INTEGER NOT NULL DEFAULT 1, + ttl_seconds INTEGER NOT NULL DEFAULT 300, + env TEXT NOT NULL DEFAULT '{}', + secrets TEXT NOT NULL DEFAULT '[]', + pool_size INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +ALTER TABLE sandboxes ADD COLUMN IF NOT EXISTS template TEXT NOT NULL DEFAULT ''; +`, + }, + { + version: 3, + sql: ` +CREATE TABLE IF NOT EXISTS environment_specs ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + name TEXT NOT NULL, + base_image TEXT NOT NULL, + python_packages TEXT NOT NULL DEFAULT '[]', + apt_packages TEXT NOT NULL DEFAULT '[]', + python_version TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(owner_id, name) +); + +CREATE INDEX IF NOT EXISTS idx_environment_specs_owner ON environment_specs(owner_id); + +CREATE TABLE IF NOT EXISTS environment_builds ( + id TEXT PRIMARY KEY, + spec_id TEXT NOT NULL REFERENCES environment_specs(id), + status TEXT NOT NULL DEFAULT 'queued', + current_step TEXT NOT NULL DEFAULT '', + log_blob TEXT NOT NULL DEFAULT '', + image_size_bytes BIGINT NOT NULL DEFAULT 0, + digest_local TEXT NOT NULL DEFAULT '', + error TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_environment_builds_spec ON environment_builds(spec_id); +CREATE INDEX IF NOT EXISTS idx_environment_builds_status ON environment_builds(status); + +CREATE TABLE IF NOT EXISTS environment_artifacts ( + id BIGSERIAL PRIMARY KEY, + build_id TEXT NOT NULL REFERENCES environment_builds(id), + target TEXT NOT NULL, + image_ref TEXT NOT NULL, + digest TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'pending', + error TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(build_id, target) +); + +CREATE INDEX IF NOT EXISTS idx_environment_artifacts_build ON environment_artifacts(build_id); +CREATE INDEX IF NOT EXISTS idx_environment_artifacts_target ON environment_artifacts(target); + +CREATE TABLE IF NOT EXISTS registry_connections ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + provider TEXT NOT NULL, + username TEXT NOT NULL, + secret_ref TEXT NOT NULL, + is_default BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(owner_id, provider, username) +); + +CREATE INDEX IF NOT EXISTS idx_registry_connections_owner ON registry_connections(owner_id); +CREATE INDEX IF NOT EXISTS idx_registry_connections_provider ON registry_connections(provider); +`, + }, + { + version: 4, + sql: ` +ALTER TABLE sandboxes ADD COLUMN IF NOT EXISTS owner_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE sandboxes ADD COLUMN IF NOT EXISTS vm_id TEXT NOT NULL DEFAULT ''; +CREATE INDEX IF NOT EXISTS idx_sandboxes_owner ON sandboxes(owner_id); +CREATE INDEX IF NOT EXISTS idx_sandboxes_vm ON sandboxes(vm_id); +`, + }, + { + version: 5, + sql: ` +CREATE TABLE IF NOT EXISTS owner_quotas ( + owner_id TEXT PRIMARY KEY, + max_sandboxes INTEGER NOT NULL DEFAULT 0, + max_ttl_seconds BIGINT NOT NULL DEFAULT 0, + max_exec_timeout_seconds BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +`, + }, + { + version: 6, + sql: ` +CREATE TABLE IF NOT EXISTS admin_audit_logs ( + id BIGSERIAL PRIMARY KEY, + actor TEXT NOT NULL DEFAULT '', + method TEXT NOT NULL, + path TEXT NOT NULL, + status INTEGER NOT NULL DEFAULT 0, + duration_ms BIGINT NOT NULL DEFAULT 0, + request_id TEXT NOT NULL DEFAULT '', + remote_addr TEXT NOT NULL DEFAULT '', + user_agent TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_admin_audit_logs_created_at ON admin_audit_logs(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_admin_audit_logs_actor ON admin_audit_logs(actor); +`, + }, + { + version: 7, + sql: ` +CREATE TABLE IF NOT EXISTS operation_audit_logs ( + id BIGSERIAL PRIMARY KEY, + actor TEXT NOT NULL DEFAULT '', + action TEXT NOT NULL, + sandbox_id TEXT NOT NULL DEFAULT '', + resource TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT '', + detail TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_created_at ON operation_audit_logs(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_actor ON operation_audit_logs(actor); +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_sandbox ON operation_audit_logs(sandbox_id); +CREATE INDEX IF NOT EXISTS idx_operation_audit_logs_action ON operation_audit_logs(action); +`, + }, + { + version: 8, + sql: ` +CREATE TABLE IF NOT EXISTS workers ( + id TEXT PRIMARY KEY, + hostname TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'online', + providers TEXT NOT NULL DEFAULT '[]', + capabilities TEXT NOT NULL DEFAULT '[]', + capacity TEXT NOT NULL DEFAULT '{}', + last_heartbeat TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_workers_status ON workers(status); +CREATE INDEX IF NOT EXISTS idx_workers_last_heartbeat ON workers(last_heartbeat DESC); +`, + }, + { + version: 9, + sql: ` +ALTER TABLE sandboxes ADD COLUMN IF NOT EXISTS worker_id TEXT NOT NULL DEFAULT 'local'; +CREATE INDEX IF NOT EXISTS idx_sandboxes_worker ON sandboxes(worker_id); +`, + }, + { + version: 10, + sql: ` +CREATE TABLE IF NOT EXISTS leases ( + resource_id TEXT PRIMARY KEY, + resource_type TEXT NOT NULL DEFAULT '', + holder_id TEXT NOT NULL, + generation BIGINT NOT NULL DEFAULT 1, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_leases_holder ON leases(holder_id); +CREATE INDEX IF NOT EXISTS idx_leases_expires_at ON leases(expires_at); +`, + }, + { + version: 11, + sql: ` +CREATE TABLE IF NOT EXISTS tenants ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + owner_id TEXT NOT NULL DEFAULT '', + settings TEXT NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_tenants_owner ON tenants(owner_id); + +CREATE TABLE IF NOT EXISTS tenant_members ( + tenant_id TEXT NOT NULL, + user_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'viewer', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (tenant_id, user_id) +); + +CREATE INDEX IF NOT EXISTS idx_tenant_members_user ON tenant_members(user_id); + +CREATE TABLE IF NOT EXISTS policies ( + id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + resource_type TEXT NOT NULL, + effect TEXT NOT NULL DEFAULT 'allow', + pattern TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 10, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_policies_tenant ON policies(tenant_id); +CREATE INDEX IF NOT EXISTS idx_policies_resource ON policies(resource_type); + +ALTER TABLE admin_audit_logs ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE operation_audit_logs ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE sandboxes ADD COLUMN IF NOT EXISTS tenant_id TEXT NOT NULL DEFAULT ''; +CREATE INDEX IF NOT EXISTS idx_sandboxes_tenant ON sandboxes(tenant_id); +`, + }, +} diff --git a/internal/store/postgres_migrations_test.go b/internal/store/postgres_migrations_test.go new file mode 100644 index 0000000..27cb320 --- /dev/null +++ b/internal/store/postgres_migrations_test.go @@ -0,0 +1,73 @@ +package store + +import ( + "strings" + "testing" +) + +func TestPostgresMigrationsTrackSQLiteVersions(t *testing.T) { + if len(postgresMigrations) != len(migrations) { + t.Fatalf("postgres migration count = %d, want %d", len(postgresMigrations), len(migrations)) + } + for i, sqliteMigration := range migrations { + if postgresMigrations[i].version != sqliteMigration.version { + t.Fatalf("postgres migration[%d] version = %d, want %d", i, postgresMigrations[i].version, sqliteMigration.version) + } + } +} + +func TestPostgresMigrationsCoverStoreTables(t *testing.T) { + sql := joinedMigrationSQL(postgresMigrations) + for _, table := range []string{ + "schema_migrations", + "sandboxes", + "exec_logs", + "provider_configs", + "templates", + "environment_specs", + "environment_builds", + "environment_artifacts", + "registry_connections", + "owner_quotas", + "admin_audit_logs", + "operation_audit_logs", + "workers", + "leases", + } { + if !strings.Contains(sql, table) { + t.Fatalf("postgres migrations missing table %s", table) + } + } +} + +func TestPostgresMigrationsUsePostgresDialect(t *testing.T) { + sql := joinedMigrationSQL(postgresMigrations) + for _, forbidden := range []string{ + "AUTOINCREMENT", + "DATETIME", + "datetime('now')", + } { + if strings.Contains(sql, forbidden) { + t.Fatalf("postgres migrations contain sqlite dialect token %q", forbidden) + } + } + for _, required := range []string{ + "TIMESTAMPTZ", + "BIGSERIAL", + "DEFAULT now()", + "ADD COLUMN IF NOT EXISTS", + } { + if !strings.Contains(sql, required) { + t.Fatalf("postgres migrations missing postgres dialect token %q", required) + } + } +} + +func joinedMigrationSQL(migrations []migration) string { + var b strings.Builder + for _, migration := range migrations { + b.WriteString(migration.sql) + b.WriteByte('\n') + } + return b.String() +} diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index b1f2ae0..c34d782 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "time" _ "modernc.org/sqlite" @@ -76,11 +77,14 @@ func (s *SQLiteStore) migrate() error { // --- Sandbox CRUD --- func (s *SQLiteStore) CreateSandbox(ctx context.Context, sb *SandboxRecord) error { + if strings.TrimSpace(sb.WorkerID) == "" { + sb.WorkerID = "local" + } _, err := s.db.ExecContext(ctx, ` - INSERT INTO sandboxes (id, state, provider, image, memory_mb, vcpus, metadata, owner_id, vm_id, created_at, expires_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + INSERT INTO sandboxes (id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, sb.ID, sb.State, sb.Provider, sb.Image, sb.MemoryMB, sb.VCPUs, sb.Metadata, - sb.OwnerID, sb.VMID, + sb.OwnerID, sb.TenantID, sb.VMID, sb.WorkerID, sb.CreatedAt.UTC(), sb.ExpiresAt.UTC(), sb.UpdatedAt.UTC(), ) return err @@ -89,19 +93,19 @@ func (s *SQLiteStore) CreateSandbox(ctx context.Context, sb *SandboxRecord) erro func (s *SQLiteStore) GetSandbox(ctx context.Context, id string) (*SandboxRecord, error) { sb := &SandboxRecord{} err := s.db.QueryRowContext(ctx, ` - SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, vm_id, created_at, expires_at, updated_at + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at FROM sandboxes WHERE id = ?`, id, ).Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, - &sb.Metadata, &sb.OwnerID, &sb.VMID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt) + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("sandbox %q not found", id) + return nil, NotFoundError("sandbox", id) } return sb, err } func (s *SQLiteStore) ListSandboxes(ctx context.Context) ([]*SandboxRecord, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, vm_id, created_at, expires_at, updated_at + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at FROM sandboxes WHERE state != 'destroyed' ORDER BY created_at DESC`) if err != nil { return nil, err @@ -112,7 +116,7 @@ func (s *SQLiteStore) ListSandboxes(ctx context.Context) ([]*SandboxRecord, erro for rows.Next() { sb := &SandboxRecord{} if err := rows.Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, - &sb.Metadata, &sb.OwnerID, &sb.VMID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { return nil, err } sandboxes = append(sandboxes, sb) @@ -130,7 +134,7 @@ func (s *SQLiteStore) UpdateSandboxState(ctx context.Context, id string, state s } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("sandbox %q not found", id) + return NotFoundError("sandbox", id) } return nil } @@ -145,7 +149,7 @@ func (s *SQLiteStore) UpdateSandboxExpiresAt(ctx context.Context, id string, exp } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("sandbox %q not found or already destroyed", id) + return NotFoundError("sandbox", id) } return nil } @@ -156,7 +160,7 @@ func (s *SQLiteStore) DeleteSandbox(ctx context.Context, id string) error { func (s *SQLiteStore) ListExpiredSandboxes(ctx context.Context, before time.Time) ([]*SandboxRecord, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, vm_id, created_at, expires_at, updated_at + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at FROM sandboxes WHERE state NOT IN ('destroyed') AND expires_at < ?`, before.UTC(), ) @@ -169,7 +173,7 @@ func (s *SQLiteStore) ListExpiredSandboxes(ctx context.Context, before time.Time for rows.Next() { sb := &SandboxRecord{} if err := rows.Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, - &sb.Metadata, &sb.OwnerID, &sb.VMID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { return nil, err } sandboxes = append(sandboxes, sb) @@ -179,7 +183,7 @@ func (s *SQLiteStore) ListExpiredSandboxes(ctx context.Context, before time.Time func (s *SQLiteStore) ListSandboxesByOwner(ctx context.Context, ownerID string) ([]*SandboxRecord, error) { rows, err := s.db.QueryContext(ctx, ` - SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, vm_id, created_at, expires_at, updated_at + SELECT id, state, provider, image, memory_mb, vcpus, metadata, owner_id, tenant_id, vm_id, worker_id, created_at, expires_at, updated_at FROM sandboxes WHERE state != 'destroyed' AND owner_id = ? ORDER BY created_at DESC`, ownerID) if err != nil { return nil, err @@ -190,7 +194,7 @@ func (s *SQLiteStore) ListSandboxesByOwner(ctx context.Context, ownerID string) for rows.Next() { sb := &SandboxRecord{} if err := rows.Scan(&sb.ID, &sb.State, &sb.Provider, &sb.Image, &sb.MemoryMB, &sb.VCPUs, - &sb.Metadata, &sb.OwnerID, &sb.VMID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { + &sb.Metadata, &sb.OwnerID, &sb.TenantID, &sb.VMID, &sb.WorkerID, &sb.CreatedAt, &sb.ExpiresAt, &sb.UpdatedAt); err != nil { return nil, err } sandboxes = append(sandboxes, sb) @@ -205,6 +209,397 @@ func (s *SQLiteStore) CountSandboxesByVM(ctx context.Context, vmID string) (int, return count, err } +func (s *SQLiteStore) SaveOwnerQuota(ctx context.Context, quota *OwnerQuotaRecord) error { + now := time.Now().UTC() + if quota.CreatedAt.IsZero() { + quota.CreatedAt = now + } + quota.UpdatedAt = now + _, err := s.db.ExecContext(ctx, ` + INSERT INTO owner_quotas (owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(owner_id) DO UPDATE SET + max_sandboxes = excluded.max_sandboxes, + max_ttl_seconds = excluded.max_ttl_seconds, + max_exec_timeout_seconds = excluded.max_exec_timeout_seconds, + updated_at = excluded.updated_at`, + quota.OwnerID, quota.MaxSandboxes, quota.MaxTTLSeconds, quota.MaxExecTimeoutSeconds, + quota.CreatedAt.UTC(), quota.UpdatedAt.UTC(), + ) + return err +} + +func (s *SQLiteStore) GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuotaRecord, error) { + quota := &OwnerQuotaRecord{} + err := s.db.QueryRowContext(ctx, ` + SELECT owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at + FROM owner_quotas WHERE owner_id = ?`, ownerID, + ).Scan("a.OwnerID, "a.MaxSandboxes, "a.MaxTTLSeconds, "a.MaxExecTimeoutSeconds, "a.CreatedAt, "a.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("owner_quota", ownerID) + } + return quota, err +} + +func (s *SQLiteStore) ListOwnerQuotas(ctx context.Context) ([]*OwnerQuotaRecord, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT owner_id, max_sandboxes, max_ttl_seconds, max_exec_timeout_seconds, created_at, updated_at + FROM owner_quotas ORDER BY owner_id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var quotas []*OwnerQuotaRecord + for rows.Next() { + quota := &OwnerQuotaRecord{} + if err := rows.Scan("a.OwnerID, "a.MaxSandboxes, "a.MaxTTLSeconds, + "a.MaxExecTimeoutSeconds, "a.CreatedAt, "a.UpdatedAt); err != nil { + return nil, err + } + quotas = append(quotas, quota) + } + return quotas, rows.Err() +} + +func (s *SQLiteStore) DeleteOwnerQuota(ctx context.Context, ownerID string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM owner_quotas WHERE owner_id = ?`, ownerID) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("owner_quota", ownerID) + } + return nil +} + +func (s *SQLiteStore) CreateAdminAudit(ctx context.Context, rec *AdminAuditRecord) error { + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now().UTC() + } + res, err := s.db.ExecContext(ctx, ` + INSERT INTO admin_audit_logs (actor, method, path, status, duration_ms, request_id, remote_addr, user_agent, tenant_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + rec.Actor, rec.Method, rec.Path, rec.Status, rec.DurationMS, rec.RequestID, + rec.RemoteAddr, rec.UserAgent, rec.TenantID, rec.CreatedAt.UTC(), + ) + if err != nil { + return err + } + rec.ID, _ = res.LastInsertId() + return nil +} + +func (s *SQLiteStore) ListAdminAudit(ctx context.Context, query AdminAuditQuery) ([]*AdminAuditRecord, error) { + if query.Limit <= 0 { + query.Limit = 100 + } + if query.Limit > 500 { + query.Limit = 500 + } + + clauses := []string{"1=1"} + args := make([]interface{}, 0, 6) + if query.Actor != "" { + clauses = append(clauses, "actor = ?") + args = append(args, query.Actor) + } + if query.Method != "" { + clauses = append(clauses, "method = ?") + args = append(args, query.Method) + } + if query.Status > 0 { + clauses = append(clauses, "status = ?") + args = append(args, query.Status) + } + if query.PathLike != "" { + clauses = append(clauses, "path LIKE ?") + args = append(args, "%"+query.PathLike+"%") + } + if query.TenantID != "" { + clauses = append(clauses, "tenant_id = ?") + args = append(args, query.TenantID) + } + args = append(args, query.Limit) + + rows, err := s.db.QueryContext(ctx, ` + SELECT id, actor, method, path, status, duration_ms, request_id, remote_addr, user_agent, tenant_id, created_at + FROM admin_audit_logs WHERE `+strings.Join(clauses, " AND ")+` + ORDER BY created_at DESC, id DESC LIMIT ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*AdminAuditRecord + for rows.Next() { + rec := &AdminAuditRecord{} + if err := rows.Scan(&rec.ID, &rec.Actor, &rec.Method, &rec.Path, &rec.Status, + &rec.DurationMS, &rec.RequestID, &rec.RemoteAddr, &rec.UserAgent, &rec.TenantID, &rec.CreatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +func (s *SQLiteStore) DeleteAdminAuditBefore(ctx context.Context, before time.Time) (int64, error) { + res, err := s.db.ExecContext(ctx, `DELETE FROM admin_audit_logs WHERE created_at < ?`, before.UTC()) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +func (s *SQLiteStore) CreateOperationAudit(ctx context.Context, rec *OperationAuditRecord) error { + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now().UTC() + } + res, err := s.db.ExecContext(ctx, ` + INSERT INTO operation_audit_logs (actor, action, sandbox_id, resource, provider, status, detail, tenant_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + rec.Actor, rec.Action, rec.SandboxID, rec.Resource, rec.Provider, rec.Status, rec.Detail, rec.TenantID, rec.CreatedAt.UTC(), + ) + if err != nil { + return err + } + rec.ID, _ = res.LastInsertId() + return nil +} + +func (s *SQLiteStore) ListOperationAudit(ctx context.Context, query OperationAuditQuery) ([]*OperationAuditRecord, error) { + if query.Limit <= 0 { + query.Limit = 100 + } + if query.Limit > 500 { + query.Limit = 500 + } + + clauses := []string{"1=1"} + args := make([]interface{}, 0, 8) + if query.Actor != "" { + clauses = append(clauses, "actor = ?") + args = append(args, query.Actor) + } + if query.Action != "" { + clauses = append(clauses, "action = ?") + args = append(args, query.Action) + } + if query.SandboxID != "" { + clauses = append(clauses, "sandbox_id = ?") + args = append(args, query.SandboxID) + } + if query.Resource != "" { + clauses = append(clauses, "resource = ?") + args = append(args, query.Resource) + } + if query.Status != "" { + clauses = append(clauses, "status = ?") + args = append(args, query.Status) + } + if query.TenantID != "" { + clauses = append(clauses, "tenant_id = ?") + args = append(args, query.TenantID) + } + args = append(args, query.Limit) + + rows, err := s.db.QueryContext(ctx, ` + SELECT id, actor, action, sandbox_id, resource, provider, status, detail, tenant_id, created_at + FROM operation_audit_logs WHERE `+strings.Join(clauses, " AND ")+` + ORDER BY created_at DESC, id DESC LIMIT ?`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*OperationAuditRecord + for rows.Next() { + rec := &OperationAuditRecord{} + if err := rows.Scan(&rec.ID, &rec.Actor, &rec.Action, &rec.SandboxID, &rec.Resource, + &rec.Provider, &rec.Status, &rec.Detail, &rec.TenantID, &rec.CreatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +// --- Workers --- + +func (s *SQLiteStore) SaveWorker(ctx context.Context, rec *WorkerRecord) error { + now := time.Now().UTC() + if rec.CreatedAt.IsZero() { + rec.CreatedAt = now + } + if rec.LastHeartbeat.IsZero() { + rec.LastHeartbeat = now + } + rec.UpdatedAt = now + _, err := s.db.ExecContext(ctx, ` + INSERT INTO workers (id, hostname, status, providers, capabilities, capacity, last_heartbeat, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + hostname = excluded.hostname, + status = excluded.status, + providers = excluded.providers, + capabilities = excluded.capabilities, + capacity = excluded.capacity, + last_heartbeat = excluded.last_heartbeat, + updated_at = excluded.updated_at`, + rec.ID, rec.Hostname, rec.Status, rec.Providers, rec.Capabilities, rec.Capacity, + rec.LastHeartbeat.UTC(), rec.CreatedAt.UTC(), rec.UpdatedAt.UTC(), + ) + return err +} + +func (s *SQLiteStore) GetWorker(ctx context.Context, id string) (*WorkerRecord, error) { + rec := &WorkerRecord{} + err := s.db.QueryRowContext(ctx, ` + SELECT id, hostname, status, providers, capabilities, capacity, last_heartbeat, created_at, updated_at + FROM workers WHERE id = ?`, id, + ).Scan(&rec.ID, &rec.Hostname, &rec.Status, &rec.Providers, &rec.Capabilities, &rec.Capacity, + &rec.LastHeartbeat, &rec.CreatedAt, &rec.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("worker", id) + } + return rec, err +} + +func (s *SQLiteStore) ListWorkers(ctx context.Context) ([]*WorkerRecord, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT id, hostname, status, providers, capabilities, capacity, last_heartbeat, created_at, updated_at + FROM workers ORDER BY id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*WorkerRecord + for rows.Next() { + rec := &WorkerRecord{} + if err := rows.Scan(&rec.ID, &rec.Hostname, &rec.Status, &rec.Providers, &rec.Capabilities, + &rec.Capacity, &rec.LastHeartbeat, &rec.CreatedAt, &rec.UpdatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +func (s *SQLiteStore) DeleteWorker(ctx context.Context, id string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM workers WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("worker", id) + } + return nil +} + +// --- Leases --- + +func (s *SQLiteStore) AcquireLease(ctx context.Context, resourceID, resourceType, holderID string, ttl time.Duration) (*LeaseRecord, error) { + if strings.TrimSpace(resourceID) == "" { + return nil, ConflictError("lease resource id is required") + } + if strings.TrimSpace(holderID) == "" { + return nil, ConflictError("lease holder id is required") + } + if ttl <= 0 { + return nil, ConflictError("lease ttl must be positive") + } + now := time.Now().UTC() + expiresAt := now.Add(ttl) + res, err := s.db.ExecContext(ctx, ` + INSERT INTO leases (resource_id, resource_type, holder_id, generation, expires_at, created_at, updated_at) + VALUES (?, ?, ?, 1, ?, ?, ?) + ON CONFLICT(resource_id) DO UPDATE SET + resource_type = excluded.resource_type, + holder_id = excluded.holder_id, + generation = leases.generation + 1, + expires_at = excluded.expires_at, + updated_at = excluded.updated_at + WHERE leases.expires_at <= ? OR leases.holder_id = excluded.holder_id`, + resourceID, resourceType, holderID, expiresAt, now, now, now, + ) + if err != nil { + return nil, err + } + rows, _ := res.RowsAffected() + if rows == 0 { + return nil, ConflictError("lease is held by another worker") + } + return s.GetLease(ctx, resourceID) +} + +func (s *SQLiteStore) RenewLease(ctx context.Context, resourceID, holderID string, ttl time.Duration) (*LeaseRecord, error) { + if ttl <= 0 { + return nil, ConflictError("lease ttl must be positive") + } + now := time.Now().UTC() + res, err := s.db.ExecContext(ctx, ` + UPDATE leases + SET generation = generation + 1, expires_at = ?, updated_at = ? + WHERE resource_id = ? AND holder_id = ? AND expires_at > ?`, + now.Add(ttl), now, resourceID, holderID, now, + ) + if err != nil { + return nil, err + } + rows, _ := res.RowsAffected() + if rows == 0 { + return nil, ConflictError("lease is not held by worker or has expired") + } + return s.GetLease(ctx, resourceID) +} + +func (s *SQLiteStore) GetLease(ctx context.Context, resourceID string) (*LeaseRecord, error) { + rec := &LeaseRecord{} + err := s.db.QueryRowContext(ctx, ` + SELECT resource_id, resource_type, holder_id, generation, expires_at, created_at, updated_at + FROM leases WHERE resource_id = ?`, resourceID, + ).Scan(&rec.ResourceID, &rec.ResourceType, &rec.HolderID, &rec.Generation, &rec.ExpiresAt, &rec.CreatedAt, &rec.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("lease", resourceID) + } + return rec, err +} + +func (s *SQLiteStore) ListLeases(ctx context.Context) ([]*LeaseRecord, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT resource_id, resource_type, holder_id, generation, expires_at, created_at, updated_at + FROM leases ORDER BY resource_id ASC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var records []*LeaseRecord + for rows.Next() { + rec := &LeaseRecord{} + if err := rows.Scan(&rec.ResourceID, &rec.ResourceType, &rec.HolderID, &rec.Generation, &rec.ExpiresAt, &rec.CreatedAt, &rec.UpdatedAt); err != nil { + return nil, err + } + records = append(records, rec) + } + return records, rows.Err() +} + +func (s *SQLiteStore) ReleaseLease(ctx context.Context, resourceID, holderID string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM leases WHERE resource_id = ? AND holder_id = ?`, resourceID, holderID) + if err != nil { + return err + } + rows, _ := res.RowsAffected() + if rows == 0 { + return NotFoundError("lease", resourceID) + } + return nil +} + // --- Exec Logs --- func (s *SQLiteStore) CreateExecLog(ctx context.Context, log *ExecLogRecord) error { @@ -245,7 +640,7 @@ func (s *SQLiteStore) GetProviderConfig(ctx context.Context, name string) (*Prov SELECT name, config, enabled, updated_at FROM provider_configs WHERE name = ?`, name, ).Scan(&cfg.Name, &cfg.Config, &cfg.Enabled, &cfg.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("provider config %q not found", name) + return nil, NotFoundError("provider config", name) } return cfg, err } @@ -287,6 +682,9 @@ func (s *SQLiteStore) CreateTemplate(ctx context.Context, t *TemplateRecord) err t.MemoryMB, t.CPUCores, t.TTLSeconds, t.Env, t.Secrets, t.PoolSize, t.CreatedAt.UTC(), t.UpdatedAt.UTC(), ) + if IsConstraintError(err) { + return ConflictError("template already exists") + } return err } @@ -299,7 +697,7 @@ func (s *SQLiteStore) GetTemplate(ctx context.Context, name string) (*TemplateRe &t.MemoryMB, &t.CPUCores, &t.TTLSeconds, &t.Env, &t.Secrets, &t.PoolSize, &t.CreatedAt, &t.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("template %q not found", name) + return nil, NotFoundError("template", name) } return t, err } @@ -340,7 +738,7 @@ func (s *SQLiteStore) UpdateTemplate(ctx context.Context, t *TemplateRecord) err } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("template %q not found", t.Name) + return NotFoundError("template", t.Name) } return nil } @@ -352,7 +750,7 @@ func (s *SQLiteStore) DeleteTemplate(ctx context.Context, name string) error { } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("template %q not found", name) + return NotFoundError("template", name) } return nil } @@ -366,6 +764,9 @@ func (s *SQLiteStore) CreateEnvironmentSpec(ctx context.Context, spec *Environme spec.ID, spec.OwnerID, spec.Name, spec.BaseImage, spec.PythonPackages, spec.AptPackages, spec.PythonVersion, spec.CreatedAt.UTC(), spec.UpdatedAt.UTC(), ) + if IsConstraintError(err) { + return ConflictError("spec name already exists for this owner") + } return err } @@ -379,7 +780,7 @@ func (s *SQLiteStore) GetEnvironmentSpec(ctx context.Context, id string) (*Envir &spec.PythonVersion, &spec.CreatedAt, &spec.UpdatedAt, ) if err == sql.ErrNoRows { - return nil, fmt.Errorf("environment spec %q not found", id) + return nil, NotFoundError("environment spec", id) } return spec, err } @@ -422,7 +823,7 @@ func (s *SQLiteStore) UpdateEnvironmentSpec(ctx context.Context, spec *Environme } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("environment spec %q not found", spec.ID) + return NotFoundError("environment spec", spec.ID) } return nil } @@ -434,7 +835,7 @@ func (s *SQLiteStore) DeleteEnvironmentSpec(ctx context.Context, id string) erro } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("environment spec %q not found", id) + return NotFoundError("environment spec", id) } return nil } @@ -464,7 +865,7 @@ func (s *SQLiteStore) GetEnvironmentBuild(ctx context.Context, id string) (*Envi &build.DigestLocal, &build.Error, &build.CreatedAt, &finishedAt, &build.UpdatedAt, ) if err == sql.ErrNoRows { - return nil, fmt.Errorf("environment build %q not found", id) + return nil, NotFoundError("environment build", id) } if err != nil { return nil, err @@ -519,7 +920,7 @@ func (s *SQLiteStore) UpdateEnvironmentBuild(ctx context.Context, build *Environ } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("environment build %q not found", build.ID) + return NotFoundError("environment build", build.ID) } return nil } @@ -583,6 +984,9 @@ func (s *SQLiteStore) SaveRegistryConnection(ctx context.Context, conn *Registry conn.ID, conn.OwnerID, conn.Provider, conn.Username, conn.SecretRef, conn.IsDefault, time.Now().UTC(), time.Now().UTC(), ) + if IsConstraintError(err) { + return ConflictError("registry connection already exists") + } return err } @@ -594,7 +998,7 @@ func (s *SQLiteStore) GetRegistryConnection(ctx context.Context, id string) (*Re WHERE id = ?`, id, ).Scan(&conn.ID, &conn.OwnerID, &conn.Provider, &conn.Username, &conn.SecretRef, &conn.IsDefault, &conn.CreatedAt, &conn.UpdatedAt) if err == sql.ErrNoRows { - return nil, fmt.Errorf("registry connection %q not found", id) + return nil, NotFoundError("registry connection", id) } return conn, err } @@ -630,7 +1034,7 @@ func (s *SQLiteStore) DeleteRegistryConnection(ctx context.Context, id string) e } n, _ := res.RowsAffected() if n == 0 { - return fmt.Errorf("registry connection %q not found", id) + return NotFoundError("registry connection", id) } return nil } @@ -642,6 +1046,213 @@ func nullableTime(t *time.Time) any { return t.UTC() } +// --- Tenants --- + +func (s *SQLiteStore) CreateTenant(ctx context.Context, t *TenantRecord) error { + now := time.Now().UTC() + if t.CreatedAt.IsZero() { + t.CreatedAt = now + } + t.UpdatedAt = now + if t.Settings == "" { + t.Settings = "{}" + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO tenants (id, name, owner_id, settings, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`, + t.ID, t.Name, t.OwnerID, t.Settings, t.CreatedAt.UTC(), t.UpdatedAt.UTC(), + ) + return err +} + +func (s *SQLiteStore) GetTenant(ctx context.Context, id string) (*TenantRecord, error) { + t := &TenantRecord{} + err := s.db.QueryRowContext(ctx, + `SELECT id, name, owner_id, settings, created_at, updated_at FROM tenants WHERE id = ?`, id, + ).Scan(&t.ID, &t.Name, &t.OwnerID, &t.Settings, &t.CreatedAt, &t.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("tenant", id) + } + return t, err +} + +func (s *SQLiteStore) ListTenants(ctx context.Context) ([]*TenantRecord, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT id, name, owner_id, settings, created_at, updated_at FROM tenants ORDER BY created_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + var tenants []*TenantRecord + for rows.Next() { + t := &TenantRecord{} + if err := rows.Scan(&t.ID, &t.Name, &t.OwnerID, &t.Settings, &t.CreatedAt, &t.UpdatedAt); err != nil { + return nil, err + } + tenants = append(tenants, t) + } + return tenants, rows.Err() +} + +func (s *SQLiteStore) UpdateTenant(ctx context.Context, t *TenantRecord) error { + t.UpdatedAt = time.Now().UTC() + res, err := s.db.ExecContext(ctx, + `UPDATE tenants SET name = ?, owner_id = ?, settings = ?, updated_at = ? WHERE id = ?`, + t.Name, t.OwnerID, t.Settings, t.UpdatedAt.UTC(), t.ID, + ) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("tenant", t.ID) + } + return nil +} + +func (s *SQLiteStore) DeleteTenant(ctx context.Context, id string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM tenants WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("tenant", id) + } + return nil +} + +// --- Tenant members --- + +func (s *SQLiteStore) SaveTenantMember(ctx context.Context, m *TenantMemberRecord) error { + now := time.Now().UTC() + if m.CreatedAt.IsZero() { + m.CreatedAt = now + } + m.UpdatedAt = now + _, err := s.db.ExecContext(ctx, ` + INSERT INTO tenant_members (tenant_id, user_id, role, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(tenant_id, user_id) DO UPDATE SET role = excluded.role, updated_at = excluded.updated_at`, + m.TenantID, m.UserID, m.Role, m.CreatedAt.UTC(), m.UpdatedAt.UTC(), + ) + return err +} + +func (s *SQLiteStore) GetTenantMember(ctx context.Context, tenantID, userID string) (*TenantMemberRecord, error) { + m := &TenantMemberRecord{} + err := s.db.QueryRowContext(ctx, + `SELECT tenant_id, user_id, role, created_at, updated_at FROM tenant_members WHERE tenant_id = ? AND user_id = ?`, + tenantID, userID, + ).Scan(&m.TenantID, &m.UserID, &m.Role, &m.CreatedAt, &m.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("tenant_member", tenantID+"/"+userID) + } + return m, err +} + +func (s *SQLiteStore) ListTenantMembers(ctx context.Context, tenantID string) ([]*TenantMemberRecord, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT tenant_id, user_id, role, created_at, updated_at FROM tenant_members WHERE tenant_id = ? ORDER BY created_at ASC`, + tenantID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var members []*TenantMemberRecord + for rows.Next() { + m := &TenantMemberRecord{} + if err := rows.Scan(&m.TenantID, &m.UserID, &m.Role, &m.CreatedAt, &m.UpdatedAt); err != nil { + return nil, err + } + members = append(members, m) + } + return members, rows.Err() +} + +func (s *SQLiteStore) DeleteTenantMember(ctx context.Context, tenantID, userID string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM tenant_members WHERE tenant_id = ? AND user_id = ?`, tenantID, userID) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("tenant_member", tenantID+"/"+userID) + } + return nil +} + +// --- Policies --- + +func (s *SQLiteStore) CreatePolicy(ctx context.Context, p *PolicyRecord) error { + now := time.Now().UTC() + if p.CreatedAt.IsZero() { + p.CreatedAt = now + } + p.UpdatedAt = now + _, err := s.db.ExecContext(ctx, + `INSERT INTO policies (id, tenant_id, resource_type, effect, pattern, priority, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + p.ID, p.TenantID, p.ResourceType, p.Effect, p.Pattern, p.Priority, p.CreatedAt.UTC(), p.UpdatedAt.UTC(), + ) + return err +} + +func (s *SQLiteStore) GetPolicy(ctx context.Context, id string) (*PolicyRecord, error) { + p := &PolicyRecord{} + err := s.db.QueryRowContext(ctx, + `SELECT id, tenant_id, resource_type, effect, pattern, priority, created_at, updated_at FROM policies WHERE id = ?`, id, + ).Scan(&p.ID, &p.TenantID, &p.ResourceType, &p.Effect, &p.Pattern, &p.Priority, &p.CreatedAt, &p.UpdatedAt) + if err == sql.ErrNoRows { + return nil, NotFoundError("policy", id) + } + return p, err +} + +func (s *SQLiteStore) ListPolicies(ctx context.Context, query PolicyQuery) ([]*PolicyRecord, error) { + q := `SELECT id, tenant_id, resource_type, effect, pattern, priority, created_at, updated_at FROM policies WHERE 1=1` + var args []any + if query.TenantID != "" { + // Return tenant-specific policies plus global (empty tenant_id) policies. + q += " AND (tenant_id = ? OR tenant_id = '')" + args = append(args, query.TenantID) + } else { + // No tenant filter: return only global policies. + q += " AND tenant_id = ''" + } + if query.ResourceType != "" { + q += " AND resource_type = ?" + args = append(args, query.ResourceType) + } + q += " ORDER BY priority ASC, created_at ASC" + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var policies []*PolicyRecord + for rows.Next() { + p := &PolicyRecord{} + if err := rows.Scan(&p.ID, &p.TenantID, &p.ResourceType, &p.Effect, &p.Pattern, &p.Priority, &p.CreatedAt, &p.UpdatedAt); err != nil { + return nil, err + } + policies = append(policies, p) + } + return policies, rows.Err() +} + +func (s *SQLiteStore) DeletePolicy(ctx context.Context, id string) error { + res, err := s.db.ExecContext(ctx, `DELETE FROM policies WHERE id = ?`, id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return NotFoundError("policy", id) + } + return nil +} + func (s *SQLiteStore) Close() error { return s.db.Close() } diff --git a/internal/store/sqlite_test.go b/internal/store/sqlite_test.go index 57fc509..0336b93 100644 --- a/internal/store/sqlite_test.go +++ b/internal/store/sqlite_test.go @@ -2,10 +2,14 @@ package store import ( "context" + "database/sql" + "errors" "os" "path/filepath" "testing" "time" + + _ "modernc.org/sqlite" ) func testStore(t *testing.T) *SQLiteStore { @@ -43,6 +47,227 @@ func TestMigrations(t *testing.T) { } } +func TestSQLiteStoreMigratesLegacyDatabase(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "legacy.db") + + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open legacy db: %v", err) + } + if _, err := db.Exec(migrations[0].sql); err != nil { + t.Fatalf("create v1 schema: %v", err) + } + if _, err := db.Exec("INSERT INTO schema_migrations (version) VALUES (1)"); err != nil { + t.Fatalf("record v1 migration: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close legacy db: %v", err) + } + + s, err := NewSQLiteStore(dbPath) + if err != nil { + t.Fatalf("migrate legacy db: %v", err) + } + + var migrated int + if err := s.db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&migrated); err != nil { + t.Fatalf("count migrations: %v", err) + } + if migrated != len(migrations) { + t.Fatalf("migration count = %d, want %d", migrated, len(migrations)) + } + + for _, col := range []string{"template", "owner_id", "vm_id", "worker_id"} { + if !sqliteColumnExists(t, s.db, "sandboxes", col) { + t.Fatalf("sandboxes missing migrated column %s", col) + } + } + + for _, table := range []string{ + "templates", + "environment_specs", + "environment_builds", + "environment_artifacts", + "registry_connections", + "owner_quotas", + "admin_audit_logs", + "operation_audit_logs", + "workers", + "leases", + } { + if !sqliteTableExists(t, s.db, table) { + t.Fatalf("missing migrated table %s", table) + } + } + + if err := s.Close(); err != nil { + t.Fatalf("close migrated db: %v", err) + } + s, err = NewSQLiteStore(dbPath) + if err != nil { + t.Fatalf("reopen migrated db: %v", err) + } + if err := s.Close(); err != nil { + t.Fatalf("close reopened db: %v", err) + } +} + +func sqliteColumnExists(t *testing.T, db *sql.DB, table, column string) bool { + t.Helper() + rows, err := db.Query("PRAGMA table_info(" + table + ")") + if err != nil { + t.Fatalf("pragma table_info(%s): %v", table, err) + } + defer rows.Close() + + for rows.Next() { + var cid int + var name, typ string + var notNull int + var defaultValue interface{} + var pk int + if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { + t.Fatalf("scan column info: %v", err) + } + if name == column { + return true + } + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate columns: %v", err) + } + return false +} + +func sqliteTableExists(t *testing.T, db *sql.DB, table string) bool { + t.Helper() + var count int + if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?", table).Scan(&count); err != nil { + t.Fatalf("check table %s: %v", table, err) + } + return count == 1 +} + +func TestWorkerRegistryCRUD(t *testing.T) { + s := testStore(t) + ctx := context.Background() + now := time.Now().UTC().Add(-time.Minute) + + rec := &WorkerRecord{ + ID: "worker-a", + Hostname: "host-a", + Status: "online", + Providers: `["mock","docker"]`, + Capabilities: `["spawn","exec"]`, + Capacity: `{"max_sandboxes":10}`, + LastHeartbeat: now, + } + if err := s.SaveWorker(ctx, rec); err != nil { + t.Fatalf("save worker: %v", err) + } + if rec.CreatedAt.IsZero() || rec.UpdatedAt.IsZero() { + t.Fatalf("expected timestamps to be populated: %+v", rec) + } + + got, err := s.GetWorker(ctx, "worker-a") + if err != nil { + t.Fatalf("get worker: %v", err) + } + if got.Hostname != "host-a" || got.Status != "online" || got.Providers != `["mock","docker"]` { + t.Fatalf("unexpected worker: %+v", got) + } + + rec.Status = "draining" + rec.Capacity = `{"max_sandboxes":5}` + if err := s.SaveWorker(ctx, rec); err != nil { + t.Fatalf("update worker: %v", err) + } + got, err = s.GetWorker(ctx, "worker-a") + if err != nil { + t.Fatalf("get updated worker: %v", err) + } + if got.Status != "draining" || got.Capacity != `{"max_sandboxes":5}` { + t.Fatalf("unexpected updated worker: %+v", got) + } + + workers, err := s.ListWorkers(ctx) + if err != nil { + t.Fatalf("list workers: %v", err) + } + if len(workers) != 1 || workers[0].ID != "worker-a" { + t.Fatalf("unexpected workers: %+v", workers) + } + + if err := s.DeleteWorker(ctx, "worker-a"); err != nil { + t.Fatalf("delete worker: %v", err) + } + if _, err := s.GetWorker(ctx, "worker-a"); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted worker err = %v, want ErrNotFound", err) + } +} + +func TestLeaseLifecycle(t *testing.T) { + s := testStore(t) + ctx := context.Background() + + lease, err := s.AcquireLease(ctx, "sb-lease", "sandbox", "worker-a", time.Minute) + if err != nil { + t.Fatalf("acquire lease: %v", err) + } + if lease.ResourceID != "sb-lease" || lease.HolderID != "worker-a" || lease.Generation != 1 { + t.Fatalf("unexpected lease: %+v", lease) + } + + if _, err := s.AcquireLease(ctx, "sb-lease", "sandbox", "worker-b", time.Minute); !errors.Is(err, ErrConflict) { + t.Fatalf("competing acquire err = %v, want ErrConflict", err) + } + + renewed, err := s.RenewLease(ctx, "sb-lease", "worker-a", 2*time.Minute) + if err != nil { + t.Fatalf("renew lease: %v", err) + } + if renewed.Generation <= lease.Generation || !renewed.ExpiresAt.After(lease.ExpiresAt) { + t.Fatalf("expected renewed generation and expiry: old=%+v new=%+v", lease, renewed) + } + + leases, err := s.ListLeases(ctx) + if err != nil { + t.Fatalf("list leases: %v", err) + } + if len(leases) != 1 || leases[0].ResourceID != "sb-lease" { + t.Fatalf("unexpected leases: %+v", leases) + } + + if err := s.ReleaseLease(ctx, "sb-lease", "worker-b"); !errors.Is(err, ErrNotFound) { + t.Fatalf("wrong holder release err = %v, want ErrNotFound", err) + } + if err := s.ReleaseLease(ctx, "sb-lease", "worker-a"); err != nil { + t.Fatalf("release lease: %v", err) + } + if _, err := s.GetLease(ctx, "sb-lease"); !errors.Is(err, ErrNotFound) { + t.Fatalf("get released lease err = %v, want ErrNotFound", err) + } +} + +func TestLeaseAcquireAfterExpiry(t *testing.T) { + s := testStore(t) + ctx := context.Background() + + if _, err := s.AcquireLease(ctx, "sb-expired", "sandbox", "worker-a", time.Nanosecond); err != nil { + t.Fatalf("acquire lease: %v", err) + } + time.Sleep(time.Millisecond) + + lease, err := s.AcquireLease(ctx, "sb-expired", "sandbox", "worker-b", time.Minute) + if err != nil { + t.Fatalf("acquire expired lease: %v", err) + } + if lease.HolderID != "worker-b" || lease.Generation != 2 { + t.Fatalf("unexpected reacquired lease: %+v", lease) + } +} + func TestSandboxCRUD(t *testing.T) { s := testStore(t) ctx := context.Background() @@ -56,6 +281,7 @@ func TestSandboxCRUD(t *testing.T) { MemoryMB: 512, VCPUs: 1, Metadata: `{"env":"test"}`, + WorkerID: "worker-a", CreatedAt: now, ExpiresAt: now.Add(30 * time.Minute), UpdatedAt: now, @@ -77,6 +303,9 @@ func TestSandboxCRUD(t *testing.T) { if got.Image != "alpine:latest" { t.Fatalf("expected alpine:latest, got %s", got.Image) } + if got.WorkerID != "worker-a" { + t.Fatalf("expected worker-a, got %s", got.WorkerID) + } // List list, err := s.ListSandboxes(ctx) @@ -144,6 +373,111 @@ func TestExecLogs(t *testing.T) { } } +func TestAdminAuditLogs(t *testing.T) { + s := testStore(t) + ctx := context.Background() + now := time.Now().UTC() + + if err := s.CreateAdminAudit(ctx, &AdminAuditRecord{ + Actor: "operator-a", + Method: "PUT", + Path: "/api/v1/admin/quotas/owner-a", + Status: 200, + DurationMS: 7, + RequestID: "req-a", + RemoteAddr: "127.0.0.1", + UserAgent: "test-agent", + CreatedAt: now, + }); err != nil { + t.Fatalf("create admin audit: %v", err) + } + + if err := s.CreateAdminAudit(ctx, &AdminAuditRecord{ + Actor: "operator-b", + Method: "GET", + Path: "/api/v1/admin/diagnostics", + Status: 200, + CreatedAt: now.Add(time.Second), + }); err != nil { + t.Fatalf("create second admin audit: %v", err) + } + + records, err := s.ListAdminAudit(ctx, AdminAuditQuery{Limit: 1}) + if err != nil { + t.Fatalf("list admin audit: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected 1 record, got %d", len(records)) + } + if records[0].Actor != "operator-b" || records[0].Path != "/api/v1/admin/diagnostics" { + t.Fatalf("unexpected latest audit record: %+v", records[0]) + } + + records, err = s.ListAdminAudit(ctx, AdminAuditQuery{Actor: "operator-a", Method: "PUT", Status: 200, PathLike: "quotas"}) + if err != nil { + t.Fatalf("filter admin audit: %v", err) + } + if len(records) != 1 || records[0].Actor != "operator-a" { + t.Fatalf("unexpected filtered audit records: %+v", records) + } + + deleted, err := s.DeleteAdminAuditBefore(ctx, now.Add(500*time.Millisecond)) + if err != nil { + t.Fatalf("delete old admin audit: %v", err) + } + if deleted != 1 { + t.Fatalf("deleted = %d, want 1", deleted) + } + records, err = s.ListAdminAudit(ctx, AdminAuditQuery{Limit: 10}) + if err != nil { + t.Fatalf("list remaining admin audit: %v", err) + } + if len(records) != 1 || records[0].Actor != "operator-b" { + t.Fatalf("unexpected remaining audit records: %+v", records) + } +} + +func TestOperationAuditLogs(t *testing.T) { + s := testStore(t) + ctx := context.Background() + now := time.Now().UTC() + + if err := s.CreateOperationAudit(ctx, &OperationAuditRecord{ + Actor: "owner-a", + Action: "file.write", + SandboxID: "sb-00000003", + Resource: "/workspace/app.py", + Provider: "mock", + Status: "success", + Detail: "mode=0644", + CreatedAt: now, + }); err != nil { + t.Fatalf("create operation audit: %v", err) + } + if err := s.CreateOperationAudit(ctx, &OperationAuditRecord{ + Actor: "owner-b", + Action: "exec", + SandboxID: "sb-00000004", + Provider: "mock", + Status: "failure", + Detail: "exit=1", + CreatedAt: now.Add(time.Second), + }); err != nil { + t.Fatalf("create second operation audit: %v", err) + } + + records, err := s.ListOperationAudit(ctx, OperationAuditQuery{Limit: 10, Action: "file.write", SandboxID: "sb-00000003"}) + if err != nil { + t.Fatalf("list operation audit: %v", err) + } + if len(records) != 1 { + t.Fatalf("expected 1 operation audit record, got %d", len(records)) + } + if records[0].Actor != "owner-a" || records[0].Resource != "/workspace/app.py" { + t.Fatalf("unexpected operation audit record: %+v", records[0]) + } +} + func TestProviderConfigs(t *testing.T) { s := testStore(t) ctx := context.Background() @@ -247,6 +581,9 @@ func TestUpdateSandboxExpiresAt_Destroyed(t *testing.T) { if err == nil { t.Fatal("expected error extending destroyed sandbox") } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } } func TestUpdateSandboxExpiresAt_NotFound(t *testing.T) { @@ -255,6 +592,9 @@ func TestUpdateSandboxExpiresAt_NotFound(t *testing.T) { if err == nil { t.Fatal("expected error for nonexistent sandbox") } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } } func TestGetSandboxNotFound(t *testing.T) { @@ -263,6 +603,9 @@ func TestGetSandboxNotFound(t *testing.T) { if err == nil { t.Fatal("expected error for nonexistent sandbox") } + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } } func TestEnvironmentSpecCRUD(t *testing.T) { @@ -285,6 +628,11 @@ func TestEnvironmentSpecCRUD(t *testing.T) { if err := s.CreateEnvironmentSpec(ctx, spec); err != nil { t.Fatalf("create spec: %v", err) } + conflicting := *spec + conflicting.ID = "envspec-conflict" + if err := s.CreateEnvironmentSpec(ctx, &conflicting); !errors.Is(err, ErrConflict) { + t.Fatalf("expected ErrConflict for duplicate owner/name, got %v", err) + } got, err := s.GetEnvironmentSpec(ctx, spec.ID) if err != nil { @@ -313,6 +661,8 @@ func TestEnvironmentSpecCRUD(t *testing.T) { } if _, err := s.GetEnvironmentSpec(ctx, spec.ID); err == nil { t.Fatal("expected get to fail after delete") + } else if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound after delete, got %v", err) } } @@ -413,3 +763,46 @@ func TestEnvironmentBuildArtifactAndRegistryCRUD(t *testing.T) { t.Fatalf("delete registry connection: %v", err) } } + +func TestOwnerQuotaStore(t *testing.T) { + s := testStore(t) + ctx := context.Background() + + quota := &OwnerQuotaRecord{ + OwnerID: "owner-1", + MaxSandboxes: 3, + MaxTTLSeconds: 3600, + MaxExecTimeoutSeconds: 60, + } + if err := s.SaveOwnerQuota(ctx, quota); err != nil { + t.Fatalf("save quota: %v", err) + } + + got, err := s.GetOwnerQuota(ctx, "owner-1") + if err != nil { + t.Fatalf("get quota: %v", err) + } + if got.MaxSandboxes != 3 || got.MaxTTLSeconds != 3600 || got.MaxExecTimeoutSeconds != 60 { + t.Fatalf("unexpected quota: %+v", got) + } + + quota.MaxSandboxes = 5 + if err := s.SaveOwnerQuota(ctx, quota); err != nil { + t.Fatalf("update quota: %v", err) + } + + quotas, err := s.ListOwnerQuotas(ctx) + if err != nil { + t.Fatalf("list quotas: %v", err) + } + if len(quotas) != 1 || quotas[0].MaxSandboxes != 5 { + t.Fatalf("unexpected quotas: %+v", quotas) + } + + if err := s.DeleteOwnerQuota(ctx, "owner-1"); err != nil { + t.Fatalf("delete quota: %v", err) + } + if _, err := s.GetOwnerQuota(ctx, "owner-1"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound after delete, got %v", err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index d11bf37..e71a84e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -14,7 +14,9 @@ type SandboxRecord struct { VCPUs int Metadata string // JSON OwnerID string + TenantID string VMID string + WorkerID string CreatedAt time.Time ExpiresAt time.Time UpdatedAt time.Time @@ -104,6 +106,120 @@ type RegistryConnectionRecord struct { UpdatedAt time.Time } +type OwnerQuotaRecord struct { + OwnerID string + MaxSandboxes int + MaxTTLSeconds int64 + MaxExecTimeoutSeconds int64 + CreatedAt time.Time + UpdatedAt time.Time +} + +type AdminAuditRecord struct { + ID int64 `json:"id" example:"42"` + Actor string `json:"actor" example:"admin"` + Method string `json:"method" example:"PUT"` + Path string `json:"path" example:"/api/v1/admin/quotas/owner-a"` + Status int `json:"status" example:"200"` + DurationMS int64 `json:"duration_ms" example:"4"` + RequestID string `json:"request_id" example:"req-abc123"` + RemoteAddr string `json:"remote_addr" example:"127.0.0.1"` + UserAgent string `json:"user_agent" example:"stacyvm-web"` + TenantID string `json:"tenant_id" example:"tenant-acme"` + CreatedAt time.Time `json:"created_at" example:"2026-05-08T10:30:00Z"` +} + +type AdminAuditQuery struct { + Limit int + Actor string + Method string + Status int + PathLike string + TenantID string +} + +type OperationAuditRecord struct { + ID int64 `json:"id"` + Actor string `json:"actor"` + Action string `json:"action"` + SandboxID string `json:"sandbox_id"` + Resource string `json:"resource"` + Provider string `json:"provider"` + Status string `json:"status"` + Detail string `json:"detail"` + TenantID string `json:"tenant_id"` + CreatedAt time.Time `json:"created_at"` +} + +type OperationAuditQuery struct { + Limit int + Actor string + Action string + SandboxID string + Resource string + Status string + TenantID string +} + +type WorkerRecord struct { + ID string `json:"id" example:"worker-local"` + Hostname string `json:"hostname" example:"stacyvm-host-1"` + Status string `json:"status" example:"online"` + Providers string `json:"providers"` // JSON array + Capabilities string `json:"capabilities"` // JSON array + Capacity string `json:"capacity"` // JSON object + LastHeartbeat time.Time `json:"last_heartbeat" example:"2026-05-09T10:30:00Z"` + CreatedAt time.Time `json:"created_at" example:"2026-05-09T10:00:00Z"` + UpdatedAt time.Time `json:"updated_at" example:"2026-05-09T10:30:00Z"` +} + +type LeaseRecord struct { + ResourceID string `json:"resource_id" example:"sb-abc123"` + ResourceType string `json:"resource_type" example:"sandbox"` + HolderID string `json:"holder_id" example:"worker-local"` + Generation int64 `json:"generation" example:"3"` + ExpiresAt time.Time `json:"expires_at" example:"2026-05-09T10:31:00Z"` + CreatedAt time.Time `json:"created_at" example:"2026-05-09T10:30:00Z"` + UpdatedAt time.Time `json:"updated_at" example:"2026-05-09T10:30:30Z"` +} + +// TenantRecord represents a tenant (project/organization) in the multi-tenant model. +type TenantRecord struct { + ID string `json:"id" example:"tenant-acme"` + Name string `json:"name" example:"Acme Corp"` + OwnerID string `json:"owner_id" example:"user-alice"` + Settings string `json:"settings"` // JSON object + CreatedAt time.Time `json:"created_at" example:"2026-05-10T10:00:00Z"` + UpdatedAt time.Time `json:"updated_at" example:"2026-05-10T10:00:00Z"` +} + +// TenantMemberRecord maps a user/subject to a role within a tenant. +type TenantMemberRecord struct { + TenantID string `json:"tenant_id" example:"tenant-acme"` + UserID string `json:"user_id" example:"user-alice"` + Role string `json:"role" example:"admin"` // viewer, operator, admin + CreatedAt time.Time `json:"created_at" example:"2026-05-10T10:00:00Z"` + UpdatedAt time.Time `json:"updated_at" example:"2026-05-10T10:00:00Z"` +} + +// PolicyRecord is a provider/image/network allow-deny rule scoped to a tenant or globally. +type PolicyRecord struct { + ID string `json:"id" example:"pol-abc123"` + TenantID string `json:"tenant_id" example:"tenant-acme"` // empty = global + ResourceType string `json:"resource_type" example:"image"` // image, provider, network + Effect string `json:"effect" example:"allow"` // allow, deny + Pattern string `json:"pattern" example:"alpine:*"` + Priority int `json:"priority" example:"10"` + CreatedAt time.Time `json:"created_at" example:"2026-05-10T10:00:00Z"` + UpdatedAt time.Time `json:"updated_at" example:"2026-05-10T10:00:00Z"` +} + +// PolicyQuery filters for listing policies. +type PolicyQuery struct { + TenantID string + ResourceType string +} + // Store defines the persistence interface. type Store interface { // Sandbox CRUD @@ -117,6 +233,34 @@ type Store interface { ListSandboxesByOwner(ctx context.Context, ownerID string) ([]*SandboxRecord, error) CountSandboxesByVM(ctx context.Context, vmID string) (int, error) + // Owner quotas + SaveOwnerQuota(ctx context.Context, quota *OwnerQuotaRecord) error + GetOwnerQuota(ctx context.Context, ownerID string) (*OwnerQuotaRecord, error) + ListOwnerQuotas(ctx context.Context) ([]*OwnerQuotaRecord, error) + DeleteOwnerQuota(ctx context.Context, ownerID string) error + + // Admin audit + CreateAdminAudit(ctx context.Context, rec *AdminAuditRecord) error + ListAdminAudit(ctx context.Context, query AdminAuditQuery) ([]*AdminAuditRecord, error) + DeleteAdminAuditBefore(ctx context.Context, before time.Time) (int64, error) + + // Operation audit + CreateOperationAudit(ctx context.Context, rec *OperationAuditRecord) error + ListOperationAudit(ctx context.Context, query OperationAuditQuery) ([]*OperationAuditRecord, error) + + // Workers + SaveWorker(ctx context.Context, rec *WorkerRecord) error + GetWorker(ctx context.Context, id string) (*WorkerRecord, error) + ListWorkers(ctx context.Context) ([]*WorkerRecord, error) + DeleteWorker(ctx context.Context, id string) error + + // Leases + AcquireLease(ctx context.Context, resourceID, resourceType, holderID string, ttl time.Duration) (*LeaseRecord, error) + RenewLease(ctx context.Context, resourceID, holderID string, ttl time.Duration) (*LeaseRecord, error) + GetLease(ctx context.Context, resourceID string) (*LeaseRecord, error) + ListLeases(ctx context.Context) ([]*LeaseRecord, error) + ReleaseLease(ctx context.Context, resourceID, holderID string) error + // Exec logs CreateExecLog(ctx context.Context, log *ExecLogRecord) error ListExecLogs(ctx context.Context, sandboxID string) ([]*ExecLogRecord, error) @@ -156,6 +300,25 @@ type Store interface { ListRegistryConnections(ctx context.Context, ownerID string) ([]*RegistryConnectionRecord, error) DeleteRegistryConnection(ctx context.Context, id string) error + // Tenants + CreateTenant(ctx context.Context, t *TenantRecord) error + GetTenant(ctx context.Context, id string) (*TenantRecord, error) + ListTenants(ctx context.Context) ([]*TenantRecord, error) + UpdateTenant(ctx context.Context, t *TenantRecord) error + DeleteTenant(ctx context.Context, id string) error + + // Tenant members (RBAC) + SaveTenantMember(ctx context.Context, m *TenantMemberRecord) error + GetTenantMember(ctx context.Context, tenantID, userID string) (*TenantMemberRecord, error) + ListTenantMembers(ctx context.Context, tenantID string) ([]*TenantMemberRecord, error) + DeleteTenantMember(ctx context.Context, tenantID, userID string) error + + // Policies + CreatePolicy(ctx context.Context, p *PolicyRecord) error + GetPolicy(ctx context.Context, id string) (*PolicyRecord, error) + ListPolicies(ctx context.Context, query PolicyQuery) ([]*PolicyRecord, error) + DeletePolicy(ctx context.Context, id string) error + // Lifecycle Close() error } diff --git a/internal/store/store_contract_test.go b/internal/store/store_contract_test.go new file mode 100644 index 0000000..b1d8219 --- /dev/null +++ b/internal/store/store_contract_test.go @@ -0,0 +1,830 @@ +package store + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +type storeContractOpenFunc func(t *testing.T) Store + +func TestSQLiteStoreContract(t *testing.T) { + runStoreContract(t, "sqlite", func(t *testing.T) Store { + t.Helper() + st, err := Open(Config{ + Driver: DriverSQLite, + Path: filepath.Join(t.TempDir(), "contract.db"), + }) + if err != nil { + t.Fatalf("open sqlite store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st + }) +} + +func TestPostgresStoreContract(t *testing.T) { + dsn := os.Getenv("STACYVM_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("set STACYVM_POSTGRES_TEST_DSN to run Postgres store contract") + } + runStoreContract(t, "postgres", func(t *testing.T) Store { + t.Helper() + st, err := Open(Config{Driver: DriverPostgres, DSN: dsn}) + if err != nil { + t.Fatalf("open postgres store: %v", err) + } + if pg, ok := st.(*PostgresStore); ok { + resetPostgresContractStore(t, pg) + } + t.Cleanup(func() { _ = st.Close() }) + return st + }) +} + +func resetPostgresContractStore(t *testing.T, st *PostgresStore) { + t.Helper() + _, err := st.db.Exec(` +TRUNCATE + sandboxes, + exec_logs, + provider_configs, + templates, + environment_artifacts, + environment_builds, + environment_specs, + registry_connections, + owner_quotas, + admin_audit_logs, + operation_audit_logs, + workers, + leases, + tenants, + tenant_members, + policies +RESTART IDENTITY CASCADE`) + if err != nil { + t.Fatalf("reset postgres contract store: %v", err) + } +} + +func runStoreContract(t *testing.T, name string, open storeContractOpenFunc) { + t.Helper() + + t.Run(name+"/sandboxes", func(t *testing.T) { + contractSandboxes(t, open(t)) + }) + t.Run(name+"/workers", func(t *testing.T) { + contractWorkers(t, open(t)) + }) + t.Run(name+"/leases", func(t *testing.T) { + contractLeases(t, open(t)) + }) + t.Run(name+"/audits_and_logs", func(t *testing.T) { + contractAuditsAndLogs(t, open(t)) + }) + t.Run(name+"/quotas_and_provider_configs", func(t *testing.T) { + contractQuotasAndProviderConfigs(t, open(t)) + }) + t.Run(name+"/templates", func(t *testing.T) { + contractTemplates(t, open(t)) + }) + t.Run(name+"/environments_and_registry", func(t *testing.T) { + contractEnvironmentsAndRegistry(t, open(t)) + }) + t.Run(name+"/tenants_and_policies", func(t *testing.T) { + contractTenantsAndPolicies(t, open(t)) + }) +} + +func contractSandboxes(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC().Add(-time.Minute) + + sb := &SandboxRecord{ + ID: "contract-sb-a", + State: "running", + Provider: "mock", + Image: "alpine:3.20", + MemoryMB: 256, + VCPUs: 1, + Metadata: `{"contract":true}`, + OwnerID: "owner-a", + VMID: "vm-a", + WorkerID: "worker-a", + CreatedAt: now, + ExpiresAt: now.Add(time.Hour), + UpdatedAt: now, + } + if err := st.CreateSandbox(ctx, sb); err != nil { + t.Fatalf("create sandbox: %v", err) + } + if err := st.CreateSandbox(ctx, &SandboxRecord{ + ID: "contract-sb-b", + State: "running", + Provider: "mock", + Image: "ubuntu:24.04", + OwnerID: "owner-b", + VMID: "vm-b", + WorkerID: "worker-b", + CreatedAt: now, + ExpiresAt: now.Add(2 * time.Hour), + UpdatedAt: now, + }); err != nil { + t.Fatalf("create second sandbox: %v", err) + } + + got, err := st.GetSandbox(ctx, sb.ID) + if err != nil { + t.Fatalf("get sandbox: %v", err) + } + if got.OwnerID != "owner-a" || got.WorkerID != "worker-a" || got.Metadata != `{"contract":true}` { + t.Fatalf("unexpected sandbox fields: %+v", got) + } + + byOwner, err := st.ListSandboxesByOwner(ctx, "owner-a") + if err != nil { + t.Fatalf("list by owner: %v", err) + } + if len(byOwner) != 1 || byOwner[0].ID != sb.ID { + t.Fatalf("unexpected owner list: %+v", byOwner) + } + + count, err := st.CountSandboxesByVM(ctx, "vm-a") + if err != nil { + t.Fatalf("count by vm: %v", err) + } + if count != 1 { + t.Fatalf("count by vm = %d, want 1", count) + } + + expiredAt := now.Add(-time.Hour) + if err := st.UpdateSandboxExpiresAt(ctx, sb.ID, expiredAt); err != nil { + t.Fatalf("update expires: %v", err) + } + expired, err := st.ListExpiredSandboxes(ctx, now) + if err != nil { + t.Fatalf("list expired: %v", err) + } + if len(expired) != 1 || expired[0].ID != sb.ID { + t.Fatalf("unexpected expired list: %+v", expired) + } + + if err := st.UpdateSandboxState(ctx, sb.ID, "destroyed"); err != nil { + t.Fatalf("update state: %v", err) + } + active, err := st.ListSandboxes(ctx) + if err != nil { + t.Fatalf("list sandboxes: %v", err) + } + if len(active) != 1 || active[0].ID != "contract-sb-b" { + t.Fatalf("destroyed sandbox should not be listed as active: %+v", active) + } + + if err := st.DeleteSandbox(ctx, "contract-sb-b"); err != nil { + t.Fatalf("delete sandbox: %v", err) + } + deleted, err := st.GetSandbox(ctx, "contract-sb-b") + if err != nil { + t.Fatalf("get soft-deleted sandbox: %v", err) + } + if deleted.State != "destroyed" { + t.Fatalf("deleted sandbox state = %q, want destroyed", deleted.State) + } +} + +func contractWorkers(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC().Add(-time.Minute) + + rec := &WorkerRecord{ + ID: "worker-a", + Hostname: "host-a", + Status: "online", + Providers: `["docker","mock"]`, + Capabilities: `["spawn","exec","files"]`, + Capacity: `{"max_sandboxes":8}`, + LastHeartbeat: now, + } + if err := st.SaveWorker(ctx, rec); err != nil { + t.Fatalf("save worker: %v", err) + } + if rec.CreatedAt.IsZero() || rec.UpdatedAt.IsZero() { + t.Fatalf("worker timestamps were not populated: %+v", rec) + } + + rec.Status = "draining" + rec.Capacity = `{"max_sandboxes":4}` + if err := st.SaveWorker(ctx, rec); err != nil { + t.Fatalf("update worker: %v", err) + } + got, err := st.GetWorker(ctx, rec.ID) + if err != nil { + t.Fatalf("get worker: %v", err) + } + if got.Status != "draining" || got.Capacity != `{"max_sandboxes":4}` { + t.Fatalf("unexpected updated worker: %+v", got) + } + + workers, err := st.ListWorkers(ctx) + if err != nil { + t.Fatalf("list workers: %v", err) + } + if len(workers) != 1 || workers[0].ID != rec.ID { + t.Fatalf("unexpected workers: %+v", workers) + } + + if err := st.DeleteWorker(ctx, rec.ID); err != nil { + t.Fatalf("delete worker: %v", err) + } + if _, err := st.GetWorker(ctx, rec.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted worker error = %v, want ErrNotFound", err) + } +} + +func contractLeases(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + + lease, err := st.AcquireLease(ctx, "sandbox-a", "sandbox", "worker-a", time.Minute) + if err != nil { + t.Fatalf("acquire lease: %v", err) + } + if lease.HolderID != "worker-a" || lease.Generation != 1 { + t.Fatalf("unexpected lease: %+v", lease) + } + if _, err := st.AcquireLease(ctx, "sandbox-a", "sandbox", "worker-b", time.Minute); !errors.Is(err, ErrConflict) { + t.Fatalf("conflicting lease error = %v, want ErrConflict", err) + } + + renewed, err := st.RenewLease(ctx, "sandbox-a", "worker-a", time.Minute) + if err != nil { + t.Fatalf("renew lease: %v", err) + } + if renewed.Generation != lease.Generation+1 { + t.Fatalf("renewed generation = %d, want %d", renewed.Generation, lease.Generation+1) + } + if err := st.ReleaseLease(ctx, "sandbox-a", "worker-b"); !errors.Is(err, ErrNotFound) { + t.Fatalf("wrong holder release error = %v, want ErrNotFound", err) + } + + leases, err := st.ListLeases(ctx) + if err != nil { + t.Fatalf("list leases: %v", err) + } + if len(leases) != 1 || leases[0].ResourceID != "sandbox-a" { + t.Fatalf("unexpected leases: %+v", leases) + } + + if err := st.ReleaseLease(ctx, "sandbox-a", "worker-a"); err != nil { + t.Fatalf("release lease: %v", err) + } + if _, err := st.GetLease(ctx, "sandbox-a"); !errors.Is(err, ErrNotFound) { + t.Fatalf("get released lease error = %v, want ErrNotFound", err) + } + + if _, err := st.AcquireLease(ctx, "sandbox-expiring", "sandbox", "worker-a", time.Nanosecond); err != nil { + t.Fatalf("acquire expiring lease: %v", err) + } + time.Sleep(2 * time.Millisecond) + taken, err := st.AcquireLease(ctx, "sandbox-expiring", "sandbox", "worker-b", time.Minute) + if err != nil { + t.Fatalf("take over expired lease: %v", err) + } + if taken.HolderID != "worker-b" || taken.Generation != 2 { + t.Fatalf("unexpected takeover lease: %+v", taken) + } +} + +func contractAuditsAndLogs(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC().Add(-time.Minute) + + if err := st.CreateSandbox(ctx, &SandboxRecord{ + ID: "sandbox-a", + State: "running", + Provider: "mock", + Image: "alpine:3.20", + MemoryMB: 256, + VCPUs: 1, + Metadata: `{}`, + OwnerID: "owner-a", + VMID: "vm-a", + WorkerID: "worker-a", + CreatedAt: now, + ExpiresAt: now.Add(time.Hour), + UpdatedAt: now, + }); err != nil { + t.Fatalf("create sandbox for exec log: %v", err) + } + + if err := st.CreateExecLog(ctx, &ExecLogRecord{ + SandboxID: "sandbox-a", + Command: "echo ok", + ExitCode: 0, + Stdout: "ok\n", + Duration: "10ms", + CreatedAt: now, + }); err != nil { + t.Fatalf("create exec log: %v", err) + } + logs, err := st.ListExecLogs(ctx, "sandbox-a") + if err != nil { + t.Fatalf("list exec logs: %v", err) + } + if len(logs) != 1 || logs[0].Command != "echo ok" || logs[0].ExitCode != 0 { + t.Fatalf("unexpected exec logs: %+v", logs) + } + + if err := st.CreateAdminAudit(ctx, &AdminAuditRecord{ + Actor: "admin", + Method: "POST", + Path: "/api/v1/admin/quotas/owner-a", + Status: 201, + DurationMS: 7, + RequestID: "req-a", + RemoteAddr: "127.0.0.1", + UserAgent: "contract", + CreatedAt: now, + }); err != nil { + t.Fatalf("create admin audit: %v", err) + } + adminLogs, err := st.ListAdminAudit(ctx, AdminAuditQuery{Actor: "admin", Status: 201, Limit: 10}) + if err != nil { + t.Fatalf("list admin audit: %v", err) + } + if len(adminLogs) != 1 || adminLogs[0].RequestID != "req-a" { + t.Fatalf("unexpected admin audit logs: %+v", adminLogs) + } + deleted, err := st.DeleteAdminAuditBefore(ctx, now.Add(time.Hour)) + if err != nil { + t.Fatalf("delete admin audit: %v", err) + } + if deleted != 1 { + t.Fatalf("deleted admin audit rows = %d, want 1", deleted) + } + + if err := st.CreateOperationAudit(ctx, &OperationAuditRecord{ + Actor: "owner-a", + Action: "sandbox.exec", + SandboxID: "sandbox-a", + Resource: "exec", + Provider: "mock", + Status: "success", + Detail: `{"exit_code":0}`, + CreatedAt: now, + }); err != nil { + t.Fatalf("create operation audit: %v", err) + } + ops, err := st.ListOperationAudit(ctx, OperationAuditQuery{Actor: "owner-a", Status: "success", Limit: 10}) + if err != nil { + t.Fatalf("list operation audit: %v", err) + } + if len(ops) != 1 || ops[0].Action != "sandbox.exec" { + t.Fatalf("unexpected operation audit logs: %+v", ops) + } +} + +func contractQuotasAndProviderConfigs(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + + if err := st.SaveOwnerQuota(ctx, &OwnerQuotaRecord{ + OwnerID: "owner-a", + MaxSandboxes: 3, + MaxTTLSeconds: 3600, + MaxExecTimeoutSeconds: 120, + }); err != nil { + t.Fatalf("save quota: %v", err) + } + if err := st.SaveOwnerQuota(ctx, &OwnerQuotaRecord{ + OwnerID: "owner-a", + MaxSandboxes: 5, + MaxTTLSeconds: 7200, + MaxExecTimeoutSeconds: 300, + }); err != nil { + t.Fatalf("update quota: %v", err) + } + quota, err := st.GetOwnerQuota(ctx, "owner-a") + if err != nil { + t.Fatalf("get quota: %v", err) + } + if quota.MaxSandboxes != 5 || quota.MaxTTLSeconds != 7200 { + t.Fatalf("unexpected quota: %+v", quota) + } + quotas, err := st.ListOwnerQuotas(ctx) + if err != nil { + t.Fatalf("list quotas: %v", err) + } + if len(quotas) != 1 || quotas[0].OwnerID != "owner-a" { + t.Fatalf("unexpected quotas: %+v", quotas) + } + if err := st.DeleteOwnerQuota(ctx, "owner-a"); err != nil { + t.Fatalf("delete quota: %v", err) + } + if _, err := st.GetOwnerQuota(ctx, "owner-a"); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted quota error = %v, want ErrNotFound", err) + } + + if err := st.SaveProviderConfig(ctx, &ProviderConfigRecord{ + Name: "docker", + Config: `{"network":"none"}`, + Enabled: true, + }); err != nil { + t.Fatalf("save provider config: %v", err) + } + cfg, err := st.GetProviderConfig(ctx, "docker") + if err != nil { + t.Fatalf("get provider config: %v", err) + } + if !cfg.Enabled || cfg.Config != `{"network":"none"}` { + t.Fatalf("unexpected provider config: %+v", cfg) + } + configs, err := st.ListProviderConfigs(ctx) + if err != nil { + t.Fatalf("list provider configs: %v", err) + } + if len(configs) != 1 || configs[0].Name != "docker" { + t.Fatalf("unexpected provider configs: %+v", configs) + } +} + +func contractTemplates(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC().Add(-time.Minute) + + tpl := &TemplateRecord{ + Name: "go-agent", + Version: 1, + Image: "golang:1.23", + Description: "Go agent template", + Setup: `["go version"]`, + AllowedHosts: `["proxy.golang.org"]`, + MemoryMB: 1024, + CPUCores: 2, + TTLSeconds: 1800, + Env: `{"GOFLAGS":"-mod=mod"}`, + Secrets: `["GITHUB_TOKEN"]`, + PoolSize: 2, + CreatedAt: now, + UpdatedAt: now, + } + if err := st.CreateTemplate(ctx, tpl); err != nil { + t.Fatalf("create template: %v", err) + } + if err := st.CreateTemplate(ctx, tpl); !errors.Is(err, ErrConflict) { + t.Fatalf("duplicate template error = %v, want ErrConflict", err) + } + tpl.Version = 2 + tpl.PoolSize = 4 + if err := st.UpdateTemplate(ctx, tpl); err != nil { + t.Fatalf("update template: %v", err) + } + got, err := st.GetTemplate(ctx, tpl.Name) + if err != nil { + t.Fatalf("get template: %v", err) + } + if got.Version != 2 || got.PoolSize != 4 { + t.Fatalf("unexpected template: %+v", got) + } + templates, err := st.ListTemplates(ctx) + if err != nil { + t.Fatalf("list templates: %v", err) + } + if len(templates) != 1 || templates[0].Name != tpl.Name { + t.Fatalf("unexpected templates: %+v", templates) + } + if err := st.DeleteTemplate(ctx, tpl.Name); err != nil { + t.Fatalf("delete template: %v", err) + } + if _, err := st.GetTemplate(ctx, tpl.Name); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted template error = %v, want ErrNotFound", err) + } +} + +func contractEnvironmentsAndRegistry(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + now := time.Now().UTC().Add(-time.Minute) + + spec := &EnvironmentSpecRecord{ + ID: "spec-a", + OwnerID: "owner-a", + Name: "py-agent", + BaseImage: "python:3.12-slim", + PythonPackages: `["pytest"]`, + AptPackages: `["git"]`, + PythonVersion: "3.12", + CreatedAt: now, + UpdatedAt: now, + } + if err := st.CreateEnvironmentSpec(ctx, spec); err != nil { + t.Fatalf("create environment spec: %v", err) + } + spec.BaseImage = "python:3.12-bookworm" + if err := st.UpdateEnvironmentSpec(ctx, spec); err != nil { + t.Fatalf("update environment spec: %v", err) + } + gotSpec, err := st.GetEnvironmentSpec(ctx, spec.ID) + if err != nil { + t.Fatalf("get environment spec: %v", err) + } + if gotSpec.BaseImage != "python:3.12-bookworm" { + t.Fatalf("unexpected environment spec: %+v", gotSpec) + } + specs, err := st.ListEnvironmentSpecs(ctx, "owner-a") + if err != nil { + t.Fatalf("list environment specs: %v", err) + } + if len(specs) != 1 || specs[0].ID != spec.ID { + t.Fatalf("unexpected specs: %+v", specs) + } + + deleteSpec := &EnvironmentSpecRecord{ + ID: "spec-delete", + OwnerID: "owner-a", + Name: "delete-me", + BaseImage: "python:3.12-slim", + PythonPackages: `[]`, + AptPackages: `[]`, + PythonVersion: "3.12", + CreatedAt: now, + UpdatedAt: now, + } + if err := st.CreateEnvironmentSpec(ctx, deleteSpec); err != nil { + t.Fatalf("create deletable environment spec: %v", err) + } + if err := st.DeleteEnvironmentSpec(ctx, deleteSpec.ID); err != nil { + t.Fatalf("delete environment spec: %v", err) + } + if _, err := st.GetEnvironmentSpec(ctx, deleteSpec.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted environment spec error = %v, want ErrNotFound", err) + } + + finishedAt := now.Add(time.Minute) + build := &EnvironmentBuildRecord{ + ID: "build-a", + SpecID: spec.ID, + Status: "building", + CurrentStep: "install", + LogBlob: "installing\n", + ImageSizeBytes: 100, + CreatedAt: now, + UpdatedAt: now, + } + if err := st.CreateEnvironmentBuild(ctx, build); err != nil { + t.Fatalf("create environment build: %v", err) + } + build.Status = "ready" + build.CurrentStep = "complete" + build.DigestLocal = "sha256:abc" + build.FinishedAt = &finishedAt + if err := st.UpdateEnvironmentBuild(ctx, build); err != nil { + t.Fatalf("update environment build: %v", err) + } + gotBuild, err := st.GetEnvironmentBuild(ctx, build.ID) + if err != nil { + t.Fatalf("get environment build: %v", err) + } + if gotBuild.Status != "ready" || gotBuild.FinishedAt == nil { + t.Fatalf("unexpected environment build: %+v", gotBuild) + } + builds, err := st.ListEnvironmentBuilds(ctx, spec.ID) + if err != nil { + t.Fatalf("list environment builds: %v", err) + } + if len(builds) != 1 || builds[0].ID != build.ID { + t.Fatalf("unexpected builds: %+v", builds) + } + + if err := st.SaveEnvironmentArtifact(ctx, &EnvironmentArtifactRecord{ + BuildID: build.ID, + Target: "linux/amd64", + ImageRef: "registry.example.com/owner/py-agent:latest", + Digest: "sha256:one", + Status: "pushed", + }); err != nil { + t.Fatalf("save artifact: %v", err) + } + if err := st.SaveEnvironmentArtifact(ctx, &EnvironmentArtifactRecord{ + BuildID: build.ID, + Target: "linux/amd64", + ImageRef: "registry.example.com/owner/py-agent:v2", + Digest: "sha256:two", + Status: "pushed", + }); err != nil { + t.Fatalf("update artifact: %v", err) + } + artifacts, err := st.ListEnvironmentArtifacts(ctx, build.ID) + if err != nil { + t.Fatalf("list artifacts: %v", err) + } + if len(artifacts) != 1 || artifacts[0].Digest != "sha256:two" { + t.Fatalf("unexpected artifacts: %+v", artifacts) + } + + conn := &RegistryConnectionRecord{ + ID: "registry-a", + OwnerID: "owner-a", + Provider: "dockerhub", + Username: "owner", + SecretRef: "secret://registry-a", + IsDefault: true, + } + if err := st.SaveRegistryConnection(ctx, conn); err != nil { + t.Fatalf("save registry connection: %v", err) + } + conn.Username = "owner-updated" + if err := st.SaveRegistryConnection(ctx, conn); err != nil { + t.Fatalf("update registry connection: %v", err) + } + gotConn, err := st.GetRegistryConnection(ctx, conn.ID) + if err != nil { + t.Fatalf("get registry connection: %v", err) + } + if gotConn.Username != "owner-updated" || !gotConn.IsDefault { + t.Fatalf("unexpected registry connection: %+v", gotConn) + } + conns, err := st.ListRegistryConnections(ctx, "owner-a") + if err != nil { + t.Fatalf("list registry connections: %v", err) + } + if len(conns) != 1 || conns[0].ID != conn.ID { + t.Fatalf("unexpected registry connections: %+v", conns) + } + if err := st.DeleteRegistryConnection(ctx, conn.ID); err != nil { + t.Fatalf("delete registry connection: %v", err) + } + if _, err := st.GetRegistryConnection(ctx, conn.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted registry connection error = %v, want ErrNotFound", err) + } + +} + +func contractTenantsAndPolicies(t *testing.T, st Store) { + t.Helper() + ctx := context.Background() + + // --- Tenants --- + tenant := &TenantRecord{ + ID: "tenant-contract-a", + Name: "Contract Tenant A", + OwnerID: "owner-contract", + } + if err := st.CreateTenant(ctx, tenant); err != nil { + t.Fatalf("create tenant: %v", err) + } + + got, err := st.GetTenant(ctx, tenant.ID) + if err != nil { + t.Fatalf("get tenant: %v", err) + } + if got.Name != "Contract Tenant A" { + t.Errorf("unexpected tenant name: %q", got.Name) + } + + got.Name = "Updated Name" + if err := st.UpdateTenant(ctx, got); err != nil { + t.Fatalf("update tenant: %v", err) + } + got2, _ := st.GetTenant(ctx, tenant.ID) + if got2.Name != "Updated Name" { + t.Errorf("tenant name not updated, got %q", got2.Name) + } + + tenants, err := st.ListTenants(ctx) + if err != nil { + t.Fatalf("list tenants: %v", err) + } + found := false + for _, te := range tenants { + if te.ID == tenant.ID { + found = true + break + } + } + if !found { + t.Errorf("tenant not found in list") + } + + // --- Tenant Members --- + member := &TenantMemberRecord{ + TenantID: tenant.ID, + UserID: "user-contract-1", + Role: "operator", + } + if err := st.SaveTenantMember(ctx, member); err != nil { + t.Fatalf("save tenant member: %v", err) + } + + gotMember, err := st.GetTenantMember(ctx, tenant.ID, member.UserID) + if err != nil { + t.Fatalf("get tenant member: %v", err) + } + if gotMember.Role != "operator" { + t.Errorf("unexpected member role: %q", gotMember.Role) + } + + // Upsert role change. + member.Role = "admin" + if err := st.SaveTenantMember(ctx, member); err != nil { + t.Fatalf("upsert tenant member: %v", err) + } + gotMember2, _ := st.GetTenantMember(ctx, tenant.ID, member.UserID) + if gotMember2.Role != "admin" { + t.Errorf("member role not updated after upsert, got %q", gotMember2.Role) + } + + members, err := st.ListTenantMembers(ctx, tenant.ID) + if err != nil { + t.Fatalf("list tenant members: %v", err) + } + if len(members) != 1 || members[0].UserID != member.UserID { + t.Errorf("unexpected member list: %+v", members) + } + + if err := st.DeleteTenantMember(ctx, tenant.ID, member.UserID); err != nil { + t.Fatalf("delete tenant member: %v", err) + } + if _, err := st.GetTenantMember(ctx, tenant.ID, member.UserID); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted member error = %v, want ErrNotFound", err) + } + + // --- Policies --- + pol := &PolicyRecord{ + ID: "pol-contract-1", + TenantID: tenant.ID, + ResourceType: "image", + Effect: "allow", + Pattern: "alpine:*", + Priority: 5, + } + if err := st.CreatePolicy(ctx, pol); err != nil { + t.Fatalf("create policy: %v", err) + } + + gotPol, err := st.GetPolicy(ctx, pol.ID) + if err != nil { + t.Fatalf("get policy: %v", err) + } + if gotPol.Pattern != "alpine:*" || gotPol.Effect != "allow" { + t.Errorf("unexpected policy: %+v", gotPol) + } + + // Global policy (no tenant). + globalPol := &PolicyRecord{ + ID: "pol-contract-global", + TenantID: "", + ResourceType: "image", + Effect: "deny", + Pattern: "evil:*", + Priority: 1, + } + if err := st.CreatePolicy(ctx, globalPol); err != nil { + t.Fatalf("create global policy: %v", err) + } + + // ListPolicies with tenant should include both tenant and global policies. + pols, err := st.ListPolicies(ctx, PolicyQuery{TenantID: tenant.ID, ResourceType: "image"}) + if err != nil { + t.Fatalf("list policies: %v", err) + } + if len(pols) < 2 { + t.Errorf("expected at least 2 policies (tenant + global), got %d", len(pols)) + } + + // ListPolicies without tenant should return only global policies. + globalPols, err := st.ListPolicies(ctx, PolicyQuery{ResourceType: "image"}) + if err != nil { + t.Fatalf("list global policies: %v", err) + } + for _, p := range globalPols { + if p.TenantID != "" { + t.Errorf("expected only global policies, got tenant-scoped: %+v", p) + } + } + + // Delete policy. + if err := st.DeletePolicy(ctx, pol.ID); err != nil { + t.Fatalf("delete policy: %v", err) + } + if _, err := st.GetPolicy(ctx, pol.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted policy error = %v, want ErrNotFound", err) + } + _ = st.DeletePolicy(ctx, globalPol.ID) // cleanup + + // --- Tenant deletion --- + if err := st.DeleteTenant(ctx, tenant.ID); err != nil { + t.Fatalf("delete tenant: %v", err) + } + if _, err := st.GetTenant(ctx, tenant.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("get deleted tenant error = %v, want ErrNotFound", err) + } +} diff --git a/internal/worker/client.go b/internal/worker/client.go new file mode 100644 index 0000000..8b94e15 --- /dev/null +++ b/internal/worker/client.go @@ -0,0 +1,176 @@ +package worker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +type Client struct { + BaseURL string + WorkerID string + Token string + TokenFunc func() (string, error) + HTTPClient *http.Client +} + +func (c Client) Heartbeat(ctx context.Context, params workerproto.HeartbeatParams) error { + if strings.TrimSpace(c.BaseURL) == "" { + return fmt.Errorf("control plane URL is required") + } + if strings.TrimSpace(c.WorkerID) == "" { + return fmt.Errorf("worker id is required") + } + token, err := c.authToken() + if err != nil { + return err + } + body, err := json.Marshal(params) + if err != nil { + return err + } + url := strings.TrimRight(c.BaseURL, "/") + "/api/v1/worker/" + c.WorkerID + "/heartbeat" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Worker-ID", c.WorkerID) + req.Header.Set("X-Worker-Token", token) + + client := c.HTTPClient + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("worker heartbeat failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + return nil +} + +func (c Client) RenewLease(ctx context.Context, resourceID, ttl string) (workerproto.LeaseToken, error) { + var zero workerproto.LeaseToken + if strings.TrimSpace(c.BaseURL) == "" { + return zero, fmt.Errorf("control plane URL is required") + } + if strings.TrimSpace(c.WorkerID) == "" { + return zero, fmt.Errorf("worker id is required") + } + token, err := c.authToken() + if err != nil { + return zero, err + } + if strings.TrimSpace(resourceID) == "" { + return zero, fmt.Errorf("lease resource id is required") + } + body, err := json.Marshal(workerproto.RenewLeaseParams{ResourceID: resourceID, TTL: ttl}) + if err != nil { + return zero, err + } + url := strings.TrimRight(c.BaseURL, "/") + "/api/v1/worker/" + c.WorkerID + "/leases/" + resourceID + "/renew" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return zero, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Worker-ID", c.WorkerID) + req.Header.Set("X-Worker-Token", token) + + client := c.HTTPClient + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return zero, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return zero, fmt.Errorf("worker lease renewal failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + var result workerproto.RenewLeaseResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return zero, err + } + return result.Lease, nil +} + +// NewIssuerTokenFunc returns a TokenFunc that fetches a short-lived signed worker +// token from the control-plane's centralized issuer endpoint. This lets workers +// authenticate without needing direct access to auth.worker_signing_key. +// +// bootstrapKey is an admin API key used solely to call POST /api/v1/admin/worker-tokens. +// ttl is the desired token lifetime (max 15 minutes). +func NewIssuerTokenFunc(controlPlaneURL, workerID, bootstrapAdminKey, ttl string) func() (string, error) { + if ttl == "" { + ttl = "5m" + } + return func() (string, error) { + base := strings.TrimRight(controlPlaneURL, "/") + url := base + "/api/v1/admin/worker-tokens" + body, _ := json.Marshal(map[string]string{ + "worker_id": workerID, + "ttl": ttl, + "audience": "worker:control-plane", + }) + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("issuer: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Admin-API-Key", bootstrapAdminKey) + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("issuer: calling control plane: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return "", fmt.Errorf("issuer: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + var result struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("issuer: decoding response: %w", err) + } + if strings.TrimSpace(result.Token) == "" { + return "", fmt.Errorf("issuer: response contained no token") + } + return result.Token, nil + } +} + +func (c Client) authToken() (string, error) { + if c.TokenFunc != nil { + token, err := c.TokenFunc() + if err != nil { + return "", err + } + if strings.TrimSpace(token) == "" { + return "", fmt.Errorf("worker token is required") + } + return token, nil + } + if strings.TrimSpace(c.Token) == "" { + return "", fmt.Errorf("worker token is required") + } + return c.Token, nil +} diff --git a/internal/worker/client_test.go b/internal/worker/client_test.go new file mode 100644 index 0000000..201819d --- /dev/null +++ b/internal/worker/client_test.go @@ -0,0 +1,128 @@ +package worker + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +func TestClientHeartbeatSendsWorkerCredentials(t *testing.T) { + var gotPath string + var gotWorkerID string + var gotToken string + var gotBody workerproto.HeartbeatParams + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotWorkerID = r.Header.Get("X-Worker-ID") + gotToken = r.Header.Get("X-Worker-Token") + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode body: %v", err) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := Client{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + err := client.Heartbeat(context.Background(), workerproto.HeartbeatParams{ + Hostname: "host-a", + Status: "online", + Providers: []string{"mock"}, + Capabilities: []string{"heartbeat"}, + }) + if err != nil { + t.Fatalf("heartbeat: %v", err) + } + if gotPath != "/api/v1/worker/worker-a/heartbeat" { + t.Fatalf("path = %q", gotPath) + } + if gotWorkerID != "worker-a" || gotToken != "worker-secret" { + t.Fatalf("unexpected credentials: worker_id=%q token=%q", gotWorkerID, gotToken) + } + if gotBody.Hostname != "host-a" || gotBody.Providers[0] != "mock" { + t.Fatalf("unexpected body: %+v", gotBody) + } +} + +func TestClientHeartbeatRejectsMissingConfig(t *testing.T) { + if err := (Client{}).Heartbeat(context.Background(), workerproto.HeartbeatParams{}); err == nil { + t.Fatal("expected missing config error") + } +} + +func TestClientHeartbeatUsesTokenFunc(t *testing.T) { + var gotToken string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotToken = r.Header.Get("X-Worker-Token") + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := Client{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "static-token", + TokenFunc: func() (string, error) { + return "signed-token", nil + }, + } + if err := client.Heartbeat(context.Background(), workerproto.HeartbeatParams{}); err != nil { + t.Fatalf("heartbeat: %v", err) + } + if gotToken != "signed-token" { + t.Fatalf("worker token = %q, want token from TokenFunc", gotToken) + } +} + +func TestClientRenewLeaseSendsWorkerCredentials(t *testing.T) { + expiresAt := time.Now().UTC().Add(time.Minute) + var gotPath string + var gotWorkerID string + var gotToken string + var gotBody workerproto.RenewLeaseParams + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotWorkerID = r.Header.Get("X-Worker-ID") + gotToken = r.Header.Get("X-Worker-Token") + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode body: %v", err) + } + _ = json.NewEncoder(w).Encode(workerproto.RenewLeaseResult{Lease: workerproto.LeaseToken{ + ResourceID: "sb-1", + HolderID: "worker-a", + Generation: 2, + ExpiresAt: expiresAt, + }}) + })) + defer server.Close() + + client := Client{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + lease, err := client.RenewLease(context.Background(), "sb-1", "30s") + if err != nil { + t.Fatalf("renew lease: %v", err) + } + if gotPath != "/api/v1/worker/worker-a/leases/sb-1/renew" { + t.Fatalf("path = %q", gotPath) + } + if gotWorkerID != "worker-a" || gotToken != "worker-secret" { + t.Fatalf("unexpected credentials: worker_id=%q token=%q", gotWorkerID, gotToken) + } + if gotBody.ResourceID != "sb-1" || gotBody.TTL != "30s" { + t.Fatalf("unexpected body: %+v", gotBody) + } + if lease.Generation != 2 || !lease.ExpiresAt.Equal(expiresAt) { + t.Fatalf("unexpected lease: %+v", lease) + } +} diff --git a/internal/worker/rpc.go b/internal/worker/rpc.go new file mode 100644 index 0000000..85433a0 --- /dev/null +++ b/internal/worker/rpc.go @@ -0,0 +1,621 @@ +package worker + +import ( + "bytes" + "context" + "crypto/subtle" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/httputil" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +type RPCServer struct { + WorkerID string + Token string + SigningKey string + SigningKeys []string + RevokedTokenIDs []string + Now func() time.Time + Registry *providers.Registry + LeaseRenewer LeaseRenewer + draining atomic.Bool +} + +type LeaseRenewer interface { + RenewLease(ctx context.Context, resourceID, ttl string) (workerproto.LeaseToken, error) +} + +func (s *RPCServer) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/rpc", s.handleRPC) + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + httputil.WriteJSON(w, http.StatusOK, map[string]string{"status": "ok", "worker_id": s.WorkerID}) + }) + return mux +} + +func (s *RPCServer) Draining() bool { + if s == nil { + return false + } + return s.draining.Load() +} + +func (s *RPCServer) handleRPC(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + httputil.WriteError(w, http.StatusMethodNotAllowed, httputil.CodeBadRequest, "method not allowed") + return + } + if !s.authenticate(r) { + httputil.WriteError(w, http.StatusUnauthorized, httputil.CodeUnauth, "invalid or missing worker RPC credentials") + return + } + var req workerproto.Request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + httputil.WriteError(w, http.StatusBadRequest, httputil.CodeBadRequest, "invalid worker RPC request") + return + } + if req.WorkerID != s.WorkerID { + httputil.WriteError(w, http.StatusForbidden, httputil.CodeUnauth, "worker RPC request targets a different worker") + return + } + if err := workerproto.ValidateRequest(req); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ + ID: req.ID, + WorkerID: s.WorkerID, + Error: err.Error(), + }) + return + } + + switch req.Method { + case workerproto.MethodStatus: + s.handleStatus(w, r.Context(), req) + case workerproto.MethodExec: + s.handleExec(w, r.Context(), req) + case workerproto.MethodExecStream: + s.handleExecStream(w, r, req) + case workerproto.MethodFileWrite, workerproto.MethodFileRead, workerproto.MethodFileList, + workerproto.MethodFileDelete, workerproto.MethodFileMove, workerproto.MethodFileChmod, + workerproto.MethodFileStat, workerproto.MethodFileGlob: + s.handleFile(w, r.Context(), req) + case workerproto.MethodLogs: + s.handleLogs(w, r.Context(), req) + case workerproto.MethodRenewLease: + s.handleRenewLease(w, r.Context(), req) + case workerproto.MethodSpawn: + if s.draining.Load() { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ + ID: req.ID, + WorkerID: s.WorkerID, + Error: "worker is draining", + }) + return + } + s.handleSpawn(w, r.Context(), req) + case workerproto.MethodDestroy: + s.handleDestroy(w, r.Context(), req) + case workerproto.MethodShutdown: + s.draining.Store(true) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID}) + default: + httputil.WriteJSON(w, http.StatusNotImplemented, workerproto.Response{ + ID: req.ID, + WorkerID: s.WorkerID, + Error: "worker RPC method is not implemented by this worker runtime", + }) + } +} + +func (s *RPCServer) handleStatus(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + var params workerproto.StatusParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID := strings.TrimSpace(params.RuntimeID) + if runtimeID == "" { + runtimeID = params.SandboxID + } + status, err := provider.Status(ctx, runtimeID) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, providers.ErrSandboxNotFound) { + code = http.StatusNotFound + } + httputil.WriteJSON(w, code, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + result, _ := json.Marshal(workerproto.StatusResult{ + SandboxID: params.SandboxID, + State: status.State, + Provider: provider.Name(), + WorkerID: s.WorkerID, + }) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func (s *RPCServer) handleExec(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + var params workerproto.ExecParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID := strings.TrimSpace(params.RuntimeID) + if runtimeID == "" { + runtimeID = params.SandboxID + } + execCtx := ctx + var cancel context.CancelFunc + if strings.TrimSpace(params.Timeout) != "" { + timeout, err := time.ParseDuration(params.Timeout) + if err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if timeout > 0 { + execCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + } + execResult, err := provider.Exec(execCtx, runtimeID, providers.ExecOptions{ + Command: params.Command, + Args: params.Args, + Mode: params.Mode, + Env: params.Env, + WorkDir: params.WorkDir, + }) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, providers.ErrSandboxNotFound) { + code = http.StatusNotFound + } + httputil.WriteJSON(w, code, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + result, _ := json.Marshal(workerproto.ExecResult{ + SandboxID: params.SandboxID, + ExitCode: execResult.ExitCode, + Stdout: execResult.Stdout, + Stderr: execResult.Stderr, + }) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func (s *RPCServer) handleExecStream(w http.ResponseWriter, r *http.Request, req workerproto.Request) { + ctx := r.Context() + var params workerproto.ExecParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID := strings.TrimSpace(params.RuntimeID) + if runtimeID == "" { + runtimeID = params.SandboxID + } + execCtx := ctx + var cancel context.CancelFunc + if strings.TrimSpace(params.Timeout) != "" { + timeout, err := time.ParseDuration(params.Timeout) + if err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if timeout > 0 { + execCtx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + } + ch, err := provider.ExecStream(execCtx, runtimeID, providers.ExecOptions{ + Command: params.Command, + Args: params.Args, + Mode: params.Mode, + Env: params.Env, + WorkDir: params.WorkDir, + }) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, providers.ErrSandboxNotFound) { + code = http.StatusNotFound + } + httputil.WriteJSON(w, code, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if strings.EqualFold(r.Header.Get("X-Worker-Stream"), "ndjson") { + s.streamExecChunks(w, execCtx, req, ch) + return + } + chunks := make([]workerproto.StreamChunk, 0, 8) + for chunk := range ch { + chunks = append(chunks, workerproto.StreamChunk{Stream: chunk.Stream, Data: chunk.Data}) + } + if execCtx.Err() != nil { + httputil.WriteJSON(w, http.StatusGatewayTimeout, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: execCtx.Err().Error()}) + return + } + result, _ := json.Marshal(workerproto.ExecStreamResult{ + SandboxID: params.SandboxID, + Chunks: chunks, + }) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func (s *RPCServer) streamExecChunks(w http.ResponseWriter, ctx context.Context, req workerproto.Request, ch <-chan providers.StreamChunk) { + w.Header().Set("Content-Type", "application/x-ndjson") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + enc := json.NewEncoder(w) + for chunk := range ch { + _ = enc.Encode(workerproto.Response{ + ID: req.ID, + WorkerID: s.WorkerID, + Result: mustJSONRaw(workerproto.StreamChunk{Stream: chunk.Stream, Data: chunk.Data}), + }) + if flusher != nil { + flusher.Flush() + } + } + if ctx.Err() != nil { + _ = enc.Encode(workerproto.Response{ + ID: req.ID, + WorkerID: s.WorkerID, + Error: ctx.Err().Error(), + }) + if flusher != nil { + flusher.Flush() + } + } +} + +func (s *RPCServer) handleFile(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + var params workerproto.FileParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID := strings.TrimSpace(params.RuntimeID) + if runtimeID == "" { + runtimeID = params.SandboxID + } + result, err := runFileOperation(ctx, provider, runtimeID, req.Method, params) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, providers.ErrSandboxNotFound) { + code = http.StatusNotFound + } + httputil.WriteJSON(w, code, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func runFileOperation(ctx context.Context, provider providers.Provider, runtimeID, method string, params workerproto.FileParams) (json.RawMessage, error) { + switch method { + case workerproto.MethodFileWrite: + if err := provider.WriteFile(ctx, runtimeID, params.Path, bytes.NewReader(params.Content), params.Mode); err != nil { + return nil, err + } + return nil, nil + case workerproto.MethodFileRead: + rc, err := provider.ReadFile(ctx, runtimeID, params.Path) + if err != nil { + return nil, err + } + defer rc.Close() + content, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + return json.Marshal(workerproto.FileReadResult{SandboxID: params.SandboxID, Content: content}) + case workerproto.MethodFileList: + files, err := provider.ListFiles(ctx, runtimeID, params.Path) + if err != nil { + return nil, err + } + return json.Marshal(workerproto.FileListResult{SandboxID: params.SandboxID, Files: toWorkerFileInfo(files)}) + case workerproto.MethodFileDelete: + return nil, provider.DeleteFile(ctx, runtimeID, params.Path, params.Recursive) + case workerproto.MethodFileMove: + return nil, provider.MoveFile(ctx, runtimeID, params.OldPath, params.NewPath) + case workerproto.MethodFileChmod: + return nil, provider.ChmodFile(ctx, runtimeID, params.Path, params.Mode) + case workerproto.MethodFileStat: + file, err := provider.StatFile(ctx, runtimeID, params.Path) + if err != nil { + return nil, err + } + return json.Marshal(workerproto.FileStatResult{SandboxID: params.SandboxID, File: toWorkerFileInfo([]providers.FileInfo{*file})[0]}) + case workerproto.MethodFileGlob: + matches, err := provider.GlobFiles(ctx, runtimeID, params.Pattern) + if err != nil { + return nil, err + } + return json.Marshal(workerproto.FileGlobResult{SandboxID: params.SandboxID, Matches: matches}) + default: + return nil, workerproto.ErrUnknownMethod + } +} + +func toWorkerFileInfo(files []providers.FileInfo) []workerproto.FileInfo { + out := make([]workerproto.FileInfo, len(files)) + for i, file := range files { + out[i] = workerproto.FileInfo{ + Path: file.Path, + Size: file.Size, + Mode: file.Mode, + IsDir: file.IsDir, + ModTime: file.ModTime, + } + } + return out +} + +func (s *RPCServer) handleLogs(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + var params workerproto.LogsParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID := strings.TrimSpace(params.RuntimeID) + if runtimeID == "" { + runtimeID = params.SandboxID + } + lines, err := provider.ConsoleLog(ctx, runtimeID, params.Lines) + if err != nil { + code := http.StatusInternalServerError + if errors.Is(err, providers.ErrSandboxNotFound) { + code = http.StatusNotFound + } + httputil.WriteJSON(w, code, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + result, _ := json.Marshal(workerproto.LogsResult{ + SandboxID: params.SandboxID, + Lines: lines, + }) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func (s *RPCServer) handleSpawn(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + if err := validateLeaseToken(req.Lease, s.WorkerID); err != nil { + httputil.WriteJSON(w, http.StatusForbidden, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + var params workerproto.SpawnParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if params.SandboxID != req.Lease.ResourceID { + httputil.WriteJSON(w, http.StatusForbidden, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "lease resource does not match spawn request"}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID, err := provider.Spawn(ctx, providers.SpawnOptions{ + Image: params.Image, + MemoryMB: params.MemoryMB, + VCPUs: params.VCPUs, + Metadata: params.Metadata, + }) + if err != nil { + httputil.WriteJSON(w, http.StatusInternalServerError, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + result, _ := json.Marshal(workerproto.SpawnResult{ + SandboxID: params.SandboxID, + RuntimeID: runtimeID, + State: "running", + Provider: provider.Name(), + WorkerID: s.WorkerID, + Metadata: params.Metadata, + }) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func (s *RPCServer) handleDestroy(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + if err := validateLeaseToken(req.Lease, s.WorkerID); err != nil { + httputil.WriteJSON(w, http.StatusForbidden, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + var params workerproto.DestroyParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if params.SandboxID != req.Lease.ResourceID { + httputil.WriteJSON(w, http.StatusForbidden, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "lease resource does not match destroy request"}) + return + } + if s.Registry == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "provider registry unavailable"}) + return + } + provider, err := s.Registry.Get(params.Provider) + if err != nil { + httputil.WriteJSON(w, http.StatusNotFound, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + runtimeID := strings.TrimSpace(params.RuntimeID) + if runtimeID == "" { + runtimeID = params.SandboxID + } + if err := provider.Destroy(ctx, runtimeID); err != nil && !errors.Is(err, providers.ErrSandboxNotFound) { + httputil.WriteJSON(w, http.StatusInternalServerError, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID}) +} + +func (s *RPCServer) handleRenewLease(w http.ResponseWriter, ctx context.Context, req workerproto.Request) { + if err := validateLeaseToken(req.Lease, s.WorkerID); err != nil { + httputil.WriteJSON(w, http.StatusForbidden, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + var params workerproto.RenewLeaseParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + httputil.WriteJSON(w, http.StatusBadRequest, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + if params.ResourceID != req.Lease.ResourceID { + httputil.WriteJSON(w, http.StatusForbidden, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "lease resource does not match renewal request"}) + return + } + if s.LeaseRenewer == nil { + httputil.WriteJSON(w, http.StatusServiceUnavailable, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: "lease renewer unavailable"}) + return + } + lease, err := s.LeaseRenewer.RenewLease(ctx, params.ResourceID, params.TTL) + if err != nil { + httputil.WriteJSON(w, http.StatusConflict, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Error: err.Error()}) + return + } + result, _ := json.Marshal(workerproto.RenewLeaseResult{Lease: lease}) + httputil.WriteJSON(w, http.StatusOK, workerproto.Response{ID: req.ID, WorkerID: s.WorkerID, Result: result}) +} + +func validateLeaseToken(token *workerproto.LeaseToken, workerID string) error { + if token == nil { + return errors.New("lease token is required") + } + if token.HolderID != workerID { + return errors.New("lease holder does not match worker") + } + if strings.TrimSpace(token.ResourceID) == "" { + return errors.New("lease resource is required") + } + if token.ExpiresAt.Before(time.Now().UTC()) { + return errors.New("lease token is expired") + } + return nil +} + +func mustJSONRaw(value interface{}) json.RawMessage { + data, _ := json.Marshal(value) + return data +} + +func (s *RPCServer) authenticate(r *http.Request) bool { + workerID := strings.TrimSpace(s.WorkerID) + if workerID == "" { + return false + } + if r.Header.Get("X-Worker-ID") != workerID { + return false + } + token := r.Header.Get("X-Worker-Token") + if token == "" { + token = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + } + if token == "" { + return false + } + if staticToken := strings.TrimSpace(s.Token); staticToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(staticToken)) == 1 { + return true + } + now := s.Now + if now == nil { + now = time.Now + } + for _, signingKey := range append([]string{s.SigningKey}, s.SigningKeys...) { + claims, ok := middleware.VerifyWorkerTokenForAudience(signingKey, token, middleware.WorkerTokenAudienceRPC, now().UTC()) + if ok && claims.WorkerID == workerID && !workerTokenRevoked(claims, s.RevokedTokenIDs) { + return true + } + } + return false +} + +func workerTokenRevoked(claims middleware.WorkerTokenClaims, revokedIDs []string) bool { + if claims.TokenID == "" || len(revokedIDs) == 0 { + return false + } + for _, id := range revokedIDs { + if strings.TrimSpace(id) == claims.TokenID { + return true + } + } + return false +} + +func NewHTTPServer(addr string, handler http.Handler) *http.Server { + return &http.Server{ + Addr: addr, + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + } +} + +func NewHTTPServerWithTLS(addr string, handler http.Handler, tlsConfig TLSConfig) (*http.Server, error) { + server := NewHTTPServer(addr, handler) + cfg, err := tlsConfig.ServerConfig() + if err != nil { + return nil, err + } + server.TLSConfig = cfg + return server, nil +} diff --git a/internal/worker/rpc_client.go b/internal/worker/rpc_client.go new file mode 100644 index 0000000..d8b20cc --- /dev/null +++ b/internal/worker/rpc_client.go @@ -0,0 +1,410 @@ +package worker + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +type RPCClient struct { + BaseURL string + WorkerID string + Token string + TokenFunc func() (string, error) + HTTPClient *http.Client + RPCTLS TLSConfig +} + +func (c RPCClient) Spawn(ctx context.Context, reqID string, lease workerproto.LeaseToken, params workerproto.SpawnParams) (workerproto.SpawnResult, error) { + var result workerproto.SpawnResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodSpawn, + WorkerID: c.WorkerID, + Lease: &lease, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) Status(ctx context.Context, reqID string, params workerproto.StatusParams) (workerproto.StatusResult, error) { + var result workerproto.StatusResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodStatus, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) Exec(ctx context.Context, reqID string, params workerproto.ExecParams) (workerproto.ExecResult, error) { + var result workerproto.ExecResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodExec, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) ExecStream(ctx context.Context, reqID string, params workerproto.ExecParams) (workerproto.ExecStreamResult, error) { + var result workerproto.ExecStreamResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodExecStream, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) ExecStreamLive(ctx context.Context, reqID string, params workerproto.ExecParams) (<-chan workerproto.StreamChunk, <-chan error, error) { + if strings.TrimSpace(c.BaseURL) == "" { + return nil, nil, fmt.Errorf("worker RPC URL is required") + } + if strings.TrimSpace(c.WorkerID) == "" { + return nil, nil, fmt.Errorf("worker id is required") + } + token, err := c.authToken() + if err != nil { + return nil, nil, err + } + body, err := json.Marshal(workerproto.Request{ + ID: reqID, + Method: workerproto.MethodExecStream, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return nil, nil, err + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/rpc", bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", "application/x-ndjson") + httpReq.Header.Set("X-Worker-Stream", "ndjson") + httpReq.Header.Set("X-Worker-ID", c.WorkerID) + httpReq.Header.Set("X-Worker-Token", token) + + client, err := c.httpClient(&http.Client{}) + if err != nil { + return nil, nil, err + } + resp, err := client.Do(httpReq) + if err != nil { + return nil, nil, err + } + if resp.StatusCode >= 300 { + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + var response workerproto.Response + if len(data) > 0 { + _ = json.Unmarshal(data, &response) + } + if response.Error != "" { + return nil, nil, fmt.Errorf("worker RPC failed: HTTP %d: %s", resp.StatusCode, response.Error) + } + return nil, nil, fmt.Errorf("worker RPC failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(data))) + } + + chunks := make(chan workerproto.StreamChunk, 64) + errs := make(chan error, 1) + go func() { + defer resp.Body.Close() + defer close(chunks) + defer close(errs) + scanner := bufio.NewScanner(resp.Body) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + var response workerproto.Response + if err := json.Unmarshal(scanner.Bytes(), &response); err != nil { + errs <- fmt.Errorf("decode worker stream response: %w", err) + return + } + if response.Error != "" { + errs <- fmt.Errorf("worker RPC stream failed: %s", response.Error) + return + } + if len(response.Result) == 0 { + continue + } + var chunk workerproto.StreamChunk + if err := json.Unmarshal(response.Result, &chunk); err != nil { + errs <- fmt.Errorf("decode worker stream chunk: %w", err) + return + } + select { + case chunks <- chunk: + case <-ctx.Done(): + errs <- ctx.Err() + return + } + } + if err := scanner.Err(); err != nil { + errs <- err + } + }() + return chunks, errs, nil +} + +func (c RPCClient) FileWrite(ctx context.Context, reqID string, params workerproto.FileParams) error { + _, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileWrite, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + return err +} + +func (c RPCClient) FileRead(ctx context.Context, reqID string, params workerproto.FileParams) (workerproto.FileReadResult, error) { + var result workerproto.FileReadResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileRead, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) FileList(ctx context.Context, reqID string, params workerproto.FileParams) (workerproto.FileListResult, error) { + var result workerproto.FileListResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileList, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) FileDelete(ctx context.Context, reqID string, params workerproto.FileParams) error { + _, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileDelete, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + return err +} + +func (c RPCClient) FileMove(ctx context.Context, reqID string, params workerproto.FileParams) error { + _, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileMove, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + return err +} + +func (c RPCClient) FileChmod(ctx context.Context, reqID string, params workerproto.FileParams) error { + _, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileChmod, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + return err +} + +func (c RPCClient) FileStat(ctx context.Context, reqID string, params workerproto.FileParams) (workerproto.FileStatResult, error) { + var result workerproto.FileStatResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileStat, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) FileGlob(ctx context.Context, reqID string, params workerproto.FileParams) (workerproto.FileGlobResult, error) { + var result workerproto.FileGlobResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodFileGlob, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) Logs(ctx context.Context, reqID string, params workerproto.LogsParams) (workerproto.LogsResult, error) { + var result workerproto.LogsResult + resp, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodLogs, + WorkerID: c.WorkerID, + Params: mustRawMessage(params), + }) + if err != nil { + return result, err + } + if err := json.Unmarshal(resp.Result, &result); err != nil { + return result, err + } + return result, nil +} + +func (c RPCClient) Destroy(ctx context.Context, reqID string, lease workerproto.LeaseToken, params workerproto.DestroyParams) error { + _, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodDestroy, + WorkerID: c.WorkerID, + Lease: &lease, + Params: mustRawMessage(params), + }) + return err +} + +func (c RPCClient) Shutdown(ctx context.Context, reqID string) error { + _, err := c.call(ctx, workerproto.Request{ + ID: reqID, + Method: workerproto.MethodShutdown, + WorkerID: c.WorkerID, + }) + return err +} + +func (c RPCClient) call(ctx context.Context, request workerproto.Request) (workerproto.Response, error) { + var zero workerproto.Response + if strings.TrimSpace(c.BaseURL) == "" { + return zero, fmt.Errorf("worker RPC URL is required") + } + if strings.TrimSpace(c.WorkerID) == "" { + return zero, fmt.Errorf("worker id is required") + } + token, err := c.authToken() + if err != nil { + return zero, err + } + body, err := json.Marshal(request) + if err != nil { + return zero, err + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/rpc", bytes.NewReader(body)) + if err != nil { + return zero, err + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("X-Worker-ID", c.WorkerID) + httpReq.Header.Set("X-Worker-Token", token) + + client, err := c.httpClient(&http.Client{Timeout: 30 * time.Second}) + if err != nil { + return zero, err + } + httpResp, err := client.Do(httpReq) + if err != nil { + return zero, err + } + defer httpResp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(httpResp.Body, 1<<20)) + var response workerproto.Response + if len(data) > 0 { + if err := json.Unmarshal(data, &response); err != nil { + return zero, fmt.Errorf("decode worker RPC response: %w", err) + } + } + if httpResp.StatusCode >= 300 { + if response.Error != "" { + return zero, fmt.Errorf("worker RPC failed: HTTP %d: %s", httpResp.StatusCode, response.Error) + } + return zero, fmt.Errorf("worker RPC failed: HTTP %d: %s", httpResp.StatusCode, strings.TrimSpace(string(data))) + } + if response.Error != "" { + return zero, fmt.Errorf("worker RPC failed: %s", response.Error) + } + return response, nil +} + +func (c RPCClient) httpClient(defaultClient *http.Client) (*http.Client, error) { + if c.HTTPClient != nil { + return c.HTTPClient, nil + } + return c.RPCTLS.HTTPClient(defaultClient) +} + +func (c RPCClient) authToken() (string, error) { + if c.TokenFunc != nil { + token, err := c.TokenFunc() + if err != nil { + return "", err + } + if strings.TrimSpace(token) == "" { + return "", fmt.Errorf("worker token is required") + } + return token, nil + } + if strings.TrimSpace(c.Token) == "" { + return "", fmt.Errorf("worker token is required") + } + return c.Token, nil +} + +func mustRawMessage(value interface{}) json.RawMessage { + data, _ := json.Marshal(value) + return data +} diff --git a/internal/worker/rpc_client_test.go b/internal/worker/rpc_client_test.go new file mode 100644 index 0000000..ca27e3b --- /dev/null +++ b/internal/worker/rpc_client_test.go @@ -0,0 +1,505 @@ +package worker + +import ( + "context" + "net/http/httptest" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/api/middleware" + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +func TestRPCClientSpawn(t *testing.T) { + registry := providers.NewRegistry() + registry.Register(providers.NewMockProvider()) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + result, err := client.Spawn(context.Background(), "req-1", workerproto.LeaseToken{ + ResourceID: "sb-control-plane", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, workerproto.SpawnParams{ + SandboxID: "sb-control-plane", + Image: "alpine:latest", + Provider: "mock", + MemoryMB: 512, + VCPUs: 1, + TTL: "5m", + }) + if err != nil { + t.Fatalf("spawn: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.RuntimeID == "" || result.WorkerID != "worker-a" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCClientStatus(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + result, err := client.Status(context.Background(), "req-1", workerproto.StatusParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + }) + if err != nil { + t.Fatalf("status: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.State == "" || result.WorkerID != "worker-a" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCClientStatusWithSignedToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + Now: func() time.Time { return now }, + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + TokenFunc: func() (string, error) { + return middleware.SignWorkerToken("worker-signing-key-with-at-least-32-bytes", middleware.WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: middleware.WorkerTokenAudienceRPC, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(time.Minute).Unix(), + }) + }, + } + result, err := client.Status(context.Background(), "req-1", workerproto.StatusParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + }) + if err != nil { + t.Fatalf("status: %v", err) + } + if result.WorkerID != "worker-a" { + t.Fatalf("worker id = %q, want worker-a", result.WorkerID) + } +} + +func TestRPCClientRejectsControlPlaneAudienceToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + registry := providers.NewRegistry() + registry.Register(providers.NewMockProvider()) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + Now: func() time.Time { return now }, + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + TokenFunc: func() (string, error) { + return middleware.SignWorkerToken("worker-signing-key-with-at-least-32-bytes", middleware.WorkerTokenClaims{ + WorkerID: "worker-a", + Audience: middleware.WorkerTokenAudienceControlPlane, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(time.Minute).Unix(), + }) + }, + } + _, err := client.Status(context.Background(), "req-1", workerproto.StatusParams{ + SandboxID: "sb-control-plane", + RuntimeID: "runtime-id", + Provider: "mock", + }) + if err == nil { + t.Fatal("expected control-plane audience token to be rejected by worker RPC") + } +} + +func TestRPCServerRejectsRevokedSignedToken(t *testing.T) { + now := time.Date(2026, 5, 9, 10, 0, 0, 0, time.UTC) + registry := providers.NewRegistry() + registry.Register(providers.NewMockProvider()) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + SigningKey: "worker-signing-key-with-at-least-32-bytes", + RevokedTokenIDs: []string{"revoked-token-id"}, + Now: func() time.Time { return now }, + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + TokenFunc: func() (string, error) { + return middleware.SignWorkerToken("worker-signing-key-with-at-least-32-bytes", middleware.WorkerTokenClaims{ + WorkerID: "worker-a", + TokenID: "revoked-token-id", + Audience: middleware.WorkerTokenAudienceRPC, + IssuedAt: now.Unix(), + ExpiresAt: now.Add(time.Minute).Unix(), + }) + }, + } + _, err := client.Status(context.Background(), "req-1", workerproto.StatusParams{ + SandboxID: "sb-control-plane", + RuntimeID: "runtime-id", + Provider: "mock", + }) + if err == nil { + t.Fatal("expected revoked signed token to be rejected by worker RPC") + } +} + +func TestRPCClientExec(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + result, err := client.Exec(context.Background(), "req-1", workerproto.ExecParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + Command: "echo client exec", + }) + if err != nil { + t.Fatalf("exec: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.ExitCode != 0 || result.Stdout != "client exec\n" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCClientExecStream(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + result, err := client.ExecStream(context.Background(), "req-1", workerproto.ExecParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + Command: "echo client stream", + }) + if err != nil { + t.Fatalf("exec stream: %v", err) + } + if result.SandboxID != "sb-control-plane" || len(result.Chunks) != 1 || result.Chunks[0].Data != "client stream\n" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCClientExecStreamLive(t *testing.T) { + registry := providers.NewRegistry() + base := providers.NewMockProvider() + runtimeID, err := base.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + release := make(chan struct{}) + registry.Register(&liveStreamProvider{Provider: base, release: release}) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + chunks, errs, err := client.ExecStreamLive(context.Background(), "req-1", workerproto.ExecParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + Command: "ignored", + }) + if err != nil { + t.Fatalf("exec stream live: %v", err) + } + select { + case chunk := <-chunks: + if chunk.Data != "first\n" { + t.Fatalf("first chunk = %+v, want first", chunk) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for first live chunk") + } + close(release) + for range chunks { + } + for err := range errs { + if err != nil { + t.Fatalf("stream error: %v", err) + } + } +} + +func TestRPCClientFileOperations(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + base := workerproto.FileParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + } + write := base + write.Path = "/workspace/client.txt" + write.Content = []byte("client file") + write.Mode = "0644" + if err := client.FileWrite(context.Background(), "req-write", write); err != nil { + t.Fatalf("write: %v", err) + } + read := base + read.Path = "/workspace/client.txt" + readResult, err := client.FileRead(context.Background(), "req-read", read) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(readResult.Content) != "client file" { + t.Fatalf("content = %q, want client file", string(readResult.Content)) + } + list := base + list.Path = "/workspace" + listResult, err := client.FileList(context.Background(), "req-list", list) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(listResult.Files) == 0 { + t.Fatal("expected listed files") + } + stat := base + stat.Path = "/workspace/client.txt" + statResult, err := client.FileStat(context.Background(), "req-stat", stat) + if err != nil { + t.Fatalf("stat: %v", err) + } + if statResult.File.Size != int64(len("client file")) { + t.Fatalf("stat size = %d, want %d", statResult.File.Size, len("client file")) + } + glob := base + glob.Pattern = "/workspace/*.txt" + globResult, err := client.FileGlob(context.Background(), "req-glob", glob) + if err != nil { + t.Fatalf("glob: %v", err) + } + if len(globResult.Matches) != 1 { + t.Fatalf("matches = %+v, want one match", globResult.Matches) + } + move := base + move.OldPath = "/workspace/client.txt" + move.NewPath = "/workspace/client-moved.txt" + if err := client.FileMove(context.Background(), "req-move", move); err != nil { + t.Fatalf("move: %v", err) + } + chmod := base + chmod.Path = "/workspace/client-moved.txt" + chmod.Mode = "0755" + if err := client.FileChmod(context.Background(), "req-chmod", chmod); err != nil { + t.Fatalf("chmod: %v", err) + } + deleteParams := base + deleteParams.Path = "/workspace/client-moved.txt" + if err := client.FileDelete(context.Background(), "req-delete", deleteParams); err != nil { + t.Fatalf("delete: %v", err) + } +} + +func TestRPCClientLogs(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + result, err := client.Logs(context.Background(), "req-logs", workerproto.LogsParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + Lines: 2, + }) + if err != nil { + t.Fatalf("logs: %v", err) + } + if result.SandboxID != "sb-control-plane" || len(result.Lines) != 2 { + t.Fatalf("unexpected logs result: %+v", result) + } +} + +func TestRPCClientDestroy(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + server := httptest.NewServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + } + err = client.Destroy(context.Background(), "req-1", workerproto.LeaseToken{ + ResourceID: "sb-control-plane", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, workerproto.DestroyParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + }) + if err != nil { + t.Fatalf("destroy: %v", err) + } + status, err := mock.Status(context.Background(), runtimeID) + if err != nil { + t.Fatalf("status after destroy: %v", err) + } + if status.State != "destroyed" { + t.Fatalf("state after destroy = %q, want destroyed", status.State) + } +} diff --git a/internal/worker/rpc_test.go b/internal/worker/rpc_test.go new file mode 100644 index 0000000..ecebc02 --- /dev/null +++ b/internal/worker/rpc_test.go @@ -0,0 +1,589 @@ +package worker + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +type liveStreamProvider struct { + providers.Provider + release chan struct{} +} + +func (p *liveStreamProvider) ExecStream(ctx context.Context, sandboxID string, opts providers.ExecOptions) (<-chan providers.StreamChunk, error) { + ch := make(chan providers.StreamChunk, 2) + go func() { + defer close(ch) + ch <- providers.StreamChunk{Stream: "stdout", Data: "first\n"} + select { + case <-ctx.Done(): + case <-p.release: + ch <- providers.StreamChunk{Stream: "stdout", Data: "second\n"} + } + }() + return ch, nil +} + +type fakeLeaseRenewer struct { + resourceID string + ttl string + lease workerproto.LeaseToken +} + +func (f *fakeLeaseRenewer) RenewLease(ctx context.Context, resourceID, ttl string) (workerproto.LeaseToken, error) { + f.resourceID = resourceID + f.ttl = ttl + return f.lease, nil +} + +func TestRPCServerStatus(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(t.Context(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.StatusParams{ + SandboxID: "sb-control-plane", + Provider: "mock", + RuntimeID: runtimeID, + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodStatus, + WorkerID: "worker-a", + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp workerproto.Response + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + var result workerproto.StatusResult + if err := json.Unmarshal(resp.Result, &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.WorkerID != "worker-a" || result.State == "" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCServerExec(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(t.Context(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.ExecParams{ + SandboxID: "sb-control-plane", + Provider: "mock", + RuntimeID: runtimeID, + Command: "echo worker exec", + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodExec, + WorkerID: "worker-a", + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp workerproto.Response + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + var result workerproto.ExecResult + if err := json.Unmarshal(resp.Result, &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.ExitCode != 0 || result.Stdout != "worker exec\n" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCServerExecStream(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(t.Context(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.ExecParams{ + SandboxID: "sb-control-plane", + Provider: "mock", + RuntimeID: runtimeID, + Command: "echo worker stream", + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodExecStream, + WorkerID: "worker-a", + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp workerproto.Response + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + var result workerproto.ExecStreamResult + if err := json.Unmarshal(resp.Result, &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.SandboxID != "sb-control-plane" || len(result.Chunks) != 1 || result.Chunks[0].Data != "worker stream\n" { + t.Fatalf("unexpected result: %+v", result) + } +} + +func TestRPCServerExecStreamNDJSONFlushesLiveChunks(t *testing.T) { + registry := providers.NewRegistry() + base := providers.NewMockProvider() + runtimeID, err := base.Spawn(t.Context(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + release := make(chan struct{}) + registry.Register(&liveStreamProvider{Provider: base, release: release}) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + server := httptest.NewServer((&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler()) + defer server.Close() + + params, _ := json.Marshal(workerproto.ExecParams{ + SandboxID: "sb-control-plane", + Provider: "mock", + RuntimeID: runtimeID, + Command: "ignored", + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodExecStream, + WorkerID: "worker-a", + Params: params, + }) + req, err := http.NewRequest(http.MethodPost, server.URL+"/rpc", bytes.NewReader(reqBody)) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + req.Header.Set("X-Worker-Stream", "ndjson") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + + lineCh := make(chan string, 2) + go func() { + buf := make([]byte, 4096) + n, err := resp.Body.Read(buf) + if err != nil { + lineCh <- fmt.Sprintf("read error: %v", err) + return + } + lineCh <- string(buf[:n]) + }() + select { + case line := <-lineCh: + if !strings.Contains(line, "first") { + t.Fatalf("first streamed line = %q, want first chunk", line) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for first live chunk") + } + close(release) +} + +func TestRPCServerRenewLease(t *testing.T) { + renewed := workerproto.LeaseToken{ + ResourceID: "sb-1", + HolderID: "worker-a", + Generation: 3, + ExpiresAt: time.Now().UTC().Add(2 * time.Minute), + } + renewer := &fakeLeaseRenewer{lease: renewed} + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: providers.NewRegistry(), LeaseRenewer: renewer}).Handler() + params, _ := json.Marshal(workerproto.RenewLeaseParams{ResourceID: "sb-1", TTL: "30s"}) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodRenewLease, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-1", + HolderID: "worker-a", + Generation: 2, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if renewer.resourceID != "sb-1" || renewer.ttl != "30s" { + t.Fatalf("unexpected renewal call: resource=%q ttl=%q", renewer.resourceID, renewer.ttl) + } + var resp workerproto.Response + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + var result workerproto.RenewLeaseResult + if err := json.Unmarshal(resp.Result, &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.Lease.Generation != 3 { + t.Fatalf("unexpected lease: %+v", result.Lease) + } +} + +func TestRPCServerSpawn(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.SpawnParams{ + SandboxID: "sb-control-plane", + Image: "alpine:latest", + Provider: "mock", + MemoryMB: 512, + VCPUs: 1, + TTL: "5m", + Metadata: map[string]string{"purpose": "test"}, + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodSpawn, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-control-plane", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + var resp workerproto.Response + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + var result workerproto.SpawnResult + if err := json.Unmarshal(resp.Result, &result); err != nil { + t.Fatalf("decode result: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.RuntimeID == "" || result.WorkerID != "worker-a" || result.State != "running" { + t.Fatalf("unexpected result: %+v", result) + } + if _, err := mock.Status(t.Context(), result.RuntimeID); err != nil { + t.Fatalf("spawned runtime status: %v", err) + } +} + +func TestRPCServerSpawnRejectsLeaseMismatch(t *testing.T) { + registry := providers.NewRegistry() + registry.Register(providers.NewMockProvider()) + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.SpawnParams{ + SandboxID: "sb-control-plane", + Image: "alpine:latest", + Provider: "mock", + MemoryMB: 512, + VCPUs: 1, + TTL: "5m", + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodSpawn, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-other", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", w.Code, http.StatusForbidden) + } +} + +func TestRPCServerShutdownDrainsWorker(t *testing.T) { + registry := providers.NewRegistry() + registry.Register(providers.NewMockProvider()) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + rpcServer := &RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry} + handler := rpcServer.Handler() + shutdownBody, _ := json.Marshal(workerproto.Request{ + ID: "req-shutdown", + Method: workerproto.MethodShutdown, + WorkerID: "worker-a", + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(shutdownBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("shutdown status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + if !rpcServer.Draining() { + t.Fatal("expected worker to be draining") + } + + params, _ := json.Marshal(workerproto.SpawnParams{ + SandboxID: "sb-control-plane", + Image: "alpine:latest", + Provider: "mock", + MemoryMB: 512, + VCPUs: 1, + TTL: "5m", + }) + spawnBody, _ := json.Marshal(workerproto.Request{ + ID: "req-spawn", + Method: workerproto.MethodSpawn, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-control-plane", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, + Params: params, + }) + req = httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(spawnBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("spawn status = %d, want %d", w.Code, http.StatusServiceUnavailable) + } +} + +func TestRPCServerDestroy(t *testing.T) { + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(t.Context(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn: %v", err) + } + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.DestroyParams{ + SandboxID: "sb-control-plane", + Provider: "mock", + RuntimeID: runtimeID, + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodDestroy, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-control-plane", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", w.Code, http.StatusOK, w.Body.String()) + } + status, err := mock.Status(t.Context(), runtimeID) + if err != nil { + t.Fatalf("status after destroy: %v", err) + } + if status.State != "destroyed" { + t.Fatalf("state after destroy = %q, want destroyed", status.State) + } +} + +func TestRPCServerDestroyRejectsLeaseMismatch(t *testing.T) { + registry := providers.NewRegistry() + registry.Register(providers.NewMockProvider()) + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: registry}).Handler() + params, _ := json.Marshal(workerproto.DestroyParams{ + SandboxID: "sb-control-plane", + Provider: "mock", + RuntimeID: "runtime-1", + }) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodDestroy, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-other", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().UTC().Add(time.Minute), + }, + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", w.Code, http.StatusForbidden) + } +} + +func TestRPCServerRenewLeaseRejectsExpiredToken(t *testing.T) { + renewer := &fakeLeaseRenewer{} + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: providers.NewRegistry(), LeaseRenewer: renewer}).Handler() + params, _ := json.Marshal(workerproto.RenewLeaseParams{ResourceID: "sb-1", TTL: "30s"}) + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodRenewLease, + WorkerID: "worker-a", + Lease: &workerproto.LeaseToken{ + ResourceID: "sb-1", + HolderID: "worker-a", + Generation: 2, + ExpiresAt: time.Now().UTC().Add(-time.Second), + }, + Params: params, + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", w.Code, http.StatusForbidden) + } + if renewer.resourceID != "" { + t.Fatalf("renewal should not be called, got resource %q", renewer.resourceID) + } +} + +func TestRPCServerRejectsWrongWorker(t *testing.T) { + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: providers.NewRegistry()}).Handler() + reqBody, _ := json.Marshal(workerproto.Request{ + ID: "req-1", + Method: workerproto.MethodShutdown, + WorkerID: "worker-b", + }) + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader(reqBody)) + req.Header.Set("X-Worker-ID", "worker-a") + req.Header.Set("X-Worker-Token", "worker-secret") + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", w.Code, http.StatusForbidden) + } +} + +func TestRPCServerRejectsMissingCredentials(t *testing.T) { + handler := (&RPCServer{WorkerID: "worker-a", Token: "worker-secret", Registry: providers.NewRegistry()}).Handler() + req := httptest.NewRequest(http.MethodPost, "/rpc", bytes.NewReader([]byte(`{}`))) + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", w.Code, http.StatusUnauthorized) + } +} diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go new file mode 100644 index 0000000..cc7f838 --- /dev/null +++ b/internal/worker/runtime.go @@ -0,0 +1,134 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/workerproto" + "github.com/rs/zerolog" +) + +type Runtime struct { + Client Client + ListenAddr string + HeartbeatInterval time.Duration + Logger zerolog.Logger + Providers []string + Capacity map[string]interface{} + Registry *providers.Registry + RPCTLS TLSConfig + SigningKey string + SigningKeys []string + RevokedTokenIDs []string +} + +func (r Runtime) Run(ctx context.Context) error { + var server *http.Server + var rpcServer *RPCServer + serverErr := make(chan error, 1) + if r.ListenAddr != "" { + rpcServer = &RPCServer{ + WorkerID: r.Client.WorkerID, + Token: r.Client.Token, + SigningKey: r.SigningKey, + SigningKeys: r.SigningKeys, + RevokedTokenIDs: r.RevokedTokenIDs, + Registry: r.Registry, + LeaseRenewer: r.Client, + } + var err error + server, err = NewHTTPServerWithTLS(r.ListenAddr, rpcServer.Handler(), r.RPCTLS) + if err != nil { + return err + } + go func() { + r.Logger.Info().Str("addr", r.ListenAddr).Msg("starting worker RPC server") + var err error + if r.RPCTLS.Enabled { + err = server.ListenAndServeTLS("", "") + } else { + err = server.ListenAndServe() + } + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + serverErr <- err + }() + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = server.Shutdown(shutdownCtx) + }() + } + if err := r.heartbeat(ctx, isDraining(rpcServer)); err != nil { + return err + } + interval := r.HeartbeatInterval + if interval == 0 { + interval = 30 * time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-serverErr: + return err + case <-ticker.C: + if err := r.heartbeat(ctx, isDraining(rpcServer)); err != nil { + r.Logger.Warn().Err(err).Msg("worker heartbeat failed") + } + } + } +} + +func isDraining(server *RPCServer) bool { + return server != nil && server.Draining() +} + +func (r Runtime) RunOnce(ctx context.Context) error { + return r.heartbeat(ctx, false) +} + +func (r Runtime) heartbeat(ctx context.Context, draining bool) error { + hostname, err := os.Hostname() + if err != nil { + hostname = r.Client.WorkerID + } + if hostname == "" { + return fmt.Errorf("worker hostname is empty") + } + status := "online" + if draining { + status = "draining" + } + params := workerproto.HeartbeatParams{ + Hostname: hostname, + Status: status, + Providers: r.Providers, + Capabilities: []string{"remote_worker", "heartbeat"}, + Capacity: r.Capacity, + } + if r.ListenAddr != "" { + if params.Capacity == nil { + params.Capacity = map[string]interface{}{} + } + rpcURL := r.ListenAddr + if !strings.HasPrefix(rpcURL, "http://") && !strings.HasPrefix(rpcURL, "https://") { + scheme := "http" + if r.RPCTLS.Enabled { + scheme = "https" + } + rpcURL = scheme + "://" + rpcURL + } + params.Capacity["rpc_url"] = rpcURL + } + return r.Client.Heartbeat(ctx, params) +} diff --git a/internal/worker/tls.go b/internal/worker/tls.go new file mode 100644 index 0000000..0c65a23 --- /dev/null +++ b/internal/worker/tls.go @@ -0,0 +1,97 @@ +package worker + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "os" + "strings" +) + +type TLSConfig struct { + Enabled bool + ServerCertFile string + ServerKeyFile string + ClientCAFile string + CAFile string + ClientCertFile string + ClientKeyFile string + ServerName string + InsecureSkipVerify bool +} + +func (c TLSConfig) ServerConfig() (*tls.Config, error) { + if !c.Enabled { + return nil, nil + } + if strings.TrimSpace(c.ServerCertFile) == "" || strings.TrimSpace(c.ServerKeyFile) == "" { + return nil, fmt.Errorf("worker RPC TLS server cert and key files are required") + } + cert, err := tls.LoadX509KeyPair(c.ServerCertFile, c.ServerKeyFile) + if err != nil { + return nil, fmt.Errorf("load worker RPC server certificate: %w", err) + } + cfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + } + if strings.TrimSpace(c.ClientCAFile) != "" { + pool, err := loadCertPool(c.ClientCAFile) + if err != nil { + return nil, fmt.Errorf("load worker RPC client CA: %w", err) + } + cfg.ClientCAs = pool + cfg.ClientAuth = tls.RequireAndVerifyClientCert + } + return cfg, nil +} + +func (c TLSConfig) HTTPClient(timeoutSource *http.Client) (*http.Client, error) { + if !c.Enabled { + return timeoutSource, nil + } + transport := http.DefaultTransport.(*http.Transport).Clone() + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: strings.TrimSpace(c.ServerName), + InsecureSkipVerify: c.InsecureSkipVerify, + } + if strings.TrimSpace(c.CAFile) != "" { + pool, err := loadCertPool(c.CAFile) + if err != nil { + return nil, fmt.Errorf("load worker RPC CA: %w", err) + } + tlsConfig.RootCAs = pool + } + if strings.TrimSpace(c.ClientCertFile) != "" || strings.TrimSpace(c.ClientKeyFile) != "" { + if strings.TrimSpace(c.ClientCertFile) == "" || strings.TrimSpace(c.ClientKeyFile) == "" { + return nil, fmt.Errorf("worker RPC TLS client cert and key files must be configured together") + } + cert, err := tls.LoadX509KeyPair(c.ClientCertFile, c.ClientKeyFile) + if err != nil { + return nil, fmt.Errorf("load worker RPC client certificate: %w", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + } + transport.TLSClientConfig = tlsConfig + client := &http.Client{Transport: transport} + if timeoutSource != nil { + client.Timeout = timeoutSource.Timeout + client.CheckRedirect = timeoutSource.CheckRedirect + client.Jar = timeoutSource.Jar + } + return client, nil +} + +func loadCertPool(path string) (*x509.CertPool, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(data) { + return nil, fmt.Errorf("no PEM certificates found in %s", path) + } + return pool, nil +} diff --git a/internal/worker/tls_test.go b/internal/worker/tls_test.go new file mode 100644 index 0000000..2815053 --- /dev/null +++ b/internal/worker/tls_test.go @@ -0,0 +1,224 @@ +package worker + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/StacyOs/stacyvm/internal/providers" + "github.com/StacyOs/stacyvm/internal/workerproto" +) + +func TestTLSConfigServerConfigRequiresCertPair(t *testing.T) { + _, err := (TLSConfig{Enabled: true}).ServerConfig() + if err == nil || !strings.Contains(err.Error(), "server cert and key") { + t.Fatalf("expected server cert/key error, got %v", err) + } +} + +func TestTLSConfigHTTPClientRequiresClientCertPair(t *testing.T) { + _, err := (TLSConfig{ + Enabled: true, + ClientCertFile: "/tmp/client.crt", + }).HTTPClient(&http.Client{Timeout: 5 * time.Second}) + if err == nil || !strings.Contains(err.Error(), "client cert and key") { + t.Fatalf("expected client cert/key error, got %v", err) + } +} + +func TestTLSConfigHTTPClientPreservesDefaultTimeout(t *testing.T) { + client, err := (TLSConfig{Enabled: true, InsecureSkipVerify: true}).HTTPClient(&http.Client{Timeout: 5 * time.Second}) + if err != nil { + t.Fatalf("http client: %v", err) + } + if client.Timeout != 5*time.Second { + t.Fatalf("timeout = %s, want 5s", client.Timeout) + } + if client.Transport == nil { + t.Fatal("transport is nil") + } +} + +func TestLoadCertPoolRejectsInvalidPEM(t *testing.T) { + path := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(path, []byte("not pem"), 0600); err != nil { + t.Fatal(err) + } + _, err := loadCertPool(path) + if err == nil || !strings.Contains(err.Error(), "no PEM certificates") { + t.Fatalf("expected PEM error, got %v", err) + } +} + +func TestRPCClientMTLSConformance(t *testing.T) { + dir := t.TempDir() + caCert, caKey := writeTestCA(t, dir, "worker-ca") + serverCert, serverKey := writeSignedTestCert(t, dir, "worker", caCert, caKey, testCertOptions{ + CommonName: "worker-a.internal", + DNSNames: []string{"worker-a.internal"}, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + }, + }) + clientCert, clientKey := writeSignedTestCert(t, dir, "control-plane", caCert, caKey, testCertOptions{ + CommonName: "control-plane", + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + }, + }) + + registry := providers.NewRegistry() + mock := providers.NewMockProvider() + registry.Register(mock) + if err := registry.SetDefault("mock"); err != nil { + t.Fatalf("set default: %v", err) + } + runtimeID, err := mock.Spawn(context.Background(), providers.SpawnOptions{Image: "alpine:latest"}) + if err != nil { + t.Fatalf("spawn mock: %v", err) + } + + serverTLS := TLSConfig{ + Enabled: true, + ServerCertFile: serverCert, + ServerKeyFile: serverKey, + ClientCAFile: caCert, + } + tlsConfig, err := serverTLS.ServerConfig() + if err != nil { + t.Fatalf("server tls config: %v", err) + } + server := httptest.NewUnstartedServer((&RPCServer{ + WorkerID: "worker-a", + Token: "worker-secret", + Registry: registry, + }).Handler()) + server.TLS = tlsConfig + server.StartTLS() + defer server.Close() + + client := RPCClient{ + BaseURL: server.URL, + WorkerID: "worker-a", + Token: "worker-secret", + RPCTLS: TLSConfig{ + Enabled: true, + CAFile: caCert, + ClientCertFile: clientCert, + ClientKeyFile: clientKey, + ServerName: "worker-a.internal", + }, + } + result, err := client.Status(context.Background(), "req-mtls", workerproto.StatusParams{ + SandboxID: "sb-control-plane", + RuntimeID: runtimeID, + Provider: "mock", + }) + if err != nil { + t.Fatalf("status over mtls: %v", err) + } + if result.SandboxID != "sb-control-plane" || result.WorkerID != "worker-a" { + t.Fatalf("unexpected result: %+v", result) + } +} + +type testCertOptions struct { + CommonName string + DNSNames []string + ExtKeyUsage []x509.ExtKeyUsage +} + +func writeTestCA(t *testing.T, dir, name string) (string, *rsa.PrivateKey) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate ca key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create ca cert: %v", err) + } + path := filepath.Join(dir, name+".crt") + writePEMFile(t, path, "CERTIFICATE", der) + return path, key +} + +func writeSignedTestCert(t *testing.T, dir, name string, caCertPath string, caKey *rsa.PrivateKey, opts testCertOptions) (string, string) { + t.Helper() + caCert := readCert(t, caCertPath) + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate %s key: %v", name, err) + } + serial, err := rand.Int(rand.Reader, big.NewInt(1<<62)) + if err != nil { + t.Fatalf("generate serial: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: opts.CommonName}, + DNSNames: opts.DNSNames, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: opts.ExtKeyUsage, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey) + if err != nil { + t.Fatalf("create %s cert: %v", name, err) + } + certPath := filepath.Join(dir, name+".crt") + keyPath := filepath.Join(dir, name+".key") + writePEMFile(t, certPath, "CERTIFICATE", der) + writePEMFile(t, keyPath, "RSA PRIVATE KEY", x509.MarshalPKCS1PrivateKey(key)) + return certPath, keyPath +} + +func readCert(t *testing.T, path string) *x509.Certificate { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read cert: %v", err) + } + block, _ := pem.Decode(data) + if block == nil { + t.Fatal("missing cert PEM block") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parse cert: %v", err) + } + return cert +} + +func writePEMFile(t *testing.T, path, typ string, der []byte) { + t.Helper() + file, err := os.Create(path) + if err != nil { + t.Fatalf("create %s: %v", path, err) + } + defer file.Close() + if err := pem.Encode(file, &pem.Block{Type: typ, Bytes: der}); err != nil { + t.Fatalf("encode pem: %v", err) + } +} diff --git a/internal/workerproto/protocol.go b/internal/workerproto/protocol.go new file mode 100644 index 0000000..4c7b899 --- /dev/null +++ b/internal/workerproto/protocol.go @@ -0,0 +1,306 @@ +// Package workerproto defines the control-plane contract used to assign +// sandbox lifecycle work to StacyVM workers. +// +// This package intentionally defines payloads only. Phase 10 makes the +// contract explicit without choosing a network transport yet. +package workerproto + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +const ( + MethodHeartbeat = "worker.heartbeat" + MethodSpawn = "worker.spawn" + MethodDestroy = "worker.destroy" + MethodStatus = "worker.status" + MethodExec = "worker.exec" + MethodExecStream = "worker.exec_stream" + MethodFileWrite = "worker.file_write" + MethodFileRead = "worker.file_read" + MethodFileList = "worker.file_list" + MethodFileDelete = "worker.file_delete" + MethodFileMove = "worker.file_move" + MethodFileChmod = "worker.file_chmod" + MethodFileStat = "worker.file_stat" + MethodFileGlob = "worker.file_glob" + MethodLogs = "worker.logs" + MethodRenewLease = "worker.renew_lease" + MethodShutdown = "worker.shutdown" +) + +const ( + ScopeHeartbeat = "worker:heartbeat" + ScopeSpawn = "worker:spawn" + ScopeDestroy = "worker:destroy" + ScopeStatus = "worker:status" + ScopeExec = "worker:exec" + ScopeFiles = "worker:files" + ScopeLogs = "worker:logs" + ScopeLease = "worker:lease" +) + +var ( + ErrInvalidMessage = errors.New("invalid worker message") + ErrUnknownMethod = errors.New("unknown worker method") +) + +// Request is the control-plane to worker envelope. +type Request struct { + ID string `json:"id"` + Method string `json:"method"` + WorkerID string `json:"worker_id"` + Lease *LeaseToken `json:"lease,omitempty"` + Params json.RawMessage `json:"params,omitempty"` +} + +// Response is the worker to control-plane envelope. +type Response struct { + ID string `json:"id"` + WorkerID string `json:"worker_id"` + Error string `json:"error,omitempty"` + Result json.RawMessage `json:"result,omitempty"` +} + +// LeaseToken carries the fencing information a worker must present when +// mutating sandbox lifecycle state. +type LeaseToken struct { + ResourceID string `json:"resource_id"` + HolderID string `json:"holder_id"` + Generation int64 `json:"generation"` + ExpiresAt time.Time `json:"expires_at"` +} + +// HeartbeatParams are sent by workers to report liveness and capacity. +type HeartbeatParams struct { + Hostname string `json:"hostname"` + Status string `json:"status"` + Providers []string `json:"providers"` + Capabilities []string `json:"capabilities"` + Capacity map[string]interface{} `json:"capacity"` +} + +// SpawnParams assigns sandbox creation to a worker. +type SpawnParams struct { + SandboxID string `json:"sandbox_id"` + Image string `json:"image"` + Provider string `json:"provider"` + MemoryMB int `json:"memory_mb"` + VCPUs int `json:"vcpus"` + OwnerID string `json:"owner_id,omitempty"` + TTL string `json:"ttl"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// SpawnResult is returned once a worker has created a runtime sandbox. +type SpawnResult struct { + SandboxID string `json:"sandbox_id"` + RuntimeID string `json:"runtime_id,omitempty"` + State string `json:"state"` + Provider string `json:"provider"` + WorkerID string `json:"worker_id"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// DestroyParams assigns sandbox destruction to a worker. +type DestroyParams struct { + SandboxID string `json:"sandbox_id"` + Provider string `json:"provider"` + RuntimeID string `json:"runtime_id,omitempty"` +} + +// StatusParams asks a worker to report runtime state for one sandbox. +type StatusParams struct { + SandboxID string `json:"sandbox_id"` + Provider string `json:"provider"` + RuntimeID string `json:"runtime_id,omitempty"` +} + +// StatusResult reports worker-observed runtime state. +type StatusResult struct { + SandboxID string `json:"sandbox_id"` + State string `json:"state"` + Provider string `json:"provider"` + WorkerID string `json:"worker_id"` + Error string `json:"error,omitempty"` +} + +// ExecParams asks a worker to run a non-streaming command in an owned runtime. +type ExecParams struct { + SandboxID string `json:"sandbox_id"` + Provider string `json:"provider"` + RuntimeID string `json:"runtime_id,omitempty"` + Command string `json:"command"` + Args []string `json:"args,omitempty"` + Mode string `json:"mode,omitempty"` + Env map[string]string `json:"env,omitempty"` + WorkDir string `json:"workdir,omitempty"` + Timeout string `json:"timeout,omitempty"` +} + +// ExecResult reports the completed command result from a worker runtime. +type ExecResult struct { + SandboxID string `json:"sandbox_id"` + ExitCode int `json:"exit_code"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` +} + +// StreamChunk is one stdout/stderr payload emitted by worker.exec_stream. +type StreamChunk struct { + Stream string `json:"stream"` + Data string `json:"data"` +} + +// ExecStreamResult reports buffered stream chunks from a worker runtime. +type ExecStreamResult struct { + SandboxID string `json:"sandbox_id"` + Chunks []StreamChunk `json:"chunks"` +} + +// FileParams asks a worker to perform a file operation in an owned runtime. +type FileParams struct { + SandboxID string `json:"sandbox_id"` + Provider string `json:"provider"` + RuntimeID string `json:"runtime_id,omitempty"` + Path string `json:"path,omitempty"` + Content []byte `json:"content,omitempty"` + Mode string `json:"mode,omitempty"` + Recursive bool `json:"recursive,omitempty"` + OldPath string `json:"old_path,omitempty"` + NewPath string `json:"new_path,omitempty"` + Pattern string `json:"pattern,omitempty"` +} + +// FileInfo describes one file returned by worker file APIs. +type FileInfo struct { + Path string `json:"path"` + Size int64 `json:"size"` + Mode string `json:"mode"` + IsDir bool `json:"is_dir"` + ModTime string `json:"mod_time"` +} + +// FileReadResult contains file content read from a worker runtime. +type FileReadResult struct { + SandboxID string `json:"sandbox_id"` + Content []byte `json:"content"` +} + +// FileListResult contains file entries returned by a worker runtime. +type FileListResult struct { + SandboxID string `json:"sandbox_id"` + Files []FileInfo `json:"files"` +} + +// FileStatResult contains one file entry returned by a worker runtime. +type FileStatResult struct { + SandboxID string `json:"sandbox_id"` + File FileInfo `json:"file"` +} + +// FileGlobResult contains glob matches returned by a worker runtime. +type FileGlobResult struct { + SandboxID string `json:"sandbox_id"` + Matches []string `json:"matches"` +} + +// LogsParams asks a worker to return console logs for an owned runtime. +type LogsParams struct { + SandboxID string `json:"sandbox_id"` + Provider string `json:"provider"` + RuntimeID string `json:"runtime_id,omitempty"` + Lines int `json:"lines,omitempty"` +} + +// LogsResult contains console log lines returned by a worker runtime. +type LogsResult struct { + SandboxID string `json:"sandbox_id"` + Lines []string `json:"lines"` +} + +// RenewLeaseParams asks a worker to confirm it still owns work and needs a +// renewed lease window. +type RenewLeaseParams struct { + ResourceID string `json:"resource_id"` + TTL string `json:"ttl"` +} + +// RenewLeaseResult returns the updated fencing token. +type RenewLeaseResult struct { + Lease LeaseToken `json:"lease"` +} + +// AuthClaims are the transport-neutral identity facts a validated worker token +// must produce. +type AuthClaims struct { + WorkerID string `json:"worker_id"` + Scopes []string `json:"scopes"` + Expires time.Time `json:"expires"` +} + +func (c AuthClaims) HasScope(scope string) bool { + for _, candidate := range c.Scopes { + if candidate == scope { + return true + } + } + return false +} + +func ValidateRequest(req Request) error { + if strings.TrimSpace(req.ID) == "" { + return fmt.Errorf("%w: id is required", ErrInvalidMessage) + } + if strings.TrimSpace(req.WorkerID) == "" { + return fmt.Errorf("%w: worker_id is required", ErrInvalidMessage) + } + switch req.Method { + case MethodHeartbeat: + return validateParams[HeartbeatParams](req.Params) + case MethodSpawn: + if req.Lease == nil { + return fmt.Errorf("%w: lease is required for spawn", ErrInvalidMessage) + } + return validateParams[SpawnParams](req.Params) + case MethodDestroy: + if req.Lease == nil { + return fmt.Errorf("%w: lease is required for destroy", ErrInvalidMessage) + } + return validateParams[DestroyParams](req.Params) + case MethodStatus: + return validateParams[StatusParams](req.Params) + case MethodExec: + return validateParams[ExecParams](req.Params) + case MethodExecStream: + return validateParams[ExecParams](req.Params) + case MethodFileWrite, MethodFileRead, MethodFileList, MethodFileDelete, MethodFileMove, MethodFileChmod, MethodFileStat, MethodFileGlob: + return validateParams[FileParams](req.Params) + case MethodLogs: + return validateParams[LogsParams](req.Params) + case MethodRenewLease: + if req.Lease == nil { + return fmt.Errorf("%w: lease is required for renew_lease", ErrInvalidMessage) + } + return validateParams[RenewLeaseParams](req.Params) + case MethodShutdown: + return nil + default: + return fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method) + } +} + +func validateParams[T any](raw json.RawMessage) error { + if len(raw) == 0 { + return fmt.Errorf("%w: params are required", ErrInvalidMessage) + } + var params T + if err := json.Unmarshal(raw, ¶ms); err != nil { + return fmt.Errorf("%w: invalid params: %v", ErrInvalidMessage, err) + } + return nil +} diff --git a/internal/workerproto/protocol_test.go b/internal/workerproto/protocol_test.go new file mode 100644 index 0000000..4890f24 --- /dev/null +++ b/internal/workerproto/protocol_test.go @@ -0,0 +1,79 @@ +package workerproto + +import ( + "encoding/json" + "errors" + "testing" + "time" +) + +func TestValidateRequestSpawnRequiresLease(t *testing.T) { + params, _ := json.Marshal(SpawnParams{ + SandboxID: "sb-1", + Image: "alpine:latest", + Provider: "mock", + MemoryMB: 512, + VCPUs: 1, + TTL: "5m", + }) + err := ValidateRequest(Request{ + ID: "req-1", + Method: MethodSpawn, + WorkerID: "worker-a", + Params: params, + }) + if !errors.Is(err, ErrInvalidMessage) { + t.Fatalf("validate err = %v, want ErrInvalidMessage", err) + } +} + +func TestValidateRequestAllowsLeasedSpawn(t *testing.T) { + params, _ := json.Marshal(SpawnParams{ + SandboxID: "sb-1", + Image: "alpine:latest", + Provider: "mock", + MemoryMB: 512, + VCPUs: 1, + TTL: "5m", + }) + err := ValidateRequest(Request{ + ID: "req-1", + Method: MethodSpawn, + WorkerID: "worker-a", + Lease: &LeaseToken{ + ResourceID: "sb-1", + HolderID: "worker-a", + Generation: 1, + ExpiresAt: time.Now().Add(time.Minute), + }, + Params: params, + }) + if err != nil { + t.Fatalf("validate spawn: %v", err) + } +} + +func TestValidateRequestRejectsUnknownMethod(t *testing.T) { + err := ValidateRequest(Request{ + ID: "req-1", + Method: "worker.nope", + WorkerID: "worker-a", + }) + if !errors.Is(err, ErrUnknownMethod) { + t.Fatalf("validate err = %v, want ErrUnknownMethod", err) + } +} + +func TestAuthClaimsHasScope(t *testing.T) { + claims := AuthClaims{ + WorkerID: "worker-a", + Scopes: []string{ScopeHeartbeat, ScopeLease}, + Expires: time.Now().Add(time.Hour), + } + if !claims.HasScope(ScopeLease) { + t.Fatal("expected lease scope") + } + if claims.HasScope(ScopeDestroy) { + t.Fatal("unexpected destroy scope") + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..ce5f540 --- /dev/null +++ b/package.json @@ -0,0 +1,42 @@ +{ + "name": "stacyvm-setup", + "version": "0.14.4", + "private": false, + "description": "Command-line bootstrapper for StacyVM.", + "type": "module", + "engines": { + "node": ">=18" + }, + "bin": { + "stacyvm-setup": "scripts/npm-setup.mjs" + }, + "files": [ + "scripts/npm-setup.mjs", + "README.md", + "LICENSE" + ], + "scripts": { + "setup": "node ./scripts/npm-setup.mjs", + "setup:check": "node ./scripts/npm-setup.mjs --check-only --skip-docker-check --skip-node-deps --no-start" + }, + "keywords": [ + "stacyvm", + "sandbox", + "agent", + "setup", + "docker", + "firecracker" + ], + "homepage": "https://github.com/StacyOS/stacyvm#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/StacyOS/stacyvm.git" + }, + "bugs": { + "url": "https://github.com/StacyOS/stacyvm/issues" + }, + "publishConfig": { + "access": "public" + }, + "license": "MIT" +} diff --git a/scripts/certify-runtime.sh b/scripts/certify-runtime.sh new file mode 100755 index 0000000..cbaac83 --- /dev/null +++ b/scripts/certify-runtime.sh @@ -0,0 +1,463 @@ +#!/usr/bin/env bash +# certify-runtime.sh — host-level runtime certification for StacyVM. +# +# Checks that the host has the required binaries, daemons, and kernel features +# for a given runtime, and optionally runs a live StacyVM sandbox smoke to +# prove end-to-end functionality. +# +# Usage: +# scripts/certify-runtime.sh [all|docker|gvisor|kata|firecracker|proot] +# [--format text|json|markdown] [--output path] +# [--stacyvm-url URL] [--stacyvm-api-key KEY] +# [--stacyvm-bin PATH] +# +# When --stacyvm-url and --stacyvm-api-key are provided, the script spawns a +# real sandbox using the target runtime, verifies it reaches running state, and +# destroys it — proving the runtime works end-to-end through StacyVM. +# +# When --stacyvm-bin is provided without --stacyvm-url, the script starts a +# temporary local StacyVM server automatically and runs the integration smoke. +# +# Environment: +# STACYVM_FIRECRACKER_KERNEL Path to kernel image (for firecracker check) +# STACYVM_PROOT_ROOTFS Path to rootfs directory (for proot check) +# STACYVM_PROOT_WORKSPACE_BASE Workspace base directory (for proot check) +set -euo pipefail + +runtime="all" +format="text" +output="" +stacyvm_url="" +stacyvm_api_key="" +stacyvm_bin="" + +usage() { + cat <<'USAGE' +usage: scripts/certify-runtime.sh [all|docker|gvisor|kata|firecracker|proot] + [--format text|json|markdown] [--output path] + [--stacyvm-url URL] [--stacyvm-api-key KEY] + [--stacyvm-bin PATH] +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + all|docker|gvisor|kata|firecracker|proot) runtime="$1"; shift ;; + --format) format="${2:-}"; shift 2 ;; + --output) output="${2:-}"; shift 2 ;; + --stacyvm-url) stacyvm_url="${2:-}"; shift 2 ;; + --stacyvm-api-key) stacyvm_api_key="${2:-}"; shift 2 ;; + --stacyvm-bin) stacyvm_bin="${2:-}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; exit 2 ;; + esac +done + +case "$format" in + text|json|markdown) ;; + *) printf 'unsupported format: %s\n' "$format" >&2; exit 2 ;; +esac + +exit_code=0 +RESULTS=() +LOCAL_SERVER_PID="" +LOCAL_WORK_DIR="" + +cleanup() { + if [[ -n "${LOCAL_SERVER_PID:-}" ]]; then + kill "$LOCAL_SERVER_PID" 2>/dev/null || true + wait "$LOCAL_SERVER_PID" 2>/dev/null || true + fi + if [[ -n "${LOCAL_WORK_DIR:-}" ]]; then + rm -rf "$LOCAL_WORK_DIR" + fi +} +trap cleanup EXIT + +# ── result recording ────────────────────────────────────────────────────────── +record() { + local status="$1" name="$2" message="$3" + RESULTS+=("${status}|${name}|${message}") + if [ "$format" = "text" ]; then + printf '[%s] %s: %s\n' "$status" "$name" "$message" + fi + if [ "$status" = "FAIL" ]; then exit_code=1; fi +} +pass() { record "PASS" "$1" "$2"; } +warn() { record "WARN" "$1" "$2"; } +fail() { record "FAIL" "$1" "$2"; } + +json_escape() { + local v="$1" + v="${v//\\/\\\\}"; v="${v//\"/\\\"}"; v="${v//$'\n'/\\n}" + v="${v//$'\r'/\\r}"; v="${v//$'\t'/\\t}" + printf '%s' "$v" +} +host_os() { uname -srm 2>/dev/null || printf 'unknown'; } +host_id() { hostname 2>/dev/null || printf 'unknown'; } +generated_at() { date -u '+%Y-%m-%dT%H:%M:%SZ'; } +docker_runtimes() { docker info --format '{{json .Runtimes}}' 2>/dev/null || true; } + +# ── host prerequisite checks ────────────────────────────────────────────────── +check_docker() { + if ! command -v docker >/dev/null 2>&1; then + fail "docker.cli" "Docker CLI not found"; return + fi + pass "docker.cli" "Docker CLI found" + + if ! docker info >/dev/null 2>&1; then + fail "docker.daemon" "Docker daemon unreachable"; return + fi + pass "docker.daemon" "Docker daemon reachable" + + if docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -Eq 'seccomp|name=seccomp'; then + pass "docker.seccomp" "seccomp advertised" + else + warn "docker.seccomp" "seccomp not advertised by docker info" + fi + + if docker run --rm --pull=never alpine:latest echo ok >/dev/null 2>&1 \ + || docker run --rm alpine:latest echo ok >/dev/null 2>&1; then + pass "docker.run" "docker run alpine echo ok succeeded" + else + warn "docker.run" "docker run alpine echo ok failed — image may not be cached on this host" + fi +} + +check_gvisor() { + if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + fail "gvisor.discovery" "Docker daemon unavailable; cannot discover gVisor runtime"; return + fi + if docker_runtimes | grep -Eq 'runsc|gvisor'; then + pass "gvisor.runtime" "runsc/gVisor runtime discovered in docker info" + if docker run --rm --runtime=runsc alpine:latest echo ok >/dev/null 2>&1; then + pass "gvisor.run" "docker run --runtime=runsc alpine echo ok succeeded" + else + fail "gvisor.run" "docker run --runtime=runsc alpine echo ok failed" + fi + else + fail "gvisor.runtime" "runsc/gVisor runtime not found — install gVisor and configure /etc/docker/daemon.json" + fi +} + +check_kata() { + if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + fail "kata.discovery" "Docker daemon unavailable; cannot discover Kata runtime"; return + fi + if docker_runtimes | grep -Eq 'kata'; then + pass "kata.runtime" "Kata runtime discovered in docker info" + if docker run --rm --runtime=kata-runtime alpine:latest echo ok >/dev/null 2>&1 \ + || docker run --rm --runtime=kata-containers alpine:latest echo ok >/dev/null 2>&1; then + pass "kata.run" "docker run --runtime=kata alpine echo ok succeeded" + else + fail "kata.run" "docker run --runtime=kata alpine echo ok failed" + fi + else + fail "kata.runtime" "Kata runtime not found — install kata-containers and configure /etc/docker/daemon.json" + fi +} + +check_firecracker() { + if command -v firecracker >/dev/null 2>&1; then + local fc_ver + fc_ver="$(firecracker --version 2>/dev/null | head -1 || printf 'unknown')" + pass "firecracker.binary" "Firecracker found ($fc_ver)" + else + fail "firecracker.binary" "Firecracker binary not found in PATH" + fi + + if [ -e /dev/kvm ]; then + pass "firecracker.kvm" "/dev/kvm present" + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + pass "firecracker.kvm_access" "/dev/kvm readable and writable" + else + warn "firecracker.kvm_access" "/dev/kvm exists but process lacks rw access; check kvm group membership" + fi + else + fail "firecracker.kvm" "/dev/kvm missing — enable KVM in BIOS or use a KVM-capable host" + fi + + if [ -n "${STACYVM_FIRECRACKER_KERNEL:-}" ]; then + if [ -f "$STACYVM_FIRECRACKER_KERNEL" ]; then + pass "firecracker.kernel" "kernel image exists at $STACYVM_FIRECRACKER_KERNEL" + else + fail "firecracker.kernel" "kernel image missing at $STACYVM_FIRECRACKER_KERNEL" + fi + else + warn "firecracker.kernel" "STACYVM_FIRECRACKER_KERNEL not set; validate kernel_path in your StacyVM config" + fi +} + +check_proot() { + if command -v proot >/dev/null 2>&1; then + pass "proot.binary" "PRoot binary found" + else + fail "proot.binary" "PRoot binary not found in PATH" + fi + + if [ -n "${STACYVM_PROOT_ROOTFS:-}" ]; then + if [ -d "$STACYVM_PROOT_ROOTFS" ]; then + pass "proot.rootfs" "rootfs directory exists at $STACYVM_PROOT_ROOTFS" + else + fail "proot.rootfs" "rootfs directory missing at $STACYVM_PROOT_ROOTFS" + fi + else + warn "proot.rootfs" "STACYVM_PROOT_ROOTFS not set; validate rootfs_path in your StacyVM config" + fi + + if [ -n "${STACYVM_PROOT_WORKSPACE_BASE:-}" ]; then + if [ -w "$STACYVM_PROOT_WORKSPACE_BASE" ]; then + pass "proot.workspace" "workspace base writable at $STACYVM_PROOT_WORKSPACE_BASE" + else + fail "proot.workspace" "workspace base at $STACYVM_PROOT_WORKSPACE_BASE is not writable" + fi + else + warn "proot.workspace" "STACYVM_PROOT_WORKSPACE_BASE not set; validate workspace_base in your StacyVM config" + fi +} + +run_checks() { + case "$runtime" in + all) check_docker; check_gvisor; check_kata; check_firecracker; check_proot ;; + docker) check_docker ;; + gvisor) check_gvisor ;; + kata) check_kata ;; + firecracker) check_firecracker ;; + proot) check_proot ;; + esac +} + +# ── optional StacyVM integration smoke ─────────────────────────────────────── +runtime_to_provider() { + case "$1" in + docker|gvisor|kata) printf 'docker' ;; + firecracker) printf 'firecracker' ;; + proot) printf 'proot' ;; + *) printf 'docker' ;; # "all" uses docker + esac +} + +runtime_to_image() { + case "$1" in + firecracker) printf 'ubuntu:22.04' ;; + *) printf 'alpine:latest' ;; + esac +} + +start_local_server() { + # Start a temporary StacyVM server with the target provider enabled. + # Sets stacyvm_url and stacyvm_api_key on success, returns 1 on failure. + local bin="$1" + if [[ ! -x "$bin" ]] && ! command -v "$bin" >/dev/null 2>&1; then + warn "stacyvm.bin" "stacyvm binary not executable at $bin; skipping integration smoke" + return 1 + fi + + LOCAL_WORK_DIR="$(mktemp -d)" + # Use a random port in the ephemeral range to avoid conflicts with prior runs. + local port; port=$(( ( RANDOM % 5000 ) + 19000 )) + local api_key="cert-smoke-api-key-32bytes-long!!" + local provider; provider="$(runtime_to_provider "$runtime")" + local cfg="$LOCAL_WORK_DIR/stacyvm.yaml" + local log="$LOCAL_WORK_DIR/server.log" + + cat >"$cfg" <"$log" 2>&1 & + LOCAL_SERVER_PID=$! + + for _ in {1..40}; do + if curl -fsS -H "X-API-Key: $api_key" "http://127.0.0.1:$port/api/v1/ready" >/dev/null 2>&1; then + stacyvm_url="http://127.0.0.1:$port" + stacyvm_api_key="$api_key" + return 0 + fi + sleep 0.3 + done + + warn "stacyvm.start" "local StacyVM server at port $port did not become ready; skipping integration smoke" + kill "$LOCAL_SERVER_PID" 2>/dev/null || true + LOCAL_SERVER_PID="" + return 1 +} + +run_stacyvm_smoke() { + local url="$1" api_key="$2" + local provider image sandbox_id state + + provider="$(runtime_to_provider "$runtime")" + image="$(runtime_to_image "$runtime")" + + # Readiness check. + if ! curl -fsS -H "X-API-Key: $api_key" "$url/api/v1/ready" >/dev/null 2>&1; then + warn "stacyvm.ready" "StacyVM at $url not reachable; skipping integration smoke" + return + fi + pass "stacyvm.ready" "StacyVM API reachable at $url" + + # Provider health. + local health + health="$(curl -fsS -H "X-API-Key: $api_key" "$url/api/v1/providers/$provider/health" 2>/dev/null || true)" + if printf '%s' "$health" | grep -qE '"healthy":true|"status":"ok"'; then + pass "stacyvm.provider_health" "$provider provider is healthy" + else + warn "stacyvm.provider_health" "$provider provider health check: ${health:-no response}" + fi + + # Spawn. Use || true to capture response body even on HTTP error. + local spawn_resp spawn_rc + spawn_resp="$(curl -sS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $api_key" \ + -d "{\"image\":\"$image\",\"provider\":\"$provider\",\"ttl\":\"2m\"}" \ + "$url/api/v1/sandboxes" 2>&1)" || spawn_rc=$? + if printf '%s' "$spawn_resp" | grep -q '"code"'; then + # HTTP error returned — provider may not be available on this host. + warn "stacyvm.spawn" "spawn returned an error (provider may be unavailable): ${spawn_resp}"; return + fi + sandbox_id="$(printf '%s' "$spawn_resp" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')" + if [[ -z "$sandbox_id" ]]; then + fail "stacyvm.spawn" "spawn response contained no sandbox id: $spawn_resp"; return + fi + pass "stacyvm.spawn" "sandbox $sandbox_id spawned via $provider ($image)" + + # Wait for running (up to 60s for Firecracker; 30s for others). + local max_wait=30 + [[ "$provider" = "firecracker" ]] && max_wait=60 + state="" + for _ in $(seq 1 "$max_wait"); do + state="$(curl -fsS -H "X-API-Key: $api_key" "$url/api/v1/sandboxes/$sandbox_id" 2>/dev/null \ + | sed -n 's/.*"state":"\([^"]*\)".*/\1/p' || true)" + [[ "$state" = "running" ]] && break + sleep 1 + done + + if [[ "$state" = "running" ]]; then + pass "stacyvm.running" "sandbox $sandbox_id reached running state" + else + fail "stacyvm.running" "sandbox $sandbox_id state is '$state' after ${max_wait}s (expected running)" + curl -fsS -X DELETE -H "X-API-Key: $api_key" "$url/api/v1/sandboxes/$sandbox_id" >/dev/null 2>&1 || true + return + fi + + # Exec a trivial command to prove the runtime executes code (skip for Firecracker + # which may require agent boot time beyond a simple exec). + if [[ "$provider" != "firecracker" ]]; then + local exec_resp + exec_resp="$(curl -fsS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $api_key" \ + -d '{"command":"echo stacyvm-runtime-ok","mode":"shell"}' \ + "$url/api/v1/sandboxes/$sandbox_id/exec" 2>/dev/null || true)" + if printf '%s' "$exec_resp" | grep -q '"exit_code":0'; then + pass "stacyvm.exec" "exec in sandbox exited 0 — runtime is executing code" + else + warn "stacyvm.exec" "exec result: ${exec_resp:-empty response}" + fi + fi + + # Destroy. + curl -fsS -X DELETE -H "X-API-Key: $api_key" "$url/api/v1/sandboxes/$sandbox_id" >/dev/null 2>&1 || true + pass "stacyvm.destroy" "sandbox $sandbox_id destroyed" +} + +# ── output writers ──────────────────────────────────────────────────────────── +write_json() { + printf '{\n' + printf ' "generated_at": "%s",\n' "$(generated_at)" + printf ' "runtime": "%s",\n' "$(json_escape "$runtime")" + printf ' "host": {\n' + printf ' "hostname": "%s",\n' "$(json_escape "$(host_id)")" + printf ' "os": "%s"\n' "$(json_escape "$(host_os)")" + printf ' },\n' + printf ' "stacyvm_url": "%s",\n' "$(json_escape "${stacyvm_url:-}")" + printf ' "status": "%s",\n' "$([ "$exit_code" -eq 0 ] && printf PASS || printf FAIL)" + printf ' "checks": [\n' + local first=1 + for entry in "${RESULTS[@]+"${RESULTS[@]}"}"; do + IFS='|' read -r s n m <<<"$entry" + [ "$first" -eq 0 ] && printf ',\n' + first=0 + printf ' {"status":"%s","name":"%s","message":"%s"}' \ + "$(json_escape "$s")" "$(json_escape "$n")" "$(json_escape "$m")" + done + printf '\n ]\n}\n' +} + +write_markdown() { + printf '%s\n\n' '# StacyVM Runtime Certification' + printf '%s\n' "- Generated at: \`$(generated_at)\`" + printf '%s\n' "- Runtime target: \`$runtime\`" + printf '%s\n' "- Host: \`$(host_id)\`" + printf '%s\n' "- OS/kernel: \`$(host_os)\`" + [[ -n "${stacyvm_url:-}" ]] && printf '%s\n' "- StacyVM URL: \`$stacyvm_url\`" + printf '%s\n\n' "- Overall status: \`$([ "$exit_code" -eq 0 ] && printf PASS || printf FAIL)\`" + printf '## Checks\n\n' + printf '| Status | Check | Message |\n|---|---|---|\n' + for entry in "${RESULTS[@]+"${RESULTS[@]}"}"; do + IFS='|' read -r s n m <<<"$entry" + printf '| `%s` | `%s` | %s |\n' "$s" "$n" "$m" + done + printf '\n## Operator Signoff\n\n' + printf '%s\n' '- StacyVM version:' + printf '%s\n' '- Config file:' + printf '%s\n' '- Provider health endpoint:' + printf '%s\n' '- Smoke script result:' + printf '%s\n' '- Provider conformance result:' + printf '%s\n' '- Known host caveats:' + printf '%s\n' '- Owner/signoff:' + printf '%s\n' '- Date:' +} + +# ── main ────────────────────────────────────────────────────────────────────── +run_checks + +# Start a local server automatically when --stacyvm-bin is given without --stacyvm-url. +if [[ -z "$stacyvm_url" ]] && [[ -n "$stacyvm_bin" ]]; then + start_local_server "$stacyvm_bin" || true +fi + +# Run integration smoke when URL + key are available. +if [[ -n "$stacyvm_url" ]] && [[ -n "$stacyvm_api_key" ]]; then + run_stacyvm_smoke "$stacyvm_url" "$stacyvm_api_key" +elif [[ -n "$stacyvm_url" ]] || [[ -n "$stacyvm_api_key" ]]; then + warn "stacyvm.config" "both --stacyvm-url and --stacyvm-api-key are required for integration smoke; skipping" +fi + +# Emit report. +case "$format" in + text) ;; + json) + if [ -n "$output" ]; then write_json >"$output"; else write_json; fi ;; + markdown) + if [ -n "$output" ]; then write_markdown >"$output"; else write_markdown; fi ;; +esac + +[ -n "$output" ] && [ "$format" != "text" ] && printf 'certification report written: %s\n' "$output" + +exit "$exit_code" diff --git a/scripts/certify-worker-identity.sh b/scripts/certify-worker-identity.sh new file mode 100755 index 0000000..8934693 --- /dev/null +++ b/scripts/certify-worker-identity.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +usage: scripts/certify-worker-identity.sh [worker-id] [--format text|json|markdown] [--output path] + +Runs a signed worker token lifecycle smoke without writing token values to the report. + +Environment: + STACYVM_WORKER_SIGNING_KEY_FILE Active worker signing key file + STACYVM_OLD_WORKER_SIGNING_KEY_FILE Previous worker signing key file + STACYVM_WORKER_IDENTITY_AUDIENCE Expected audience, default worker:control-plane + STACYVM_WORKER_IDENTITY_TTL Token lifetime, default 5m + STACYVM_WORKER_IDENTITY_TOKEN_ID Token ID, default certification-token-id +USAGE +} + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +worker_id="worker-a" +format="text" +output="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --format) + format="${2:-}" + shift 2 + ;; + --output) + output="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + --*) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + worker_id="$1" + shift + ;; + esac +done + +case "$format" in + text|json|markdown) ;; + *) + echo "format must be text, json, or markdown" >&2 + exit 2 + ;; +esac + +tmpdir="$(mktemp -d)" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +export GOCACHE="${GOCACHE:-$tmpdir/go-build}" + +audience="${STACYVM_WORKER_IDENTITY_AUDIENCE:-worker:control-plane}" +ttl="${STACYVM_WORKER_IDENTITY_TTL:-5m}" +token_id="${STACYVM_WORKER_IDENTITY_TOKEN_ID:-certification-token-id}" +signing_key_file="${STACYVM_WORKER_SIGNING_KEY_FILE:-$tmpdir/worker-signing-key}" +old_signing_key_file="${STACYVM_OLD_WORKER_SIGNING_KEY_FILE:-$tmpdir/worker-signing-key-old}" + +if [[ ! -f "$signing_key_file" ]]; then + printf '%s\n' "worker-signing-key-with-at-least-32-bytes" >"$signing_key_file" +fi +if [[ ! -f "$old_signing_key_file" ]]; then + printf '%s\n' "old-worker-signing-key-with-at-least-32-bytes" >"$old_signing_key_file" +fi + +echo "==> Issuing signed worker token" >&2 +token="$( + go run ./cmd/stacyvm worker token "$worker_id" \ + --signing-key-file "$signing_key_file" \ + --ttl "$ttl" \ + --audience "$audience" \ + --token-id "$token_id" +)" + +echo "==> Inspecting signed worker token metadata" >&2 +inspect_output="$( + go run ./cmd/stacyvm worker token inspect "$token" +)" +if [[ "$inspect_output" != *"\"signature_verified\": false"* || "$inspect_output" != *"\"token_id\": \"$token_id\""* ]]; then + echo "$inspect_output" + echo "expected unverified inspect output with token_id $token_id" >&2 + exit 1 +fi + +echo "==> Verifying signed worker token" >&2 +verify_output="$( + go run ./cmd/stacyvm worker token verify "$token" \ + --signing-key-file "$signing_key_file" \ + --worker-id "$worker_id" \ + --audience "$audience" +)" +if [[ "$verify_output" != *"\"signature_verified\": true"* || "$verify_output" != *"\"worker_id\": \"$worker_id\""* ]]; then + echo "$verify_output" + echo "expected verified token output for worker $worker_id" >&2 + exit 1 +fi + +echo "==> Confirming revoked token IDs are rejected" >&2 +if go run ./cmd/stacyvm worker token verify "$token" \ + --signing-key-file "$signing_key_file" \ + --worker-id "$worker_id" \ + --audience "$audience" \ + --revoked-token-id "$token_id" >"$tmpdir/revoked-token.out" 2>&1; then + cat "$tmpdir/revoked-token.out" + echo "expected revoked token verification to fail" >&2 + exit 1 +fi + +echo "==> Generating no-secret rotation plan" >&2 +rotation_plan="$( + go run ./cmd/stacyvm worker token rotation-plan \ + --new-key-ref "$signing_key_file" \ + --previous-key-ref "$old_signing_key_file" \ + --ttl "$ttl" +)" + +echo "==> Worker identity certification passed for $worker_id" >&2 + +generated_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +report_file="$tmpdir/report" +case "$format" in + text) + cat >"$report_file" <"$report_file" <"$report_file" < Running always-on SQLite store contract" +go test ./internal/store -run TestSQLiteStoreContract + +echo "==> Running worker identity and worker route auth checks" +go test ./internal/api/middleware ./internal/api \ + -run 'TestWorkerAuth|TestWorkerHeartbeatUsesPerWorkerToken|TestWorkerRenewLeaseUsesWorkerToken' + +echo "==> Running worker RPC mTLS conformance" +go test ./internal/worker -run TestRPCClientMTLSConformance -count=1 + +echo "==> Running signed worker RPC conformance" +go test ./internal/worker ./internal/orchestrator \ + -run 'TestRPCClientStatusWithSignedToken|TestRPCServerRejectsRevokedSignedToken|TestManager_RemoteSpawnUsesSignedWorkerRPC' -count=1 + +cluster_config="$tmpdir/stacyvm.cluster.yaml" +cat >"$cluster_config" < Linting production-aligned cluster config" +go run ./cmd/stacyvm config lint --production --file "$cluster_config" + +echo "==> Linting signed-token worker identity config" +signed_worker_config="$tmpdir/stacyvm.signed-worker.yaml" +sed \ + -e '/worker_tokens:/,/admin_fallback_enabled:/c\ + worker_signing_key: "worker-signing-key-with-at-least-32-bytes"\ + worker_signing_keys: ["old-worker-signing-key-with-at-least-32-bytes"]\ + admin_fallback_enabled: false' \ + "$cluster_config" >"$signed_worker_config" +go run ./cmd/stacyvm config lint --production --file "$signed_worker_config" + +echo "==> Running worker identity certification smoke" +worker_identity_report="$tmpdir/worker-identity-certification.md" +scripts/certify-worker-identity.sh worker-a --format markdown --output "$worker_identity_report" +if [[ ! -s "$worker_identity_report" ]]; then + echo "expected worker identity certification report to be written" >&2 + exit 1 +fi +if [[ "$(cat "$worker_identity_report")" != *"Worker Identity Certification"* ]]; then + cat "$worker_identity_report" + echo "expected worker identity certification markdown report" >&2 + exit 1 +fi +if grep -q "stacyvm-worker-v1" "$worker_identity_report"; then + cat "$worker_identity_report" + echo "worker identity certification report must not include token values" >&2 + exit 1 +fi + +echo "==> Linting signed-token worker identity migration warnings" +mixed_worker_config="$tmpdir/stacyvm.signed-worker-mixed.yaml" +sed \ + -e '/worker_tokens:/,/admin_fallback_enabled:/c\ + worker_token: "shared-worker-token-with-at-least-32-bytes"\ + worker_signing_key: "worker-signing-key-with-at-least-32-bytes"\ + admin_fallback_enabled: false' \ + "$cluster_config" >"$mixed_worker_config" +mixed_lint_output="$(go run ./cmd/stacyvm config lint --production --file "$mixed_worker_config")" +if [[ "$mixed_lint_output" != *"shared worker token still configured with signed worker tokens"* ]]; then + echo "$mixed_lint_output" + echo "expected signed-token migration warning for shared worker token" >&2 + exit 1 +fi + +invalid_rotation_config="$tmpdir/stacyvm.invalid-rotation.yaml" +sed \ + -e '/worker_tokens:/,/admin_fallback_enabled:/c\ + worker_signing_key: "worker-signing-key-with-at-least-32-bytes"\ + worker_signing_keys: ["worker-signing-key-with-at-least-32-bytes"]\ + admin_fallback_enabled: false' \ + "$cluster_config" >"$invalid_rotation_config" +rotation_lint_output="$(go run ./cmd/stacyvm config lint --production --file "$invalid_rotation_config")" +if [[ "$rotation_lint_output" != *"rotation keys include the active worker signing key"* ]]; then + echo "$rotation_lint_output" + echo "expected signed-token migration warning for invalid rotation keys" >&2 + exit 1 +fi + +echo "==> Linting production-aligned Postgres cluster config" +postgres_config="$tmpdir/stacyvm.postgres.yaml" +sed \ + -e 's/driver: "sqlite"/driver: "postgres"/' \ + -e 's|path: "'"$tmpdir"'/stacyvm-cluster.db"|dsn: "postgres://stacyvm:stacyvm@127.0.0.1:5432/stacyvm?sslmode=disable"|' \ + "$cluster_config" >"$postgres_config" +go run ./cmd/stacyvm config lint --production --file "$postgres_config" + +if [[ -n "${STACYVM_POSTGRES_TEST_DSN:-}" ]]; then + echo "==> Running live Postgres store contract" + go test ./internal/store -run 'TestPostgresStoreContract|TestPostgresLeaseConcurrency|TestPostgresMigrationRehearsal' -count=1 +else + echo "==> Skipping live Postgres store contract; STACYVM_POSTGRES_TEST_DSN is not set" +fi + +echo "==> Linting OIDC-enabled cluster config" +oidc_config="$tmpdir/stacyvm.oidc.yaml" +cat >"$oidc_config" <&1)" +if [[ "$oidc_lint_output" != *"auth.oidc_enabled: enabled"* ]]; then + echo "$oidc_lint_output" + echo "expected OIDC enabled lint pass" >&2 + exit 1 +fi +if [[ "$oidc_lint_output" != *"auth.oidc_issuer"* ]]; then + echo "$oidc_lint_output" + echo "expected OIDC issuer lint check" >&2 + exit 1 +fi +if [[ "$oidc_lint_output" != *"auth.oidc_groups"* ]]; then + echo "$oidc_lint_output" + echo "expected OIDC group mapping lint check" >&2 + exit 1 +fi +echo " OIDC config lint: OK" + +echo "==> Running remote worker mTLS smoke with ephemeral certificates" +# Build the binary into the tmpdir if not already built. +SMOKE_BIN="$tmpdir/stacyvm" +if [[ ! -x "$SMOKE_BIN" ]]; then + go build -o "$SMOKE_BIN" ./cmd/stacyvm +fi +scripts/smoke-remote-worker.sh "$SMOKE_BIN" --mtls + +echo "==> Cluster conformance CI checks passed" diff --git a/scripts/ci-public-release-sanity.sh b/scripts/ci-public-release-sanity.sh new file mode 100755 index 0000000..0c8bba3 --- /dev/null +++ b/scripts/ci-public-release-sanity.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +DIST_DIR="$(mktemp -d)" +trap 'rm -rf "$DIST_DIR"' EXIT + +echo "==> Checking public install and verification scripts" +bash -n scripts/install.sh +bash -n scripts/verify-release.sh +bash -n scripts/post-release-validate.sh +bash -n scripts/public-readiness-evidence.sh + +echo "==> Checking public production config posture" +STACYVM_AUTH_API_KEY=public-ci-api-key-with-at-least-32-bytes \ + STACYVM_AUTH_ADMIN_API_KEY=public-ci-admin-key-with-at-least-32-bytes \ + go run ./cmd/stacyvm config lint --production --file deploy/stacyvm.production.yaml + +echo "==> Building release artifacts" +make release-build-all VERSION=phase-9-ci DIST_DIR="$DIST_DIR" + +echo "==> Verifying release checksums" +( + cd "$DIST_DIR" + sha256sum -c checksums.txt +) + +echo "==> Public release sanity checks passed" diff --git a/scripts/ci-smoke-deployment.sh b/scripts/ci-smoke-deployment.sh new file mode 100755 index 0000000..553f2d7 --- /dev/null +++ b/scripts/ci-smoke-deployment.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PORT="${STACYVM_SMOKE_PORT:-17423}" +API_KEY="${STACYVM_API_KEY:-ci-smoke-key}" +DB_PATH="${STACYVM_DATABASE_PATH:-${TMPDIR:-/tmp}/stacyvm-ci-smoke.db}" +LOG_PATH="${STACYVM_SMOKE_LOG:-${TMPDIR:-/tmp}/stacyvm-ci-smoke.log}" + +cd "$ROOT" + +rm -f "$DB_PATH" "$DB_PATH-shm" "$DB_PATH-wal" "$LOG_PATH" + +cleanup() { + if [[ -n "${server_pid:-}" ]]; then + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + fi +} +trap cleanup EXIT + +STACYVM_SERVER_PORT="$PORT" \ +STACYVM_PROVIDERS_DEFAULT=mock \ +STACYVM_PROVIDERS_MOCK_ENABLED=true \ +STACYVM_PROVIDERS_DOCKER_ENABLED=false \ +STACYVM_PROVIDERS_FIRECRACKER_ENABLED=false \ +STACYVM_AUTH_API_KEY="$API_KEY" \ +STACYVM_DATABASE_PATH="$DB_PATH" \ + ./stacyvm serve >"$LOG_PATH" 2>&1 & +server_pid="$!" + +for _ in $(seq 1 50); do + if curl --silent --fail --max-time 1 -H "X-API-Key: $API_KEY" "http://127.0.0.1:$PORT/api/v1/live" >/dev/null; then + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + echo "StacyVM server exited before becoming live. Logs:" >&2 + cat "$LOG_PATH" >&2 + exit 1 + fi + sleep 0.2 +done + +scripts/smoke-deployment.sh "http://127.0.0.1:$PORT" "$API_KEY" diff --git a/scripts/ci-upgrade-migration.sh b/scripts/ci-upgrade-migration.sh new file mode 100755 index 0000000..5223d28 --- /dev/null +++ b/scripts/ci-upgrade-migration.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +echo "==> Running upgrade, config, and SQLite migration tests" +go test ./internal/config ./internal/store ./cmd/stacyvm \ + -run 'TestLoadAcceptsPhaseThreeConfig|TestSQLiteStoreMigratesLegacyDatabase|TestRunUpgradeRehearsal|TestLintConfigProductionBaselinePasses' + +echo "==> Linting production deployment config with CI secrets" +STACYVM_AUTH_API_KEY="regular-api-key-with-at-least-32-bytes" \ +STACYVM_AUTH_ADMIN_API_KEY="admin-api-key-with-at-least-32-bytesxx" \ +go run ./cmd/stacyvm config lint --production --file deploy/stacyvm.production.yaml + +echo "==> Upgrade and migration CI checks passed" diff --git a/scripts/dev.sh b/scripts/dev.sh new file mode 100755 index 0000000..2a06618 --- /dev/null +++ b/scripts/dev.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Local StacyVM developer bootstrap. +# +# Usage: +# ./scripts/dev.sh +# make dev +# +# This script intentionally does not install Docker Desktop or system packages. +# It checks the local host, prints OS-specific remediation, builds StacyVM, and +# starts the API server. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PORT="${STACYVM_SERVER_PORT:-7423}" + +red() { printf '\033[0;31m%s\033[0m\n' "$*"; } +green() { printf '\033[0;32m%s\033[0m\n' "$*"; } +yellow() { printf '\033[1;33m%s\033[0m\n' "$*"; } +step() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +fail() { + red "x $*" + exit 1 +} + +host_name() { + case "$(uname -s)" in + Darwin) echo "macOS" ;; + Linux) + if grep -qi microsoft /proc/version 2>/dev/null; then + echo "Windows WSL" + else + echo "Linux" + fi + ;; + *) uname -s ;; + esac +} + +print_go_help() { + case "$(host_name)" in + macOS) + cat <<'EOF' +Install Go with: + brew install go +EOF + ;; + "Windows WSL"|Linux) + cat <<'EOF' +Install Go with: + sudo apt update + sudo apt install -y golang-go +EOF + ;; + *) + echo "Install Go from https://go.dev/dl/" + ;; + esac +} + +print_docker_help() { + case "$(host_name)" in + macOS) + cat <<'EOF' +Install and start Docker Desktop: + https://docs.docker.com/desktop/setup/install/mac-install/ + +Then verify: + docker run --rm hello-world +EOF + ;; + "Windows WSL") + cat <<'EOF' +Install Docker Desktop, enable WSL integration for your Ubuntu distro, then run: + docker run --rm hello-world +EOF + ;; + Linux) + cat <<'EOF' +Install Docker, start it, and add your user to the docker group: + sudo apt update + sudo apt install -y docker.io + sudo systemctl enable --now docker + sudo usermod -aG docker "$USER" + +Log out and back in, then verify: + docker run --rm hello-world +EOF + ;; + *) + echo "Install Docker from https://docs.docker.com/get-docker/" + ;; + esac +} + +check_command() { + local name="$1" + local remediation="$2" + if ! command -v "$name" >/dev/null 2>&1; then + yellow "! Missing ${name}." + eval "$remediation" + exit 1 + fi +} + +step "StacyVM local setup" +echo "Host: $(host_name)" +echo "Repo: ${ROOT_DIR}" + +step "Checking tools" +check_command go print_go_help +green "+ Go: $(go version)" + +check_command docker print_docker_help +green "+ Docker CLI: $(docker --version)" + +if ! docker info >/dev/null 2>&1; then + yellow "! Docker CLI is installed, but the Docker daemon is not reachable." + print_docker_help + exit 1 +fi +green "+ Docker daemon is reachable" + +if command -v lsof >/dev/null 2>&1 && lsof -iTCP:"${PORT}" -sTCP:LISTEN >/dev/null 2>&1; then + yellow "! Port ${PORT} is already in use." + echo "Stop the process using port ${PORT}, or run StacyVM with a different server port in your config." + exit 1 +fi +green "+ Port ${PORT} is available" + +step "Building StacyVM" +cd "$ROOT_DIR" +make build + +step "Starting StacyVM" +echo "API: http://localhost:${PORT}" +echo "Health check: curl http://localhost:${PORT}/api/v1/live" +echo "" +exec ./stacyvm serve diff --git a/scripts/install.sh b/scripts/install.sh index 2a04dd3..8fe19bf 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -7,6 +7,8 @@ # Environment variables: # STACYVM_VERSION — specific version to install (default: latest) # STACYVM_INSTALL_DIR — where to put binaries (default: /usr/local/bin) +# STACYVM_REQUIRE_SIGNATURES — require Sigstore verification with cosign (default: false) +# STACYVM_VERIFY_ONLY — download and verify release assets, then exit before installing # set -euo pipefail @@ -24,6 +26,8 @@ INSTALL_DIR="${STACYVM_INSTALL_DIR:-/usr/local/bin}" DATA_DIR="/var/lib/stacyvm" KERNEL_URL="https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin" REPO="StacyOs/stacyvm" +COSIGN_IDENTITY_REGEXP="${STACYVM_COSIGN_IDENTITY_REGEXP:-https://github.com/StacyOS/stacyvm/.github/workflows/release.yml@refs/tags/v.*}" +COSIGN_ISSUER="${STACYVM_COSIGN_ISSUER:-https://token.actions.githubusercontent.com}" echo "" echo " ╔═══════════════════════════════════╗" @@ -68,6 +72,58 @@ curl -fSL -o "$TMPDIR/stacyvm" "${RELEASE_URL}/stacyvm-linux-${ARCH_SUFFIX}" curl -fSL -o "$TMPDIR/stacyvm-agent" "${RELEASE_URL}/stacyvm-agent-linux-${ARCH_SUFFIX}" curl -fSL -o "$TMPDIR/checksums.txt" "${RELEASE_URL}/checksums.txt" +# ── Verify Sigstore signatures when available ─────────── +verify_signature() { + local file="$1" + local release_name="$2" + if ! curl -fSL -o "${file}.sig" "${RELEASE_URL}/${release_name}.sig"; then + return 2 + fi + if ! curl -fSL -o "${file}.pem" "${RELEASE_URL}/${release_name}.pem"; then + return 2 + fi + cosign verify-blob "$file" \ + --signature "${file}.sig" \ + --certificate "${file}.pem" \ + --certificate-identity-regexp "$COSIGN_IDENTITY_REGEXP" \ + --certificate-oidc-issuer "$COSIGN_ISSUER" >/dev/null +} + +if command -v cosign >/dev/null 2>&1; then + info "Verifying Sigstore signatures..." + signatures_missing=false + for item in \ + "$TMPDIR/stacyvm:stacyvm-linux-${ARCH_SUFFIX}" \ + "$TMPDIR/stacyvm-agent:stacyvm-agent-linux-${ARCH_SUFFIX}" \ + "$TMPDIR/checksums.txt:checksums.txt" + do + file="${item%%:*}" + release_name="${item#*:}" + set +e + verify_signature "$file" "$release_name" + verify_status=$? + set -e + if [[ "$verify_status" -eq 2 ]]; then + signatures_missing=true + elif [[ "$verify_status" -ne 0 ]]; then + fail "Sigstore signature verification failed for ${release_name}" + fi + done + if [[ "$signatures_missing" == "true" ]]; then + if [[ "${STACYVM_REQUIRE_SIGNATURES:-false}" == "true" ]]; then + fail "Sigstore signature assets are missing for this release." + fi + warn "Sigstore signature assets are missing for this release; relying on checksums only." + else + info "Signatures verified" + fi +elif [[ "${STACYVM_REQUIRE_SIGNATURES:-false}" == "true" ]]; then + fail "cosign is required because STACYVM_REQUIRE_SIGNATURES=true. Install cosign and rerun." +else + warn "cosign not found; skipping Sigstore signature verification and relying on checksums only." + warn "For public installs, install cosign or set STACYVM_REQUIRE_SIGNATURES=true." +fi + # ── Verify checksums ──────────────────────────────────── info "Verifying checksums..." cd "$TMPDIR" @@ -89,6 +145,11 @@ sha256sum -c checksums.txt --ignore-missing 2>/dev/null || { info "Checksums verified" cd - > /dev/null +if [[ "${STACYVM_VERIFY_ONLY:-false}" == "true" ]]; then + info "Verify-only mode complete; skipping install and host setup." + exit 0 +fi + # ── Install binaries ───────────────────────────────────── info "Installing to ${INSTALL_DIR}..." chmod +x "$TMPDIR/stacyvm" "$TMPDIR/stacyvm-agent" diff --git a/scripts/npm-setup.mjs b/scripts/npm-setup.mjs new file mode 100644 index 0000000..dedb3f9 --- /dev/null +++ b/scripts/npm-setup.mjs @@ -0,0 +1,287 @@ +#!/usr/bin/env node +import { existsSync, readdirSync, statSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +const REPO_URL = "https://github.com/StacyOS/stacyvm.git"; +const DEFAULT_BRANCH = process.env.STACYVM_SETUP_BRANCH ?? "main"; +const DEFAULT_DIR = "stacyvm"; +const PACKAGE_DIRS = ["web", "sdk/js", "examples/code-runner-typescript"]; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const bundledRepoRoot = resolve(scriptDir, ".."); + +function usage() { + console.log(`StacyVM one-command setup + +Usage: + npx stacyvm-setup@latest + npx github:StacyOS/stacyvm#phase-14-worker-identity-hardening stacyvm-setup --branch phase-14-worker-identity-hardening + node scripts/npm-setup.mjs + +Options: + --dir Directory to use or create. Default: ./stacyvm outside a repo, current repo inside a repo. + --branch Branch to clone when --dir is not already a StacyVM checkout. Default: ${DEFAULT_BRANCH} + --repo Git repository URL. Default: ${REPO_URL} + --no-start Set up and build, but do not start the server. + --skip-docker-check Do not require Docker daemon access during setup checks. + --skip-node-deps Do not run npm install in web/sdk/example packages. + --check-only Only check the host and repo; do not download deps, build, or start. + --help Show this help. + +Environment: + STACYVM_SETUP_BRANCH Default clone branch. + STACYVM_SERVER_PORT Port expected by the local server. Default: 7423. +`); +} + +function parseArgs(argv) { + if (argv[0] === "setup" || argv[0] === "dev" || argv[0] === "start") { + argv = argv.slice(1); + } + + const options = { + branch: DEFAULT_BRANCH, + repo: REPO_URL, + dir: "", + start: true, + dockerCheck: true, + nodeDeps: true, + checkOnly: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + switch (arg) { + case "--help": + case "-h": + usage(); + process.exit(0); + break; + case "--dir": + options.dir = argv[++i] ?? ""; + break; + case "--branch": + options.branch = argv[++i] ?? ""; + break; + case "--repo": + options.repo = argv[++i] ?? ""; + break; + case "--no-start": + options.start = false; + break; + case "--skip-docker-check": + options.dockerCheck = false; + break; + case "--skip-node-deps": + options.nodeDeps = false; + break; + case "--check-only": + options.checkOnly = true; + options.start = false; + break; + default: + throw new Error(`Unknown option or command: ${arg}`); + } + } + + return options; +} + +function logStep(message) { + console.log(`\n\x1b[1m${message}\x1b[0m`); +} + +function logOk(message) { + console.log(`\x1b[32m+\x1b[0m ${message}`); +} + +function logWarn(message) { + console.log(`\x1b[33m!\x1b[0m ${message}`); +} + +function fail(message) { + console.error(`\x1b[31mx\x1b[0m ${message}`); + process.exit(1); +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + env: { ...process.env, ...(options.env ?? {}) }, + stdio: options.capture ? "pipe" : "inherit", + encoding: "utf8", + }); + + if (result.error) { + throw result.error; + } + if (result.status !== 0 && !options.allowFailure) { + throw new Error(`${command} ${args.join(" ")} exited with ${result.status}`); + } + return result; +} + +function commandExists(command) { + const probe = process.platform === "win32" ? "where" : "command"; + const args = process.platform === "win32" ? [command] : ["-v", command]; + const result = spawnSync(probe, args, { + shell: process.platform !== "win32", + stdio: "ignore", + }); + return result.status === 0; +} + +function isStacyRepo(dir) { + return existsSync(join(dir, "go.mod")) && existsSync(join(dir, "cmd", "stacyvm")); +} + +function isEmptyDir(dir) { + if (!existsSync(dir)) return true; + return statSync(dir).isDirectory() && readdirSync(dir).length === 0; +} + +function hostHelp() { + if (process.platform === "darwin") { + return "macOS: install Docker Desktop, then run `brew install go git make`."; + } + if (process.platform === "win32") { + return "Windows: use WSL 2 with Ubuntu and Docker Desktop WSL integration, then run this command inside Ubuntu."; + } + return "Linux/Ubuntu: install Docker and Go, start Docker, and ensure your user can run `docker ps`."; +} + +function resolveTargetDir(options) { + if (options.dir) return resolve(options.dir); + if (isStacyRepo(process.cwd())) return process.cwd(); + if (isStacyRepo(bundledRepoRoot)) return bundledRepoRoot; + return resolve(process.cwd(), DEFAULT_DIR); +} + +async function ensureRepo(targetDir, options) { + if (isStacyRepo(targetDir)) { + logOk(`Using StacyVM checkout: ${targetDir}`); + return; + } + + if (existsSync(targetDir) && !isEmptyDir(targetDir)) { + fail(`${targetDir} exists but is not an empty directory or StacyVM checkout.`); + } + + if (!commandExists("git")) { + fail(`git is required to clone StacyVM. ${hostHelp()}`); + } + + await mkdir(dirname(targetDir), { recursive: true }); + logStep(`Cloning StacyVM into ${targetDir}`); + run("git", ["clone", "--depth", "1", "--branch", options.branch, options.repo, targetDir]); +} + +function checkHost(options) { + logStep("Checking host"); + + if (process.platform === "win32") { + fail("Run StacyVM setup inside WSL 2 Ubuntu instead of native Windows PowerShell."); + } + + if (!commandExists("go")) { + fail(`Go is required for source setup. ${hostHelp()}`); + } + const goVersion = run("go", ["version"], { capture: true }).stdout.trim(); + logOk(goVersion); + + if (!commandExists("docker")) { + fail(`Docker is required for the default local provider. ${hostHelp()}`); + } + const dockerVersion = run("docker", ["--version"], { capture: true }).stdout.trim(); + logOk(dockerVersion); + + if (options.dockerCheck) { + const dockerInfo = run("docker", ["info"], { capture: true, allowFailure: true }); + if (dockerInfo.status !== 0) { + fail(`Docker CLI is installed, but the daemon is not reachable. ${hostHelp()}`); + } + logOk("Docker daemon is reachable"); + } else { + logWarn("Skipping Docker daemon check"); + } +} + +function installNodeDeps(repoDir, options) { + if (!options.nodeDeps) { + logWarn("Skipping npm install for repo packages"); + return; + } + if (!commandExists("npm")) { + logWarn("npm is not installed; skipping web/sdk/example package installs."); + return; + } + + logStep("Installing Node package dependencies"); + for (const packageDir of PACKAGE_DIRS) { + const fullDir = join(repoDir, packageDir); + if (!existsSync(join(fullDir, "package.json"))) continue; + console.log(`npm install (${packageDir})`); + run("npm", ["install"], { cwd: fullDir }); + } +} + +function downloadGoDeps(repoDir) { + logStep("Downloading Go dependencies"); + run("go", ["mod", "download"], { cwd: repoDir }); +} + +function buildStacyVM(repoDir) { + logStep("Building StacyVM"); + const output = process.platform === "win32" ? "stacyvm.exe" : "stacyvm"; + run("go", [ + "build", + "-ldflags=-s -w -X main.version=dev", + "-o", + output, + "./cmd/stacyvm", + ], { cwd: repoDir }); +} + +function startServer(repoDir) { + const port = process.env.STACYVM_SERVER_PORT ?? "7423"; + const binary = process.platform === "win32" ? "stacyvm.exe" : "./stacyvm"; + logStep("Starting StacyVM"); + console.log(`API: http://localhost:${port}`); + console.log(`Health check: curl http://localhost:${port}/api/v1/live`); + console.log(""); + run(binary, ["serve"], { cwd: repoDir }); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const targetDir = resolveTargetDir(options); + + console.log("StacyVM setup"); + console.log(`Target: ${targetDir}`); + + await ensureRepo(targetDir, options); + checkHost(options); + + if (options.checkOnly) { + logOk("Check-only mode complete"); + return; + } + + installNodeDeps(targetDir, options); + downloadGoDeps(targetDir); + buildStacyVM(targetDir); + + if (options.start) { + startServer(targetDir); + } else { + logOk("Setup complete"); + console.log(`Run next: cd ${targetDir} && ./stacyvm serve`); + } +} + +main().catch((error) => { + fail(error instanceof Error ? error.message : String(error)); +}); diff --git a/scripts/post-release-validate.sh b/scripts/post-release-validate.sh new file mode 100755 index 0000000..f877369 --- /dev/null +++ b/scripts/post-release-validate.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="${1:-${STACYVM_VERSION:-}}" +repo="${STACYVM_REPO:-StacyOs/stacyvm}" + +usage() { + cat <<'USAGE' +usage: scripts/post-release-validate.sh + +Validates a published StacyVM GitHub release after a real version tag exists. +The release must include binaries, checksums, Sigstore signatures, and +certificates for amd64 and arm64. + +Environment: + STACYVM_REPO GitHub repo, default StacyOs/stacyvm + STACYVM_VALIDATE_INSTALLER Run install.sh in verify-only mode when on Linux +USAGE +} + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +if [ -z "$version" ]; then + printf 'version is required\n' >&2 + usage >&2 + exit 2 +fi + +if ! command -v gh >/dev/null 2>&1; then + printf 'gh is required to inspect GitHub release assets\n' >&2 + exit 1 +fi + +required_assets=( + checksums.txt + checksums.txt.sig + checksums.txt.pem + stacyvm-linux-amd64 + stacyvm-linux-amd64.sig + stacyvm-linux-amd64.pem + stacyvm-agent-linux-amd64 + stacyvm-agent-linux-amd64.sig + stacyvm-agent-linux-amd64.pem + stacyvm-linux-arm64 + stacyvm-linux-arm64.sig + stacyvm-linux-arm64.pem + stacyvm-agent-linux-arm64 + stacyvm-agent-linux-arm64.sig + stacyvm-agent-linux-arm64.pem +) + +assets="$(gh release view "$version" --repo "$repo" --json assets --jq '.assets[].name')" +missing=0 +for asset in "${required_assets[@]}"; do + if ! grep -Fxq "$asset" <<<"$assets"; then + printf '[FAIL] missing release asset: %s\n' "$asset" >&2 + missing=$((missing + 1)) + else + printf '[PASS] release asset: %s\n' "$asset" + fi +done +if [ "$missing" -gt 0 ]; then + printf 'release %s is missing %d required asset(s)\n' "$version" "$missing" >&2 + exit 1 +fi + +scripts/verify-release.sh "$version" amd64 +scripts/verify-release.sh "$version" arm64 + +if [ "${STACYVM_VALIDATE_INSTALLER:-false}" = "true" ]; then + if [ "$(uname -s)" != "Linux" ]; then + printf '[WARN] installer verify-only check skipped: install.sh is Linux-only\n' >&2 + else + tmpdir="$(mktemp -d)" + trap 'rm -rf "$tmpdir"' EXIT + STACYVM_VERSION="$version" \ + STACYVM_INSTALL_DIR="$tmpdir" \ + STACYVM_REQUIRE_SIGNATURES=true \ + STACYVM_VERIFY_ONLY=true \ + scripts/install.sh + fi +fi + +printf '[PASS] post-release validation complete for %s\n' "$version" diff --git a/scripts/public-readiness-evidence.sh b/scripts/public-readiness-evidence.sh new file mode 100755 index 0000000..1251ab9 --- /dev/null +++ b/scripts/public-readiness-evidence.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +OUTPUT="${STACYVM_PUBLIC_READINESS_OUTPUT:-public-readiness-evidence.md}" +CONFIG_FILE="${STACYVM_PUBLIC_READINESS_CONFIG:-deploy/stacyvm.production.yaml}" +RUN_CLUSTER="${STACYVM_RUN_CLUSTER_CONFORMANCE:-false}" +POST_RELEASE_VERSION="${STACYVM_POST_RELEASE_VERSION:-}" +RUNTIMES="${STACYVM_RUNTIME_CERTIFY:-}" + +usage() { + cat <<'USAGE' +usage: scripts/public-readiness-evidence.sh [--output file] [--config file] + +Generates a Markdown evidence report for the public self-serve go-live gate. +The report records local CI-equivalent checks and clearly marks tag/host-gated +items that must be captured before announcement. + +Required environment for production config lint: + STACYVM_AUTH_API_KEY + STACYVM_AUTH_ADMIN_API_KEY + +Optional environment: + STACYVM_PUBLIC_READINESS_OUTPUT Report path, default public-readiness-evidence.md + STACYVM_PUBLIC_READINESS_CONFIG Config path, default deploy/stacyvm.production.yaml + STACYVM_RUN_CLUSTER_CONFORMANCE true to run scripts/ci-cluster-conformance.sh + STACYVM_POST_RELEASE_VERSION Version tag for scripts/post-release-validate.sh + STACYVM_VALIDATE_INSTALLER true to include install.sh verify-only in post-release gate + STACYVM_RUNTIME_CERTIFY Comma-separated runtimes: docker,gvisor,kata,firecracker,proot + +The script exits non-zero when required local checks fail. Skipped tag/host-gated +items are recorded as SKIP in the report instead of pretending the release is +fully ready. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --output) OUTPUT="$2"; shift 2 ;; + --config) CONFIG_FILE="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) printf 'unknown argument: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ ! -f "$CONFIG_FILE" ]]; then + printf 'config file not found: %s\n' "$CONFIG_FILE" >&2 + exit 2 +fi + +if [[ -z "${STACYVM_AUTH_API_KEY:-}" || -z "${STACYVM_AUTH_ADMIN_API_KEY:-}" ]]; then + printf 'STACYVM_AUTH_API_KEY and STACYVM_AUTH_ADMIN_API_KEY are required for production config lint evidence\n' >&2 + exit 2 +fi + +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +failures=0 +skips=0 +rows=() + +now_utc="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" +git_sha="$(git rev-parse HEAD 2>/dev/null || printf 'unknown')" +git_branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || printf 'unknown')" +git_status="$(git status --short 2>/dev/null || true)" + +record_row() { + local status="$1" + local gate="$2" + local detail="$3" + rows+=("| $status | $gate | $detail |") +} + +run_gate() { + local gate="$1" + shift + local log="$tmpdir/$(printf '%s' "$gate" | tr -cs 'A-Za-z0-9' '-').log" + printf '==> %s\n' "$gate" + if "$@" >"$log" 2>&1; then + record_row "PASS" "$gate" "See \`$log\` during this run." + else + failures=$((failures + 1)) + record_row "FAIL" "$gate" "Command failed; last 40 log lines are included below." + { + printf '\n### %s Failure Log\n\n' "$gate" + printf '```text\n' + tail -40 "$log" + printf '\n```\n' + } >>"$tmpdir/failures.md" + fi +} + +skip_gate() { + local gate="$1" + local detail="$2" + skips=$((skips + 1)) + record_row "SKIP" "$gate" "$detail" +} + +run_gate "Shell script syntax" bash -n \ + scripts/install.sh \ + scripts/verify-release.sh \ + scripts/post-release-validate.sh \ + scripts/public-readiness-evidence.sh \ + scripts/ci-public-release-sanity.sh \ + scripts/ci-upgrade-migration.sh \ + scripts/ci-cluster-conformance.sh \ + scripts/certify-runtime.sh \ + scripts/smoke-remote-worker.sh + +run_gate "Production config lint" go run ./cmd/stacyvm config lint --production --file "$CONFIG_FILE" +run_gate "Go test suite" env -u STACYVM_AUTH_API_KEY -u STACYVM_AUTH_ADMIN_API_KEY go test ./... +run_gate "Web production build" npm --prefix web run build +run_gate "Public release sanity" scripts/ci-public-release-sanity.sh +run_gate "Upgrade and migration sanity" env -u STACYVM_AUTH_API_KEY -u STACYVM_AUTH_ADMIN_API_KEY scripts/ci-upgrade-migration.sh + +if [[ "$RUN_CLUSTER" == "true" ]]; then + run_gate "Cluster conformance" env -u STACYVM_AUTH_API_KEY -u STACYVM_AUTH_ADMIN_API_KEY scripts/ci-cluster-conformance.sh +else + skip_gate "Cluster conformance" "Set \`STACYVM_RUN_CLUSTER_CONFORMANCE=true\` to capture this gate; it opens local listener ports." +fi + +if [[ -n "$POST_RELEASE_VERSION" ]]; then + run_gate "Post-release asset validation" scripts/post-release-validate.sh "$POST_RELEASE_VERSION" +else + skip_gate "Post-release asset validation" "Set \`STACYVM_POST_RELEASE_VERSION=\` after publishing a real GitHub release." +fi + +if [[ -n "$RUNTIMES" ]]; then + IFS=',' read -r -a runtime_list <<<"$RUNTIMES" + for runtime in "${runtime_list[@]}"; do + runtime="$(printf '%s' "$runtime" | xargs)" + [[ -z "$runtime" ]] && continue + run_gate "Runtime certification: $runtime" scripts/certify-runtime.sh "$runtime" --format markdown --output "$tmpdir/$runtime-certification.md" + done +else + skip_gate "Runtime certification" "Set \`STACYVM_RUNTIME_CERTIFY=docker,gvisor,kata,firecracker,proot\` for the runtime claims in this launch." +fi + +readiness="PUBLIC SELF-SERVE CANDIDATE" +if [[ "$failures" -gt 0 ]]; then + readiness="NOT READY" +elif [[ "$skips" -eq 0 ]]; then + readiness="PUBLIC SELF-SERVE READY" +fi + +{ + printf '# StacyVM Public Readiness Evidence\n\n' + printf '%s\n' "- Generated: \`$now_utc\`" + printf '%s\n' "- Branch: \`$git_branch\`" + printf '%s\n' "- Commit: \`$git_sha\`" + printf '%s\n' "- Config: \`$CONFIG_FILE\`" + printf '%s\n\n' "- Verdict: **$readiness**" + + printf '## Gate Results\n\n' + printf '| Status | Gate | Detail |\n' + printf '|---|---|---|\n' + for row in "${rows[@]}"; do + printf '%s\n' "$row" + done + + printf '\n## Interpretation\n\n' + if [[ "$failures" -gt 0 ]]; then + printf 'This report is **not ready** for public announcement because one or more required gates failed.\n' + elif [[ "$skips" -gt 0 ]]; then + printf 'This report is a **public self-serve candidate**. All required local gates passed, but skipped tag/host-gated evidence must be captured before announcement.\n' + else + printf 'This report is **public self-serve ready** for the tested release, host, runtime, and network scope.\n' + fi + + printf '\n## External Evidence Required Before Announcement\n\n' + printf '%s\n' '- Real tagged release validated by `scripts/post-release-validate.sh `.' + printf '%s\n' '- Runtime certification reports for every runtime claimed publicly.' + printf '%s\n' '- Live Postgres contract evidence for cluster/multi-worker claims.' + printf '%s\n' '- Target-network worker RPC mTLS smoke with deployment-issued certificates for enterprise/multi-worker claims.' + printf '%s\n' '- Staging install rehearsal from published artifacts, including `doctor --production`, smoke deployment, backup/restore, upgrade rehearsal, and support bundle redaction.' + + if [[ -n "$git_status" ]]; then + printf '\n## Working Tree Note\n\n' + printf 'The working tree had local changes when this report was generated:\n\n' + printf '```text\n%s\n```\n' "$git_status" + fi + + if [[ -f "$tmpdir/failures.md" ]]; then + cat "$tmpdir/failures.md" + fi +} >"$OUTPUT" + +printf 'public readiness evidence written: %s\n' "$OUTPUT" + +if [[ "$failures" -gt 0 ]]; then + exit 1 +fi diff --git a/scripts/smoke-deployment.sh b/scripts/smoke-deployment.sh new file mode 100755 index 0000000..a58f054 --- /dev/null +++ b/scripts/smoke-deployment.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${STACYVM_SMOKE_URL:-${1:-http://127.0.0.1:7423}}" +API_KEY="${STACYVM_API_KEY:-${2:-}}" +TIMEOUT_SECONDS="${STACYVM_SMOKE_TIMEOUT:-5}" + +BASE_URL="${BASE_URL%/}" + +headers=() +if [[ -n "$API_KEY" ]]; then + headers=(-H "X-API-Key: $API_KEY") +fi + +curl_base=(curl --silent --show-error --fail --max-time "$TIMEOUT_SECONDS" "${headers[@]}") + +probe_json() { + local path="$1" + local expected="$2" + local url="$BASE_URL$path" + + printf 'Checking %s ... ' "$url" + local body + body="$("${curl_base[@]}" "$url")" + if [[ "$body" != *"$expected"* ]]; then + printf 'failed\n' + printf 'Expected response to contain: %s\n' "$expected" >&2 + printf 'Response:\n%s\n' "$body" >&2 + return 1 + fi + printf 'ok\n' +} + +probe_metrics() { + local path="/api/v1/metrics/prometheus" + local url="$BASE_URL$path" + + printf 'Checking %s ... ' "$url" + local body + body="$("${curl_base[@]}" "$url")" + if [[ "$body" != *"stacyvm_uptime_seconds"* ]]; then + printf 'failed\n' + printf 'Expected Prometheus metrics to contain stacyvm_uptime_seconds.\n' >&2 + printf 'Response:\n%s\n' "$body" >&2 + return 1 + fi + printf 'ok\n' +} + +probe_json "/api/v1/live" '"status":"alive"' +probe_json "/api/v1/health" '"status":"ok"' +probe_json "/api/v1/ready" '"status":"ready"' +probe_metrics + +printf 'StacyVM deployment smoke checks passed for %s\n' "$BASE_URL" diff --git a/scripts/smoke-remote-worker.sh b/scripts/smoke-remote-worker.sh new file mode 100755 index 0000000..f585bf6 --- /dev/null +++ b/scripts/smoke-remote-worker.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +# smoke-remote-worker.sh — end-to-end remote-worker smoke test. +# +# Plain mode (default): uses shared worker token over HTTP. +# mTLS mode (--mtls): generates an ephemeral CA + server/client certs via +# openssl and runs the same smoke over HTTPS with mutual +# TLS authentication between control plane and worker RPC. +# +# Usage: +# scripts/smoke-remote-worker.sh [binary] [--mtls] [--ca-cert f] [--ca-key f] +# [--server-cert f] [--server-key f] +# [--client-cert f] [--client-key f] +# +# Environment: +# STACYVM_REMOTE_SMOKE_DATABASE_DRIVER sqlite (default) or postgres +# STACYVM_REMOTE_SMOKE_DATABASE_DSN required when driver=postgres +# STACYVM_REMOTE_SMOKE_CONTROL_PORT default 17423 +# STACYVM_REMOTE_SMOKE_WORKER_PORT default 17430 +set -euo pipefail + +# ── argument parsing ───────────────────────────────────────────────────────── +BIN="${1:-./stacyvm}" +shift || true + +MTLS=false +CA_CERT="" CA_KEY="" +SRV_CERT="" SRV_KEY="" +CLI_CERT="" CLI_KEY="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --mtls) MTLS=true; shift ;; + --ca-cert) CA_CERT="$2"; shift 2 ;; + --ca-key) CA_KEY="$2"; shift 2 ;; + --server-cert) SRV_CERT="$2"; shift 2 ;; + --server-key) SRV_KEY="$2"; shift 2 ;; + --client-cert) CLI_CERT="$2"; shift 2 ;; + --client-key) CLI_KEY="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ "$BIN" != /* ]]; then BIN="$ROOT_DIR/${BIN#./}"; fi +if [[ ! -x "$BIN" ]]; then + echo "stacyvm binary is not executable: $BIN" >&2 + echo "build it first, or pass the path to a built stacyvm binary" >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d)" +CONTROL_DIR="$WORK_DIR/control-plane" +WORKER_DIR="$WORK_DIR/worker" +CERT_DIR="$WORK_DIR/certs" +CONTROL_CONFIG="$CONTROL_DIR/stacyvm.yaml" +WORKER_CONFIG="$WORKER_DIR/stacyvm.yaml" +SERVER_LOG="$WORK_DIR/server.log" +WORKER_LOG="$WORK_DIR/worker.log" +DB_PATH="$WORK_DIR/stacyvm.db" + +DATABASE_DRIVER="${STACYVM_REMOTE_SMOKE_DATABASE_DRIVER:-sqlite}" +DATABASE_DSN="${STACYVM_REMOTE_SMOKE_DATABASE_DSN:-}" +# Default ports differ between plain and mTLS so sequential runs don't race for the same port. +if $MTLS; then + CONTROL_PORT="${STACYVM_REMOTE_SMOKE_CONTROL_PORT:-17433}" + WORKER_PORT="${STACYVM_REMOTE_SMOKE_WORKER_PORT:-17440}" +else + CONTROL_PORT="${STACYVM_REMOTE_SMOKE_CONTROL_PORT:-17423}" + WORKER_PORT="${STACYVM_REMOTE_SMOKE_WORKER_PORT:-17430}" +fi +API_KEY="dev-api-key-dev-api-key-dev-api-key" +ADMIN_KEY="dev-admin-key-dev-admin-key-dev" +WORKER_TOKEN="dev-worker-token-dev-worker-token" + +cleanup() { + kill "${WORKER_PID:-}" 2>/dev/null || true + kill "${SERVER_PID:-}" 2>/dev/null || true + wait "${WORKER_PID:-}" 2>/dev/null || true + wait "${SERVER_PID:-}" 2>/dev/null || true + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +mkdir -p "$CONTROL_DIR" "$WORKER_DIR" "$CERT_DIR" + +# ── mTLS cert generation ───────────────────────────────────────────────────── +if $MTLS; then + if [[ -z "$CA_CERT" ]]; then + # No external certs provided — generate an ephemeral PKI. + if ! command -v openssl >/dev/null 2>&1; then + echo "openssl is required for --mtls ephemeral cert generation" >&2 + exit 1 + fi + + echo "==> Generating ephemeral CA and mTLS certificates" + + # CA + openssl genrsa -out "$CERT_DIR/ca.key" 2048 2>/dev/null + openssl req -new -x509 -days 1 \ + -key "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \ + -subj "/CN=stacyvm-smoke-ca" 2>/dev/null + + # Worker RPC server cert (SAN must include 127.0.0.1 for TLS hostname check). + openssl genrsa -out "$CERT_DIR/server.key" 2048 2>/dev/null + openssl req -new -key "$CERT_DIR/server.key" -out "$CERT_DIR/server.csr" \ + -subj "/CN=stacyvm-worker-rpc" 2>/dev/null + openssl x509 -req -days 1 \ + -in "$CERT_DIR/server.csr" \ + -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" -CAcreateserial \ + -out "$CERT_DIR/server.crt" \ + -extfile <(printf 'subjectAltName=IP:127.0.0.1') 2>/dev/null + + # Control-plane client cert (authenticates to worker's mTLS server). + openssl genrsa -out "$CERT_DIR/client.key" 2048 2>/dev/null + openssl req -new -key "$CERT_DIR/client.key" -out "$CERT_DIR/client.csr" \ + -subj "/CN=stacyvm-control-plane" 2>/dev/null + openssl x509 -req -days 1 \ + -in "$CERT_DIR/client.csr" \ + -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" -CAcreateserial \ + -out "$CERT_DIR/client.crt" 2>/dev/null + + CA_CERT="$CERT_DIR/ca.crt" + SRV_CERT="$CERT_DIR/server.crt" + SRV_KEY="$CERT_DIR/server.key" + CLI_CERT="$CERT_DIR/client.crt" + CLI_KEY="$CERT_DIR/client.key" + else + # Validate all cert files were supplied. + for f in "$CA_CERT" "$SRV_CERT" "$SRV_KEY" "$CLI_CERT" "$CLI_KEY"; do + if [[ ! -f "$f" ]]; then + echo "cert file not found: $f" >&2 + exit 1 + fi + done + fi +fi + +# ── control-plane config ───────────────────────────────────────────────────── +cat >"$CONTROL_CONFIG" <&2; exit 1; } + cat >>"$CONTROL_CONFIG" <>"$CONTROL_CONFIG" <>"$CONTROL_CONFIG" <"$WORKER_CONFIG" + +# ── smoke ───────────────────────────────────────────────────────────────────── +MODE="plain-HTTP" +if $MTLS; then MODE="mTLS"; fi +# Wait for ports to be free (guards against in-use ports from sequential runs). +# Uses curl --max-time so we don't hang if the port check itself blocks. +wait_port_free() { + local port="$1" + for _ in {1..25}; do + if command -v nc >/dev/null 2>&1; then + if ! nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then return 0; fi + elif ! curl -fsS --max-time 0.3 "http://127.0.0.1:$port/" >/dev/null 2>&1 \ + && ! curl -kfsS --max-time 0.3 "https://127.0.0.1:$port/" >/dev/null 2>&1; then + return 0 + fi + sleep 0.3 + done + echo "port $port is still in use; stop the existing process or override the smoke port" >&2 + exit 1 +} +wait_port_free "$CONTROL_PORT" +wait_port_free "$WORKER_PORT" + +echo "==> Starting control plane [${MODE}]" +(cd "$CONTROL_DIR"; "$BIN" serve) >"$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +for _ in {1..50}; do + if curl -fsS -H "X-API-Key: $API_KEY" "http://127.0.0.1:$CONTROL_PORT/api/v1/ready" >/dev/null 2>&1; then break; fi + sleep 0.2 +done +if ! curl -fsS -H "X-API-Key: $API_KEY" "http://127.0.0.1:$CONTROL_PORT/api/v1/ready" >/dev/null; then + echo "control plane did not become ready; log:" >&2; cat "$SERVER_LOG" >&2; exit 1 +fi + +echo "==> Starting remote worker [${MODE}]" +(cd "$WORKER_DIR"; "$BIN" worker) >"$WORKER_LOG" 2>&1 & +WORKER_PID=$! + +for _ in {1..50}; do + if curl -fsS -H "X-API-Key: $API_KEY" "http://127.0.0.1:$CONTROL_PORT/api/v1/workers" \ + | grep -q '"id":"worker-a"'; then break; fi + sleep 0.2 +done +if ! curl -fsS -H "X-API-Key: $API_KEY" "http://127.0.0.1:$CONTROL_PORT/api/v1/workers" \ + | grep -q '"id":"worker-a"'; then + echo "worker did not register; log:" >&2; cat "$WORKER_LOG" >&2; exit 1 +fi + +if $MTLS; then + echo "==> Verifying worker advertised rpc_url uses HTTPS" + RPC_URL="$(curl -fsS -H "X-API-Key: $API_KEY" \ + "http://127.0.0.1:$CONTROL_PORT/api/v1/workers/worker-a" \ + | sed -n 's/.*"rpc_url":"\([^"]*\)".*/\1/p')" + if [[ "$RPC_URL" != https://* ]]; then + echo "worker rpc_url is not HTTPS: '${RPC_URL}'" >&2 + cat "$WORKER_LOG" >&2 + exit 1 + fi + echo " rpc_url=$RPC_URL [OK]" +fi + +echo "==> Preferring remote worker for this smoke" +curl -fsS -X DELETE -H "X-Admin-API-Key: $ADMIN_KEY" \ + "http://127.0.0.1:$CONTROL_PORT/api/v1/admin/workers/local" >/dev/null || true + +echo "==> Spawning remote mock sandbox" +SPAWN_RESPONSE_FILE="$WORK_DIR/spawn-response.json" +SPAWN_STATUS="$(curl -sS -o "$SPAWN_RESPONSE_FILE" -w '%{http_code}' -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $API_KEY" \ + -d '{"image":"alpine:latest","provider":"mock","ttl":"5m"}' \ + "http://127.0.0.1:$CONTROL_PORT/api/v1/sandboxes")" +SPAWN_RESPONSE="$(cat "$SPAWN_RESPONSE_FILE")" +if [[ "$SPAWN_STATUS" -lt 200 || "$SPAWN_STATUS" -ge 300 ]]; then + echo "spawn failed with HTTP $SPAWN_STATUS: $SPAWN_RESPONSE" >&2 + echo "control plane log:" >&2 + cat "$SERVER_LOG" >&2 + echo "worker log:" >&2 + cat "$WORKER_LOG" >&2 + exit 1 +fi +SANDBOX_ID="$(printf '%s' "$SPAWN_RESPONSE" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')" +if [[ -z "$SANDBOX_ID" ]]; then + echo "spawn response had no sandbox id: $SPAWN_RESPONSE" >&2; exit 1 +fi +if ! printf '%s' "$SPAWN_RESPONSE" | grep -q '"worker_id":"worker-a"'; then + echo "spawn did not route to worker-a: $SPAWN_RESPONSE" >&2; exit 1 +fi +echo " sandbox=$SANDBOX_ID worker=worker-a [OK]" + +echo "==> Verifying sandbox running" +STATUS="$(curl -fsS -H "X-API-Key: $API_KEY" \ + "http://127.0.0.1:$CONTROL_PORT/api/v1/sandboxes/$SANDBOX_ID")" +if ! printf '%s' "$STATUS" | grep -q '"state":"running"'; then + echo "sandbox not running: $STATUS" >&2; exit 1 +fi + +if $MTLS; then + echo "==> Executing command over remote worker mTLS RPC" + EXEC_RESULT="$(curl -fsS -X POST \ + -H "Content-Type: application/json" \ + -H "X-API-Key: $API_KEY" \ + -d '{"command":"echo stacyvm-mtls-ok","mode":"shell"}' \ + "http://127.0.0.1:$CONTROL_PORT/api/v1/sandboxes/$SANDBOX_ID/exec")" + if ! printf '%s' "$EXEC_RESULT" | grep -q '"exit_code":0'; then + echo "exec over mTLS RPC failed: $EXEC_RESULT" >&2; exit 1 + fi + echo " exec exit_code=0 [OK]" +fi + +echo "==> Destroying remote sandbox" +curl -fsS -X DELETE -H "X-API-Key: $API_KEY" \ + "http://127.0.0.1:$CONTROL_PORT/api/v1/sandboxes/$SANDBOX_ID" >/dev/null + +echo "" +echo "==> Remote worker smoke PASSED [${MODE}]" +if $MTLS; then + echo " mTLS certs used:" + echo " CA: ${CA_CERT}" + echo " server: ${SRV_CERT}" + echo " client: ${CLI_CERT}" +fi diff --git a/scripts/verify-release.sh b/scripts/verify-release.sh new file mode 100755 index 0000000..8dfa9cb --- /dev/null +++ b/scripts/verify-release.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="${STACYVM_VERSION:-}" +arch="${STACYVM_ARCH:-}" +repo="${STACYVM_REPO:-StacyOs/stacyvm}" +workdir="${STACYVM_VERIFY_DIR:-}" +identity_regexp="${STACYVM_COSIGN_IDENTITY_REGEXP:-https://github.com/StacyOS/stacyvm/.github/workflows/release.yml@refs/tags/v.*}" +issuer="${STACYVM_COSIGN_ISSUER:-https://token.actions.githubusercontent.com}" + +usage() { + cat <<'USAGE' +usage: scripts/verify-release.sh [amd64|arm64] + +Downloads StacyVM release checksums, binaries, Sigstore signatures, and +certificates, then verifies artifact authenticity and SHA-256 integrity. + +Environment: + STACYVM_REPO GitHub repo, default StacyOs/stacyvm + STACYVM_VERIFY_DIR Reuse/download into this directory + STACYVM_COSIGN_IDENTITY_REGEXP Expected keyless signer identity regexp + STACYVM_COSIGN_ISSUER Expected certificate issuer +USAGE +} + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + usage + exit 0 +fi + +if [ -n "${1:-}" ]; then + version="$1" +fi +if [ -n "${2:-}" ]; then + arch="$2" +fi +if [ -z "$version" ]; then + printf 'version is required\n' >&2 + usage >&2 + exit 2 +fi +if [ -z "$arch" ]; then + case "$(uname -m)" in + x86_64) arch="amd64" ;; + aarch64|arm64) arch="arm64" ;; + *) printf 'unsupported architecture: %s\n' "$(uname -m)" >&2; exit 2 ;; + esac +fi +case "$arch" in + amd64|arm64) ;; + *) printf 'unsupported arch: %s\n' "$arch" >&2; exit 2 ;; +esac + +if ! command -v cosign >/dev/null 2>&1; then + printf 'cosign is required for release verification: https://docs.sigstore.dev/cosign/installation/\n' >&2 + exit 1 +fi +if ! command -v sha256sum >/dev/null 2>&1; then + printf 'sha256sum is required for checksum verification\n' >&2 + exit 1 +fi + +if [ -z "$workdir" ]; then + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT +else + mkdir -p "$workdir" +fi + +release_url="https://github.com/${repo}/releases/download/${version}" +artifacts=( + "checksums.txt" + "stacyvm-linux-${arch}" + "stacyvm-agent-linux-${arch}" +) + +download() { + local name="$1" + curl -fsSL -o "${workdir}/${name}" "${release_url}/${name}" +} + +for artifact in "${artifacts[@]}"; do + download "$artifact" + download "${artifact}.sig" + download "${artifact}.pem" +done + +for artifact in "${artifacts[@]}"; do + cosign verify-blob "${workdir}/${artifact}" \ + --signature "${workdir}/${artifact}.sig" \ + --certificate "${workdir}/${artifact}.pem" \ + --certificate-identity-regexp "$identity_regexp" \ + --certificate-oidc-issuer "$issuer" >/dev/null + printf '[PASS] signature: %s\n' "$artifact" +done + +( + cd "$workdir" + sha256sum -c checksums.txt --ignore-missing +) + +printf '[PASS] checksums verified for %s %s\n' "$version" "$arch" +printf 'verified artifacts in %s\n' "$workdir" diff --git a/sdk/js/README.md b/sdk/js/README.md index 9a18e9b..def8cd2 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -112,8 +112,18 @@ All fields on `SpawnOptions` are optional. Server defaults apply when fields are | `memory_mb` | `number` | RAM in MB | | `vcpus` | `number` | Virtual CPUs | | `ttl` | `string` | Auto-destroy after this duration | +| `owner_id` | `string` | Owner ID for per-owner quotas when no `userId` header is set | | `metadata` | `Record` | Free-form labels | +Preflight quota and scheduler admission without creating a sandbox: + +```typescript +const decision = await client.admission({ image: "python:3.12", ttl: "1h" }); +if (!decision.allowed && decision.queueable) { + console.log(`Request would queue because ${decision.reason}`); +} +``` + --- ## Executing commands @@ -305,6 +315,7 @@ await client.health(); // { status: "ok", version: "0.5.1", uptime: "2h13 await client.list(); // SandboxInfo[] — all active sandboxes await client.providers(); // [{ name: "docker", healthy: true, default: true }, ...] await client.poolStatus(); // pool VM and user counts +await client.quotaSummary(); // redacted owner quota policy counts await client.prune(); // returns count of expired sandboxes destroyed ``` @@ -358,7 +369,7 @@ import { Sandbox, TemplateManager, // Types - SandboxState, // "creating" | "running" | "stopped" | "destroyed" | "error" + SandboxState, // "creating" | "running" | "unhealthy" | "expired" | "stopped" | "destroyed" | "error" SandboxInfo, SpawnOptions, ExecOptions, @@ -371,6 +382,9 @@ import { ProviderInfo, HealthInfo, VMPoolStatus, + SpawnAdmissionDecision, + QuotaSummary, + ForgevmClientConfig, ForgevmClientOptions, } from "stacyvm"; ``` diff --git a/sdk/js/src/client.ts b/sdk/js/src/client.ts index 81b92e8..176f60a 100644 --- a/sdk/js/src/client.ts +++ b/sdk/js/src/client.ts @@ -11,10 +11,13 @@ import { import { Sandbox } from "./sandbox.js"; import { TemplateManager } from "./templates.js"; import type { + ForgevmClientConfig, ForgevmClientOptions, HealthInfo, ProviderInfo, + QuotaSummary, SandboxInfo, + SpawnAdmissionDecision, SpawnOptions, VMPoolStatus, } from "./types.js"; @@ -112,8 +115,10 @@ export class Client { * }); * ``` */ - constructor(options?: ForgevmClientOptions) { - const opts = options ?? {}; + constructor(options?: ForgevmClientConfig) { + const opts: ForgevmClientOptions = typeof options === "string" + ? { baseUrl: options } + : options ?? {}; if (opts.baseUrl) { // Strip trailing slash for consistent URL construction. @@ -173,6 +178,8 @@ export class Client { if (opts?.memory_mb !== undefined) body["memory_mb"] = opts.memory_mb; if (opts?.vcpus !== undefined) body["vcpus"] = opts.vcpus; if (opts?.ttl) body["ttl"] = opts.ttl; + if (opts?.owner_id) body["owner_id"] = opts.owner_id; + if (opts?.template) body["template"] = opts.template; if (opts?.metadata) body["metadata"] = opts.metadata; const response = await this._fetch("/api/v1/sandboxes", { @@ -186,6 +193,31 @@ export class Client { return new Sandbox(this._baseUrl, this._headers, this._timeout, data); } + /** + * Preflight a spawn request against quota and scheduler limits. + * + * @param opts - Sandbox configuration to evaluate. + * @returns Admission decision without creating a sandbox. + */ + async admission(opts?: SpawnOptions): Promise { + const body: Record = {}; + if (opts?.image) body["image"] = opts.image; + if (opts?.provider) body["provider"] = opts.provider; + if (opts?.memory_mb !== undefined) body["memory_mb"] = opts.memory_mb; + if (opts?.vcpus !== undefined) body["vcpus"] = opts.vcpus; + if (opts?.ttl) body["ttl"] = opts.ttl; + if (opts?.owner_id) body["owner_id"] = opts.owner_id; + if (opts?.metadata) body["metadata"] = opts.metadata; + + const response = await this._fetch("/api/v1/sandboxes/admission", { + method: "POST", + body: JSON.stringify(body), + }); + + await handleResponse(response); + return (await response.json()) as SpawnAdmissionDecision; + } + /** * Retrieve an existing sandbox by its ID. * @@ -319,6 +351,18 @@ export class Client { return (await response.json()) as VMPoolStatus; } + /** + * Get redacted quota policy coverage counts. + */ + async quotaSummary(): Promise { + const response = await this._fetch("/api/v1/quotas/summary", { + method: "GET", + }); + + await handleResponse(response); + return (await response.json()) as QuotaSummary; + } + // ----------------------------------------------------------------------- // Convenience patterns // ----------------------------------------------------------------------- diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 6452266..f1ffe58 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -46,9 +46,12 @@ export type { TemplateConfig, TemplateSpawnOverrides, ProviderInfo, + QuotaSummary, HealthInfo, + SpawnAdmissionDecision, StreamChunk, FileInfo, + ForgevmClientConfig, ForgevmClientOptions, ApiErrorBody, } from "./types.js"; diff --git a/sdk/js/src/sandbox.ts b/sdk/js/src/sandbox.ts index abdcbd9..59f9eed 100644 --- a/sdk/js/src/sandbox.ts +++ b/sdk/js/src/sandbox.ts @@ -127,8 +127,8 @@ export class Sandbox { /** * Execute a command inside the sandbox and wait for the result. * - * @param command - The command string to execute (interpreted by the - * sandbox's shell). + * @param command - The command to execute. By default this is interpreted by + * the sandbox's shell; pass `mode: "argv"` to run it directly with args. * @param opts - Optional execution parameters. * @returns The execution result including exit code, stdout, stderr, and * duration. @@ -147,6 +147,7 @@ export class Sandbox { async exec(command: string, opts?: ExecOptions): Promise { const body: Record = { command }; if (opts?.args) body["args"] = opts.args; + if (opts?.mode) body["mode"] = opts.mode; if (opts?.env) body["env"] = opts.env; if (opts?.workdir) body["workdir"] = opts.workdir; if (opts?.timeout) body["timeout"] = opts.timeout; @@ -189,6 +190,7 @@ export class Sandbox { ): AsyncGenerator { const body: Record = { command, stream: true }; if (opts?.args) body["args"] = opts.args; + if (opts?.mode) body["mode"] = opts.mode; if (opts?.env) body["env"] = opts.env; if (opts?.workdir) body["workdir"] = opts.workdir; diff --git a/sdk/js/src/types.ts b/sdk/js/src/types.ts index 07bcbba..9ed71ce 100644 --- a/sdk/js/src/types.ts +++ b/sdk/js/src/types.ts @@ -13,6 +13,8 @@ * * - `creating` -- the sandbox is being provisioned * - `running` -- the sandbox is up and accepting commands + * - `unhealthy` -- the owning worker/runtime needs operator attention + * - `expired` -- the sandbox TTL elapsed before cleanup could complete * - `stopped` -- the sandbox has been gracefully stopped * - `destroyed` -- the sandbox has been torn down * - `error` -- the sandbox encountered a fatal error @@ -20,6 +22,8 @@ export type SandboxState = | "creating" | "running" + | "unhealthy" + | "expired" | "stopped" | "destroyed" | "error"; @@ -57,10 +61,36 @@ export interface SpawnOptions { vcpus?: number; /** Time-to-live duration string. */ ttl?: string; + /** Owner ID used for per-owner quotas when no X-User-ID header is set. */ + owner_id?: string; + /** Template name to apply when spawning this sandbox. */ + template?: string; /** Arbitrary key-value metadata. */ metadata?: Record; } +/** + * Admission result for a spawn preflight request. + */ +export interface SpawnAdmissionDecision { + /** Whether the request can be spawned immediately. */ + allowed: boolean; + /** Whether the request can wait in the spawn queue. */ + queueable: boolean; + /** Denial reason, when allowed is false. */ + reason?: string; + /** Current active sandbox count. */ + active_sandboxes: number; + /** Configured global sandbox limit. */ + max_sandboxes: number; + /** Current active sandbox count for the owner. */ + active_owner_sandboxes?: number; + /** Effective sandbox limit for the owner. */ + max_owner_sandboxes?: number; + /** Effective maximum TTL for the request. */ + max_ttl?: string; +} + /** * Full information about a sandbox as returned by list / get endpoints. */ @@ -97,6 +127,8 @@ export interface SandboxInfo { export interface ExecOptions { /** Positional arguments appended to the command. */ args?: string[]; + /** Execution mode. `shell` runs through /bin/sh -c; `argv` runs direct arguments. */ + mode?: "shell" | "argv"; /** Environment variables injected into the command. */ env?: Record; /** Working directory inside the sandbox. */ @@ -276,6 +308,11 @@ export interface ForgevmClientOptions { timeout?: number; } +/** + * Client constructor input: either a full base URL string or an options object. + */ +export type ForgevmClientConfig = string | ForgevmClientOptions; + /** * VM pool status information. */ @@ -287,6 +324,16 @@ export interface VMPoolStatus { max_users_per_vm: number; } +/** + * Redacted quota policy coverage counts. + */ +export interface QuotaSummary { + total: number; + with_max_sandboxes: number; + with_max_ttl: number; + with_max_exec_timeout: number; +} + // --------------------------------------------------------------------------- // API error envelope // --------------------------------------------------------------------------- diff --git a/sdk/js/test/client_parity.test.ts b/sdk/js/test/client_parity.test.ts new file mode 100644 index 0000000..71f43f2 --- /dev/null +++ b/sdk/js/test/client_parity.test.ts @@ -0,0 +1,144 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { Client } from "../src/client.js"; + +type RecordedRequest = { + url: string; + method: string; + headers: Record; + body?: unknown; +}; + +const originalFetch = globalThis.fetch; +const requests: RecordedRequest[] = []; + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +function installFetchStub(): void { + requests.length = 0; + globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + const headers = new Headers(init?.headers); + const body = typeof init?.body === "string" ? JSON.parse(init.body) : undefined; + requests.push({ + url, + method, + headers: Object.fromEntries(headers.entries()), + body, + }); + + if (url.endsWith("/api/v1/sandboxes") && method === "POST") { + return jsonResponse({ + id: "sb-parity", + state: "running", + provider: "mock", + image: body?.image ?? "alpine:latest", + memory_mb: body?.memory_mb ?? 512, + vcpus: body?.vcpus ?? 1, + created_at: "2026-05-08T00:00:00Z", + expires_at: "2026-05-08T00:30:00Z", + metadata: body?.metadata ?? {}, + preview_domain: "localhost", + }); + } + + if (url.endsWith("/api/v1/sandboxes/admission") && method === "POST") { + return jsonResponse({ + allowed: true, + queueable: false, + active_sandboxes: 1, + max_sandboxes: 100, + }); + } + + if (url.endsWith("/api/v1/providers") && method === "GET") { + return jsonResponse([{ name: "mock", healthy: true, default: true }]); + } + + if (url.endsWith("/api/v1/quotas/summary") && method === "GET") { + return jsonResponse({ + total: 1, + with_max_sandboxes: 1, + with_max_ttl: 0, + with_max_exec_timeout: 0, + }); + } + + return jsonResponse({ status: "ok", version: "test", uptime: "1s" }); + }; +} + +afterEach(() => { + globalThis.fetch = originalFetch; + requests.length = 0; +}); + +describe("Client public API parity", () => { + test("spawn sends the same control-plane fields as the Python SDK", async () => { + installFetchStub(); + const client = new Client({ + baseUrl: "http://stacyvm.test/", + apiKey: "api-key", + userId: "team-a", + }); + + const sandbox = await client.spawn({ + image: "python:3.12-slim", + provider: "mock", + memory_mb: 1024, + vcpus: 2, + ttl: "1h", + owner_id: "team-a", + template: "python-dev", + metadata: { purpose: "parity" }, + }); + + expect(sandbox.id).toBe("sb-parity"); + expect(requests[0]).toEqual({ + url: "http://stacyvm.test/api/v1/sandboxes", + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": "api-key", + "x-user-id": "team-a", + }, + body: { + image: "python:3.12-slim", + provider: "mock", + memory_mb: 1024, + vcpus: 2, + ttl: "1h", + owner_id: "team-a", + template: "python-dev", + metadata: { purpose: "parity" }, + }, + }); + }); + + test("exposes admission, providers, quota summary, and health helpers", async () => { + installFetchStub(); + const client = new Client("http://stacyvm.test"); + + await expect(client.admission({ image: "alpine", owner_id: "team-a" })).resolves.toMatchObject({ + allowed: true, + }); + await expect(client.providers()).resolves.toEqual([ + { name: "mock", healthy: true, default: true }, + ]); + await expect(client.quotaSummary()).resolves.toMatchObject({ total: 1 }); + await expect(client.health()).resolves.toMatchObject({ status: "ok" }); + + expect(requests.map((request) => request.url)).toEqual([ + "http://stacyvm.test/api/v1/sandboxes/admission", + "http://stacyvm.test/api/v1/providers", + "http://stacyvm.test/api/v1/quotas/summary", + "http://stacyvm.test/api/v1/health", + ]); + }); +}); diff --git a/sdk/python/README.md b/sdk/python/README.md index 7914255..ac02619 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -97,6 +97,7 @@ sandbox = client.spawn( memory_mb=1024, vcpus=2, ttl="1h", # "30s", "5m", "2h" — Go duration syntax + owner_id="team-a", # optional per-owner quota identity metadata={"user": "alice", "task": "data-analysis"}, ) ``` @@ -110,6 +111,7 @@ All parameters are optional. Server defaults apply when omitted. | `memory_mb` | `int \| None` | RAM in MB | | `vcpus` | `int \| None` | Virtual CPUs | | `ttl` | `str \| None` | Auto-destroy after this duration | +| `owner_id` | `str \| None` | Owner ID for per-owner quotas when no `user_id` header is set | | `template` | `str \| None` | Spawn from a server-side template by name | | `metadata` | `dict[str, str] \| None` | Free-form labels | @@ -119,6 +121,14 @@ Spawn from a template directly: sandbox = client.spawn_template("python-dev") ``` +Preflight quota and scheduler admission without creating a sandbox: + +```python +decision = client.admission(image="python:3.12", ttl="1h") +if not decision.allowed and decision.queueable: + print(f"Request would queue because {decision.reason}") +``` + --- ## Executing commands @@ -349,6 +359,7 @@ Behavioural notes: client.health() # {"status": "ok", "version": "0.5.1", "uptime": "2h13m"} client.list() # list[SandboxInfo] — all active sandboxes client.pool_status() # pool VM and user counts +client.quota_summary() # QuotaSummary — redacted owner quota policy counts client.prune() # int — count of expired sandboxes destroyed ``` @@ -401,7 +412,9 @@ from stacyvm import ( TemplateManager, # Models ExecResult, + QuotaSummary, SandboxInfo, + SpawnAdmissionDecision, StreamChunk, # Exceptions ForgevmError, diff --git a/sdk/python/stacyvm/__init__.py b/sdk/python/stacyvm/__init__.py index 9ae21c1..1081d1a 100644 --- a/sdk/python/stacyvm/__init__.py +++ b/sdk/python/stacyvm/__init__.py @@ -4,7 +4,13 @@ from stacyvm.sandbox import Sandbox from stacyvm.async_client import AsyncClient from stacyvm.async_sandbox import AsyncSandbox -from stacyvm.models import ExecResult, SandboxInfo, Template +from stacyvm.models import ( + ExecResult, + QuotaSummary, + SandboxInfo, + SpawnAdmissionDecision, + Template, +) from stacyvm.exceptions import ( ForgevmError, SandboxNotFound, @@ -19,7 +25,9 @@ "AsyncClient", "AsyncSandbox", "ExecResult", + "QuotaSummary", "SandboxInfo", + "SpawnAdmissionDecision", "Template", "ForgevmError", "SandboxNotFound", diff --git a/sdk/python/stacyvm/async_client.py b/sdk/python/stacyvm/async_client.py index e1d672d..a993393 100644 --- a/sdk/python/stacyvm/async_client.py +++ b/sdk/python/stacyvm/async_client.py @@ -6,7 +6,7 @@ from stacyvm.async_sandbox import AsyncSandbox from stacyvm.exceptions import ConnectionError, handle_response -from stacyvm.models import SandboxInfo +from stacyvm.models import QuotaSummary, SandboxInfo, SpawnAdmissionDecision class AsyncClient: @@ -46,6 +46,7 @@ async def spawn( memory_mb: int | None = None, vcpus: int | None = None, ttl: str | None = None, + owner_id: str | None = None, template: str | None = None, metadata: dict[str, str] | None = None, ) -> AsyncSandbox: @@ -59,6 +60,8 @@ async def spawn( body["vcpus"] = vcpus if ttl: body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id if template: body["template"] = template if metadata: @@ -73,6 +76,37 @@ async def spawn( data = resp.json() return AsyncSandbox(self._http, data["id"], info=data) + async def admission( + self, + image: str | None = None, + provider: str | None = None, + memory_mb: int | None = None, + vcpus: int | None = None, + ttl: str | None = None, + owner_id: str | None = None, + metadata: dict[str, str] | None = None, + ) -> SpawnAdmissionDecision: + """Preflight a spawn request without creating a sandbox.""" + body: dict = {} + if image: + body["image"] = image + if provider: + body["provider"] = provider + if memory_mb: + body["memory_mb"] = memory_mb + if vcpus: + body["vcpus"] = vcpus + if ttl: + body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id + if metadata: + body["metadata"] = metadata + + resp = await self._http.post("/api/v1/sandboxes/admission", json=body) + handle_response(resp) + return SpawnAdmissionDecision(**resp.json()) + async def spawn_template(self, template_name: str) -> AsyncSandbox: """Spawn a sandbox from a saved template.""" try: @@ -122,12 +156,24 @@ async def pool_status(self) -> dict: handle_response(resp) return resp.json() + async def quota_summary(self) -> QuotaSummary: + """Get redacted quota policy coverage counts.""" + resp = await self._http.get("/api/v1/quotas/summary") + handle_response(resp) + return QuotaSummary(**resp.json()) + async def health(self) -> dict: """Check server health.""" resp = await self._http.get("/api/v1/health") handle_response(resp) return resp.json() + async def providers(self) -> list[dict]: + """List registered providers and their health status.""" + resp = await self._http.get("/api/v1/providers") + handle_response(resp) + return resp.json() + async def close(self) -> None: """Close the HTTP client.""" await self._http.aclose() diff --git a/sdk/python/stacyvm/async_sandbox.py b/sdk/python/stacyvm/async_sandbox.py index ff1dce7..ba0800d 100644 --- a/sdk/python/stacyvm/async_sandbox.py +++ b/sdk/python/stacyvm/async_sandbox.py @@ -35,6 +35,7 @@ async def exec( self, command: str, args: list[str] | None = None, + mode: str | None = None, env: dict[str, str] | None = None, workdir: str | None = None, timeout: str | None = None, @@ -43,6 +44,8 @@ async def exec( body: dict = {"command": command} if args: body["args"] = args + if mode: + body["mode"] = mode if env: body["env"] = env if workdir: @@ -64,6 +67,7 @@ async def exec_stream( self, command: str, args: list[str] | None = None, + mode: str | None = None, env: dict[str, str] | None = None, workdir: str | None = None, ) -> AsyncIterator[StreamChunk]: @@ -73,6 +77,8 @@ async def exec_stream( body: dict = {"command": command, "stream": True} if args: body["args"] = args + if mode: + body["mode"] = mode if env: body["env"] = env if workdir: diff --git a/sdk/python/stacyvm/client.py b/sdk/python/stacyvm/client.py index 064dd2a..7c339ed 100644 --- a/sdk/python/stacyvm/client.py +++ b/sdk/python/stacyvm/client.py @@ -5,8 +5,9 @@ import httpx from stacyvm.exceptions import ConnectionError, handle_response -from stacyvm.models import SandboxInfo +from stacyvm.models import QuotaSummary, SandboxInfo, SpawnAdmissionDecision from stacyvm.sandbox import Sandbox +from stacyvm.templates import TemplateManager class Client: @@ -38,6 +39,7 @@ def __init__( headers=headers, timeout=timeout, ) + self.templates = TemplateManager(self._http) def spawn( self, @@ -46,6 +48,7 @@ def spawn( memory_mb: int | None = None, vcpus: int | None = None, ttl: str | None = None, + owner_id: str | None = None, template: str | None = None, metadata: dict[str, str] | None = None, ) -> Sandbox: @@ -59,6 +62,8 @@ def spawn( body["vcpus"] = vcpus if ttl: body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id if template: body["template"] = template if metadata: @@ -73,6 +78,38 @@ def spawn( data = resp.json() return Sandbox(self._http, data["id"], info=data) + def admission( + self, + image: str | None = None, + provider: str | None = None, + memory_mb: int | None = None, + vcpus: int | None = None, + ttl: str | None = None, + owner_id: str | None = None, + metadata: dict[str, str] | None = None, + ) -> SpawnAdmissionDecision: + """Preflight a spawn request without creating a sandbox.""" + body: dict = {} + if image: + body["image"] = image + if provider: + body["provider"] = provider + if memory_mb: + body["memory_mb"] = memory_mb + if vcpus: + body["vcpus"] = vcpus + if ttl: + body["ttl"] = ttl + if owner_id: + body["owner_id"] = owner_id + if metadata: + body["metadata"] = metadata + + resp = self._http.post("/api/v1/sandboxes/admission", json=body) + handle_response(resp) + data = resp.json() + return SpawnAdmissionDecision(**data) + def get(self, sandbox_id: str) -> Sandbox: """Get an existing sandbox by ID.""" resp = self._http.get(f"/api/v1/sandboxes/{sandbox_id}") @@ -122,12 +159,24 @@ def pool_status(self) -> dict: handle_response(resp) return resp.json() + def quota_summary(self) -> QuotaSummary: + """Get redacted quota policy coverage counts.""" + resp = self._http.get("/api/v1/quotas/summary") + handle_response(resp) + return QuotaSummary(**resp.json()) + def health(self) -> dict: """Check server health.""" resp = self._http.get("/api/v1/health") handle_response(resp) return resp.json() + def providers(self) -> list[dict]: + """List registered providers and their health status.""" + resp = self._http.get("/api/v1/providers") + handle_response(resp) + return resp.json() + def close(self) -> None: """Close the HTTP client.""" self._http.close() diff --git a/sdk/python/stacyvm/models.py b/sdk/python/stacyvm/models.py index 6c70abd..f438386 100644 --- a/sdk/python/stacyvm/models.py +++ b/sdk/python/stacyvm/models.py @@ -31,6 +31,30 @@ class SandboxInfo: preview_domain: str = "localhost" +@dataclass +class SpawnAdmissionDecision: + """Admission result for a spawn preflight request.""" + + allowed: bool + queueable: bool + reason: str = "" + active_sandboxes: int = 0 + max_sandboxes: int = 0 + active_owner_sandboxes: int = 0 + max_owner_sandboxes: int = 0 + max_ttl: str = "" + + +@dataclass +class QuotaSummary: + """Redacted quota policy coverage counts.""" + + total: int = 0 + with_max_sandboxes: int = 0 + with_max_ttl: int = 0 + with_max_exec_timeout: int = 0 + + @dataclass class Template: """Sandbox template configuration.""" diff --git a/sdk/python/stacyvm/sandbox.py b/sdk/python/stacyvm/sandbox.py index 5f601f3..fd70772 100644 --- a/sdk/python/stacyvm/sandbox.py +++ b/sdk/python/stacyvm/sandbox.py @@ -35,6 +35,7 @@ def exec( self, command: str, args: list[str] | None = None, + mode: str | None = None, env: dict[str, str] | None = None, workdir: str | None = None, timeout: str | None = None, @@ -43,6 +44,8 @@ def exec( body: dict = {"command": command} if args: body["args"] = args + if mode: + body["mode"] = mode if env: body["env"] = env if workdir: @@ -64,6 +67,7 @@ def exec_stream( self, command: str, args: list[str] | None = None, + mode: str | None = None, env: dict[str, str] | None = None, workdir: str | None = None, ) -> Iterator[StreamChunk]: @@ -71,6 +75,8 @@ def exec_stream( body: dict = {"command": command, "stream": True} if args: body["args"] = args + if mode: + body["mode"] = mode if env: body["env"] = env if workdir: diff --git a/sdk/python/tests/test_client_parity.py b/sdk/python/tests/test_client_parity.py new file mode 100644 index 0000000..eca3056 --- /dev/null +++ b/sdk/python/tests/test_client_parity.py @@ -0,0 +1,136 @@ +"""Mocked SDK parity checks that do not require a live StacyVM server.""" + +from __future__ import annotations + +import unittest + +import httpx + +from stacyvm import Client + + +class FakeHTTP: + def __init__(self): + self.requests: list[dict] = [] + self.headers = {} + + def post(self, path: str, json: dict | None = None): + self.requests.append({"method": "POST", "path": path, "json": json}) + if path == "/api/v1/sandboxes": + body = json or {} + return response( + { + "id": "sb-parity", + "state": "running", + "provider": body.get("provider", "mock"), + "image": body.get("image", "alpine:latest"), + "memory_mb": body.get("memory_mb", 512), + "vcpus": body.get("vcpus", 1), + "created_at": "2026-05-08T00:00:00Z", + "expires_at": "2026-05-08T00:30:00Z", + "metadata": body.get("metadata", {}), + "preview_domain": "localhost", + } + ) + if path == "/api/v1/sandboxes/admission": + return response( + { + "allowed": True, + "queueable": False, + "active_sandboxes": 1, + "max_sandboxes": 100, + } + ) + raise AssertionError(f"unexpected POST {path}") + + def get(self, path: str): + self.requests.append({"method": "GET", "path": path}) + if path == "/api/v1/providers": + return response([{"name": "mock", "healthy": True, "default": True}]) + if path == "/api/v1/quotas/summary": + return response( + { + "total": 1, + "with_max_sandboxes": 1, + "with_max_ttl": 0, + "with_max_exec_timeout": 0, + } + ) + if path == "/api/v1/health": + return response({"status": "ok", "version": "test", "uptime": "1s"}) + raise AssertionError(f"unexpected GET {path}") + + def close(self): + pass + + +def response(body): + request = httpx.Request("GET", "http://stacyvm.test") + return httpx.Response(200, json=body, request=request) + + +def client_with_fake_http() -> tuple[Client, FakeHTTP]: + client = Client("http://stacyvm.test", api_key="api-key", user_id="team-a") + fake = FakeHTTP() + client._http = fake + client.templates._http = fake + return client, fake + + +class ClientParityTests(unittest.TestCase): + def test_spawn_sends_control_plane_fields(self): + client, fake = client_with_fake_http() + + sandbox = client.spawn( + image="python:3.12-slim", + provider="mock", + memory_mb=1024, + vcpus=2, + ttl="1h", + owner_id="team-a", + template="python-dev", + metadata={"purpose": "parity"}, + ) + + self.assertEqual(sandbox.id, "sb-parity") + self.assertEqual( + fake.requests[0], + { + "method": "POST", + "path": "/api/v1/sandboxes", + "json": { + "image": "python:3.12-slim", + "provider": "mock", + "memory_mb": 1024, + "vcpus": 2, + "ttl": "1h", + "owner_id": "team-a", + "template": "python-dev", + "metadata": {"purpose": "parity"}, + }, + }, + ) + + def test_exposes_admission_providers_quota_summary_and_health_helpers(self): + client, fake = client_with_fake_http() + + self.assertTrue(client.admission(image="alpine", owner_id="team-a").allowed) + self.assertEqual( + client.providers(), [{"name": "mock", "healthy": True, "default": True}] + ) + self.assertEqual(client.quota_summary().total, 1) + self.assertEqual(client.health()["status"], "ok") + + self.assertEqual( + [request["path"] for request in fake.requests], + [ + "/api/v1/sandboxes/admission", + "/api/v1/providers", + "/api/v1/quotas/summary", + "/api/v1/health", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/style.css b/style.css new file mode 100644 index 0000000..35e6888 --- /dev/null +++ b/style.css @@ -0,0 +1,76 @@ +:root { + --stacy-orange: #ff7038; + --stacy-ink: #1c1c1c; + --stacy-panel: #262626; + --stacy-line: #3a3832; + --stacy-paper: #f0e7da; + --stacy-sand: #d1c2a5; + --stacy-muted: #d0c6b5; +} + +html.dark, +.dark body { + background: + linear-gradient(rgba(255, 255, 255, 0.025) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.025) 1px, transparent 1px), + #1c1c1c; + background-size: 56px 56px; +} + +body { + letter-spacing: 0; +} + +#navbar, +#sidebar, +#table-of-contents { + backdrop-filter: blur(18px); +} + +.dark #navbar, +.dark #sidebar, +.dark #table-of-contents { + background-color: rgba(28, 28, 28, 0.88); + border-color: var(--stacy-line); +} + +.dark .card, +.dark [class*="card"] { + border-color: rgba(209, 194, 165, 0.18); +} + +.stacy-proof-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 14px; + margin: 28px 0; +} + +.stacy-proof { + min-height: 132px; + padding: 18px; + border: 1px solid rgba(209, 194, 165, 0.28); + background: rgba(240, 231, 218, 0.08); +} + +.stacy-proof strong { + display: block; + color: var(--stacy-orange); + font-family: "IBM Plex Mono", monospace; + font-size: 12px; + text-transform: uppercase; +} + +.stacy-proof span { + display: block; + margin-top: 18px; + color: inherit; + font-size: 18px; + line-height: 1.35; +} + +@media (max-width: 900px) { + .stacy-proof-grid { + grid-template-columns: 1fr; + } +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 5a59c5e..014244c 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -6,10 +6,12 @@ import { FileCode2, Server, FlaskConical, + Gauge, Settings as SettingsIcon, Menu, X, Egg, + Building2, } from 'lucide-react'; import Dashboard from './pages/Dashboard'; import Sandboxes from './pages/Sandboxes'; @@ -17,6 +19,8 @@ import Templates from './pages/Templates'; import Providers from './pages/Providers'; import Settings from './pages/Settings'; import Environments from './pages/Environments'; +import Operations from './pages/Operations'; +import Tenants from './pages/Tenants'; import { ToastProvider } from './hooks/useToast'; import ToastContainer from './components/Toast'; @@ -26,6 +30,8 @@ const navItems = [ { to: '/templates', label: 'Templates', icon: FileCode2 }, { to: '/environments', label: 'Environments', icon: FlaskConical }, { to: '/providers', label: 'Providers', icon: Server }, + { to: '/tenants', label: 'Tenants', icon: Building2 }, + { to: '/operations', label: 'Operations', icon: Gauge }, { to: '/settings', label: 'Settings', icon: SettingsIcon }, ]; @@ -156,6 +162,8 @@ export default function App() { } /> } /> } /> + } /> + } /> } /> diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 2aa8891..ee719a1 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -74,8 +74,14 @@ export interface CreateTemplateRequest { export interface Provider { name: string; + default?: boolean; is_default: boolean; healthy: boolean; + latency_ms?: number; + last_checked?: string; + error?: string; + capabilities?: string[]; + runtime_count?: number; } export interface HealthResponse { @@ -87,8 +93,93 @@ export interface HealthResponse { export interface MetricsResponse { goroutines: number; memory_alloc: number; + memory_sys?: number; + memory_heap_alloc?: number; + gc_cycles?: number; active_sandboxes: number; total_sandboxes: number; + sandboxes?: { + total: number; + active: number; + by_state: Record; + by_provider: Record; + }; + providers?: { + total: number; + healthy: number; + items: Provider[]; + }; +} + +export interface OwnerQuota { + owner_id: string; + max_sandboxes: number; + max_ttl: string; + max_exec_timeout: string; + created_at?: string; + updated_at?: string; +} + +export interface OwnerUsage { + owner_id: string; + active_sandboxes: number; + max_sandboxes: number; + max_ttl: string; + max_exec_timeout: string; + quota_configured: boolean; +} + +export interface QuotaSummary { + total: number; + with_max_sandboxes: number; + with_max_ttl: number; + with_max_exec_timeout: number; +} + +export interface DiagnosticsResponse { + generated_at: string; + build: Record; + process: Record; + store: { + healthy?: boolean; + latency_ms?: number; + error?: string; + }; + limits: Record; + scheduler: Record; + quotas: QuotaSummary; + rate_limit: Record; + providers: Provider[]; + sandboxes: { + total: number; + active: number; + by_state: Record; + by_provider: Record; + }; + events: Record; + operations: Array>; + redactions: string[]; +} + +export interface AdminAuditRecord { + id: number; + actor: string; + method: string; + path: string; + status: number; + duration_ms: number; + request_id: string; + remote_addr: string; + user_agent: string; + created_at: string; +} + +export interface AdminAuditQuery { + limit?: number; + actor?: string; + method?: string; + status?: number; + path?: string; } export interface SSEEvent { @@ -189,6 +280,12 @@ export interface EnvironmentSuggestionsResponse { suggestions: string[]; } +interface StoredAppSettings { + authEnabled?: boolean; + authToken?: string; + adminToken?: string; +} + // --------------------------------------------------------------------------- // API Error // --------------------------------------------------------------------------- @@ -210,21 +307,72 @@ export class ApiError extends Error { const BASE = '/api/v1'; +interface RequestOptions extends RequestInit { + admin?: boolean; +} + +function loadStoredSettings(): StoredAppSettings { + if (typeof window === 'undefined') return {}; + + try { + const stored = window.localStorage.getItem('stacyvm-settings'); + return stored ? (JSON.parse(stored) as StoredAppSettings) : {}; + } catch { + return {}; + } +} + +function normalizeHeaders(headers?: HeadersInit): Record { + if (!headers) return {}; + if (headers instanceof Headers) return Object.fromEntries(headers.entries()); + if (Array.isArray(headers)) return Object.fromEntries(headers); + return { ...headers }; +} + +function authHeaders(admin: boolean): Record { + const settings = loadStoredSettings(); + if (!settings.authEnabled) return {}; + + const headers: Record = {}; + const apiKey = settings.authToken?.trim(); + const adminKey = settings.adminToken?.trim(); + + if (apiKey) { + headers['X-API-Key'] = apiKey; + } + if (admin && adminKey) { + headers['X-Admin-API-Key'] = adminKey; + } + + return headers; +} + +function normalizeProvider(provider: Provider): Provider { + const isDefault = provider.is_default ?? provider.default ?? false; + return { + ...provider, + default: provider.default ?? isDefault, + is_default: isDefault, + }; +} + async function request( path: string, - options: RequestInit = {}, + options: RequestOptions = {}, ): Promise { const url = `${BASE}${path}`; + const { admin = false, headers: optionHeaders, ...fetchOptions } = options; const headers: Record = { - ...(options.headers as Record), + ...authHeaders(admin), + ...normalizeHeaders(optionHeaders), }; - if (options.body && typeof options.body === 'string') { + if (fetchOptions.body && typeof fetchOptions.body === 'string') { headers['Content-Type'] = 'application/json'; } const res = await fetch(url, { - ...options, + ...fetchOptions, headers, }); @@ -317,7 +465,7 @@ export async function execStreamNDJSON( const url = `${BASE}/sandboxes/${encodeURIComponent(sandboxId)}/exec`; const res = await fetch(url, { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { ...authHeaders(false), 'Content-Type': 'application/json' }, body: JSON.stringify({ command, stream: true }), signal, }); @@ -475,8 +623,10 @@ export async function spawnFromTemplate( // --------------------------------------------------------------------------- export async function listProviders(): Promise { - const result = await request('/providers'); - return result ?? []; + const result = await request('/admin/providers', { + admin: true, + }); + return (result ?? []).map(normalizeProvider); } export interface ProviderDetail { @@ -484,11 +634,22 @@ export interface ProviderDetail { healthy: boolean; default: boolean; sandbox_count: number; + health?: Provider; config: Record; } export async function getProviderDetail(name: string): Promise { - return request(`/providers/${encodeURIComponent(name)}`); + return request( + `/admin/providers/${encodeURIComponent(name)}`, + { admin: true }, + ); +} + +export async function testProviders(): Promise> { + return request>('/admin/providers/test', { + method: 'POST', + admin: true, + }); } // --------------------------------------------------------------------------- @@ -598,7 +759,74 @@ export async function getHealth(): Promise { } export async function getMetrics(): Promise { - return request('/metrics'); + const metrics = await request('/admin/metrics', { admin: true }); + return { + ...metrics, + active_sandboxes: metrics.active_sandboxes ?? metrics.sandboxes?.active ?? 0, + total_sandboxes: metrics.total_sandboxes ?? metrics.sandboxes?.total ?? 0, + }; +} + +export async function getDiagnostics(): Promise { + return request('/admin/diagnostics', { admin: true }); +} + +export async function listOwnerQuotas(): Promise { + const result = await request('/admin/quotas', { admin: true }); + return result ?? []; +} + +export async function getQuotaSummary(): Promise { + return request('/admin/quotas/summary', { admin: true }); +} + +export async function saveOwnerQuota(quota: OwnerQuota): Promise { + return request( + `/admin/quotas/${encodeURIComponent(quota.owner_id)}`, + { + method: 'PUT', + body: JSON.stringify(quota), + admin: true, + }, + ); +} + +export async function deleteOwnerQuota(ownerId: string): Promise { + await request(`/admin/quotas/${encodeURIComponent(ownerId)}`, { + method: 'DELETE', + admin: true, + }); +} + +export async function getOwnerUsage(ownerId: string): Promise { + return request( + `/admin/quotas/${encodeURIComponent(ownerId)}/usage`, + { admin: true }, + ); +} + +function auditQueryParams(query: AdminAuditQuery): string { + const params = new URLSearchParams(); + params.set('limit', String(query.limit ?? 100)); + if (query.actor) params.set('actor', query.actor); + if (query.method) params.set('method', query.method); + if (query.status) params.set('status', String(query.status)); + if (query.path) params.set('path', query.path); + return params.toString(); +} + +export async function listAdminAudit(query: AdminAuditQuery = {}): Promise { + const result = await request( + `/admin/audit?${auditQueryParams(query)}`, + { admin: true }, + ); + return result ?? []; +} + +export async function exportAdminAuditCsv(query: AdminAuditQuery = {}): Promise { + const params = new URLSearchParams(auditQueryParams(query)); + params.set('format', 'csv'); + return request(`/admin/audit?${params.toString()}`, { admin: true }); } // --------------------------------------------------------------------------- diff --git a/web/src/pages/Operations.tsx b/web/src/pages/Operations.tsx new file mode 100644 index 0000000..afab7f7 --- /dev/null +++ b/web/src/pages/Operations.tsx @@ -0,0 +1,794 @@ +import { useEffect, useState } from 'react'; +import { + Activity, + AlertCircle, + CheckCircle2, + Database, + Gauge, + Loader2, + RefreshCw, + Save, + Search, + Shield, + Trash2, +} from 'lucide-react'; +import { + type AdminAuditRecord, + type AdminAuditQuery, + type DiagnosticsResponse, + type OwnerQuota, + type OwnerUsage, + type QuotaSummary, + deleteOwnerQuota, + exportAdminAuditCsv, + getDiagnostics, + getOwnerUsage, + getQuotaSummary, + listAdminAudit, + listOwnerQuotas, + saveOwnerQuota, +} from '../api/client'; +import { useToast } from '../hooks/useToast'; + +type OperationsTab = 'quotas' | 'diagnostics' | 'audit'; + +interface QuotaForm { + owner_id: string; + max_sandboxes: string; + max_ttl: string; + max_exec_timeout: string; +} + +const EMPTY_FORM: QuotaForm = { + owner_id: '', + max_sandboxes: '0', + max_ttl: '', + max_exec_timeout: '', +}; + +function formatUnknown(value: unknown): string { + if (value === null || value === undefined || value === '') return 'none'; + if (typeof value === 'object') return JSON.stringify(value); + return String(value); +} + +function adminErrorMessage(err: unknown, fallback: string): string { + const message = err instanceof Error ? err.message : fallback; + if (message.includes('401') || message.includes('403')) { + return `${message}. Check Settings and make sure the Admin API Key is configured.`; + } + return message; +} + +function toQuotaForm(quota: OwnerQuota): QuotaForm { + return { + owner_id: quota.owner_id, + max_sandboxes: String(quota.max_sandboxes || 0), + max_ttl: quota.max_ttl || '', + max_exec_timeout: quota.max_exec_timeout || '', + }; +} + +export default function Operations() { + const [activeTab, setActiveTab] = useState('quotas'); + + return ( +
+
+
+

Operations

+

+ Admin quota controls and redacted platform diagnostics +

+
+
+ setActiveTab('quotas')} + /> + setActiveTab('diagnostics')} + /> + setActiveTab('audit')} + /> +
+
+ + {activeTab === 'quotas' && } + {activeTab === 'diagnostics' && } + {activeTab === 'audit' && } +
+ ); +} + +function TabButton({ + label, + active, + onClick, +}: { + label: string; + active: boolean; + onClick: () => void; +}) { + return ( + + ); +} + +function QuotaPanel() { + const { addToast } = useToast(); + const [quotas, setQuotas] = useState([]); + const [summary, setSummary] = useState(null); + const [usage, setUsage] = useState(null); + const [form, setForm] = useState(EMPTY_FORM); + const [usageOwner, setUsageOwner] = useState(''); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [checkingUsage, setCheckingUsage] = useState(false); + const [error, setError] = useState(null); + + const refresh = async () => { + setLoading(true); + try { + const [quotaList, quotaSummary] = await Promise.all([ + listOwnerQuotas(), + getQuotaSummary(), + ]); + setQuotas(quotaList); + setSummary(quotaSummary); + setError(null); + } catch (err) { + setError(adminErrorMessage(err, 'Failed to load quotas')); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + refresh(); + }, []); + + const submitQuota = async () => { + const ownerId = form.owner_id.trim(); + if (!ownerId) { + addToast({ type: 'warning', title: 'Owner ID is required' }); + return; + } + + setSaving(true); + try { + await saveOwnerQuota({ + owner_id: ownerId, + max_sandboxes: Math.max(0, Number(form.max_sandboxes) || 0), + max_ttl: form.max_ttl.trim(), + max_exec_timeout: form.max_exec_timeout.trim(), + }); + addToast({ type: 'success', title: 'Quota saved', message: ownerId }); + setForm(EMPTY_FORM); + await refresh(); + } catch (err) { + addToast({ + type: 'error', + title: 'Quota save failed', + message: err instanceof Error ? err.message : 'Unknown error', + }); + } finally { + setSaving(false); + } + }; + + const removeQuota = async (ownerId: string) => { + try { + await deleteOwnerQuota(ownerId); + addToast({ type: 'success', title: 'Quota deleted', message: ownerId }); + await refresh(); + } catch (err) { + addToast({ + type: 'error', + title: 'Delete failed', + message: err instanceof Error ? err.message : 'Unknown error', + }); + } + }; + + const checkUsage = async (ownerId = usageOwner.trim()) => { + if (!ownerId) { + addToast({ type: 'warning', title: 'Owner ID is required' }); + return; + } + + setCheckingUsage(true); + try { + const result = await getOwnerUsage(ownerId); + setUsage(result); + setUsageOwner(ownerId); + } catch (err) { + addToast({ + type: 'error', + title: 'Usage check failed', + message: err instanceof Error ? err.message : 'Unknown error', + }); + } finally { + setCheckingUsage(false); + } + }; + + return ( +
+
+ + + + +
+ +
+
+
+
+

Owner Quotas

+

+ Persisted quota overrides for tenant owners +

+
+ +
+ + {loading ? ( +
+ +
+ ) : error ? ( +
+ +

{error}

+
+ ) : quotas.length === 0 ? ( +
+ +

No owner quota overrides

+
+ ) : ( +
+ + + + + + + + + + + + {quotas.map((quota) => ( + + + + + + + + ))} + +
OwnerSandboxesTTLExec TimeoutActions
+ {quota.owner_id} + {quota.max_sandboxes}{quota.max_ttl || 'none'} + {quota.max_exec_timeout || 'none'} + +
+ + + +
+
+
+ )} +
+ +
+
+

Save Quota

+ + setForm((prev) => ({ ...prev, owner_id: e.target.value }))} + placeholder="owner-a" + /> + + + + setForm((prev) => ({ ...prev, max_sandboxes: e.target.value })) + } + /> + + + setForm((prev) => ({ ...prev, max_ttl: e.target.value }))} + placeholder="30m" + /> + + + + setForm((prev) => ({ ...prev, max_exec_timeout: e.target.value })) + } + placeholder="30s" + /> + + +
+ +
+

Owner Usage

+
+ setUsageOwner(e.target.value)} + placeholder="owner-a" + /> + +
+ {usage && ( +
+ + + + +
+ )} +
+
+
+
+ ); +} + +function DiagnosticsPanel() { + const [diagnostics, setDiagnostics] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const refresh = async () => { + setLoading(true); + try { + const result = await getDiagnostics(); + setDiagnostics(result); + setError(null); + } catch (err) { + setError(adminErrorMessage(err, 'Failed to load diagnostics')); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + refresh(); + }, []); + + if (loading) { + return ( +
+ +
+ ); + } + + if (error || !diagnostics) { + return ( +
+ +

{error || 'Diagnostics unavailable'}

+ +
+ ); + } + + return ( +
+
+ +
+ +
+ + + + +
+ +
+ + + + +
+ +
+

Redactions

+
+ {diagnostics.redactions.map((item) => ( + + {item} + + ))} +
+
+
+ ); +} + +function AuditPanel() { + const { addToast } = useToast(); + const [records, setRecords] = useState([]); + const [filters, setFilters] = useState({ + limit: '100', + actor: '', + method: '', + status: '', + path: '', + }); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const currentQuery = (): AdminAuditQuery => ({ + limit: Math.max(1, Number(filters.limit) || 100), + actor: filters.actor.trim() || undefined, + method: filters.method || undefined, + status: filters.status ? Number(filters.status) : undefined, + path: filters.path.trim() || undefined, + }); + const hasActiveFilters = Boolean( + filters.actor.trim() || filters.method || filters.status || filters.path.trim(), + ); + + const refresh = async () => { + setLoading(true); + try { + const result = await listAdminAudit(currentQuery()); + setRecords(result); + setError(null); + } catch (err) { + setError(adminErrorMessage(err, 'Failed to load audit logs')); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + refresh(); + }, []); + + const exportAudit = async () => { + try { + const csv = await exportAdminAuditCsv(currentQuery()); + const url = URL.createObjectURL(new Blob([csv], { type: 'text/csv;charset=utf-8' })); + const link = document.createElement('a'); + link.href = url; + link.download = 'stacyvm-admin-audit.csv'; + link.click(); + URL.revokeObjectURL(url); + } catch (err) { + addToast({ + type: 'error', + title: 'Audit export failed', + message: adminErrorMessage(err, 'Unknown error'), + }); + } + }; + + const clearFilters = async () => { + setFilters({ + limit: '100', + actor: '', + method: '', + status: '', + path: '', + }); + setLoading(true); + try { + const result = await listAdminAudit({ limit: 100 }); + setRecords(result); + setError(null); + } catch (err) { + setError(adminErrorMessage(err, 'Failed to load audit logs')); + } finally { + setLoading(false); + } + }; + + return ( +
+
+
+

Admin Audit

+

+ Recent redacted admin route access records +

+
+
+ setFilters((prev) => ({ ...prev, limit: e.target.value }))} + title="Limit" + /> + setFilters((prev) => ({ ...prev, actor: e.target.value }))} + placeholder="actor" + /> + + setFilters((prev) => ({ ...prev, status: e.target.value }))} + placeholder="status" + /> + setFilters((prev) => ({ ...prev, path: e.target.value }))} + placeholder="path contains" + /> + + + +
+
+ + {loading ? ( +
+ +
+ ) : error ? ( +
+ +

{error}

+
+ ) : records.length === 0 ? ( +
+ +

+ {hasActiveFilters ? 'No audit records match these filters' : 'No admin audit records yet'} +

+ {hasActiveFilters && ( + + )} +
+ ) : ( +
+ + + + + + + + + + + + + {records.map((record) => ( + + + + + + + + + ))} + +
TimeActorRequestStatusDurationClient
+ {new Date(record.created_at).toLocaleString()} + + {record.actor || 'admin'} + +
+ {record.method} + + {record.path} + +
+ {record.request_id && ( +

+ {record.request_id} +

+ )} + 		+ + {record.status} + + + {record.duration_ms}ms + +

{record.remote_addr || 'n/a'}

+

+ {record.user_agent || 'unknown'} +

+
+
+ )} +
+ ); +} + +function SummaryCard({ + label, + value, + icon: Icon, +}: { + label: string; + value: string | number; + icon: typeof Shield; +}) { + return ( +
+
+ +
+
+

{value}

+

{label}

+
+
+ ); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( + + ); +} + +function UsageStat({ label, value }: { label: string; value: string | number }) { + return ( +
+

{label}

+

{value}

+
+ ); +} + +function DiagnosticsCard({ + title, + data, +}: { + title: string; + data: Record; +}) { + return ( +
+
+

{title}

+
+
+ {Object.entries(data).map(([key, value]) => ( +
+ {key} + + {formatUnknown(value)} + +
+ ))} +
+
+ ); +} diff --git a/web/src/pages/Providers.tsx b/web/src/pages/Providers.tsx index eb9af27..c725bb7 100644 --- a/web/src/pages/Providers.tsx +++ b/web/src/pages/Providers.tsx @@ -15,7 +15,13 @@ import { Settings, Hash, } from 'lucide-react'; -import { type Provider, type ProviderDetail, listProviders, getHealth, getProviderDetail } from '../api/client'; +import { + type Provider, + type ProviderDetail, + listProviders, + getProviderDetail, + testProviders, +} from '../api/client'; import { ProviderCardSkeleton } from '../components/Skeleton'; import { useToast } from '../hooks/useToast'; @@ -83,16 +89,15 @@ export default function Providers() { const handleTestConnection = async (providerName: string) => { setTestingProvider(providerName); try { - // Test via the health endpoint (which validates backend connectivity) - const health = await getHealth(); - const ok = health.status === 'ok'; + const results = await testProviders(); + const ok = results[providerName] ?? false; const message = ok - ? `Connected successfully (uptime: ${health.uptime})` - : `Health check returned: ${health.status}`; + ? 'Provider health check passed' + : 'Provider health check failed'; setTestResults((prev) => ({ ...prev, [providerName]: { ok, message } })); addToast({ type: ok ? 'success' : 'warning', - title: `${providerName}: ${ok ? 'Connected' : 'Unhealthy'}`, + title: `${providerName}: ${ok ? 'Healthy' : 'Unhealthy'}`, message, }); } catch (err) { @@ -277,6 +282,17 @@ function ProviderCard({

{info.description}

+
+ {provider.latency_ms !== undefined && ( + {provider.latency_ms}ms health + )} + {provider.runtime_count !== undefined && ( + {provider.runtime_count} runtime{provider.runtime_count !== 1 ? 's' : ''} + )} + {provider.error && ( + {provider.error} + )} +
{/* Test button + expand */} @@ -336,6 +352,13 @@ function ProviderCard({ {detail.sandbox_count} active sandbox{detail.sandbox_count !== 1 ? 'es' : ''} + {detail.health?.latency_ms !== undefined && ( +
+ + {detail.health.latency_ms}ms + health latency +
+ )} {/* Config table */} diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 16eb393..bed45ef 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -20,6 +20,7 @@ interface AppSettings { poolSize: number; authEnabled: boolean; authToken: string; + adminToken: string; serverPort: number; serverHost: string; theme: 'dark' | 'light' | 'system'; @@ -32,6 +33,7 @@ const DEFAULT_SETTINGS: AppSettings = { poolSize: 5, authEnabled: false, authToken: '', + adminToken: '', serverPort: 7423, serverHost: 'localhost', theme: 'dark', @@ -293,11 +295,11 @@ export default function Settings() { {activeSection === 'auth' && (