From 14f1712afdd6634497c2f15f05fd6a88839a752f Mon Sep 17 00:00:00 2001 From: Shubham-Rasal Date: Sun, 16 Aug 2026 01:19:58 +0530 Subject: [PATCH] feat: harden RL runtime and refresh dashboard Co-Authored-By: Claude Opus 4.8 --- .github/workflows/validate-slime.yml | 47 ++ README.md | 16 +- cmd/daemon/daemon.go | 17 +- cmd/daemon/daemon_test.go | 16 +- control-plane/api/rl_slime.go | 80 ++- control-plane/api/rl_slime_test.go | 43 ++ control-plane/contracts/v1.go | 27 +- control-plane/k8s/render.go | 91 ++- control-plane/k8s/render_test.go | 39 +- dashboard/app/benchmarks/page.tsx | 574 ------------------ dashboard/app/faas/page.tsx | 33 +- dashboard/app/globals.css | 143 +++-- dashboard/app/gpus/page.tsx | 44 -- dashboard/app/layout.tsx | 11 +- dashboard/app/login/page.tsx | 2 +- dashboard/app/page.tsx | 10 +- dashboard/app/settings/page.tsx | 2 +- dashboard/app/templates/page.tsx | 150 +++-- dashboard/components/Sidebar.tsx | 31 +- dashboard/components/SubmitJobDialog.tsx | 6 +- .../components/dashboard/dashboard-shell.tsx | 22 +- .../components/dashboard/page-header.tsx | 14 +- .../components/faas/container-detail.tsx | 6 +- dashboard/components/marketing/features.tsx | 41 +- dashboard/components/marketing/footer.tsx | 10 +- dashboard/components/marketing/hero.tsx | 16 +- dashboard/components/marketing/logo-mark.tsx | 12 +- dashboard/components/marketing/logo.tsx | 6 +- dashboard/components/marketing/nav.tsx | 45 +- dashboard/components/theme-provider.tsx | 17 + dashboard/components/theme-switcher.tsx | 54 ++ .../components/training/rl-dashboard.tsx | 20 +- .../training/rl-run-detail-panel.tsx | 25 +- .../components/training/rl-runs-panel.tsx | 57 +- dashboard/components/ui/avatar.tsx | 6 +- dashboard/components/ui/card.tsx | 12 +- dashboard/components/ui/dialog.tsx | 12 +- dashboard/components/ui/input.tsx | 2 +- dashboard/components/ui/label.tsx | 2 +- dashboard/components/ui/scroll-area.tsx | 8 +- dashboard/components/ui/select.tsx | 14 +- dashboard/components/ui/sheet.tsx | 20 +- dashboard/components/ui/table.tsx | 12 +- dashboard/components/ui/tabs.tsx | 4 +- dashboard/components/ui/textarea.tsx | 2 +- dashboard/components/ui/tooltip.tsx | 16 +- dashboard/lib/api.ts | 28 +- dashboard/package-lock.json | 11 + dashboard/package.json | 1 + deploy/k8s/control-plane.yaml | 15 + .../devlog-modal-sdk.md | 0 perf_notes.md => docs/perf_notes.md | 0 docs/slime-kuberay-production.md | 23 +- todo.md => docs/todo.md | 0 scripts/validation/aws_remote_e2e_train.sh | 6 +- scripts/validation/validate_slime_local.sh | 5 +- training/slime/configs/async_qwen3_0_6b.sh | 12 +- .../slime/configs/disaggregated_qwen3_0_6b.sh | 12 +- training/slime/configs/one_gpu_qwen3_0_6b.sh | 27 +- .../slime/configs/qwen3_0_6b_model_args.sh | 20 + training/slime/skyscale/adapters.py | 93 +++ training/slime/skyscale/reporter.py | 89 +++ training/slime/tests/test_adapters.py | 47 +- training/slime/tests/test_reporter.py | 50 ++ 64 files changed, 1288 insertions(+), 988 deletions(-) create mode 100644 .github/workflows/validate-slime.yml create mode 100644 control-plane/api/rl_slime_test.go delete mode 100644 dashboard/app/benchmarks/page.tsx delete mode 100644 dashboard/app/gpus/page.tsx create mode 100644 dashboard/components/theme-provider.tsx create mode 100644 dashboard/components/theme-switcher.tsx rename devlog-modal-sdk.md => docs/devlog-modal-sdk.md (100%) rename perf_notes.md => docs/perf_notes.md (100%) rename todo.md => docs/todo.md (100%) create mode 100644 training/slime/configs/qwen3_0_6b_model_args.sh create mode 100644 training/slime/skyscale/reporter.py create mode 100644 training/slime/tests/test_reporter.py diff --git a/.github/workflows/validate-slime.yml b/.github/workflows/validate-slime.yml new file mode 100644 index 0000000..7922291 --- /dev/null +++ b/.github/workflows/validate-slime.yml @@ -0,0 +1,47 @@ +name: Validate Slime Integration + +on: + pull_request: + paths: + - "control-plane/**" + - "training/slime/**" + - "deploy/k8s/**" + - "dashboard/components/training/**" + - "dashboard/lib/api.ts" + - "dashboard/package*.json" + - "scripts/validation/validate_slime_local.sh" + - ".github/workflows/validate-slime.yml" + push: + paths: + - "control-plane/**" + - "training/slime/**" + - "deploy/k8s/**" + - "dashboard/components/training/**" + - "dashboard/lib/api.ts" + - "dashboard/package*.json" + - "scripts/validation/validate_slime_local.sh" + - ".github/workflows/validate-slime.yml" + workflow_dispatch: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: control-plane/go.mod + cache-dependency-path: control-plane/go.sum + - uses: azure/setup-kubectl@v4 + with: + version: latest + - name: Validate control plane, runtime, and manifests + run: bash scripts/validation/validate_slime_local.sh + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: dashboard/package-lock.json + - name: Build dashboard + working-directory: dashboard + run: npm ci && npm run build diff --git a/README.md b/README.md index e0c3f2e..c657097 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Reinforcement Learning as a Service.** Post-train any LLM on any task using distributed async RL — without managing clusters, scheduling workers, or provisioning GPUs. One API call starts the whole pipeline. -> **Currently supports:** distributed GRPO RL runs (`POST /api/rl/runs`) with Modal GPU sandboxes (vLLM policy server, CPU rollout workers, GRPO trainer), Firecracker microVMs as code-execution RL environments (`reset` / `step` / `close`), an async experience buffer, live metrics in the Next.js dashboard, and Prometheus + Grafana observability. Also: Firecracker sandboxes, Railway-backed container sandboxes, on-demand GPU job submission (Modal / Akash / HuggingFace), and sign-in-gated GPU spend when `SKYSCALE_DASHBOARD_TOKEN` is set. +> **Currently supports:** first-class slime RL runs on KubeRay and the legacy distributed GRPO path on Modal. Slime runs use SGLang rollouts, Megatron training, sandbox rewards, grouped-sample persistence, checkpoint reporting, and immutable run contracts. The dashboard submits and monitors either backend through `POST /api/rl/runs`. > > **Not yet:** closed-loop policy weight hot-swap on Modal, multi-turn episodes, custom problem-set uploads, and permissionless workers. See [What's not built yet](#whats-not-built-yet). @@ -199,8 +199,6 @@ The Next.js dashboard (`dashboard/`) is the primary operator UI: | **Training** | `/` | RL runs list, live reward/loss charts, run detail drawer (logs, metadata, events), start/stop runs | | **Templates** | `/templates` | Job templates for quick submission | | **Sandboxes** | `/faas` | Deploy and manage isolated container sandboxes (Railway-backed) | -| **On-Demand GPUs** | `/gpus` | GPU inventory and job queue | -| **Load Speed** | `/benchmarks` | FaaS cold-start and throughput benchmarks | RL run detail includes stage, policy URL, buffer size, worker/trainer status, in-app Recharts metrics, activity log streaming, and an optional Grafana link. @@ -223,13 +221,15 @@ HF_TOKEN= python3 scripts/modal_pipeline_test.py It runs through all four stages — policy server health, RL run creation, buffer fill from 2 workers, and 3 GRPO training steps — and prints a pass/fail report with final metrics. -**Load testing sandboxes:** +### Slime on KubeRay + +Set `SKYSCALE_RL_KUBERNETES=1`, configure the pinned runtime images and runtime token, prepare the model PVC, then choose **Slime on KubeRay** in the dashboard. The control plane snapshots the run contract, creates a RayJob and rollout services, records grouped samples and optimizer progress, and resumes retries from the latest checkpoint on the model PVC. ```bash -k6 run perf/faas_load_test.js +bash scripts/validation/validate_slime_local.sh ``` -Set `API_URL` to your control plane origin. See `perf/faas_load_test.js` for VUs and duration defaults. +Production prerequisites and manifests are documented in [`docs/slime-kuberay-production.md`](docs/slime-kuberay-production.md). The numerical one-GPU validation path is `AWS_SLIME_E2E=1 bash scripts/validation/aws_slime_gpu.sh`. --- @@ -406,6 +406,10 @@ This design focused on `skyscale deploy` / `skyscale invoke` with a warm VM pool | `HF_TOKEN` | HuggingFace token for model downloads | | `ARTIFACT_LOCAL_DIR` | Local checkpoint fallback when S3 is not configured | | `S3_ENDPOINT` / `S3_BUCKET` / `S3_ACCESS_KEY` / `S3_SECRET_KEY` | S3-compatible artifact store (optional) | +| `SKYSCALE_RL_KUBERNETES` | Enable the slime KubeRay reconciler (`1`) | +| `SKYSCALE_SLIME_IMAGE` / `SKYSCALE_SGLANG_IMAGE` | Immutable runtime image references | +| `SKYSCALE_MODEL_PVC` / `SKYSCALE_MODEL_MOUNT_PATH` | Prepared Hugging Face and Megatron model artifact volume | +| `SKYSCALE_RUNTIME_TOKEN` | Shared bearer token for slime runtime callbacks | | `FAAS_VM_KERNEL_PATH` | Firecracker kernel path (auto-downloaded if absent) | | `FAAS_VM_ROOTFS_PATH` | VM rootfs path (auto-downloaded if absent) | | `FAAS_VM_MEMORY_MB` | Memory per VM in MB (default `128`) | diff --git a/cmd/daemon/daemon.go b/cmd/daemon/daemon.go index 9f5a9a2..5aa36ce 100644 --- a/cmd/daemon/daemon.go +++ b/cmd/daemon/daemon.go @@ -21,7 +21,6 @@ const ( daemonPort = "8081" // Port for the daemon to listen on codeDir = "/tmp/faas/code" logDir = "/var/log/faas" - sandboxWorkspace = "/sandbox/workspace" // Persistent workspace for sandbox sessions // Endpoints functionEndpoint = "/api/functions" @@ -70,12 +69,22 @@ type VMInfo struct { var vmInfo VMInfo var httpClient *http.Client var controlPlaneURL string +var sandboxWorkspace = envOrDefault("SANDBOX_WORKSPACE", "/sandbox/workspace") + +func envOrDefault(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} func init() { // Create necessary directories - os.MkdirAll(codeDir, 0755) - os.MkdirAll(logDir, 0755) - os.MkdirAll(sandboxWorkspace, 0755) + for _, directory := range []string{codeDir, logDir, sandboxWorkspace} { + if err := os.MkdirAll(directory, 0755); err != nil { + log.Printf("Failed to create directory %s: %v", directory, err) + } + } // Read control plane URL from env, fall back to Firecracker CNI gateway controlPlaneURL = os.Getenv("CONTROL_PLANE_URL") diff --git a/cmd/daemon/daemon_test.go b/cmd/daemon/daemon_test.go index c340f0c..67f8ddb 100644 --- a/cmd/daemon/daemon_test.go +++ b/cmd/daemon/daemon_test.go @@ -14,6 +14,11 @@ import ( func setupWorkspace(t *testing.T) string { t.Helper() dir := t.TempDir() + original := sandboxWorkspace + sandboxWorkspace = dir + t.Cleanup(func() { + sandboxWorkspace = original + }) return dir } @@ -21,10 +26,6 @@ func setupWorkspace(t *testing.T) string { func TestSandboxExecPython(t *testing.T) { workspace := setupWorkspace(t) - origWS := sandboxWorkspace - // Can't reassign a const; use a package-var approach via the test build. - // We test the handler directly, overriding the global via init in the test binary. - _ = origWS _ = workspace body, _ := json.Marshal(sandboxExecRequest{ @@ -57,6 +58,7 @@ func TestSandboxExecPython(t *testing.T) { } func TestSandboxExecBash(t *testing.T) { + setupWorkspace(t) body, _ := json.Marshal(sandboxExecRequest{ ExecID: "test-bash", Code: "echo hello_bash", @@ -84,6 +86,7 @@ func TestSandboxExecBash(t *testing.T) { } func TestSandboxExecFilePersistence(t *testing.T) { + setupWorkspace(t) // Write a file in one exec, read it back in a second exec. // Both execs use sandboxWorkspace as cwd so the file is accessible. writeBody, _ := json.Marshal(sandboxExecRequest{ @@ -130,6 +133,7 @@ func TestSandboxExecFilePersistence(t *testing.T) { } func TestSandboxExecTimeout(t *testing.T) { + setupWorkspace(t) body, _ := json.Marshal(sandboxExecRequest{ ExecID: "timeout-exec", Code: "import time; time.sleep(60)", @@ -151,6 +155,7 @@ func TestSandboxExecTimeout(t *testing.T) { } func TestSandboxExecUnsupportedLanguage(t *testing.T) { + setupWorkspace(t) body, _ := json.Marshal(sandboxExecRequest{ Code: "console.log('hi')", Language: "javascript", @@ -167,6 +172,7 @@ func TestSandboxExecUnsupportedLanguage(t *testing.T) { // ─── /sandbox/files/ tests ─────────────────────────────────────────────────── func TestFileUploadDownload(t *testing.T) { + setupWorkspace(t) content := []byte("hello sandbox file") // Upload @@ -193,6 +199,7 @@ func TestFileUploadDownload(t *testing.T) { } func TestFileDownloadMissing(t *testing.T) { + setupWorkspace(t) req := httptest.NewRequest(http.MethodGet, "/sandbox/files/nonexistent_file.txt", nil) rr := httptest.NewRecorder() handleSandboxFile(rr, req) @@ -202,6 +209,7 @@ func TestFileDownloadMissing(t *testing.T) { } func TestFileMethodNotAllowed(t *testing.T) { + setupWorkspace(t) req := httptest.NewRequest(http.MethodDelete, "/sandbox/files/somefile.txt", nil) rr := httptest.NewRecorder() handleSandboxFile(rr, req) diff --git a/control-plane/api/rl_slime.go b/control-plane/api/rl_slime.go index 1be2610..45ecd2c 100644 --- a/control-plane/api/rl_slime.go +++ b/control-plane/api/rl_slime.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/bluequbit/faas/control-plane/contracts" @@ -15,6 +16,13 @@ import ( "github.com/gorilla/mux" ) +type slimeRunPreset struct { + BaseModel string `json:"base_model"` + NumWorkers int `json:"num_workers"` + GPUModel string `json:"gpu_model"` + ProblemSet string `json:"problem_set"` +} + func (h *APIHandler) rlStartSlimeRunHandler(w http.ResponseWriter, r *http.Request, raw []byte) { if h.rlReconciler == nil { http.Error(w, "slime backend requires SKYSCALE_RL_KUBERNETES=1 and Kubernetes credentials", http.StatusServiceUnavailable) @@ -30,9 +38,14 @@ func (h *APIHandler) rlStartSlimeRunHandler(w http.ResponseWriter, r *http.Reque } if wrapper.Spec != nil { spec = *wrapper.Spec - } else if err := json.Unmarshal(raw, &spec); err != nil { - http.Error(w, "invalid run contract", http.StatusBadRequest) - return + } else { + var preset slimeRunPreset + if err := json.Unmarshal(raw, &preset); err == nil && preset.BaseModel != "" { + spec = slimeSpecFromPreset(preset, r) + } else if err := json.Unmarshal(raw, &spec); err != nil { + http.Error(w, "invalid run contract", http.StatusBadRequest) + return + } } spec.Normalize() spec.Backend = "slime" @@ -88,6 +101,67 @@ func (h *APIHandler) rlStartSlimeRunHandler(w http.ResponseWriter, r *http.Reque }) } +func slimeSpecFromPreset(preset slimeRunPreset, r *http.Request) contracts.RLRunSpec { + spec := contracts.DefaultRunSpec() + spec.Metadata.TenantID = headerEnvOrDefault(r, "X-Skyscale-Tenant", "SKYSCALE_DEFAULT_TENANT", "default") + spec.Metadata.ProjectID = headerEnvOrDefault(r, "X-Skyscale-Project", "SKYSCALE_DEFAULT_PROJECT", "default") + spec.Model.Source = preset.BaseModel + spec.Model.Revision = envOrDefault("SKYSCALE_MODEL_REVISION", "main") + spec.Model.VolumeClaim = envOrDefault("SKYSCALE_MODEL_PVC", "qwen3-0-6b-models") + spec.Model.MountPath = envOrDefault("SKYSCALE_MODEL_MOUNT_PATH", "/models") + if preset.ProblemSet != "" { + spec.Data.SourceURI = "skyscale://problems/" + preset.ProblemSet + } + if preset.GPUModel != "" { + spec.Topology.Trainer.Resources.GPUType = preset.GPUModel + spec.Topology.Rollout.Resources.GPUType = preset.GPUModel + } + if preset.NumWorkers > 1 { + spec.Topology.Mode = "disaggregated" + spec.Topology.Rollout.External = true + spec.Topology.Rollout.Replicas = preset.NumWorkers + spec.Topology.Rollout.MinReplicas = preset.NumWorkers + spec.Topology.Rollout.MaxReplicas = preset.NumWorkers + spec.Topology.Rollout.Resources.GPUs = 1 + } else { + spec.Topology.Mode = "colocated" + spec.Topology.Rollout.External = false + spec.Topology.Rollout.Replicas = 1 + spec.Topology.Rollout.MinReplicas = 1 + spec.Topology.Rollout.MaxReplicas = 1 + } + spec.Image.Slime = envOrDefault("SKYSCALE_SLIME_IMAGE", spec.Image.Slime) + spec.Image.SGLang = envOrDefault("SKYSCALE_SGLANG_IMAGE", spec.Image.Slime) + spec.Image.Digest = os.Getenv("SKYSCALE_SLIME_IMAGE_DIGEST") + spec.Security.ImageAllowlist = splitNonEmpty(envOrDefault("SKYSCALE_RL_IMAGE_ALLOWLIST", "ghcr.io/skyscale/")) + spec.Security.SecretRefs = splitNonEmpty(envOrDefault("SKYSCALE_RUNTIME_SECRET", "skyscale-runtime")) + return spec +} + +func headerEnvOrDefault(r *http.Request, header, env, fallback string) string { + if value := r.Header.Get(header); value != "" { + return value + } + return envOrDefault(env, fallback) +} + +func envOrDefault(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func splitNonEmpty(value string) []string { + var values []string + for _, item := range strings.Split(value, ",") { + if item = strings.TrimSpace(item); item != "" { + values = append(values, item) + } + } + return values +} + func envInt(name string, fallback int) int { raw := os.Getenv(name) if raw == "" { diff --git a/control-plane/api/rl_slime_test.go b/control-plane/api/rl_slime_test.go new file mode 100644 index 0000000..fff0d71 --- /dev/null +++ b/control-plane/api/rl_slime_test.go @@ -0,0 +1,43 @@ +package api + +import ( + "net/http/httptest" + "testing" +) + +func TestSlimeSpecFromPresetBuildsProductionContract(t *testing.T) { + t.Setenv("SKYSCALE_SLIME_IMAGE", "ghcr.io/skyscale/slime-runtime") + t.Setenv("SKYSCALE_SLIME_IMAGE_DIGEST", "sha256:deadbeef") + t.Setenv("SKYSCALE_SGLANG_IMAGE", "ghcr.io/skyscale/sglang@sha256:cafe") + request := httptest.NewRequest("POST", "/api/rl/runs", nil) + request.Header.Set("X-Skyscale-Tenant", "tenant-a") + request.Header.Set("X-Skyscale-Project", "project-a") + + spec := slimeSpecFromPreset(slimeRunPreset{ + BaseModel: "Qwen/Qwen3-0.6B", NumWorkers: 2, GPUModel: "l4", ProblemSet: "default", + }, request) + + if spec.Metadata.TenantID != "tenant-a" || spec.Metadata.ProjectID != "project-a" { + t.Fatalf("unexpected tenancy: %#v", spec.Metadata) + } + if spec.Topology.Mode != "disaggregated" || !spec.Topology.Rollout.External || spec.Topology.Rollout.Replicas != 2 { + t.Fatalf("unexpected rollout topology: %#v", spec.Topology) + } + if spec.Model.VolumeClaim != "qwen3-0-6b-models" || spec.Image.Digest != "sha256:deadbeef" { + t.Fatalf("runtime artifacts are not configured: model=%#v image=%#v", spec.Model, spec.Image) + } + if err := spec.Validate(); err != nil { + t.Fatalf("generated preset must be valid: %v", err) + } +} + +func TestSlimeSingleWorkerPresetUsesOneGPUColocated(t *testing.T) { + t.Setenv("SKYSCALE_SLIME_IMAGE", "ghcr.io/skyscale/slime-runtime@sha256:deadbeef") + request := httptest.NewRequest("POST", "/api/rl/runs", nil) + spec := slimeSpecFromPreset(slimeRunPreset{ + BaseModel: "Qwen/Qwen3-0.6B", NumWorkers: 1, GPUModel: "l4", + }, request) + if spec.Topology.Mode != "colocated" || spec.Topology.Rollout.External { + t.Fatalf("single-GPU preset must colocate trainer and rollout: %#v", spec.Topology) + } +} diff --git a/control-plane/contracts/v1.go b/control-plane/contracts/v1.go index d0b849f..80dcdec 100644 --- a/control-plane/contracts/v1.go +++ b/control-plane/contracts/v1.go @@ -33,6 +33,8 @@ type ModelSpec struct { MegatronFormat string `json:"megatron_format,omitempty"` TokenizerRevision string `json:"tokenizer_revision,omitempty"` ArtifactManifestURI string `json:"artifact_manifest_uri,omitempty"` + VolumeClaim string `json:"volume_claim,omitempty"` + MountPath string `json:"mount_path,omitempty"` } type DataSpec struct { @@ -171,13 +173,16 @@ func DefaultRunSpec() RLRunSpec { APIVersion: APIVersion, Kind: "RLRun", Backend: "slime", - Model: ModelSpec{Source: "Qwen/Qwen3-0.6B", Revision: "main", MegatronFormat: "torch_dist"}, - Data: DataSpec{SourceURI: "skyscale://problems/default", Revision: "v1", EnvironmentVersion: "code-v1", Seed: 42}, - Reward: RewardSpec{Name: "sandbox-tests", Version: "v1", MaxEvaluationSec: 120}, - Algorithm: AlgorithmSpec{Name: "grpo", Strategy: "synchronous", MaxPolicyLagSteps: 0, MaxQueueAgeSeconds: 900, OffPolicyAction: "discard"}, + Model: ModelSpec{ + Source: "Qwen/Qwen3-0.6B", Revision: "main", MegatronFormat: "torch_dist", + VolumeClaim: "qwen3-0-6b-models", MountPath: "/models", + }, + Data: DataSpec{SourceURI: "skyscale://problems/default", Revision: "v1", EnvironmentVersion: "code-v1", Seed: 42}, + Reward: RewardSpec{Name: "sandbox-tests", Version: "v1", MaxEvaluationSec: 120}, + Algorithm: AlgorithmSpec{Name: "grpo", Strategy: "synchronous", MaxPolicyLagSteps: 0, MaxQueueAgeSeconds: 900, OffPolicyAction: "discard"}, Topology: TopologySpec{ Mode: "colocated", - Trainer: TrainerTopology{Nodes: 1, TP: 1, PP: 1, CP: 1, Resources: ResourceSpec{CPU: "8", Memory: "32Gi", GPUs: 1}}, + Trainer: TrainerTopology{Nodes: 1, TP: 1, PP: 1, CP: 1, Resources: ResourceSpec{CPU: "6", Memory: "24Gi", GPUs: 1}}, Rollout: RolloutTopology{Replicas: 1, MinReplicas: 1, MaxReplicas: 1, TP: 1, DP: 1, MemoryFraction: 0.35, Resources: ResourceSpec{GPUs: 1}}, }, Checkpoint: CheckpointPolicy{EverySteps: 1, KeepLast: 3, ServingFormat: "full", MinimumAcks: 1, PublishTimeoutSec: 600}, @@ -224,6 +229,15 @@ func (s RLRunSpec) Validate() error { if s.Model.Source == "" || s.Model.Revision == "" { problems = append(problems, "model source and immutable revision are required") } + if s.Model.VolumeClaim != "" && !dnsLabel.MatchString(s.Model.VolumeClaim) { + problems = append(problems, "model volume_claim must be a DNS label") + } + if s.Model.MountPath != "" && !strings.HasPrefix(s.Model.MountPath, "/") { + problems = append(problems, "model mount_path must be absolute") + } + if (s.Model.VolumeClaim == "") != (s.Model.MountPath == "") { + problems = append(problems, "model volume_claim and mount_path must be set together") + } algorithms := map[string]bool{"grpo": true, "gspo": true, "cispo": true, "reinforce++": true, "ppo": true, "sft": true, "custom": true} if !algorithms[strings.ToLower(s.Algorithm.Name)] { problems = append(problems, "unsupported algorithm") @@ -258,6 +272,9 @@ func (s RLRunSpec) Validate() error { if s.Topology.Mode == "colocated" && r.External { problems = append(problems, "colocated rollout cannot be external") } + if s.Algorithm.Strategy == "asynchronous" && s.Topology.Mode != "disaggregated" { + problems = append(problems, "asynchronous strategy requires disaggregated topology") + } if s.Retention.LowWatermarkGroups < 0 || s.Retention.HighWatermarkGroups <= s.Retention.LowWatermarkGroups { problems = append(problems, "high watermark must exceed low watermark") } diff --git a/control-plane/k8s/render.go b/control-plane/k8s/render.go index 6cee6c2..8cf412f 100644 --- a/control-plane/k8s/render.go +++ b/control-plane/k8s/render.go @@ -67,16 +67,18 @@ func image(spec contracts.RLRunSpec) string { func slimeArgs(spec contracts.RLRunSpec) []any { args := []string{ - "python", strategyEntrypoint(spec.Algorithm.Strategy), - "--model", spec.Model.Source, - "--tensor-model-parallel-size", strconv.Itoa(spec.Topology.Trainer.TP), - "--pipeline-model-parallel-size", strconv.Itoa(spec.Topology.Trainer.PP), - "--context-parallel-size", strconv.Itoa(spec.Topology.Trainer.CP), + "env", "MODEL_ROOT=" + modelMountPath(spec), } if spec.Topology.Mode == "disaggregated" { address := fmt.Sprintf("%s-rollout.%s.svc.cluster.local:8000", spec.Metadata.RunID, NamespaceFor(spec.Metadata.TenantID, spec.Metadata.ProjectID)) - args = append(args, "--rollout-external-engine-addrs", address) + args = append(args, "ROLLOUT_EXTERNAL_ENGINE_ADDRS="+address) } + args = append(args, + "bash", runtimeProfile(spec), + "--tensor-model-parallel-size", strconv.Itoa(spec.Topology.Trainer.TP), + "--pipeline-model-parallel-size", strconv.Itoa(spec.Topology.Trainer.PP), + "--context-parallel-size", strconv.Itoa(spec.Topology.Trainer.CP), + ) if spec.Checkpoint.ResumeFrom != "" { args = append(args, "--load", spec.Checkpoint.ResumeFrom) } @@ -98,11 +100,28 @@ func slimeArgs(spec contracts.RLRunSpec) []any { return out } -func strategyEntrypoint(strategy string) string { - if strategy == "synchronous" { - return "train.py" +func runtimeProfile(spec contracts.RLRunSpec) string { + if spec.Algorithm.Strategy == "asynchronous" { + return "/opt/skyscale/slime/configs/async_qwen3_0_6b.sh" + } + if spec.Topology.Mode == "disaggregated" { + return "/opt/skyscale/slime/configs/disaggregated_qwen3_0_6b.sh" + } + return "/opt/skyscale/slime/configs/one_gpu_qwen3_0_6b.sh" +} + +func modelMountPath(spec contracts.RLRunSpec) string { + if spec.Model.MountPath != "" { + return spec.Model.MountPath } - return "train_async.py" + return "/models" +} + +func modelRuntimePath(spec contracts.RLRunSpec) string { + if spec.Model.VolumeClaim != "" { + return modelMountPath(spec) + "/hf" + } + return spec.Model.Source } func resources(r contracts.ResourceSpec) map[string]any { @@ -130,6 +149,8 @@ func podTemplate(spec contracts.RLRunSpec, role string, resource contracts.Resou map[string]any{"name": "SKYSCALE_TENANT_ID", "value": spec.Metadata.TenantID}, map[string]any{"name": "SKYSCALE_PROJECT_ID", "value": spec.Metadata.ProjectID}, map[string]any{"name": "SKYSCALE_CONTROL_PLANE_URL", "value": "http://skyscale-control-plane.skyscale-system.svc.cluster.local:8080"}, + map[string]any{"name": "MODEL_ROOT", "value": modelMountPath(spec)}, + map[string]any{"name": "PYTHONPATH", "value": "/opt/skyscale/slime:/root/slime:/root/Megatron-LM"}, // Credentials are mounted through explicit secret references, never serialized into samples. } container := map[string]any{ @@ -143,10 +164,22 @@ func podTemplate(spec contracts.RLRunSpec, role string, resource contracts.Resou "containers": []any{container}, "nodeSelector": mapStringAny(resource.Selectors), "securityContext": map[string]any{"runAsNonRoot": true, "seccompProfile": map[string]any{"type": "RuntimeDefault"}}, + "volumes": []any{map[string]any{ + "name": "slime-runtime", + "configMap": map[string]any{"name": "skyscale-slime-runtime", "defaultMode": int64(493)}, + }}, } + container["volumeMounts"] = []any{map[string]any{"name": "slime-runtime", "mountPath": "/opt/skyscale/slime"}} for _, secret := range spec.Security.SecretRefs { container["envFrom"] = appendAny(container["envFrom"], map[string]any{"secretRef": map[string]any{"name": secret}}) } + if spec.Model.VolumeClaim != "" { + container["volumeMounts"] = append(container["volumeMounts"].([]any), map[string]any{"name": "models", "mountPath": modelMountPath(spec)}) + podSpec["volumes"] = appendAny(podSpec["volumes"], map[string]any{ + "name": "models", + "persistentVolumeClaim": map[string]any{"claimName": spec.Model.VolumeClaim}, + }) + } return map[string]any{"metadata": map[string]any{"labels": labels(spec)}, "spec": podSpec} } @@ -169,10 +202,38 @@ func appendAny(value any, item any) []any { // deploymentStatus/jobStatus fields remain authoritative for reconciliation. func RenderRayJob(spec contracts.RLRunSpec, attemptID string) *unstructured.Unstructured { name := spec.Metadata.RunID + "-" + attemptID - head := podTemplate(spec, "head", contracts.ResourceSpec{CPU: "2", Memory: "8Gi", Selectors: spec.Topology.Trainer.Resources.Selectors}, []any{}) + head := podTemplate(spec, "head", contracts.ResourceSpec{CPU: "1", Memory: "2Gi", Selectors: spec.Topology.Trainer.Resources.Selectors}, []any{}) worker := podTemplate(spec, "trainer", spec.Topology.Trainer.Resources, []any{}) - submitter := podTemplate(spec, "submitter", contracts.ResourceSpec{CPU: "1", Memory: "2Gi"}, []any{}) + submitter := podTemplate(spec, "submitter", contracts.ResourceSpec{CPU: "250m", Memory: "512Mi"}, []any{}) submitter["spec"].(map[string]any)["restartPolicy"] = "Never" + for _, template := range []map[string]any{head, worker, submitter} { + container := template["spec"].(map[string]any)["containers"].([]any)[0].(map[string]any) + container["env"] = append(container["env"].([]any), map[string]any{"name": "SKYSCALE_ATTEMPT_ID", "value": attemptID}) + } + if spec.Model.VolumeClaim != "" { + reporter := map[string]any{ + "name": "skyscale-reporter", + "image": image(spec), + "args": []any{"python", "-m", "skyscale.reporter"}, + "env": []any{ + map[string]any{"name": "SKYSCALE_RUN_ID", "value": spec.Metadata.RunID}, + map[string]any{"name": "SKYSCALE_ATTEMPT_ID", "value": attemptID}, + map[string]any{"name": "SKYSCALE_CONTROL_PLANE_URL", "value": "http://skyscale-control-plane.skyscale-system.svc.cluster.local:8080"}, + map[string]any{"name": "SKYSCALE_CHECKPOINT_DIR", "value": modelMountPath(spec) + "/resume"}, + }, + "resources": map[string]any{ + "requests": map[string]any{"cpu": "50m", "memory": "64Mi"}, + "limits": map[string]any{"cpu": "500m", "memory": "256Mi"}, + }, + "securityContext": map[string]any{"allowPrivilegeEscalation": false, "capabilities": map[string]any{"drop": []any{"ALL"}}}, + "volumeMounts": []any{map[string]any{"name": "models", "mountPath": modelMountPath(spec)}}, + } + for _, secret := range spec.Security.SecretRefs { + reporter["envFrom"] = appendAny(reporter["envFrom"], map[string]any{"secretRef": map[string]any{"name": secret}}) + } + headSpec := head["spec"].(map[string]any) + headSpec["containers"] = append(headSpec["containers"].([]any), reporter) + } workerReplicas := int64(spec.Topology.Trainer.Nodes) object := map[string]any{ "apiVersion": "ray.io/v1", "kind": "RayJob", "metadata": metadata(spec, name), @@ -220,7 +281,7 @@ func RenderRolloutDeployment(spec contracts.RLRunSpec, policyVersion string) *un func RenderRolloutDeploymentState(spec contracts.RLRunSpec, policyVersion string, replicas int, draining bool) *unstructured.Unstructured { name := spec.Metadata.RunID + "-rollout" r := spec.Topology.Rollout - args := []any{"python", "-m", "sglang.launch_server", "--model-path", spec.Model.Source, + args := []any{"python", "-m", "sglang.launch_server", "--model-path", modelRuntimePath(spec), "--tp-size", strconv.Itoa(r.TP), "--dp-size", strconv.Itoa(r.DP), "--mem-fraction-static", strconv.FormatFloat(r.MemoryFraction, 'f', 2, 64), "--host", "0.0.0.0", "--port", "8000"} template := podTemplate(spec, "rollout", r.Resources, args) @@ -234,7 +295,7 @@ func RenderRolloutDeploymentState(spec contracts.RLRunSpec, policyVersion string c["readinessProbe"] = map[string]any{"httpGet": map[string]any{"path": "/server_info", "port": int64(8000)}, "periodSeconds": int64(5), "failureThreshold": int64(12)} c["lifecycle"] = map[string]any{"preStop": map[string]any{"exec": map[string]any{"command": []any{"sh", "-c", "sleep 15"}}}} c["env"] = append(c["env"].([]any), map[string]any{"name": "SKYSCALE_POLICY_VERSION", "value": policyVersion}) - c["volumeMounts"] = []any{map[string]any{"name": "weights", "mountPath": "/var/lib/skyscale/weights"}} + c["volumeMounts"] = appendAny(c["volumeMounts"], map[string]any{"name": "weights", "mountPath": "/var/lib/skyscale/weights"}) registrar := map[string]any{ "name": "engine-registrar", "image": image(spec), @@ -256,7 +317,7 @@ func RenderRolloutDeploymentState(spec contracts.RLRunSpec, policyVersion string registrar["envFrom"] = appendAny(registrar["envFrom"], map[string]any{"secretRef": map[string]any{"name": secret}}) } templateSpec["containers"] = append(containers, registrar) - templateSpec["volumes"] = []any{map[string]any{"name": "weights", "emptyDir": map[string]any{}}} + templateSpec["volumes"] = appendAny(templateSpec["volumes"], map[string]any{"name": "weights", "emptyDir": map[string]any{}}) object := map[string]any{ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": metadata(spec, name), "spec": map[string]any{ diff --git a/control-plane/k8s/render_test.go b/control-plane/k8s/render_test.go index 49a8bed..23b988f 100644 --- a/control-plane/k8s/render_test.go +++ b/control-plane/k8s/render_test.go @@ -43,9 +43,12 @@ func TestRenderColocatedRayJobUsesWholeGPU(t *testing.T) { func TestDisaggregatedTrainerRegistersExternalEngine(t *testing.T) { job := RenderRayJob(renderSpec("disaggregated"), "attempt-1") entrypoint, _, _ := unstructured.NestedString(job.Object, "spec", "entrypoint") - if !strings.Contains(entrypoint, "--rollout-external-engine-addrs") || !strings.Contains(entrypoint, "run-a-rollout") { + if !strings.Contains(entrypoint, "ROLLOUT_EXTERNAL_ENGINE_ADDRS=") || !strings.Contains(entrypoint, "run-a-rollout") { t.Fatalf("missing external engine registration: %s", entrypoint) } + if !strings.Contains(entrypoint, "disaggregated_qwen3_0_6b.sh") { + t.Fatalf("trainer does not use the validated disaggregated profile: %s", entrypoint) + } deployment := RenderRolloutDeployment(renderSpec("disaggregated"), "policy-7") if deployment.GetKind() != "Deployment" { t.Fatalf("unexpected kind %s", deployment.GetKind()) @@ -64,13 +67,45 @@ func TestAsyncAndRecoveryEntrypoint(t *testing.T) { spec.Checkpoint.ResumeFrom = "s3://checkpoints/run/step-7" job := RenderRayJob(spec, "attempt-2") entrypoint, _, _ := unstructured.NestedString(job.Object, "spec", "entrypoint") - for _, expected := range []string{"train_async.py", "--load", "s3://checkpoints/run/step-7"} { + for _, expected := range []string{"async_qwen3_0_6b.sh", "--load", "s3://checkpoints/run/step-7"} { if !strings.Contains(entrypoint, expected) { t.Fatalf("entrypoint missing %q: %s", expected, entrypoint) } } } +func TestRuntimePodsMountPreparedModelArtifacts(t *testing.T) { + spec := renderSpec("disaggregated") + job := RenderRayJob(spec, "attempt-1") + raw := job.Object["spec"].(map[string]any)["rayClusterSpec"].(map[string]any)["workerGroupSpecs"].([]any)[0].(map[string]any) + podSpec := raw["template"].(map[string]any)["spec"].(map[string]any) + volumes := podSpec["volumes"].([]any) + foundModelVolume := false + for _, volume := range volumes { + volumeObject := volume.(map[string]any) + if claim, ok := volumeObject["persistentVolumeClaim"].(map[string]any); ok { + if claim["claimName"] != "qwen3-0-6b-models" { + t.Fatalf("unexpected model claim: %#v", claim) + } + foundModelVolume = true + } + } + if !foundModelVolume { + t.Fatalf("model PVC volume is missing: %#v", volumes) + } + head := job.Object["spec"].(map[string]any)["rayClusterSpec"].(map[string]any)["headGroupSpec"].(map[string]any) + headContainers := head["template"].(map[string]any)["spec"].(map[string]any)["containers"].([]any) + if len(headContainers) != 2 || headContainers[1].(map[string]any)["name"] != "skyscale-reporter" { + t.Fatalf("checkpoint reporter is not attached to the Ray head: %#v", headContainers) + } + deployment := RenderRolloutDeployment(spec, "policy-7") + deploymentSpec := deployment.Object["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any) + args := deploymentSpec["containers"].([]any)[0].(map[string]any)["args"].([]any) + if !strings.Contains(strings.Join(anyStrings(args), " "), "/models/hf") { + t.Fatalf("rollout does not use prepared model artifacts: %v", args) + } +} + func TestEvaluatorAndCanaryResourcesAreIsolated(t *testing.T) { spec := renderSpec("disaggregated") spec.Evaluation = contracts.EvaluationPolicy{ diff --git a/dashboard/app/benchmarks/page.tsx b/dashboard/app/benchmarks/page.tsx deleted file mode 100644 index f17294c..0000000 --- a/dashboard/app/benchmarks/page.tsx +++ /dev/null @@ -1,574 +0,0 @@ -'use client' - -import { useMemo, useState } from 'react' -import { Gauge, Timer, Zap } from 'lucide-react' -import { DashboardShell } from '@/components/dashboard/dashboard-shell' -import { PageHeader } from '@/components/dashboard/page-header' -import { StatBadge } from '@/components/dashboard/stat-badge' -import { ContentPanel } from '@/components/dashboard/content-panel' -import { useRealtimeStream } from '@/hooks/useRealtimeStream' -import { api } from '@/lib/api' -import { Button } from '@/components/ui/button' -import { Card, CardContent } from '@/components/ui/card' -import { Badge } from '@/components/ui/badge' -import { cn } from '@/lib/utils' - -type ColumnProfile = { - name: string - type: 'number' | 'string' - null_count: number - null_pct: number - unique_count: number - stats?: { min: number; max: number; mean: number; p50: number } -} - -type ProfileReport = { - ok: boolean - row_count: number - column_count: number - quality_score: number - columns: ColumnProfile[] - issues: Array<{ severity: 'high' | 'medium'; column: string; message: string }> -} - -type InvokeResponse = { - request_id?: string - function_id?: string - status_code: number - output?: ProfileReport - error_message?: string - duration_ms?: number -} - -type LiveRun = { - functionName: string - httpStatus: 'ok' | 'error' - wallMs: number - completedAt: string - response: InvokeResponse -} - -const SAMPLE_CSV = `customer_id,plan,monthly_spend,events_last_30d,churn_risk,region -1001,pro,129.50,84,0.12,NA -1002,starter,19.00,12,0.43,EU -1003,enterprise,899.00,412,0.04,US -1004,pro,,55,0.20,US -1005,starter,19.00,,0.62, -1006,enterprise,1200.00,530,0.03,APAC -1007,pro,149.00,73,,EU -1008,starter,9.00,4,0.81, -` - -const CSV_PROFILER_CODE = `import base64 -import csv -import io -import math -import time - -NULLS = {"", "null", "none", "nan", "n/a", "na"} - -def is_null(value): - return value is None or str(value).strip().lower() in NULLS - -def to_float(value): - if is_null(value): - return None - try: - n = float(str(value).strip()) - return None if math.isnan(n) or math.isinf(n) else n - except Exception: - return None - -def handle(event, context): - time.sleep(3) - csv_text = base64.b64decode(event.get("csv_b64", "")).decode("utf-8") - reader = csv.DictReader(io.StringIO(csv_text)) - rows = list(reader) - columns = reader.fieldnames or [] - report = [] - for column_name in columns: - values = [row.get(column_name, "") for row in rows] - null_count = sum(1 for value in values if is_null(value)) - non_null = [value for value in values if not is_null(value)] - nums = [to_float(value) for value in non_null] - nums = [value for value in nums if value is not None] - numeric = bool(non_null) and len(nums) == len(non_null) - item = { - "name": column_name, - "type": "number" if numeric else "string", - "null_count": null_count, - "null_pct": round(null_count * 100 / len(rows), 2) if rows else 0, - "unique_count": len({str(value).strip() for value in non_null}), - } - if numeric and nums: - ordered = sorted(nums) - item["stats"] = { - "min": min(nums), - "max": max(nums), - "mean": round(sum(nums) / len(nums), 3), - "p50": ordered[len(ordered) // 2], - } - report.append(item) - issues = [] - for col in report: - if col["null_pct"] >= 30: - issues.append({"severity": "high", "column": col["name"], "message": str(col["null_pct"]) + "% missing"}) - elif col["null_count"]: - issues.append({"severity": "medium", "column": col["name"], "message": str(col["null_count"]) + " missing values"}) - score = max(0, round(100 - sum(c["null_pct"] for c in report) / max(len(report), 1), 1)) - return {"ok": True, "row_count": len(rows), "column_count": len(columns), "quality_score": score, "columns": report, "issues": issues} -` - -const VERIFIED_LOAD_TEST = { - p95Ms: 1680, - avgMs: 821, - throughputRps: 0.5, - successRate: '100/100', - scenario: '100 concurrent users, 100 invocations', - pool: '2 warm Firecracker VMs', -} - -const SPEED_STATS = [ - { - label: 'p95 latency', - value: '1.68s', - tone: 'running' as const, - sub: '95% of runs finish inside the VM within this time', - icon: Gauge, - }, - { - label: 'Average latency', - value: '821ms', - tone: 'primary' as const, - sub: 'Typical execution time once a warm VM is ready', - icon: Timer, - }, - { - label: 'Throughput', - value: '0.50 req/s', - tone: 'warning' as const, - sub: 'Sustained rate under 100 concurrent users (k6)', - icon: Zap, - }, - { - label: 'Success rate', - value: '100/100', - tone: 'success' as const, - sub: 'All invocations completed without error', - icon: Gauge, - }, -] - -export default function BenchmarksPage() { - const { data, connected } = useRealtimeStream() - const [running, setRunning] = useState(false) - const [liveRun, setLiveRun] = useState(null) - const [runError, setRunError] = useState(null) - const [showWorkloadOutput, setShowWorkloadOutput] = useState(false) - - const runResult = liveRun?.response ?? null - const report = runResult?.output - - const headlineStats = useMemo(() => { - if (!liveRun) return SPEED_STATS - - const ok = liveRun.httpStatus === 'ok' && liveRun.response.status_code === 200 - const vmMs = liveRun.response.duration_ms - const wallMs = liveRun.wallMs - - return [ - { - label: 'Your last run', - value: ok ? 'Success' : 'Failed', - tone: ok ? 'success' as const : 'primary' as const, - sub: `Completed at ${liveRun.completedAt}`, - icon: Gauge, - }, - { - label: 'End-to-end time', - value: wallMs ? formatMs(wallMs) : '—', - tone: 'running' as const, - sub: 'Browser click → response (includes register + invoke)', - icon: Timer, - }, - { - label: 'VM execution', - value: vmMs ? formatMs(vmMs) : '—', - tone: 'primary' as const, - sub: 'Time spent running Python inside Firecracker', - icon: Zap, - }, - { - label: 'vs verified p95', - value: vmMs ? compareToP95(vmMs) : '—', - tone: vmMs && vmMs <= VERIFIED_LOAD_TEST.p95Ms ? 'success' as const : 'warning' as const, - sub: `Verified p95 is ${formatMs(VERIFIED_LOAD_TEST.p95Ms)} under load`, - icon: Gauge, - }, - ] - }, [liveRun]) - - const readyWarm = useMemo( - () => data.vm_pool.filter(vm => (vm.HardwareType === 'cpu' || !vm.HardwareType) && vm.IsWarm && vm.Status === 'ready' && vm.ID !== 'host-vm-test').length, - [data.vm_pool], - ) - const busyCPU = useMemo( - () => data.vm_pool.filter(vm => (vm.HardwareType === 'cpu' || !vm.HardwareType) && vm.Status === 'busy').length, - [data.vm_pool], - ) - - async function runSpeedTest() { - setRunning(true) - setRunError(null) - try { - const name = `speed-test-${Date.now()}` - await api.registerFunction({ - name, - runtime: 'python3', - memory: 128, - timeout: 30, - code: CSV_PROFILER_CODE, - requirements: '', - config: '', - }) - const startedAt = performance.now() - const result = await api.invoke(name, { - input: { csv_b64: btoa(SAMPLE_CSV) }, - sync: true, - job_type: 'faas_function', - hardware_type: 'cpu', - }) as InvokeResponse - setLiveRun({ - functionName: name, - httpStatus: 'ok', - wallMs: Math.round(performance.now() - startedAt), - completedAt: new Date().toLocaleTimeString(), - response: result, - }) - if (result.status_code !== 200) { - setRunError(result.error_message ?? 'Speed test returned a non-200 status') - } - } catch (err) { - const message = err instanceof Error ? err.message : 'Failed to run speed test' - setRunError(message) - setLiveRun({ - functionName: 'speed-test', - httpStatus: 'error', - wallMs: 0, - completedAt: new Date().toLocaleTimeString(), - response: { status_code: 500, error_message: message }, - }) - } finally { - setRunning(false) - } - } - - return ( - - - - - - } - /> - -
- - - - What this measures - -

- Time from request to finished execution -

-

- We stress-test the platform with 100 concurrent users hitting a warm VM pool. - Each request runs the same Python workload so results are comparable run to run. - Lower latency and higher throughput mean faster load handling. -

-
- {VERIFIED_LOAD_TEST.scenario} - {VERIFIED_LOAD_TEST.pool} - Fixed CSV workload (~8 rows) -
- - - -
-

- {liveRun ? 'Your latest run' : 'Verified load test results'} -

-
- {headlineStats.map(stat => ( - - ))} -
-
- -
- - - - - -

- Registers a fresh function and invokes it once from your browser. - This includes setup overhead, so it is usually slower than the verified pool benchmark above. -

- - {runError && ( -

- {runError} -

- )} -
- -
- -
- -
- -

- Automated load test against a pre-warmed two-VM pool. These numbers are the baseline for platform speed. -

- - - - -

- Every invocation parses a small CSV and returns JSON. The workload is fixed so timing differences reflect platform speed, not varying input size. -

-
- - - - -
- - {showWorkloadOutput && report && ( -
- - -
- )} - {showWorkloadOutput && !report && ( -

- Run a speed test to see the workload output from your invocation. -

- )} - -
-
- - ) -} - -function formatMs(ms: number) { - if (ms < 1000) return `${ms}ms` - return `${(ms / 1000).toFixed(2)}s` -} - -function compareToP95(vmMs: number) { - const diff = vmMs - VERIFIED_LOAD_TEST.p95Ms - if (diff <= 0) return `${Math.abs(Math.round(diff / VERIFIED_LOAD_TEST.p95Ms * 100))}% faster` - return `${Math.round(diff / VERIFIED_LOAD_TEST.p95Ms * 100)}% slower` -} - -function SpeedMetricCard({ - label, - value, - tone, - sub, - icon: Icon, -}: { - label: string - value: string - tone: 'primary' | 'success' | 'warning' | 'running' - sub: string - icon: typeof Gauge -}) { - const toneClass = { - primary: 'text-primary', - success: 'text-[var(--success)]', - warning: 'text-[var(--warning)]', - running: 'text-[var(--running)]', - }[tone] - - return ( - - -
- - {label} -
-
{value}
-
{sub}
- - - ) -} - -function LatencyBreakdown({ liveRun }: { liveRun: LiveRun | null }) { - const vmMs = liveRun?.response.duration_ms ?? VERIFIED_LOAD_TEST.avgMs - const p95Ms = VERIFIED_LOAD_TEST.p95Ms - const maxMs = Math.max(p95Ms, vmMs) * 1.15 - - const bars = liveRun - ? [ - { label: 'VM execution', ms: vmMs, tone: 'bg-[var(--running)]', hint: 'Python running inside Firecracker' }, - { label: 'End-to-end', ms: liveRun.wallMs, tone: 'bg-primary', hint: 'Includes function registration + network' }, - { label: 'Verified p95', ms: p95Ms, tone: 'bg-muted-foreground/50', hint: 'Baseline under 100 concurrent users' }, - ] - : [ - { label: 'Average (warm)', ms: VERIFIED_LOAD_TEST.avgMs, tone: 'bg-[var(--running)]', hint: 'Typical VM execution time' }, - { label: 'p95 (warm)', ms: p95Ms, tone: 'bg-primary', hint: '95th percentile under load' }, - { label: 'HTTP p95', ms: 186000, tone: 'bg-muted-foreground/50', hint: 'Full round-trip including queueing at peak load' }, - ] - - return ( -
-

- {liveRun - ? 'Compare your single browser run against the verified load-test baseline.' - : 'Verified numbers from a warm pool under concurrent load. HTTP times include queueing when all VMs are busy.'} -

-
- {bars.map(bar => ( -
-
- {bar.label} - {formatMs(bar.ms)} -
-
-
-
-

{bar.hint}

-
- ))} -
-
- ) -} - -function LoadTestTable() { - const rows = [ - ['Test setup', VERIFIED_LOAD_TEST.scenario], - ['VM pool', VERIFIED_LOAD_TEST.pool], - ['Invocations succeeded', VERIFIED_LOAD_TEST.successRate], - ['Avg VM execution', formatMs(VERIFIED_LOAD_TEST.avgMs)], - ['p95 VM execution', formatMs(VERIFIED_LOAD_TEST.p95Ms)], - ['Throughput', `${VERIFIED_LOAD_TEST.throughputRps} req/s`], - ['Avg HTTP round-trip', '1m 33s (includes queue wait)'], - ['p95 HTTP round-trip', '3m 06s (includes queue wait)'], - ] - - return ( -
- {rows.map(([label, value]) => ( - - ))} -
- ) -} - -function LiveRunSummary({ run, running }: { run: LiveRun | null; running: boolean }) { - if (running) { - return ( -

- Registering function and measuring execution time… -

- ) - } - - if (!run) { - return ( -

- No live run yet. Hit “Run speed test” to measure latency from your browser. -

- ) - } - - const ok = run.httpStatus === 'ok' && run.response.status_code === 200 - - return ( -
-
- Result - - {ok ? 'Success' : 'Failed'} - -
- - - -
- ) -} - -function Row({ label, value }: { label: string; value: string }) { - return ( -
- {label} - {value} -
- ) -} - -function ReportSummary({ report, duration }: { report: ProfileReport; duration?: number }) { - return ( -
- - - - -
- ) -} - -function TinyStat({ label, value }: { label: string; value: string }) { - return ( -
-
{value}
-
{label}
-
- ) -} - -function ColumnTable({ columns }: { columns: ColumnProfile[] }) { - return ( -
-
- ColumnTypeNullsStats -
- {columns.map(column => ( -
- {column.name} - {column.type} - {column.null_pct}% - - {column.stats ? `mean ${column.stats.mean}` : `${column.unique_count} unique`} - -
- ))} -
- ) -} diff --git a/dashboard/app/faas/page.tsx b/dashboard/app/faas/page.tsx index aa68fe8..9bbbb0f 100644 --- a/dashboard/app/faas/page.tsx +++ b/dashboard/app/faas/page.tsx @@ -28,9 +28,14 @@ export default function FaasPage() { const [deployOpen, setDeployOpen] = useState(false) const [query, setQuery] = useState('') + const sortedContainers = useMemo( + () => [...containers].sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime()), + [containers], + ) + const selected = useMemo( - () => containers.find((c) => c.id === selectedId) ?? null, - [containers, selectedId], + () => containers.find((c) => c.id === selectedId) ?? sortedContainers[0] ?? null, + [containers, selectedId, sortedContainers], ) const selectedTemplate = useMemo( @@ -38,11 +43,6 @@ export default function FaasPage() { [templates, selected?.templateId], ) - const sortedContainers = useMemo( - () => [...containers].sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime()), - [containers], - ) - const filteredContainers = useMemo(() => { const q = query.trim().toLowerCase() if (!q) return sortedContainers @@ -91,15 +91,6 @@ export default function FaasPage() { return () => { cancelled = true } }, []) - useEffect(() => { - if (selectedId && containers.some((c) => c.id === selectedId)) return - if (sortedContainers.length > 0) { - setSelectedId(sortedContainers[0].id) - } else { - setSelectedId(null) - } - }, [containers, selectedId, sortedContainers]) - useEffect(() => { const needsPoll = containers.some( (c) => c.status === 'deploying' || c.status === 'stopping' || c.status === 'running' || c.status === 'creating', @@ -184,13 +175,13 @@ export default function FaasPage() { /> {error && ( -
+
{error}
)} {containers.length === 0 ? ( -
+
} title="No sandboxes deployed" @@ -214,8 +205,8 @@ export default function FaasPage() {
) : ( -
-