feat: harden RL runtime and refresh dashboard

.github/workflows/validate-slime.yml

@@ -0,0 +1,47 @@
+name: Validate Slime Integration
+
+on:
+  pull_request:
+    paths:
+      - "control-plane/**"
+      - "training/slime/**"
+      - "deploy/k8s/**"
+      - "dashboard/components/training/**"
+      - "dashboard/lib/api.ts"
+      - "dashboard/package*.json"
+      - "scripts/validation/validate_slime_local.sh"
+      - ".github/workflows/validate-slime.yml"
+  push:
+    paths:
+      - "control-plane/**"
+      - "training/slime/**"
+      - "deploy/k8s/**"
+      - "dashboard/components/training/**"
+      - "dashboard/lib/api.ts"
+      - "dashboard/package*.json"
+      - "scripts/validation/validate_slime_local.sh"
+      - ".github/workflows/validate-slime.yml"
+  workflow_dispatch:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: control-plane/go.mod
+          cache-dependency-path: control-plane/go.sum
+      - uses: azure/setup-kubectl@v4
+        with:
+          version: latest
+      - name: Validate control plane, runtime, and manifests
+        run: bash scripts/validation/validate_slime_local.sh
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: dashboard/package-lock.json
+      - name: Build dashboard
+        working-directory: dashboard
+        run: npm ci && npm run build

README.md

@@ -2,7 +2,7 @@
 
 **Reinforcement Learning as a Service.** Post-train any LLM on any task using distributed async RL — without managing clusters, scheduling workers, or provisioning GPUs. One API call starts the whole pipeline.
 
-> **Currently supports:** distributed GRPO RL runs (`POST /api/rl/runs`) with Modal GPU sandboxes (vLLM policy server, CPU rollout workers, GRPO trainer), Firecracker microVMs as code-execution RL environments (`reset` / `step` / `close`), an async experience buffer, live metrics in the Next.js dashboard, and Prometheus + Grafana observability. Also: Firecracker sandboxes, Railway-backed container sandboxes, on-demand GPU job submission (Modal / Akash / HuggingFace), and sign-in-gated GPU spend when `SKYSCALE_DASHBOARD_TOKEN` is set.
+> **Currently supports:** first-class slime RL runs on KubeRay and the legacy distributed GRPO path on Modal. Slime runs use SGLang rollouts, Megatron training, sandbox rewards, grouped-sample persistence, checkpoint reporting, and immutable run contracts. The dashboard submits and monitors either backend through `POST /api/rl/runs`.
 >
 > **Not yet:** closed-loop policy weight hot-swap on Modal, multi-turn episodes, custom problem-set uploads, and permissionless workers. See [What's not built yet](#whats-not-built-yet).
 
@@ -199,8 +199,6 @@ The Next.js dashboard (`dashboard/`) is the primary operator UI:
 | **Training** | `/` | RL runs list, live reward/loss charts, run detail drawer (logs, metadata, events), start/stop runs |
 | **Templates** | `/templates` | Job templates for quick submission |
 | **Sandboxes** | `/faas` | Deploy and manage isolated container sandboxes (Railway-backed) |
-| **On-Demand GPUs** | `/gpus` | GPU inventory and job queue |
-| **Load Speed** | `/benchmarks` | FaaS cold-start and throughput benchmarks |
 
 RL run detail includes stage, policy URL, buffer size, worker/trainer status, in-app Recharts metrics, activity log streaming, and an optional Grafana link.
 
@@ -223,13 +221,15 @@ HF_TOKEN=<token> python3 scripts/modal_pipeline_test.py
 
 It runs through all four stages — policy server health, RL run creation, buffer fill from 2 workers, and 3 GRPO training steps — and prints a pass/fail report with final metrics.
 
-**Load testing sandboxes:**
+### Slime on KubeRay
+
+Set `SKYSCALE_RL_KUBERNETES=1`, configure the pinned runtime images and runtime token, prepare the model PVC, then choose **Slime on KubeRay** in the dashboard. The control plane snapshots the run contract, creates a RayJob and rollout services, records grouped samples and optimizer progress, and resumes retries from the latest checkpoint on the model PVC.
 
 ```bash
-k6 run perf/faas_load_test.js
+bash scripts/validation/validate_slime_local.sh
 ```
 
-Set `API_URL` to your control plane origin. See `perf/faas_load_test.js` for VUs and duration defaults.
+Production prerequisites and manifests are documented in [`docs/slime-kuberay-production.md`](docs/slime-kuberay-production.md). The numerical one-GPU validation path is `AWS_SLIME_E2E=1 bash scripts/validation/aws_slime_gpu.sh`.
 
 ---
 
@@ -406,6 +406,10 @@ This design focused on `skyscale deploy` / `skyscale invoke` with a warm VM pool
 | `HF_TOKEN` | HuggingFace token for model downloads |
 | `ARTIFACT_LOCAL_DIR` | Local checkpoint fallback when S3 is not configured |
 | `S3_ENDPOINT` / `S3_BUCKET` / `S3_ACCESS_KEY` / `S3_SECRET_KEY` | S3-compatible artifact store (optional) |
+| `SKYSCALE_RL_KUBERNETES` | Enable the slime KubeRay reconciler (`1`) |
+| `SKYSCALE_SLIME_IMAGE` / `SKYSCALE_SGLANG_IMAGE` | Immutable runtime image references |
+| `SKYSCALE_MODEL_PVC` / `SKYSCALE_MODEL_MOUNT_PATH` | Prepared Hugging Face and Megatron model artifact volume |
+| `SKYSCALE_RUNTIME_TOKEN` | Shared bearer token for slime runtime callbacks |
 | `FAAS_VM_KERNEL_PATH` | Firecracker kernel path (auto-downloaded if absent) |
 | `FAAS_VM_ROOTFS_PATH` | VM rootfs path (auto-downloaded if absent) |
 | `FAAS_VM_MEMORY_MB` | Memory per VM in MB (default `128`) |

cmd/daemon/daemon.go

@@ -21,7 +21,6 @@ const (
 	daemonPort = "8081" // Port for the daemon to listen on
 	codeDir          = "/tmp/faas/code"
 	logDir           = "/var/log/faas"
-	sandboxWorkspace = "/sandbox/workspace" // Persistent workspace for sandbox sessions
 
 	// Endpoints
 	functionEndpoint = "/api/functions"
@@ -70,12 +69,22 @@ type VMInfo struct {
 var vmInfo VMInfo
 var httpClient *http.Client
 var controlPlaneURL string
+var sandboxWorkspace = envOrDefault("SANDBOX_WORKSPACE", "/sandbox/workspace")
+
+func envOrDefault(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}
 
 func init() {
 	// Create necessary directories
-	os.MkdirAll(codeDir, 0755)
-	os.MkdirAll(logDir, 0755)
-	os.MkdirAll(sandboxWorkspace, 0755)
+	for _, directory := range []string{codeDir, logDir, sandboxWorkspace} {
+		if err := os.MkdirAll(directory, 0755); err != nil {
+			log.Printf("Failed to create directory %s: %v", directory, err)
+		}
+	}
 
 	// Read control plane URL from env, fall back to Firecracker CNI gateway
 	controlPlaneURL = os.Getenv("CONTROL_PLANE_URL")

cmd/daemon/daemon_test.go

@@ -14,17 +14,18 @@ import (
 func setupWorkspace(t *testing.T) string {
 	t.Helper()
 	dir := t.TempDir()
+	original := sandboxWorkspace
+	sandboxWorkspace = dir
+	t.Cleanup(func() {
+		sandboxWorkspace = original
+	})
 	return dir
 }
 
 // ─── /sandbox/exec tests ─────────────────────────────────────────────────────
 
 func TestSandboxExecPython(t *testing.T) {
 	workspace := setupWorkspace(t)
-	origWS := sandboxWorkspace
-	// Can't reassign a const; use a package-var approach via the test build.
-	// We test the handler directly, overriding the global via init in the test binary.
-	_ = origWS
 	_ = workspace
 
 	body, _ := json.Marshal(sandboxExecRequest{
@@ -57,6 +58,7 @@ func TestSandboxExecPython(t *testing.T) {
 }
 
 func TestSandboxExecBash(t *testing.T) {
+	setupWorkspace(t)
 	body, _ := json.Marshal(sandboxExecRequest{
 		ExecID:   "test-bash",
 		Code:     "echo hello_bash",
@@ -84,6 +86,7 @@ func TestSandboxExecBash(t *testing.T) {
 }
 
 func TestSandboxExecFilePersistence(t *testing.T) {
+	setupWorkspace(t)
 	// Write a file in one exec, read it back in a second exec.
 	// Both execs use sandboxWorkspace as cwd so the file is accessible.
 	writeBody, _ := json.Marshal(sandboxExecRequest{
@@ -130,6 +133,7 @@ func TestSandboxExecFilePersistence(t *testing.T) {
 }
 
 func TestSandboxExecTimeout(t *testing.T) {
+	setupWorkspace(t)
 	body, _ := json.Marshal(sandboxExecRequest{
 		ExecID:   "timeout-exec",
 		Code:     "import time; time.sleep(60)",
@@ -151,6 +155,7 @@ func TestSandboxExecTimeout(t *testing.T) {
 }
 
 func TestSandboxExecUnsupportedLanguage(t *testing.T) {
+	setupWorkspace(t)
 	body, _ := json.Marshal(sandboxExecRequest{
 		Code:     "console.log('hi')",
 		Language: "javascript",
@@ -167,6 +172,7 @@ func TestSandboxExecUnsupportedLanguage(t *testing.T) {
 // ─── /sandbox/files/ tests ───────────────────────────────────────────────────
 
 func TestFileUploadDownload(t *testing.T) {
+	setupWorkspace(t)
 	content := []byte("hello sandbox file")
 
 	// Upload
@@ -193,6 +199,7 @@ func TestFileUploadDownload(t *testing.T) {
 }
 
 func TestFileDownloadMissing(t *testing.T) {
+	setupWorkspace(t)
 	req := httptest.NewRequest(http.MethodGet, "/sandbox/files/nonexistent_file.txt", nil)
 	rr := httptest.NewRecorder()
 	handleSandboxFile(rr, req)
@@ -202,6 +209,7 @@ func TestFileDownloadMissing(t *testing.T) {
 }
 
 func TestFileMethodNotAllowed(t *testing.T) {
+	setupWorkspace(t)
 	req := httptest.NewRequest(http.MethodDelete, "/sandbox/files/somefile.txt", nil)
 	rr := httptest.NewRecorder()
 	handleSandboxFile(rr, req)

control-plane/api/rl_slime.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/bluequbit/faas/control-plane/contracts"
@@ -15,6 +16,13 @@ import (
 	"github.com/gorilla/mux"
 )
 
+type slimeRunPreset struct {
+	BaseModel  string `json:"base_model"`
+	NumWorkers int    `json:"num_workers"`
+	GPUModel   string `json:"gpu_model"`
+	ProblemSet string `json:"problem_set"`
+}
+
 func (h *APIHandler) rlStartSlimeRunHandler(w http.ResponseWriter, r *http.Request, raw []byte) {
 	if h.rlReconciler == nil {
 		http.Error(w, "slime backend requires SKYSCALE_RL_KUBERNETES=1 and Kubernetes credentials", http.StatusServiceUnavailable)
@@ -30,9 +38,14 @@ func (h *APIHandler) rlStartSlimeRunHandler(w http.ResponseWriter, r *http.Reque
 	}
 	if wrapper.Spec != nil {
 		spec = *wrapper.Spec
-	} else if err := json.Unmarshal(raw, &spec); err != nil {
-		http.Error(w, "invalid run contract", http.StatusBadRequest)
-		return
+	} else {
+		var preset slimeRunPreset
+		if err := json.Unmarshal(raw, &preset); err == nil && preset.BaseModel != "" {
+			spec = slimeSpecFromPreset(preset, r)
+		} else if err := json.Unmarshal(raw, &spec); err != nil {
+			http.Error(w, "invalid run contract", http.StatusBadRequest)
+			return
+		}
 	}
 	spec.Normalize()
 	spec.Backend = "slime"
@@ -88,6 +101,67 @@ func (h *APIHandler) rlStartSlimeRunHandler(w http.ResponseWriter, r *http.Reque
 	})
 }
 
+func slimeSpecFromPreset(preset slimeRunPreset, r *http.Request) contracts.RLRunSpec {
+	spec := contracts.DefaultRunSpec()
+	spec.Metadata.TenantID = headerEnvOrDefault(r, "X-Skyscale-Tenant", "SKYSCALE_DEFAULT_TENANT", "default")
+	spec.Metadata.ProjectID = headerEnvOrDefault(r, "X-Skyscale-Project", "SKYSCALE_DEFAULT_PROJECT", "default")
+	spec.Model.Source = preset.BaseModel
+	spec.Model.Revision = envOrDefault("SKYSCALE_MODEL_REVISION", "main")
+	spec.Model.VolumeClaim = envOrDefault("SKYSCALE_MODEL_PVC", "qwen3-0-6b-models")
+	spec.Model.MountPath = envOrDefault("SKYSCALE_MODEL_MOUNT_PATH", "/models")
+	if preset.ProblemSet != "" {
+		spec.Data.SourceURI = "skyscale://problems/" + preset.ProblemSet
+	}
+	if preset.GPUModel != "" {
+		spec.Topology.Trainer.Resources.GPUType = preset.GPUModel
+		spec.Topology.Rollout.Resources.GPUType = preset.GPUModel
+	}
+	if preset.NumWorkers > 1 {
+		spec.Topology.Mode = "disaggregated"
+		spec.Topology.Rollout.External = true
+		spec.Topology.Rollout.Replicas = preset.NumWorkers
+		spec.Topology.Rollout.MinReplicas = preset.NumWorkers
+		spec.Topology.Rollout.MaxReplicas = preset.NumWorkers
+		spec.Topology.Rollout.Resources.GPUs = 1
+	} else {
+		spec.Topology.Mode = "colocated"
+		spec.Topology.Rollout.External = false
+		spec.Topology.Rollout.Replicas = 1
+		spec.Topology.Rollout.MinReplicas = 1
+		spec.Topology.Rollout.MaxReplicas = 1
+	}
+	spec.Image.Slime = envOrDefault("SKYSCALE_SLIME_IMAGE", spec.Image.Slime)
+	spec.Image.SGLang = envOrDefault("SKYSCALE_SGLANG_IMAGE", spec.Image.Slime)
+	spec.Image.Digest = os.Getenv("SKYSCALE_SLIME_IMAGE_DIGEST")
+	spec.Security.ImageAllowlist = splitNonEmpty(envOrDefault("SKYSCALE_RL_IMAGE_ALLOWLIST", "ghcr.io/skyscale/"))
+	spec.Security.SecretRefs = splitNonEmpty(envOrDefault("SKYSCALE_RUNTIME_SECRET", "skyscale-runtime"))
+	return spec
+}
+
+func headerEnvOrDefault(r *http.Request, header, env, fallback string) string {
+	if value := r.Header.Get(header); value != "" {
+		return value
+	}
+	return envOrDefault(env, fallback)
+}
+
+func envOrDefault(name, fallback string) string {
+	if value := os.Getenv(name); value != "" {
+		return value
+	}
+	return fallback
+}
+
+func splitNonEmpty(value string) []string {
+	var values []string
+	for _, item := range strings.Split(value, ",") {
+		if item = strings.TrimSpace(item); item != "" {
+			values = append(values, item)
+		}
+	}
+	return values
+}
+
 func envInt(name string, fallback int) int {
 	raw := os.Getenv(name)
 	if raw == "" {

control-plane/api/rl_slime_test.go

@@ -0,0 +1,43 @@
+package api
+
+import (
+	"net/http/httptest"
+	"testing"
+)
+
+func TestSlimeSpecFromPresetBuildsProductionContract(t *testing.T) {
+	t.Setenv("SKYSCALE_SLIME_IMAGE", "ghcr.io/skyscale/slime-runtime")
+	t.Setenv("SKYSCALE_SLIME_IMAGE_DIGEST", "sha256:deadbeef")
+	t.Setenv("SKYSCALE_SGLANG_IMAGE", "ghcr.io/skyscale/sglang@sha256:cafe")
+	request := httptest.NewRequest("POST", "/api/rl/runs", nil)
+	request.Header.Set("X-Skyscale-Tenant", "tenant-a")
+	request.Header.Set("X-Skyscale-Project", "project-a")
+
+	spec := slimeSpecFromPreset(slimeRunPreset{
+		BaseModel: "Qwen/Qwen3-0.6B", NumWorkers: 2, GPUModel: "l4", ProblemSet: "default",
+	}, request)
+
+	if spec.Metadata.TenantID != "tenant-a" || spec.Metadata.ProjectID != "project-a" {
+		t.Fatalf("unexpected tenancy: %#v", spec.Metadata)
+	}
+	if spec.Topology.Mode != "disaggregated" || !spec.Topology.Rollout.External || spec.Topology.Rollout.Replicas != 2 {
+		t.Fatalf("unexpected rollout topology: %#v", spec.Topology)
+	}
+	if spec.Model.VolumeClaim != "qwen3-0-6b-models" || spec.Image.Digest != "sha256:deadbeef" {
+		t.Fatalf("runtime artifacts are not configured: model=%#v image=%#v", spec.Model, spec.Image)
+	}
+	if err := spec.Validate(); err != nil {
+		t.Fatalf("generated preset must be valid: %v", err)
+	}
+}
+
+func TestSlimeSingleWorkerPresetUsesOneGPUColocated(t *testing.T) {
+	t.Setenv("SKYSCALE_SLIME_IMAGE", "ghcr.io/skyscale/slime-runtime@sha256:deadbeef")
+	request := httptest.NewRequest("POST", "/api/rl/runs", nil)
+	spec := slimeSpecFromPreset(slimeRunPreset{
+		BaseModel: "Qwen/Qwen3-0.6B", NumWorkers: 1, GPUModel: "l4",
+	}, request)
+	if spec.Topology.Mode != "colocated" || spec.Topology.Rollout.External {
+		t.Fatalf("single-GPU preset must colocate trainer and rollout: %#v", spec.Topology)
+	}
+}