From ff4079f1d8a32dc409f21570601fbd83504848f9 Mon Sep 17 00:00:00 2001 From: Riley Date: Sun, 19 Jul 2026 21:10:15 +0000 Subject: [PATCH 1/2] Add build-time per-experiment images + runner security hardening Runner security hardening: - runner.py: run user-submitted trials with a sanitized environment (_build_trial_env allowlist) so untrusted code can no longer read the runner's injected secrets (GMAIL_CREDS, MONGODB_PORT, BACKEND_PORT) via os.environ. - job-runner.yaml: add a securityContext (allowPrivilegeEscalation:false, drop ALL capabilities, RuntimeDefault seccomp) and CPU/memory requests+limits so one experiment cannot starve its node. Build-time per-experiment images (Phases 0-2): - Phase 0: runner-base.Dockerfile (interpreter + harness base image); kubernetes_init/registry/ documents the in-cluster ctlptl registry; Tiltfile builds runner-base (consumed via the backend RUNNER_BASE_IMAGE env). - Phase 2: build_image.py + job-builder.yaml render a Dockerfile from the declared deps, build a per-experiment image FROM runner-base via an in-cluster Kaniko Job (content-hash cached in the registry), and return the ref. app.py spawns the build off the request thread then runs the runner Job from the built image; spawn_runner.create_job_object takes an image_override. Returns None when no deps are declared, so current experiments are unchanged. - Phase 1: frontend declares pipRequirements / aptPackages (db_types.ts, InformationStep.tsx, NewExperiment.tsx); experiment.py mirrors the fields; app.py reads them from Mongo before building. Supporting changes: - RBAC (tilt + backend cluster roles): grant jobs get/list/watch and configmaps create/delete/get for the build orchestration. - deployment-backend.yaml: REGISTRY_HOST / RUNNER_BASE_IMAGE env. - app.py: log background spawn_job failures via a Future done-callback instead of silently swallowing them. Verified end-to-end via tilt ci in dev/minikube: stack green, Kaniko build FROM runner-base pushes glados-exp:, runner Job spawned from it, build cache hit on repeat. Co-Authored-By: Claude Opus 4.8 --- Tiltfile | 24 +- apps/backend/app.py | 51 +++- apps/backend/build_image.py | 249 ++++++++++++++++++ apps/backend/job-builder.yaml | 48 ++++ apps/backend/job-runner.yaml | 22 ++ apps/backend/spawn_runner.py | 18 +- .../flows/AddExperiment/NewExperiment.tsx | 14 +- .../stepComponents/InformationStep.tsx | 20 ++ apps/frontend/lib/db_types.ts | 5 + apps/runner/modules/data/experiment.py | 6 + apps/runner/modules/runner.py | 29 +- apps/runner/runner-base.Dockerfile | 54 ++++ .../backend/cluster-role-job-creator.yaml | 7 +- kubernetes_init/registry/README.md | 97 +++++++ kubernetes_init/registry/registry.yaml | 26 ++ .../tilt/cluster-role-job-creator.yaml | 7 +- kubernetes_init/tilt/deployment-backend.yaml | 8 + 17 files changed, 663 insertions(+), 22 deletions(-) create mode 100644 apps/backend/build_image.py create mode 100644 apps/backend/job-builder.yaml create mode 100644 apps/runner/runner-base.Dockerfile create mode 100644 kubernetes_init/registry/README.md create mode 100644 kubernetes_init/registry/registry.yaml diff --git a/Tiltfile b/Tiltfile index 1d593e57..06edc48b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -71,12 +71,32 @@ docker_build("backend", ], dockerfile='./apps/backend/backend-dev.Dockerfile') +# In-cluster registry (DEV / minikube) +# ------------------------------------- +# No default_registry() call is needed here: the cluster is created by ctlptl +# (see .devcontainer/post-start.sh -> `ctlptl create cluster minikube +# --registry=ctlptl-registry`), which publishes the `local-registry-hosting` +# ConfigMap in kube-public. Tilt auto-detects that and pushes all docker_build +# images to the ctlptl registry; in-cluster pods pull them from +# `ctlptl-registry:5000`. See kubernetes_init/registry/ for the rationale, +# the declarative Registry manifest, and verification commands. + # Build the runner -docker_build("runner", - context='./apps/runner', +docker_build("runner", + context='./apps/runner', dockerfile='./apps/runner/runner.Dockerfile', match_in_env_vars=True) +# Build the runner base image (interpreter + dependency harness layers). +# The Phase-2 Kaniko build layers user dependencies FROM this base and pushes the +# result to the in-cluster registry. It is consumed via the backend Deployment's +# RUNNER_BASE_IMAGE env var (match_in_env_vars below rewrites the bare `runner-base` +# ref to the pushed image), which is what marks this image as used. +docker_build("runner-base", + context='./apps/runner', + dockerfile='./apps/runner/runner-base.Dockerfile', + match_in_env_vars=True) + # add a command that will run on tilt down to cleanup the pv's that are made by helm if config.tilt_subcommand == 'down': local("helm uninstall glados-mongodb") diff --git a/apps/backend/app.py b/apps/backend/app.py index 33b62125..2cf8cb9e 100644 --- a/apps/backend/app.py +++ b/apps/backend/app.py @@ -2,7 +2,7 @@ import io import threading import base64 -from concurrent.futures import ProcessPoolExecutor +from concurrent.futures import ThreadPoolExecutor import os from bson.binary import Binary from flask import Flask, Response, request, jsonify, send_file @@ -11,13 +11,18 @@ from modules.mongo import upload_experiment_aggregated_results, upload_experiment_zip, upload_log_file, verify_mongo_connection, check_insert_default_experiments, download_experiment_file, get_experiment, update_exp_value from spawn_runner import create_job, create_job_object +from build_image import build_experiment_image flaskApp = Flask(__name__) config.load_incluster_config() BATCH_API = client.BatchV1Api() +CORE_API = client.CoreV1Api() -MAX_WORKERS = 1 -executor = ProcessPoolExecutor(MAX_WORKERS) +# Experiment submission runs the (possibly long) build + Job spawn off the request +# thread. Threads (not processes) so the shared k8s clients above are reused and +# no experiment payload needs pickling. +MAX_WORKERS = int(os.getenv("MAX_WORKERS", "4")) +executor = ThreadPoolExecutor(MAX_WORKERS) # Mongo Setup # create the mongo client @@ -58,12 +63,46 @@ def get_queue(): def recv_experiment(): """The query to run an experiment""" data = request.get_json() - executor.submit(spawn_job, data) + future = executor.submit(spawn_job, data) + future.add_done_callback(_log_spawn_result) return Response(status=200) +def _log_spawn_result(future): + """Surface exceptions raised by the background spawn_job worker. + + ThreadPoolExecutor stores a worker's exception on the Future and never + re-raises it, so without this callback a failed experiment launch (e.g. an + image build error or a Kubernetes API 403) would vanish silently -- the + /experiment request has already returned 200 by the time it runs. + """ + error = future.exception() + if error is not None: + flaskApp.logger.error("Experiment launch failed in background worker", exc_info=error) + def spawn_job(experiment_data): - """Function for creating a job""" - job = create_job_object(experiment_data) + """Build the per-experiment image (if dependencies are declared) then spawn the runner Job. + + The /experiment payload only carries the experiment id, so the declared + dependencies are read from MongoDB and attached before building. If that + lookup fails or no dependencies are declared, build_experiment_image returns + None and create_job_object falls back to the default runner image (unchanged + behaviour for experiments without declared dependencies). + """ + try: + stored = get_experiment(experiment_data['experiment']['id'], mongoClient) + experiment_data['experiment']['pipRequirements'] = stored.get('pipRequirements') + experiment_data['experiment']['aptPackages'] = stored.get('aptPackages') + except Exception: + # Non-fatal: fall back to the default runner image (no per-experiment build), + # but log it so a Mongo/lookup problem is visible rather than silent. + flaskApp.logger.warning( + "Could not read declared dependencies for experiment %s; proceeding without a per-experiment image", + experiment_data.get('experiment', {}).get('id'), + exc_info=True, + ) + + image_override = build_experiment_image(experiment_data, BATCH_API, CORE_API) + job = create_job_object(experiment_data, image_override=image_override) create_job(BATCH_API, job) @flaskApp.post("/cancelExperiment") diff --git a/apps/backend/build_image.py b/apps/backend/build_image.py new file mode 100644 index 00000000..d3f3c627 --- /dev/null +++ b/apps/backend/build_image.py @@ -0,0 +1,249 @@ +"""Phase 2: per-experiment image build orchestration. + +When an experiment declares dependencies, the backend builds a small image +FROM the runner base image with those pip/apt dependencies layered on top, via +an in-cluster Kaniko Job, and pushes it to the in-cluster registry. The backend +then spawns the runner Job FROM the built image (see spawn_runner.py). + +Key design points: +- **Backward compatible:** ``build_experiment_image`` returns ``None`` when the + experiment declares no dependencies, so the caller falls back to the existing + ``IMAGE_RUNNER`` image and behaviour is unchanged for current experiments. +- **Cached by content:** the built image tag is a hash of the base image plus + the rendered build context, so identical dependency sets are built once and + reused (the registry is checked before building). + +Input contract (populated by the frontend in Phase 1, read defensively here): + experiment_data['experiment']['pipRequirements'] : str -> requirements.txt contents + experiment_data['experiment']['aptPackages'] : list[str] | str -> apt packages +""" +import hashlib +import os +import time + +import requests +import yaml +from kubernetes import client, config +from kubernetes.client.rest import ApiException + +# ---- Configuration (env-overridable; defaults target the dev/minikube registry) ---- +REGISTRY_HOST = os.getenv("REGISTRY_HOST", "ctlptl-registry:5000") +RUNNER_BASE_IMAGE = os.getenv("RUNNER_BASE_IMAGE", f"{REGISTRY_HOST}/runner-base:latest") +# The dev ctlptl registry serves plain HTTP; production (TLS) should set this false. +REGISTRY_INSECURE = os.getenv("REGISTRY_INSECURE", "true").lower() in ("1", "true", "yes") +EXP_IMAGE_REPO = os.getenv("EXP_IMAGE_REPO", "glados-exp") +BUILD_NAMESPACE = os.getenv("BUILD_NAMESPACE", "default") +KANIKO_BUILD_TIMEOUT_SECONDS = int(os.getenv("KANIKO_BUILD_TIMEOUT_SECONDS", "900")) + +BUILDER_TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), "job-builder.yaml") + + +def _normalize_apt_packages(apt_packages): + """Accept a list or a whitespace/newline-separated string, return a clean list.""" + if not apt_packages: + return [] + if isinstance(apt_packages, str): + return [pkg for pkg in apt_packages.split() if pkg] + return [str(pkg).strip() for pkg in apt_packages if str(pkg).strip()] + + +def render_dockerfile(pip_requirements, apt_packages): + """Render the build Dockerfile from the declared dependencies. + + Returns ``(dockerfile_text, context_files)`` where context_files is a dict of + filename -> contents that must be present in the build context. + """ + apt_list = _normalize_apt_packages(apt_packages) + lines = [f"FROM {RUNNER_BASE_IMAGE}"] + context_files = {} + + if apt_list: + context_files["apt-packages.txt"] = "\n".join(apt_list) + "\n" + lines += [ + "COPY apt-packages.txt /tmp/apt-packages.txt", + "RUN apt-get update && " + "xargs -a /tmp/apt-packages.txt apt-get install -y --no-install-recommends && " + "rm -rf /var/lib/apt/lists/*", + ] + + if pip_requirements and pip_requirements.strip(): + context_files["requirements.txt"] = pip_requirements + lines += [ + "COPY requirements.txt /tmp/requirements.txt", + "RUN pip install --no-cache-dir -r /tmp/requirements.txt", + ] + + dockerfile_text = "\n".join(lines) + "\n" + context_files["Dockerfile"] = dockerfile_text + return dockerfile_text, context_files + + +def _content_tag(context_files): + """Deterministic image tag from the base image + full build context.""" + digest = hashlib.sha256() + digest.update(RUNNER_BASE_IMAGE.encode("utf-8")) + for name in sorted(context_files): + digest.update(b"\0") + digest.update(name.encode("utf-8")) + digest.update(b"\0") + digest.update(context_files[name].encode("utf-8")) + return digest.hexdigest()[:16] + + +def image_exists(tag): + """Return True if EXP_IMAGE_REPO:tag already exists in the registry.""" + scheme = "http" if REGISTRY_INSECURE else "https" + url = f"{scheme}://{REGISTRY_HOST}/v2/{EXP_IMAGE_REPO}/manifests/{tag}" + headers = { + "Accept": "application/vnd.docker.distribution.manifest.v2+json, " + "application/vnd.oci.image.manifest.v1+json" + } + try: + resp = requests.get(url, headers=headers, timeout=10) + return resp.status_code == 200 + except requests.RequestException: + # Treat an unreachable registry as "not cached"; the build will surface the real error. + return False + + +def _kaniko_args(destination): + args = [ + "--dockerfile=/workspace/Dockerfile", + "--context=dir:///workspace", + f"--destination={destination}", + "--verbosity=info", + ] + if REGISTRY_INSECURE: + args += ["--insecure", "--insecure-pull", "--skip-tls-verify", "--skip-tls-verify-pull"] + return args + + +def _build_job_body(job_name, configmap_name, destination): + with open(BUILDER_TEMPLATE_PATH, encoding="utf-8") as template_file: + body = yaml.safe_load(template_file) + body["metadata"]["name"] = job_name + spec = body["spec"]["template"]["spec"] + spec["containers"][0]["args"] = _kaniko_args(destination) + for volume in spec["volumes"]: + if volume["name"] == "cm": + volume["configMap"]["name"] = configmap_name + return body + + +def _delete_configmap(core_api, name): + try: + core_api.delete_namespaced_config_map(name, BUILD_NAMESPACE) + except ApiException as err: + if err.status != 404: + raise + + +def _delete_job(batch_api, name): + try: + batch_api.delete_namespaced_job(name, BUILD_NAMESPACE, propagation_policy="Background") + except ApiException as err: + if err.status != 404: + raise + + +def _wait_for_job(batch_api, job_name, timeout_seconds): + """Poll the build Job until it completes or fails. Returns 'Complete'/'Failed'/'Timeout'.""" + deadline = time.time() + timeout_seconds + while time.time() < deadline: + # read_namespaced_job (not ..._status): the full Job carries .status and only + # needs the "jobs" get permission, avoiding the separate jobs/status subresource. + status = batch_api.read_namespaced_job(job_name, BUILD_NAMESPACE).status + for condition in (status.conditions or []): + if condition.type == "Complete" and condition.status == "True": + return "Complete" + if condition.type == "Failed" and condition.status == "True": + return "Failed" + time.sleep(3) + return "Timeout" + + +def _kaniko_logs(core_api, job_name): + """Best-effort fetch of the kaniko container logs for diagnostics.""" + try: + pods = core_api.list_namespaced_pod( + BUILD_NAMESPACE, label_selector=f"job-name={job_name}" + ) + if not pods.items: + return "(no build pod found)" + pod_name = pods.items[0].metadata.name + logs = core_api.read_namespaced_pod_log(pod_name, BUILD_NAMESPACE, container="kaniko") + return "\n".join(logs.splitlines()[-20:]) + except ApiException: + return "(could not read build logs)" + + +def build_experiment_image(experiment_data, batch_api, core_api): + """Build (or reuse) the per-experiment image and return its full reference. + + Returns ``None`` when the experiment declares no dependencies, signalling the + caller to use the default runner image (unchanged current behaviour). + """ + experiment = experiment_data.get("experiment", {}) + pip_requirements = experiment.get("pipRequirements") + apt_packages = experiment.get("aptPackages") + + if not (pip_requirements and pip_requirements.strip()) and not _normalize_apt_packages(apt_packages): + # No declared dependencies -> nothing to build; use the default runner image. + return None + + _, context_files = render_dockerfile(pip_requirements, apt_packages) + tag = _content_tag(context_files) + destination = f"{REGISTRY_HOST}/{EXP_IMAGE_REPO}:{tag}" + + # Cache hit: identical dependency set already built. + if image_exists(tag): + return destination + + exp_id = experiment.get("id", tag) + job_name = f"builder-{exp_id}" + configmap_name = f"builder-ctx-{exp_id}" + + # Clean any leftovers from a previous attempt for this experiment. + _delete_job(batch_api, job_name) + _delete_configmap(core_api, configmap_name) + + configmap = client.V1ConfigMap( + metadata=client.V1ObjectMeta(name=configmap_name), + data=context_files, + ) + core_api.create_namespaced_config_map(BUILD_NAMESPACE, configmap) + batch_api.create_namespaced_job( + BUILD_NAMESPACE, _build_job_body(job_name, configmap_name, destination) + ) + + try: + result = _wait_for_job(batch_api, job_name, KANIKO_BUILD_TIMEOUT_SECONDS) + if result != "Complete": + logs = _kaniko_logs(core_api, job_name) + raise RuntimeError( + f"Image build for experiment {exp_id} did not complete ({result}). " + f"Kaniko logs (tail):\n{logs}" + ) + return destination + finally: + _delete_configmap(core_api, configmap_name) + _delete_job(batch_api, job_name) + + +def _ensure_config(): + """Load in-cluster config when running as a pod, else fall back to local kubeconfig.""" + try: + config.load_incluster_config() + except config.ConfigException: + config.load_kube_config() + + +if __name__ == "__main__": + # Local smoke test against the current kube context (e.g. from the devcontainer): + # python build_image.py + _ensure_config() + _batch = client.BatchV1Api() + _core = client.CoreV1Api() + _data = {"experiment": {"id": "localtest", "pipRequirements": "cowsay==6.1\n"}} + print("Building test image...") + print("Built image reference:", build_experiment_image(_data, _batch, _core)) diff --git a/apps/backend/job-builder.yaml b/apps/backend/job-builder.yaml new file mode 100644 index 00000000..8894bf7d --- /dev/null +++ b/apps/backend/job-builder.yaml @@ -0,0 +1,48 @@ +# Template for the per-experiment image BUILD Job (Phase 2). +# +# The backend (build_image.py) loads this, fills in the Job/ConfigMap names and +# the Kaniko args (destination + insecure flags), and creates it. A Kaniko Job +# builds a small image FROM the runner base with the user's declared pip/apt +# dependencies layered on top, and pushes it to the in-cluster registry. The +# backend then spawns the runner Job (job-runner.yaml) FROM the built image. +# +# The initContainer stages the build context from a ConfigMap into an emptyDir +# using `cp -L`: ConfigMap volumes expose their files as symlinks, and Kaniko's +# COPY would otherwise copy a dangling symlink (verified during the Phase 2 +# spike). Kaniko must see real files. +apiVersion: batch/v1 +kind: Job +metadata: + name: builder # replaced by build_image.py + namespace: default +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app: GLADOS-builder + spec: + restartPolicy: Never + initContainers: + - name: stage-context + image: busybox:1.36 + command: ["sh", "-c", "cp -L /cm/* /workspace/ && ls -la /workspace"] + volumeMounts: + - name: cm + mountPath: /cm + - name: context + mountPath: /workspace + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:latest + args: [] # replaced by build_image.py (dockerfile/context/destination + insecure flags) + volumeMounts: + - name: context + mountPath: /workspace + volumes: + - name: cm + configMap: + name: builder-context # replaced by build_image.py + - name: context + emptyDir: {} diff --git a/apps/backend/job-runner.yaml b/apps/backend/job-runner.yaml index 7212a09f..a1e5b573 100644 --- a/apps/backend/job-runner.yaml +++ b/apps/backend/job-runner.yaml @@ -14,6 +14,28 @@ spec: image: gladospipeline/glados-runner:main imagePullPolicy: Always command: [] + # Hardening for a pod that executes arbitrary user-submitted code. + # NOTE: runAsNonRoot / readOnlyRootFilesystem are intentionally omitted + # because the runner image runs as root and writes to its working dir on + # the root filesystem; enabling those requires a non-root user in the + # runner Dockerfile plus a writable volume mount for the work dir. + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + # Bound resource usage so a single experiment cannot starve its node. + # Tune these to your workloads; too-low limits will kill legitimate runs + # (the memory limit in particular triggers an OOM-kill). + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + cpu: "4" + memory: "10Gi" env: - name: MONGODB_PORT valueFrom: diff --git a/apps/backend/spawn_runner.py b/apps/backend/spawn_runner.py index 6e417ce7..935286be 100644 --- a/apps/backend/spawn_runner.py +++ b/apps/backend/spawn_runner.py @@ -10,19 +10,27 @@ batch_v1 = client.BatchV1Api() RUNNER_PATH = "./job-runner.yaml" -def create_job_object(experiment_data): - """Function that creates the job object for the runner""" +def create_job_object(experiment_data, image_override=None): + """Function that creates the job object for the runner. + + If image_override is provided (a per-experiment image built with the user's + declared dependencies, see build_image.py), the runner Job runs from it. + Otherwise the behaviour is unchanged: the IMAGE_RUNNER env image is used if + set, else the default image baked into job-runner.yaml. + """ # Configure Pod template container job_name = "runner-" + experiment_data['experiment']['id'] - + job_command = ["python3", "runner.py", json.dumps(experiment_data)] runner_body = get_yaml_file_body(RUNNER_PATH) runner_body['metadata']['name'] = job_name runner_body['spec']['template']['spec']['containers'][0]['command'] = job_command - - if os.getenv("IMAGE_RUNNER"): + + if image_override: + runner_body['spec']['template']['spec']['containers'][0]['image'] = image_override + elif os.getenv("IMAGE_RUNNER"): # Get the image name image_name = str(os.getenv("IMAGE_RUNNER")) runner_body['spec']['template']['spec']['containers'][0]['image'] = image_name diff --git a/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx b/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx index 4df63494..5749938d 100644 --- a/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx +++ b/apps/frontend/app/components/flows/AddExperiment/NewExperiment.tsx @@ -124,6 +124,8 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, isDefault, status: 'CREATED', experimentExecutable: '', configFileFormat: '', + pipRequirements: '', + aptPackages: '', }, validate: joiResolver(experimentSchema), }); @@ -156,7 +158,9 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, isDefault, file: newFileId, status: 'CREATED', experimentExecutable: expInfo['experimentExecutable'], - configFileFormat: expInfo['configFileFormat'] + configFileFormat: expInfo['configFileFormat'], + pipRequirements: expInfo['pipRequirements'] ?? '', + aptPackages: expInfo['aptPackages'] ?? '' }); setCopyId(null); setStatus(FormStates.Info); @@ -183,7 +187,9 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, isDefault, file: expInfo['file'], status: 'CREATED', experimentExecutable: expInfo['experimentExecutable'], - configFileFormat: expInfo['configFileFormat'] + configFileFormat: expInfo['configFileFormat'], + pipRequirements: expInfo['pipRequirements'] ?? '', + aptPackages: expInfo['aptPackages'] ?? '' }); setFileId(expInfo['file']); setCopyId(null); @@ -233,7 +239,9 @@ const NewExperiment = ({ formState, setFormState, copyID, setCopyId, isDefault, status: 'CREATED', sendEmail: expInfo['sendEmail'], experimentExecutable: expInfo['experimentExecutable'], - configFileFormat: expInfo['configFileFormat'] + configFileFormat: expInfo['configFileFormat'], + pipRequirements: expInfo['pipRequirements'] ?? '', + aptPackages: expInfo['aptPackages'] ?? '' }); setCopyId(null); diff --git a/apps/frontend/app/components/flows/AddExperiment/stepComponents/InformationStep.tsx b/apps/frontend/app/components/flows/AddExperiment/stepComponents/InformationStep.tsx index 124f2891..bdb236ef 100644 --- a/apps/frontend/app/components/flows/AddExperiment/stepComponents/InformationStep.tsx +++ b/apps/frontend/app/components/flows/AddExperiment/stepComponents/InformationStep.tsx @@ -163,6 +163,26 @@ export const InformationStep = ({ form, validationErrors, setValidationErrors, . /> + +
+