From 82b97ce73946bec7b3a02927824df69beb953656 Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Mon, 17 Aug 2026 20:12:33 +0600 Subject: [PATCH 1/7] Update image registry references to nukehub-dev --- backend/tests/api/environments/test_environments.py | 4 ++-- docs/operations/GPU-SETUP.md | 4 ++-- environments/conda-base/Dockerfile | 3 ++- environments/workspace/Dockerfile | 3 ++- frontend/src/routes/admin.environments.tsx | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/tests/api/environments/test_environments.py b/backend/tests/api/environments/test_environments.py index ecf5026d..778bf870 100644 --- a/backend/tests/api/environments/test_environments.py +++ b/backend/tests/api/environments/test_environments.py @@ -135,7 +135,7 @@ async def test_create_environment_with_toolchain(self, client, admin_token): "name": "Toolchain Environment", "slug": "toolchain-env", "image": "nukelab/workspace:latest", - "tool_image": "ghcr.io/nukelab/radiation-transport:v1.0.0", + "tool_image": "ghcr.io/nukehub-dev/radiation-transport:v1.0.0", "tool_mounts": ["/opt/nuke"], "category": "simulation", }, @@ -143,7 +143,7 @@ async def test_create_environment_with_toolchain(self, client, admin_token): assert response.status_code == 201 data = response.json()["data"] - assert data["tool_image"] == "ghcr.io/nukelab/radiation-transport:v1.0.0" + assert data["tool_image"] == "ghcr.io/nukehub-dev/radiation-transport:v1.0.0" assert data["tool_mounts"] == ["/opt/nuke"] @pytest.mark.asyncio diff --git a/docs/operations/GPU-SETUP.md b/docs/operations/GPU-SETUP.md index 657f9426..45575f91 100644 --- a/docs/operations/GPU-SETUP.md +++ b/docs/operations/GPU-SETUP.md @@ -86,11 +86,11 @@ GPU's memory pool and can OOM each other. Or pull a published image: ```bash - docker pull ghcr.io/nukelab/gpu:v1.0.0 + docker pull ghcr.io/nukehub-dev/gpu:v1.0.0 ``` 2. In the admin UI, create an **Environment** record pointing at the image - (e.g. `ghcr.io/nukelab/gpu:v1.0.0`). + (e.g. `ghcr.io/nukehub-dev/gpu:v1.0.0`). 3. Create or edit a **Plan** with **GPU** (`gpu_limit`) greater than 0. With exclusive allocation enabled, `gpu_limit` counts whole physical GPUs per server; keep the sum of concurrent GPU usage within the `GPU_DEVICES` pool. diff --git a/environments/conda-base/Dockerfile b/environments/conda-base/Dockerfile index 63052193..d77b079c 100644 --- a/environments/conda-base/Dockerfile +++ b/environments/conda-base/Dockerfile @@ -5,7 +5,8 @@ # domain-specific scientific toolchain images. Keeping this separate from # workspace means IDE updates do not force a rebuild of heavy C++/Fortran # scientific stacks. -ARG BASE_IMAGE=ghcr.io/nukelab/base:v1.0.0 +ARG REGISTRY=ghcr.io/nukehub-dev +ARG BASE_IMAGE=${REGISTRY}/base:v1.0.0 FROM $BASE_IMAGE diff --git a/environments/workspace/Dockerfile b/environments/workspace/Dockerfile index 6a2f0df4..298dc390 100644 --- a/environments/workspace/Dockerfile +++ b/environments/workspace/Dockerfile @@ -1,8 +1,9 @@ # Copyright (c) NukeLab Development Team. # Distributed under the terms of the BSD-2-Clause license. +ARG REGISTRY=ghcr.io/nukehub-dev ARG BASE_TAG=v1.0.0 -ARG BASE_IMAGE=ghcr.io/nukelab/conda-base:${BASE_TAG} +ARG BASE_IMAGE=${REGISTRY}/conda-base:${BASE_TAG} FROM $BASE_IMAGE diff --git a/frontend/src/routes/admin.environments.tsx b/frontend/src/routes/admin.environments.tsx index a90e8b1e..95e58ca9 100644 --- a/frontend/src/routes/admin.environments.tsx +++ b/frontend/src/routes/admin.environments.tsx @@ -666,7 +666,7 @@ function EnvironmentsPage() { type="text" value={formData.tool_image} onChange={(e) => setFormData({ ...formData, tool_image: e.target.value })} - placeholder="ghcr.io/nukelab/radiation-transport:v1.0.0" + placeholder="ghcr.io/nukehub-dev/radiation-transport:v1.0.0" />

Optional scientific toolchain mounted read-only into the runtime container at From ce278b771e32b83066eefd65f27c78620354c42c Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Fri, 21 Aug 2026 20:49:47 +0600 Subject: [PATCH 2/7] Inject resolved CI image version into backend builds --- .github/workflows/ci.yml | 3 ++ AGENTS.md | 13 ++++++ CHANGELOG.md | 47 +++++++++++++++++++++ VERSION | 1 + backend/AGENTS.md | 1 + backend/Dockerfile | 6 +++ backend/app/api/health.py | 2 +- backend/app/config.py | 24 ++++++++++- backend/app/main.py | 4 +- backend/app/version.py | 13 ++++++ backend/tests/core/test_config.py | 29 +++++++++++++ backend/tests/main/test_main.py | 4 +- docs/plan/DECISION-LOG.md | 1 + frontend/package-lock.json | 4 +- frontend/package.json | 2 +- scripts/AGENTS.md | 1 + scripts/bump-version.sh | 70 +++++++++++++++++++++++++++++++ 17 files changed, 216 insertions(+), 9 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 VERSION create mode 100644 backend/app/version.py create mode 100644 scripts/bump-version.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0d5bf5e..62f9377a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,6 +225,9 @@ jobs: target: ${{ matrix.target }} push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta.outputs.tags }} + # Inject the resolved version into the backend image so the running + # API reports the exact image tag (APP_VERSION env, see backend/Dockerfile). + build-args: ${{ matrix.name == 'backend' && format('APP_VERSION={0}', steps.meta.outputs.version) || '' }} labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.revision=${{ github.sha }} diff --git a/AGENTS.md b/AGENTS.md index 0b919bef..f57dc429 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,6 +124,19 @@ Notes: path/to/file.test.ts` directly. See `frontend/AGENTS.md` for frontend conventions. +## Releases + +- Version source of truth: git tags (`vX.Y.Z`). `scripts/ci-version.sh` turns + them into semver image tags plus `latest` in CI, and CI injects the resolved + version into the backend image via the `APP_VERSION` build arg (runtime + resolution: `settings.app_version`, fallback `backend/app/version.py`). +- Cut a release with `scripts/bump-version.sh X.Y.Z` — it syncs `VERSION`, + `frontend/package.json`, and `CHANGELOG.md` (the backend version is dynamic + via `APP_VERSION`; `backend/app/version.py` stays `0.0.0-dev`), then prints + the commit/tag/push commands (it never runs them). +- Record notable changes in the root `CHANGELOG.md` (Keep a Changelog format, + `[Unreleased]` section). + ## Architecture pointer High-level layout; see the Child NAD Index below for domain-specific details. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..81593b1a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog + +All notable changes to the NukeLab platform are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Releases are cut with `scripts/bump-version.sh X.Y.Z`, which stamps the +`[Unreleased]` section below and syncs the version across `VERSION`, +`backend/app/version.py`, and `frontend/package.json`. Git tags (`vX.Y.Z`) +are the release source of truth; CI builds container images tagged with the +release version. + +## [Unreleased] + +### Added + +- Release versioning: git tags (`vX.Y.Z`) are the single source of truth. + `scripts/bump-version.sh` syncs the version across `VERSION`, + `backend/app/version.py`, `frontend/package.json`, and this changelog. +- Dynamic version injection: CI image builds pass the resolved tag as the + `APP_VERSION` build arg (`backend/Dockerfile`), so running containers + report the exact image tag (`2.1.0`, `main`, `pr-42`). The API root, + `/health/status`, and the OpenTelemetry service version resolve via + `settings.app_version`; empty or unset `APP_VERSION` falls back to the + static literal in `app/version.py`. + +### Changed + +- Backend version string is no longer hardcoded: `app/main.py`, + `app/api/health.py`, and the OpenTelemetry service version in + `app/config.py` all resolve through `settings.app_version` (env-first, + static fallback in `app/version.py`). +- The static fallback in `app/version.py` is now the fixed sentinel + `0.0.0-dev` instead of a release-looking literal — a version that can + never be mistaken for a release, and which `bump-version.sh` no longer + touches (only `VERSION`, `frontend/package.json` + lockfile, and this + changelog are bumped at release time). +- `frontend/package.json` version set from `0.0.0` placeholder to the + platform version. + +## [2.0.0] + +The 2.0 platform was delivered phase-by-phase without git tags. See +[docs/plan/IMPLEMENTATION-PHASES.md](docs/plan/IMPLEMENTATION-PHASES.md) +for the full delivery record and [docs/plan/ROADMAP.md](docs/plan/ROADMAP.md) +for recent milestones. diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..46b105a3 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v2.0.0 diff --git a/backend/AGENTS.md b/backend/AGENTS.md index f02746f7..a51ea9ac 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -12,6 +12,7 @@ All files under `backend/` except generated artifacts (`.venv-dev`, `__pycache__ - Python 3.13; formatting and linting configured in `pyproject.toml`. - `app/main.py` is the ASGI entry point. +- `app/version.py` holds the static fallback `__version__` — permanently `0.0.0-dev`, never bumped. The effective runtime version is `settings.app_version` (`app/config.py`): the `APP_VERSION` env var — injected as a Docker build arg by CI from the image tag — wins, with empty/unset falling back to the literal. `main.py` and `app/api/health.py` read `settings.app_version` — never hardcode the version string. - `app/api/` owns route definitions; `app/services/` owns business logic; `app/models/` owns SQLAlchemy models; `app/db/` owns session/connection logic; `app/core/` owns cross-cutting utilities; `app/middleware/` owns ASGI middleware; `app/container/` owns container-runtime orchestration; `app/tasks.py` and `app/worker.py` own Celery. - `app/container/` is a driver layer: `driver.py` defines the `ContainerDriver` ABC + `ContainerDriverError` (plain-data returns only — no runtime objects escape), `docker_driver.py` is the Docker/Podman implementation, `factory.py` selects the driver via `CONTAINER_RUNTIME` (default `docker`), `client.py` is a compatibility shim for legacy imports/test seams, and `spawner.py` (server lifecycle) talks only to driver methods. To add a runtime (e.g. Kubernetes): implement `ContainerDriver` (synthesizing the documented return shapes, e.g. Docker-stats for `get_container_stats`) and register it in `factory.py`. Container health is driver-level config, not image metadata: `docker_driver.py` injects a uniform `Healthcheck` (`/usr/local/bin/nukelab-healthcheck.sh`) into every create config, because OCI images drop Dockerfile `HEALTHCHECK` and Kubernetes ignores it — a k8s driver must translate the same definition into pod liveness/startup probes and surface failures as the same `State.Health.Status` shape `HealthCheckService` consumes. `spawner.py` performs two readiness probes before marking a server `running`: (1) the container's own `/health` endpoint over the Docker network alias, and (2) the public server path through the internal Traefik load balancer (`TRAEFIK_INTERNAL_URL`, default `http://traefik:80`) with a `healthy` body check. This ensures the browser-facing route exists before the frontend is told to redirect. - Notifications: `app/services/notification_service.py` owns notification creation and delivery. Channels are `in_app`, `email`, `webhook`, and (when VAPID keys are configured) `push`. Push payloads are short previews only; dead subscriptions are removed on `404`/`410`. VAPID config and the `push_subscriptions` model live in `app/config.py` and `app/models/push_subscription.py`; pass `VAPID_*` env vars to both the `backend` and `celery-worker` containers. diff --git a/backend/Dockerfile b/backend/Dockerfile index 7ca7f870..e4f1a541 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -33,6 +33,12 @@ CMD ["python", "-m", "pytest"] # ── Production / runtime target (default) ─────────────────────────────────── FROM base AS runtime +# Platform version injected by CI (from scripts/ci-version.sh). Empty for +# local builds; the app then falls back to the static version in +# app/version.py (see app/config.py: app_version). +ARG APP_VERSION="" +ENV APP_VERSION=${APP_VERSION} + # Expose port EXPOSE 8000 diff --git a/backend/app/api/health.py b/backend/app/api/health.py index 9d0bcef3..d058d4c6 100644 --- a/backend/app/api/health.py +++ b/backend/app/api/health.py @@ -190,7 +190,7 @@ async def platform_status(): from app.services.oauth_service import oauth_service return { - "version": "2.0.0", + "version": settings.app_version, "features": { "auth_mode": settings.auth_mode, "oauth_enabled": oauth_service.is_configured diff --git a/backend/app/config.py b/backend/app/config.py index 84621ca7..9549d39e 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -8,6 +8,8 @@ from pydantic import field_validator, model_validator from pydantic_settings import BaseSettings +from app.version import __version__ + class Settings(BaseSettings): app_name: str = "NukeLab" @@ -18,6 +20,11 @@ class Settings(BaseSettings): frontend_url: str = "" # Defaults to public_url if not set app_timezone: str = "UTC" + # Platform version. Defaults to the checked-in fallback in app/version.py; + # CI image builds inject APP_VERSION (from scripts/ci-version.sh) so + # containers report the exact image tag (e.g. 2.1.0, main, pr-42). + app_version: str = __version__ + maintenance_mode: bool = False maintenance_message: str = "System under maintenance" @@ -240,7 +247,7 @@ def gpu_device_list(self) -> list[str]: otel_exporter_otlp_endpoint: str = "http://otel-collector:4317" otel_exporter_otlp_protocol: str = "grpc" # grpc | http otel_service_name: str = "nukelab-backend" - otel_service_version: str = "2.0.0" + otel_service_version: str = "" # Defaults to app_version; override via OTEL_SERVICE_VERSION otel_log_correlation: bool = True otel_sampler_ratio: float = 1.0 @@ -290,6 +297,14 @@ def gpu_device_list(self) -> list[str]: user_auth_denylist_fail_closed: bool = True user_auth_key_rotation_grace_seconds: int | None = None + @field_validator("app_version", mode="before") + @classmethod + def _empty_app_version_to_fallback(cls, value: Any) -> Any: + """Treat an empty APP_VERSION env value as "use the static fallback".""" + if value == "" or value is None: + return __version__ + return value + @field_validator("user_auth_key_rotation_grace_seconds", mode="before") @classmethod def _empty_rotation_grace_to_none(cls, value: Any) -> Any: @@ -327,6 +342,13 @@ def set_key_paths(self) -> "Settings": ) return self + @model_validator(mode="after") + def set_otel_service_version(self) -> "Settings": + """Default the OTEL service version to the resolved app version.""" + if not self.otel_service_version: + self.otel_service_version = self.app_version + return self + @model_validator(mode="after") def set_user_auth_rotation_grace(self) -> "Settings": """Default key rotation grace period to 2× access-token lifetime.""" diff --git a/backend/app/main.py b/backend/app/main.py index 2ce49d32..b441c037 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -143,7 +143,7 @@ async def lifespan(app: FastAPI): _app_kwargs = { "title": settings.app_name, "description": "NukeLab Platform v2.0 API", - "version": "2.0.0", + "version": settings.app_version, "debug": settings.app_debug, "root_path": "/api", "lifespan": lifespan, @@ -296,7 +296,7 @@ async def websocket_endpoint(websocket: WebSocket): @app.get("/") async def root(): - return {"message": f"Welcome to {settings.app_name} API", "version": "2.0.0"} + return {"message": f"Welcome to {settings.app_name} API", "version": settings.app_version} @app.get("/health") diff --git a/backend/app/version.py b/backend/app/version.py new file mode 100644 index 00000000..77ccb02e --- /dev/null +++ b/backend/app/version.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2023-2026 NukeHub Developers +# SPDX-License-Identifier: BSD-2-Clause + +"""NukeLab platform version (static fallback). + +This is the checked-in fallback for builds that did not receive a version +injection (local dev, tests). It is intentionally "0.0.0-dev" — unmistakably +not a release. Real releases are identified dynamically: CI-built images get +the exact image tag via the APP_VERSION build arg, resolved through +`settings.app_version` (app/config.py). +""" + +__version__ = "0.0.0-dev" diff --git a/backend/tests/core/test_config.py b/backend/tests/core/test_config.py index bc09f4c6..3636e3ad 100644 --- a/backend/tests/core/test_config.py +++ b/backend/tests/core/test_config.py @@ -8,6 +8,7 @@ import pytest from app.config import Settings +from app.version import __version__ class TestProductionUserAuthKeyValidation: @@ -120,3 +121,31 @@ def test_explicit_override_wins_in_production(self, tmp_path): def test_explicit_disable_in_development(self): assert Settings(app_env="development", api_docs_enabled=False).api_docs_enabled is False + + +class TestAppVersion: + """APP_VERSION (injected as a Docker build arg in CI) overrides the static + fallback from app/version.py; empty values fall back.""" + + def test_default_is_static_fallback(self, monkeypatch): + monkeypatch.delenv("APP_VERSION", raising=False) + assert Settings().app_version == __version__ + + def test_env_override_wins(self, monkeypatch): + monkeypatch.setenv("APP_VERSION", "9.9.9-ci") + assert Settings().app_version == "9.9.9-ci" + + def test_empty_env_falls_back(self, monkeypatch): + """Local Docker builds set APP_VERSION to an empty string.""" + monkeypatch.setenv("APP_VERSION", "") + assert Settings().app_version == __version__ + + def test_otel_service_version_defaults_to_app_version(self, monkeypatch): + monkeypatch.setenv("APP_VERSION", "9.9.9-ci") + monkeypatch.delenv("OTEL_SERVICE_VERSION", raising=False) + assert Settings().otel_service_version == "9.9.9-ci" + + def test_otel_service_version_explicit_override(self, monkeypatch): + monkeypatch.setenv("APP_VERSION", "9.9.9-ci") + monkeypatch.setenv("OTEL_SERVICE_VERSION", "custom") + assert Settings().otel_service_version == "custom" diff --git a/backend/tests/main/test_main.py b/backend/tests/main/test_main.py index 20eb4ef2..36bb4ec3 100644 --- a/backend/tests/main/test_main.py +++ b/backend/tests/main/test_main.py @@ -20,7 +20,7 @@ def test_app_title(self): assert app.title == settings.app_name def test_app_version(self): - assert app.version == "2.0.0" + assert app.version == settings.app_version def test_app_root_path(self): assert app.root_path == "/api" @@ -106,7 +106,7 @@ async def test_root_returns_welcome(self): result = await root() assert "message" in result assert settings.app_name in result["message"] - assert result["version"] == "2.0.0" + assert result["version"] == settings.app_version class TestHealthEndpoint: diff --git a/docs/plan/DECISION-LOG.md b/docs/plan/DECISION-LOG.md index 4b458811..84b132a7 100644 --- a/docs/plan/DECISION-LOG.md +++ b/docs/plan/DECISION-LOG.md @@ -24,3 +24,4 @@ Reversible architecture and process decisions for NukeLab v2.0. | 2026-08-14 | Separate `nukelab-environments` repository + runtime composition for scientific images | Keeps platform repo focused on runtime; toolchain images mount into `nukelab-workspace` at spawn time so workspace/IDE updates do not force MOAB/Geant4/OpenMC rebuilds; k3s-compatible via init-container volume population | Approved | | 2026-08-14 | Toolchain volume locking, stamp invalidation, and hardened helper containers | Cross-process lock container prevents racing populates; image-ID stamp invalidates stale volumes on tag re-push; hash-suffixed volume names prevent truncation collisions; helper containers drop capabilities so shared-volume content cannot be poisoned via privileged helpers | Approved | | 2026-08-15 | Toolchain datasets under `/opt/nuke/data/`; separate shared data volume deferred | OpenMC cross-sections/chain files move from `/opt/nuke/openmc_data` to `/opt/nuke/data/openmc`; Geant4 datasets stay under `/opt/nuke/geant4/share/data` (Geant4-native discovery via `geant4.sh`/compiled datadir). A dedicated data volume (independent code/data refresh, sharing datasets across toolchain versions) is deferred until dataset size/cadence justifies multi-volume manifests. Open follow-up: verify Geant4 per-dataset `G4*` env vars reach composed runtime containers (source `geant4.sh` from `toolchain-env.sh` if missing) at the next nuclear-base rebuild | Deferred | +| 2026-08-21 | Git tags as the version source of truth; `scripts/bump-version.sh` + root `CHANGELOG.md`; CI injects `APP_VERSION` build arg | Versions were scattered (`frontend/package.json` at `0.0.0`, backend `2.0.0` hardcoded in three files) and no tags existed, so `ci-version.sh` never produced semver image tags. The bumper syncs `VERSION`, `frontend/package.json` (+lockfile), and the changelog. Runtime version is fully dynamic: CI passes the resolved image tag as the backend's `APP_VERSION` build arg and `settings.app_version` resolves env-first, so `backend/app/version.py` is a fixed `0.0.0-dev` sentinel for local builds that is never bumped. The changelog stays manual (Keep a Changelog, same format as nuke-ide) because the commit history does not follow conventional commits, which auto-generators require | Approved | diff --git a/frontend/package-lock.json b/frontend/package-lock.json index df10de64..5a6f889f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend", - "version": "0.0.0", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "frontend", - "version": "0.0.0", + "version": "2.0.0", "dependencies": { "@sentry/browser": "^10.57.0", "@sentry/react": "^10.57.0", diff --git a/frontend/package.json b/frontend/package.json index 4757c348..ffe21d72 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.0.0", + "version": "2.0.0", "type": "module", "engines": { "node": ">=24.0.0" diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index ac4c387f..8184e3d0 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -25,6 +25,7 @@ All files under `scripts/`, plus the top-level `nukelabctl` dispatcher. - `set -E` ERR trap is active; append `|| true` when invoking tools that legitimately return non-zero (e.g., `shfmt -l`, `git describe`). - `_acquire_lock` uses `flock` on a persistent fd (noclobber pidfile fallback); modules must not replace the dispatcher's EXIT/INT/TERM traps — lock cleanup chains through `_release_lock` from the existing traps. - Do not hardcode the version string or names of named volumes/services; use `_nukelab_version` and `_backend_services`. Discover compose-managed volumes via the `com.docker.compose.project` label rather than hardcoded name prefixes. +- Release versioning: git tags (`vX.Y.Z`) are the source of truth. `scripts/bump-version.sh X.Y.Z` syncs `VERSION`, `frontend/package.json` + `frontend/package-lock.json` (via `npm version --no-git-tag-version`), and `CHANGELOG.md`; it never commits or tags. It deliberately does not touch `backend/app/version.py` — the backend version is dynamic (CI injects the image tag via the `APP_VERSION` build arg) and `version.py` is a fixed `0.0.0-dev` fallback. - `_backend_services` returns a space-separated string meant to word-split; do not quote it at the call site (`# shellcheck disable=SC2086`). - Environment build order matters: `manage.d/build.sh` builds `services/build-auth-sidecar.sh` before any `env base` build (base embeds the sidecar binary), then `conda-base`, then `workspace`/`dev`. `build-all.sh` mirrors that order. Keep the sidecar first when touching build orchestration. - When adding or changing `nukelabctl` commands, targets, or flags, update diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh new file mode 100644 index 00000000..387b789b --- /dev/null +++ b/scripts/bump-version.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2023-2026 NukeHub Developers +# SPDX-License-Identifier: BSD-2-Clause + +# Bump the NukeLab platform version across all version-bearing files. +# +# The git tag (vX.Y.Z) is the release source of truth; this script syncs +# every checked-in copy of the version so a release is one command: +# +# scripts/bump-version.sh 2.1.0 +# +# Updates: +# VERSION - CLI/publish artifact read by _nukelab_version +# frontend/package.json - frontend package version (via npm, which also +# frontend/package-lock.json keeps the lockfile in sync) +# CHANGELOG.md - stamps [Unreleased] with the new version + date +# +# Deliberately NOT updated: backend/app/version.py. The backend reports its +# version dynamically (APP_VERSION build arg injected by CI); version.py is a +# fixed "0.0.0-dev" fallback for local builds and is never bumped. +# +# The script never commits or tags; it prints the follow-up git commands. + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +die() { + echo "error: $*" >&2 + exit 1 +} + +_version="${1:-}" +[[ "$_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ + || die "usage: scripts/bump-version.sh (e.g. scripts/bump-version.sh 2.1.0)" + +_date="$(date +%F)" + +# VERSION file (v-prefixed, matching `git describe --tags` output format). +echo "v$_version" > "$DIR/VERSION" + +# Frontend: package.json + package-lock.json (npm keeps both in sync; +# --no-git-tag-version prevents npm from committing or tagging). +# Tolerate "Version not changed" so re-running a bump stays idempotent. +_npm_out="$(cd "$DIR/frontend" && npm version "$_version" --no-git-tag-version 2>&1)" \ + || [[ "$_npm_out" == *"Version not changed"* ]] \ + || die "npm version failed: $_npm_out" + +# CHANGELOG: stamp [Unreleased] with the new version and date (skip when the +# version heading already exists, so re-runs stay idempotent). +if grep -q "^## \[$_version\]" "$DIR/CHANGELOG.md"; then + echo "note: CHANGELOG.md already has [$_version]; left unchanged" +elif grep -q '^## \[Unreleased\]' "$DIR/CHANGELOG.md"; then + sed -i "s/^## \[Unreleased\]$/## [Unreleased]\n\n## [$_version] - $_date/" \ + "$DIR/CHANGELOG.md" +else + echo "warning: no [Unreleased] section in CHANGELOG.md; left unchanged" >&2 +fi + +echo "Bumped to $_version:" +echo " VERSION -> v$_version" +echo " frontend/package.json -> $_version (lockfile synced)" +echo " CHANGELOG.md -> [$_version] - $_date" +echo " backend -> dynamic (APP_VERSION build arg); no file to bump" +echo +echo "Next steps:" +echo " git add VERSION frontend/package.json frontend/package-lock.json CHANGELOG.md" +echo " git commit -m \"chore: bump version to $_version\"" +echo " git tag v$_version" +echo " git push origin main --tags # CI tags images: $_version, sha-, latest" From 98a048e798ac79b31ad4535607746beff807a976 Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Fri, 21 Aug 2026 20:59:35 +0600 Subject: [PATCH 3/7] Bake platform version into local backend images --- .env.example | 3 +++ AGENTS.md | 3 +++ CHANGELOG.md | 6 ++++++ backend/AGENTS.md | 2 +- compose.yml | 6 ++++++ docs/plan/DECISION-LOG.md | 3 ++- docs/plan/ROADMAP.md | 1 + scripts/AGENTS.md | 1 + scripts/bump-version.sh | 0 scripts/lib.sh | 10 ++++++++++ 10 files changed, 33 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/bump-version.sh diff --git a/.env.example b/.env.example index 3c30aa6c..7564a47b 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,9 @@ APP_DEBUG=true # false in production APP_URL=http://localhost:8080 # Your application URL # FRONTEND_URL=http://localhost:5173 # Optional: Set when frontend runs separately (e.g., Vite dev server) APP_TIMEZONE=UTC +# NUKELAB_VERSION=2.1.0 # Optional: version baked into locally built backend + # images (APP_VERSION build arg). Defaults to the VERSION + # file / git tag via nukelabctl. CI-built images ignore this. # ============================================================================= # SECURITY ⚠️ CHANGE SECRETS FOR PRODUCTION diff --git a/AGENTS.md b/AGENTS.md index f57dc429..533b7132 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,6 +130,9 @@ Notes: them into semver image tags plus `latest` in CI, and CI injects the resolved version into the backend image via the `APP_VERSION` build arg (runtime resolution: `settings.app_version`, fallback `backend/app/version.py`). + Local compose builds get the same treatment: `nukelabctl` exports + `NUKELAB_VERSION` (VERSION file / git describe) and `compose.yml` passes it + as the `APP_VERSION` build arg. - Cut a release with `scripts/bump-version.sh X.Y.Z` — it syncs `VERSION`, `frontend/package.json`, and `CHANGELOG.md` (the backend version is dynamic via `APP_VERSION`; `backend/app/version.py` stays `0.0.0-dev`), then prints diff --git a/CHANGELOG.md b/CHANGELOG.md index 81593b1a..a5796269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,12 @@ release version. `/health/status`, and the OpenTelemetry service version resolve via `settings.app_version`; empty or unset `APP_VERSION` falls back to the static literal in `app/version.py`. +- Compose builds bake the platform version into locally built backend images: + `nukelabctl` exports `NUKELAB_VERSION` (from the `VERSION` file / git tag, + overridable via the env file) and `compose.yml` passes it as the + `APP_VERSION` build arg to the backend and celery images, so stacks + deployed with `nukelabctl` report the checkout version instead of + `0.0.0-dev`. ### Changed diff --git a/backend/AGENTS.md b/backend/AGENTS.md index a51ea9ac..d607f720 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -12,7 +12,7 @@ All files under `backend/` except generated artifacts (`.venv-dev`, `__pycache__ - Python 3.13; formatting and linting configured in `pyproject.toml`. - `app/main.py` is the ASGI entry point. -- `app/version.py` holds the static fallback `__version__` — permanently `0.0.0-dev`, never bumped. The effective runtime version is `settings.app_version` (`app/config.py`): the `APP_VERSION` env var — injected as a Docker build arg by CI from the image tag — wins, with empty/unset falling back to the literal. `main.py` and `app/api/health.py` read `settings.app_version` — never hardcode the version string. +- `app/version.py` holds the static fallback `__version__` — permanently `0.0.0-dev`, never bumped. The effective runtime version is `settings.app_version` (`app/config.py`): the `APP_VERSION` env var wins, with empty/unset falling back to the literal. `APP_VERSION` is injected as a Docker build arg both by CI (the image tag) and by local compose builds (`NUKELAB_VERSION` exported by `nukelabctl`). `main.py` and `app/api/health.py` read `settings.app_version` — never hardcode the version string. - `app/api/` owns route definitions; `app/services/` owns business logic; `app/models/` owns SQLAlchemy models; `app/db/` owns session/connection logic; `app/core/` owns cross-cutting utilities; `app/middleware/` owns ASGI middleware; `app/container/` owns container-runtime orchestration; `app/tasks.py` and `app/worker.py` own Celery. - `app/container/` is a driver layer: `driver.py` defines the `ContainerDriver` ABC + `ContainerDriverError` (plain-data returns only — no runtime objects escape), `docker_driver.py` is the Docker/Podman implementation, `factory.py` selects the driver via `CONTAINER_RUNTIME` (default `docker`), `client.py` is a compatibility shim for legacy imports/test seams, and `spawner.py` (server lifecycle) talks only to driver methods. To add a runtime (e.g. Kubernetes): implement `ContainerDriver` (synthesizing the documented return shapes, e.g. Docker-stats for `get_container_stats`) and register it in `factory.py`. Container health is driver-level config, not image metadata: `docker_driver.py` injects a uniform `Healthcheck` (`/usr/local/bin/nukelab-healthcheck.sh`) into every create config, because OCI images drop Dockerfile `HEALTHCHECK` and Kubernetes ignores it — a k8s driver must translate the same definition into pod liveness/startup probes and surface failures as the same `State.Health.Status` shape `HealthCheckService` consumes. `spawner.py` performs two readiness probes before marking a server `running`: (1) the container's own `/health` endpoint over the Docker network alias, and (2) the public server path through the internal Traefik load balancer (`TRAEFIK_INTERNAL_URL`, default `http://traefik:80`) with a `healthy` body check. This ensures the browser-facing route exists before the frontend is told to redirect. - Notifications: `app/services/notification_service.py` owns notification creation and delivery. Channels are `in_app`, `email`, `webhook`, and (when VAPID keys are configured) `push`. Push payloads are short previews only; dead subscriptions are removed on `404`/`410`. VAPID config and the `push_subscriptions` model live in `app/config.py` and `app/models/push_subscription.py`; pass `VAPID_*` env vars to both the `backend` and `celery-worker` containers. diff --git a/compose.yml b/compose.yml index 01c6df83..3084f948 100644 --- a/compose.yml +++ b/compose.yml @@ -73,6 +73,8 @@ services: build: context: ./backend dockerfile: Dockerfile + args: + - APP_VERSION=${NUKELAB_VERSION:-} container_name: nukelab-backend environment: - APP_NAME=${APP_NAME:-NukeLab} @@ -344,6 +346,8 @@ services: build: context: ./backend dockerfile: Dockerfile + args: + - APP_VERSION=${NUKELAB_VERSION:-} container_name: nukelab-celery-worker command: celery -A app.worker worker --loglevel=info -P threads -c 4 environment: @@ -418,6 +422,8 @@ services: build: context: ./backend dockerfile: Dockerfile + args: + - APP_VERSION=${NUKELAB_VERSION:-} container_name: nukelab-celery-beat command: celery -A app.worker beat --loglevel=info --schedule /tmp/celerybeat-schedule environment: diff --git a/docs/plan/DECISION-LOG.md b/docs/plan/DECISION-LOG.md index 84b132a7..52d84f74 100644 --- a/docs/plan/DECISION-LOG.md +++ b/docs/plan/DECISION-LOG.md @@ -24,4 +24,5 @@ Reversible architecture and process decisions for NukeLab v2.0. | 2026-08-14 | Separate `nukelab-environments` repository + runtime composition for scientific images | Keeps platform repo focused on runtime; toolchain images mount into `nukelab-workspace` at spawn time so workspace/IDE updates do not force MOAB/Geant4/OpenMC rebuilds; k3s-compatible via init-container volume population | Approved | | 2026-08-14 | Toolchain volume locking, stamp invalidation, and hardened helper containers | Cross-process lock container prevents racing populates; image-ID stamp invalidates stale volumes on tag re-push; hash-suffixed volume names prevent truncation collisions; helper containers drop capabilities so shared-volume content cannot be poisoned via privileged helpers | Approved | | 2026-08-15 | Toolchain datasets under `/opt/nuke/data/`; separate shared data volume deferred | OpenMC cross-sections/chain files move from `/opt/nuke/openmc_data` to `/opt/nuke/data/openmc`; Geant4 datasets stay under `/opt/nuke/geant4/share/data` (Geant4-native discovery via `geant4.sh`/compiled datadir). A dedicated data volume (independent code/data refresh, sharing datasets across toolchain versions) is deferred until dataset size/cadence justifies multi-volume manifests. Open follow-up: verify Geant4 per-dataset `G4*` env vars reach composed runtime containers (source `geant4.sh` from `toolchain-env.sh` if missing) at the next nuclear-base rebuild | Deferred | -| 2026-08-21 | Git tags as the version source of truth; `scripts/bump-version.sh` + root `CHANGELOG.md`; CI injects `APP_VERSION` build arg | Versions were scattered (`frontend/package.json` at `0.0.0`, backend `2.0.0` hardcoded in three files) and no tags existed, so `ci-version.sh` never produced semver image tags. The bumper syncs `VERSION`, `frontend/package.json` (+lockfile), and the changelog. Runtime version is fully dynamic: CI passes the resolved image tag as the backend's `APP_VERSION` build arg and `settings.app_version` resolves env-first, so `backend/app/version.py` is a fixed `0.0.0-dev` sentinel for local builds that is never bumped. The changelog stays manual (Keep a Changelog, same format as nuke-ide) because the commit history does not follow conventional commits, which auto-generators require | Approved | +| 2026-08-21 | Git tags as the version source of truth; `scripts/bump-version.sh` + root `CHANGELOG.md`; `APP_VERSION` build arg everywhere | Versions were scattered (`frontend/package.json` at `0.0.0`, backend `2.0.0` hardcoded in three files) and no tags existed, so `ci-version.sh` never produced semver image tags. The bumper syncs `VERSION`, `frontend/package.json` (+lockfile), and the changelog. Runtime version is fully dynamic: `settings.app_version` resolves `APP_VERSION` env-first, injected as a build arg by CI (the image tag) and by compose builds (`NUKELAB_VERSION` exported by `nukelabctl`), so `backend/app/version.py` is a fixed `0.0.0-dev` sentinel that is never bumped. The changelog stays manual (Keep a Changelog, same format as nuke-ide) because the commit history does not follow conventional commits, which auto-generators require | Approved | +| 2026-08-21 | Prod still builds from source; pull-based prod deploys deferred | `compose.yml` has no `image:` references to the CI-built GHCR images, so prod deploys (`git pull` + `nukelabctl up/update`) rebuild from source on the host and the published registry images have no consumer. Making prod pull tagged images (`image:` + `${NUKELAB_VERSION}` pins, rollback = re-pin) is the planned upgrade, deferred until release cadence justifies it | Deferred | diff --git a/docs/plan/ROADMAP.md b/docs/plan/ROADMAP.md index 2ed494ea..dece4b8a 100644 --- a/docs/plan/ROADMAP.md +++ b/docs/plan/ROADMAP.md @@ -61,6 +61,7 @@ See [IMPLEMENTATION-PHASES.md](IMPLEMENTATION-PHASES.md) for the full phase-by-p - Kubernetes migration (Helm, HPA, PVCs, Network Policies, Pod Security Standards) - Blue-green/rollback deployment automation +- Pull-based production deploys — compose consumes tagged GHCR images instead of rebuilding from source on the prod host (see DECISION-LOG.md, 2026-08-21) - Marketplace / plugin system Pursue Kubernetes only after saturating a single large server (32+ cores, 128GB+ RAM) and proving distribution is required. diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 8184e3d0..979fb7e5 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -13,6 +13,7 @@ All files under `scripts/`, plus the top-level `nukelabctl` dispatcher. - Bash 4+; modules in `scripts/manage.d/*.sh` are sourced by the dispatcher, not executed directly. - Tracked shell scripts (`nukelabctl`, `**/*.sh`) must be mode `100755` in the git index — prod pulls rely on it. Filesystems without Unix permissions (NTFS) record new files as `100644`; fix with `git update-index --chmod=+x `. `selftest` enforces this. On such filesystems, invoke via `bash nukelabctl ...` when the on-disk exec bit cannot be set. - `scripts/lib.sh` is the single source of truth for shared helpers (env loading, engine detection, state persistence, logging, venv provisioning). +- `init_env` exports `NUKELAB_VERSION` (from `_nukelab_version`, leading `v` stripped to match bare-semver image tags) when unset; `compose.yml` build args (`APP_VERSION`) consume it so locally built backend images report the checkout version. An explicit `NUKELAB_VERSION` in the environment or env file wins. - Each management command exposes `cmd_`, `help_`, and `parse__args` when it accepts flags. - Security scanning helpers live in `scripts/security/`. diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh old mode 100644 new mode 100755 diff --git a/scripts/lib.sh b/scripts/lib.sh index 4fc1e7fc..63d0ecc4 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -186,6 +186,16 @@ init_env() { else die "No environment file found.\n\n cp .env.example .env.development" fi + + # Default the platform version used by compose build args (APP_VERSION in + # backend/Dockerfile) to the resolved NukeLab version, stripped of the + # leading "v" so it matches the bare-semver image tags CI produces. An + # explicit NUKELAB_VERSION in the environment or an env file wins. + if [ -z "${NUKELAB_VERSION:-}" ]; then + local _nv + _nv="$(_nukelab_version)" + export NUKELAB_VERSION="${_nv#v}" + fi } # ─── Container Engine ─────────────────────────────────────────────────────- From 281f6c762955a79311810ce340f117bfe6e52649 Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Fri, 21 Aug 2026 21:47:34 +0600 Subject: [PATCH 4/7] Implement pull-based production deploys with image pinning --- .env.example | 13 ++++-- AGENTS.md | 4 ++ CHANGELOG.md | 6 +++ compose.yml | 4 ++ docs/operations/PRODUCTION-DEPLOYMENT.md | 47 +++++++++++++++++++ docs/plan/DECISION-LOG.md | 2 +- docs/plan/ROADMAP.md | 2 +- docs/reference/ENV-VARS.md | 3 ++ scripts/AGENTS.md | 3 ++ scripts/lib.sh | 57 ++++++++++++++++++++++++ scripts/manage.d/pull.sh | 17 ++++++- scripts/manage.d/start.sh | 5 +++ scripts/manage.d/update.sh | 40 ++++++++++++++--- scripts/nukelabctl-completion.bash | 5 ++- 14 files changed, 195 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 7564a47b..2f3450f9 100644 --- a/.env.example +++ b/.env.example @@ -19,9 +19,16 @@ APP_DEBUG=true # false in production APP_URL=http://localhost:8080 # Your application URL # FRONTEND_URL=http://localhost:5173 # Optional: Set when frontend runs separately (e.g., Vite dev server) APP_TIMEZONE=UTC -# NUKELAB_VERSION=2.1.0 # Optional: version baked into locally built backend - # images (APP_VERSION build arg). Defaults to the VERSION - # file / git tag via nukelabctl. CI-built images ignore this. + +# Version pinning / pull-based deploys +# NUKELAB_VERSION is baked into locally built images as the APP_VERSION build +# arg; it defaults to the VERSION file / git tag resolved by nukelabctl. +# Setting NUKELAB_VERSION or NUKELAB_IMAGE_TAG switches nukelabctl to pull +# pre-built images from ghcr.io/nukehub-dev instead of building from source. +# The registry must be readable on the deploy host (public packages, or run +# `podman login ghcr.io` / `docker login ghcr.io`). +# NUKELAB_VERSION=2.1.0 # Pin platform version (also switches to pull-based deploy) +# NUKELAB_IMAGE_TAG=2.1.0 # Exact registry tag; defaults to pinned version or `latest` # ============================================================================= # SECURITY ⚠️ CHANGE SECRETS FOR PRODUCTION diff --git a/AGENTS.md b/AGENTS.md index 533b7132..e7c3b0bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -133,6 +133,10 @@ Notes: Local compose builds get the same treatment: `nukelabctl` exports `NUKELAB_VERSION` (VERSION file / git describe) and `compose.yml` passes it as the `APP_VERSION` build arg. +- Production hosts can deploy by pinning `NUKELAB_VERSION` or + `NUKELAB_IMAGE_TAG` so `nukelabctl up` / `update` pull tagged images from + `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` instead of rebuilding + from source. Unpinned deploys keep the source-build path. - Cut a release with `scripts/bump-version.sh X.Y.Z` — it syncs `VERSION`, `frontend/package.json`, and `CHANGELOG.md` (the backend version is dynamic via `APP_VERSION`; `backend/app/version.py` stays `0.0.0-dev`), then prints diff --git a/CHANGELOG.md b/CHANGELOG.md index a5796269..38583794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ release version. ### Added +- Pull-based production deploys: pin `NUKELAB_VERSION` or `NUKELAB_IMAGE_TAG` + to switch `nukelabctl up` / `update` from source builds to pulling tagged + `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` images. The three + backend services share one backend image; `update --build` forces a source + rebuild even in pull mode. Unpinned deploys keep today's source-build + behavior. - Release versioning: git tags (`vX.Y.Z`) are the single source of truth. `scripts/bump-version.sh` syncs the version across `VERSION`, `backend/app/version.py`, `frontend/package.json`, and this changelog. diff --git a/compose.yml b/compose.yml index 3084f948..21ab7005 100644 --- a/compose.yml +++ b/compose.yml @@ -75,6 +75,7 @@ services: dockerfile: Dockerfile args: - APP_VERSION=${NUKELAB_VERSION:-} + image: ghcr.io/nukehub-dev/nukelab-backend:${NUKELAB_IMAGE_TAG:-latest} container_name: nukelab-backend environment: - APP_NAME=${APP_NAME:-NukeLab} @@ -306,6 +307,7 @@ services: dockerfile: Dockerfile args: - VITE_CDN_URL=${VITE_CDN_URL:-} + image: ghcr.io/nukehub-dev/nukelab-frontend:${NUKELAB_IMAGE_TAG:-latest} container_name: nukelab-frontend environment: - VITE_API_URL=${VITE_API_URL:-/api} @@ -348,6 +350,7 @@ services: dockerfile: Dockerfile args: - APP_VERSION=${NUKELAB_VERSION:-} + image: ghcr.io/nukehub-dev/nukelab-backend:${NUKELAB_IMAGE_TAG:-latest} container_name: nukelab-celery-worker command: celery -A app.worker worker --loglevel=info -P threads -c 4 environment: @@ -424,6 +427,7 @@ services: dockerfile: Dockerfile args: - APP_VERSION=${NUKELAB_VERSION:-} + image: ghcr.io/nukehub-dev/nukelab-backend:${NUKELAB_IMAGE_TAG:-latest} container_name: nukelab-celery-beat command: celery -A app.worker beat --loglevel=info --schedule /tmp/celerybeat-schedule environment: diff --git a/docs/operations/PRODUCTION-DEPLOYMENT.md b/docs/operations/PRODUCTION-DEPLOYMENT.md index 095c9785..8047507c 100644 --- a/docs/operations/PRODUCTION-DEPLOYMENT.md +++ b/docs/operations/PRODUCTION-DEPLOYMENT.md @@ -30,6 +30,53 @@ NukeLab server containers enforce resource limits via Linux cgroups: **Key insight:** Cgroups *enforce* limits but `free`/`top`/`nproc` read `/proc` which shows host values by default. **lxcfs** virtualizes `/proc` to show cgroup-aware values. +## Deploying from the GitHub Container Registry + +By default, production deploys build the backend and frontend images from source +on the host. You can switch to pulling pre-built images from the GitHub +Container Registry instead: + +1. Ensure the deploy host can read packages from `ghcr.io/nukehub-dev`. + The packages are public on GitHub; if your host requires authentication, + log in first: + + ```bash + podman login ghcr.io # or docker login ghcr.io + ``` + +2. Pin the version you want to deploy in `.env`: + + ```env + NUKELAB_VERSION=2.1.0 + ``` + + Or pin only the image tag: + + ```env + NUKELAB_IMAGE_TAG=2.1.0 + ``` + + See `.env.example` for the full description of both variables. + +3. Run the normal lifecycle commands. `nukelabctl` detects the pin and pulls + the tagged `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` images + instead of rebuilding: + + ```bash + ./nukelabctl up prod + ./nukelabctl update + ``` + + To force a source rebuild even in pull mode, use: + + ```bash + ./nukelabctl update --build + ``` + +4. Roll back by changing the pin and running `./nukelabctl update` again. + +When neither variable is pinned, deploys keep the source-build behavior. + --- ## Cgroup Controllers diff --git a/docs/plan/DECISION-LOG.md b/docs/plan/DECISION-LOG.md index 52d84f74..4cad870f 100644 --- a/docs/plan/DECISION-LOG.md +++ b/docs/plan/DECISION-LOG.md @@ -25,4 +25,4 @@ Reversible architecture and process decisions for NukeLab v2.0. | 2026-08-14 | Toolchain volume locking, stamp invalidation, and hardened helper containers | Cross-process lock container prevents racing populates; image-ID stamp invalidates stale volumes on tag re-push; hash-suffixed volume names prevent truncation collisions; helper containers drop capabilities so shared-volume content cannot be poisoned via privileged helpers | Approved | | 2026-08-15 | Toolchain datasets under `/opt/nuke/data/`; separate shared data volume deferred | OpenMC cross-sections/chain files move from `/opt/nuke/openmc_data` to `/opt/nuke/data/openmc`; Geant4 datasets stay under `/opt/nuke/geant4/share/data` (Geant4-native discovery via `geant4.sh`/compiled datadir). A dedicated data volume (independent code/data refresh, sharing datasets across toolchain versions) is deferred until dataset size/cadence justifies multi-volume manifests. Open follow-up: verify Geant4 per-dataset `G4*` env vars reach composed runtime containers (source `geant4.sh` from `toolchain-env.sh` if missing) at the next nuclear-base rebuild | Deferred | | 2026-08-21 | Git tags as the version source of truth; `scripts/bump-version.sh` + root `CHANGELOG.md`; `APP_VERSION` build arg everywhere | Versions were scattered (`frontend/package.json` at `0.0.0`, backend `2.0.0` hardcoded in three files) and no tags existed, so `ci-version.sh` never produced semver image tags. The bumper syncs `VERSION`, `frontend/package.json` (+lockfile), and the changelog. Runtime version is fully dynamic: `settings.app_version` resolves `APP_VERSION` env-first, injected as a build arg by CI (the image tag) and by compose builds (`NUKELAB_VERSION` exported by `nukelabctl`), so `backend/app/version.py` is a fixed `0.0.0-dev` sentinel that is never bumped. The changelog stays manual (Keep a Changelog, same format as nuke-ide) because the commit history does not follow conventional commits, which auto-generators require | Approved | -| 2026-08-21 | Prod still builds from source; pull-based prod deploys deferred | `compose.yml` has no `image:` references to the CI-built GHCR images, so prod deploys (`git pull` + `nukelabctl up/update`) rebuild from source on the host and the published registry images have no consumer. Making prod pull tagged images (`image:` + `${NUKELAB_VERSION}` pins, rollback = re-pin) is the planned upgrade, deferred until release cadence justifies it | Deferred | +| 2026-08-21 | Pull-based production deploys implemented | `compose.yml` now declares `image:` references to the CI-built GHCR images for `backend`, `celery-worker`, `celery-beat`, and `frontend`. Pinning `NUKELAB_VERSION` or `NUKELAB_IMAGE_TAG` switches `nukelabctl up` / `update` to pull those tagged images instead of rebuilding from source; rollback = re-pin. Unpinned deploys keep the source-build path | Approved | diff --git a/docs/plan/ROADMAP.md b/docs/plan/ROADMAP.md index dece4b8a..35e8b642 100644 --- a/docs/plan/ROADMAP.md +++ b/docs/plan/ROADMAP.md @@ -36,6 +36,7 @@ - **NVIDIA GPU Support** — Plan-based GPU passthrough (Podman CDI / Docker DeviceRequests), quota accounting, GPU metrics, CUDA environment image (`docs/operations/GPU-SETUP.md`) - **Exclusive GPU Allocator** — Whole-GPU reservations (`gpu_allocations`) with race-safe booking and recreate-on-start (`GPU_DEVICES`) - **Container Runtime Driver Layer** — `ContainerDriver` ABC with Docker/Podman as the first driver (`CONTAINER_RUNTIME`); prepares the k3s/Kubernetes migration +- **Pull-based Production Deploys** — Pin `NUKELAB_VERSION` or `NUKELAB_IMAGE_TAG` to deploy tagged `ghcr.io/nukehub-dev/nukelab-backend` / `-frontend` images without rebuilding from source See [IMPLEMENTATION-PHASES.md](IMPLEMENTATION-PHASES.md) for the full phase-by-phase record. @@ -61,7 +62,6 @@ See [IMPLEMENTATION-PHASES.md](IMPLEMENTATION-PHASES.md) for the full phase-by-p - Kubernetes migration (Helm, HPA, PVCs, Network Policies, Pod Security Standards) - Blue-green/rollback deployment automation -- Pull-based production deploys — compose consumes tagged GHCR images instead of rebuilding from source on the prod host (see DECISION-LOG.md, 2026-08-21) - Marketplace / plugin system Pursue Kubernetes only after saturating a single large server (32+ cores, 128GB+ RAM) and proving distribution is required. diff --git a/docs/reference/ENV-VARS.md b/docs/reference/ENV-VARS.md index 5a79f520..2976af32 100644 --- a/docs/reference/ENV-VARS.md +++ b/docs/reference/ENV-VARS.md @@ -23,6 +23,9 @@ Both `.env` and `.env.development` are gitignored. `.env.example` is the only en | `APP_URL` | Public application URL | | `FRONTEND_URL` | Optional separate frontend URL for Vite dev server | | `APP_TIMEZONE` | Default timezone | +| `NUKELAB_VERSION` | Version baked into locally built backend images (`APP_VERSION` build arg); defaults to `VERSION` file / git tag. Pinning this switches `nukelabctl` to pull-based deploys | +| `NUKELAB_IMAGE_TAG` | Exact registry image tag to deploy (`ghcr.io/nukehub-dev/nukelab-backend` / `-frontend`); defaults to the pinned `NUKELAB_VERSION` in pull mode, or `latest` in source-build mode | +| `NUKELAB_PULL_DEPLOY` | Set automatically by `nukelabctl`: `true` when `NUKELAB_VERSION` or `NUKELAB_IMAGE_TAG` is pinned, otherwise `false` | ### Security diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 979fb7e5..e06a3f63 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -14,6 +14,9 @@ All files under `scripts/`, plus the top-level `nukelabctl` dispatcher. - Tracked shell scripts (`nukelabctl`, `**/*.sh`) must be mode `100755` in the git index — prod pulls rely on it. Filesystems without Unix permissions (NTFS) record new files as `100644`; fix with `git update-index --chmod=+x `. `selftest` enforces this. On such filesystems, invoke via `bash nukelabctl ...` when the on-disk exec bit cannot be set. - `scripts/lib.sh` is the single source of truth for shared helpers (env loading, engine detection, state persistence, logging, venv provisioning). - `init_env` exports `NUKELAB_VERSION` (from `_nukelab_version`, leading `v` stripped to match bare-semver image tags) when unset; `compose.yml` build args (`APP_VERSION`) consume it so locally built backend images report the checkout version. An explicit `NUKELAB_VERSION` in the environment or env file wins. +- `init_env` also exports `NUKELAB_IMAGE_TAG` (defaults to the pinned `NUKELAB_VERSION` in pull mode, or `latest` in source-build mode) and `NUKELAB_PULL_DEPLOY`. Setting `NUKELAB_VERSION` or `NUKELAB_IMAGE_TAG` explicitly switches `nukelabctl up` / `update` to pull pre-built images from `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` instead of building from source. +- `update` has a `--build` escape hatch that forces a source rebuild even when `NUKELAB_PULL_DEPLOY=true`. In source-build mode `update` and `pull` pull base images for the pullable infra services only, via `_pullable_infra_services`. +- `_pullable_infra_services` returns the pullable (non-buildable) infra services (`traefik postgres redis` + enabled overlay services) as a word-split list, mirroring `_backend_services`. App services (`backend`, `celery-worker`, `celery-beat`, `frontend`) are excluded so unpinned ghcr.io images are never pulled without registry auth; pull mode pulls them explicitly. - Each management command exposes `cmd_`, `help_`, and `parse__args` when it accepts flags. - Security scanning helpers live in `scripts/security/`. diff --git a/scripts/lib.sh b/scripts/lib.sh index 63d0ecc4..4d08cd03 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -165,6 +165,14 @@ load_env_file() { # Exports NUKELAB_ENV_FILE so compose services can reference the active env file. init_env() { local dev_mode="${1:-false}" + local _explicit_version=false + local _explicit_image_tag=false + + # Capture explicit operator pinning before env files are loaded. + # load_env_file never overwrites an already-set variable, so a value from + # the real shell environment wins over both .env and .env.development. + [ -n "${NUKELAB_VERSION:-}" ] && _explicit_version=true + [ -n "${NUKELAB_IMAGE_TAG:-}" ] && _explicit_image_tag=true if $dev_mode && [ -f .env.development ]; then # Dev mode: load the dev file first so dev values win over .env. @@ -187,6 +195,11 @@ init_env() { die "No environment file found.\n\n cp .env.example .env.development" fi + # Env files may also carry explicit pins; an already-set value is never + # overwritten, so non-empty here means the operator pinned it somewhere. + [ -n "${NUKELAB_VERSION:-}" ] && _explicit_version=true + [ -n "${NUKELAB_IMAGE_TAG:-}" ] && _explicit_image_tag=true + # Default the platform version used by compose build args (APP_VERSION in # backend/Dockerfile) to the resolved NukeLab version, stripped of the # leading "v" so it matches the bare-semver image tags CI produces. An @@ -196,6 +209,28 @@ init_env() { _nv="$(_nukelab_version)" export NUKELAB_VERSION="${_nv#v}" fi + + # Default the registry image tag consumed by compose.yml image: + # substitutions. When the operator explicitly pinned NUKELAB_VERSION, use + # it as the image tag unless NUKELAB_IMAGE_TAG was pinned separately. + # Unpinned deploys float on :latest, matching the pre-existing source-build + # behavior. + if [ -z "${NUKELAB_IMAGE_TAG:-}" ]; then + if $_explicit_version; then + export NUKELAB_IMAGE_TAG="$NUKELAB_VERSION" + else + export NUKELAB_IMAGE_TAG="latest" + fi + fi + + # Pinning NUKELAB_VERSION or NUKELAB_IMAGE_TAG switches nukelabctl into + # pull-based deploys from the GitHub Container Registry. + if $_explicit_version || $_explicit_image_tag; then + export NUKELAB_PULL_DEPLOY=true + log "Pull-based deploy enabled ${DIM}(image tag: ${NUKELAB_IMAGE_TAG})${RESET}" + else + export NUKELAB_PULL_DEPLOY=false + fi } # ─── Container Engine ─────────────────────────────────────────────────────- @@ -1075,6 +1110,28 @@ _backend_services() { echo "$services" } +_pullable_infra_services() { + # Print pullable (non-buildable) infra services, including overlay services + # when enabled. The app services (backend, celery-worker, celery-beat, + # frontend) have both image: and build: blocks; in source-build mode their + # ghcr.io tags may be unavailable without registry auth, so infra pulls + # must exclude them. Pull-based deploy mode pulls them explicitly. + local services="traefik postgres redis" + if _has_overlay "compose.pgbouncer.yml"; then + services="$services pgbouncer" + fi + if _has_overlay "compose.monitoring.yml"; then + services="$services prometheus grafana postgres-exporter redis-exporter node-exporter celery-exporter" + fi + if _has_overlay "compose.alertmanager.yml"; then + services="$services alertmanager" + fi + if _has_overlay "compose.tracing.yml"; then + services="$services otel-collector jaeger" + fi + echo "$services" +} + _stop_dev_stack() { # Dev-mode Ctrl+C handler: stop Vite and all backend/monitoring/tracing containers. echo "" diff --git a/scripts/manage.d/pull.sh b/scripts/manage.d/pull.sh index b1dcc7d2..de1973fe 100755 --- a/scripts/manage.d/pull.sh +++ b/scripts/manage.d/pull.sh @@ -4,7 +4,16 @@ cmd_pull() { step "Pulling latest images..." - $COMPOSE "${COMPOSE_ARGS[@]}" pull + if [ "${NUKELAB_PULL_DEPLOY:-false}" = "true" ]; then + # Pull-based deploys: pull everything, including the pinned app images. + $COMPOSE "${COMPOSE_ARGS[@]}" pull + else + # Source-build mode: app services are built locally, so pull only the + # infra images — the ghcr.io app tags may be unavailable without + # registry auth. Intentional word-split of the service list. + # shellcheck disable=SC2086 + $COMPOSE "${COMPOSE_ARGS[@]}" pull $(_pullable_infra_services) + fi ok "Images pulled" } @@ -12,7 +21,11 @@ help_pull() { cat <<- EOF ${BOLD}Usage:${RESET} ./nukelabctl pull -Pull the latest base images used by compose services. +Pull the latest images used by compose services. In source-build mode +(default) only infra images (postgres, redis, traefik, enabled overlays) are +pulled — app images are built locally. In pull-based deploy mode +(NUKELAB_VERSION or NUKELAB_IMAGE_TAG pinned) the pinned ghcr.io app images +are pulled too. ${BOLD}Examples:${RESET} ./nukelabctl pull diff --git a/scripts/manage.d/start.sh b/scripts/manage.d/start.sh index 2a74ed75..0293f9ef 100755 --- a/scripts/manage.d/start.sh +++ b/scripts/manage.d/start.sh @@ -144,6 +144,11 @@ cmd_start() { else step "Starting production stack..." + if [ "${NUKELAB_PULL_DEPLOY:-false}" = "true" ]; then + step "Pulling pinned registry images..." + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" pull backend frontend + fi + local _prod_backend_services _prod_backend_services=$(_backend_services) diff --git a/scripts/manage.d/update.sh b/scripts/manage.d/update.sh index 726d64e2..3ed7969f 100755 --- a/scripts/manage.d/update.sh +++ b/scripts/manage.d/update.sh @@ -5,6 +5,7 @@ # Default: rebuild without the layer cache to always pick up changes from # base images. --cache lets users reuse layers when iterating. UPDATE_BUILD_ARGS=(--no-cache) +UPDATE_FORCE_BUILD=false parse_update_args() { while [[ ${#EXTRA_ARGS[@]} -gt 0 ]]; do @@ -13,6 +14,10 @@ parse_update_args() { UPDATE_BUILD_ARGS=() EXTRA_ARGS=("${EXTRA_ARGS[@]:1}") ;; + --build) + UPDATE_FORCE_BUILD=true + EXTRA_ARGS=("${EXTRA_ARGS[@]:1}") + ;; --help | -h) help_update exit 0 @@ -30,27 +35,52 @@ parse_update_args() { cmd_update() { step "Updating NukeLab..." - log "Pulling latest images..." - _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" pull + if [ "${NUKELAB_PULL_DEPLOY:-false}" = "true" ]; then + step "Pull-based update: pulling pinned registry images..." + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" pull backend frontend + # Pinning the app images must not freeze the infra base images. + # Intentional word-split of the space-separated service list. + # shellcheck disable=SC2086 + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" pull $(_pullable_infra_services) + if $UPDATE_FORCE_BUILD; then + log "Force source rebuild requested (--build)..." + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" build "${UPDATE_BUILD_ARGS[@]}" + fi + else + # Source-build path: refresh base images for the pullable infra + # services only. The app services carry ghcr.io image: references for + # pull-based deploys; pulling them here would fail without registry + # auth, so they are excluded via _pullable_infra_services. + log "Pulling latest base images..." + # Intentional word-split of the space-separated service list. + # shellcheck disable=SC2086 + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" pull $(_pullable_infra_services) - log "Rebuilding containers..." - _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" build "${UPDATE_BUILD_ARGS[@]}" + log "Rebuilding containers..." + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" build "${UPDATE_BUILD_ARGS[@]}" + fi ok "Update complete! Run './nukelabctl restart' to apply changes." } help_update() { cat <<- EOF -${BOLD}Usage:${RESET} ./nukelabctl update [--cache] +${BOLD}Usage:${RESET} ./nukelabctl update [--cache] [--build] Pull latest base images and rebuild all containers. +In pull-based deploy mode (NUKELAB_VERSION or NUKELAB_IMAGE_TAG is pinned), +this pulls the pinned ghcr.io images and skips the source build. Use --build +in pull mode to force a source rebuild anyway. + ${BOLD}Options:${RESET} --cache Reuse Docker/Podman layer cache instead of forcing --no-cache. Faster on repeat runs; may miss changes from updated base images. + --build Force a source rebuild even in pull-based deploy mode. ${BOLD}Examples:${RESET} ./nukelabctl update ./nukelabctl update --cache + ./nukelabctl update --build EOF } diff --git a/scripts/nukelabctl-completion.bash b/scripts/nukelabctl-completion.bash index 5a20f7ca..fd9e602c 100644 --- a/scripts/nukelabctl-completion.bash +++ b/scripts/nukelabctl-completion.bash @@ -119,7 +119,10 @@ _manage_sh_complete() { esac fi ;; - update | pull | e2e | db-migrate | db-shell | backup | selftest | install-completion | help | security | init-user-auth-keys | rotate-user-auth-key | cleanup-user-auth-keys) + update) + COMPREPLY=($(compgen -W "--cache --build ${global_flags[*]}" -- "$cur")) + ;; + pull | e2e | db-migrate | db-shell | backup | selftest | install-completion | help | security | init-user-auth-keys | rotate-user-auth-key | cleanup-user-auth-keys) COMPREPLY=($(compgen -W "${global_flags[*]}" -- "$cur")) ;; *) From 1b775c4f9294813c043bd2d8b34ddcdd44cac77e Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Fri, 21 Aug 2026 22:37:04 +0600 Subject: [PATCH 5/7] Add automatic pre-migration backups and schema guard Add a pre-migration pg_dump snapshot to `db-migrate` with a `--no-backup` override, and a startup schema-compatibility guard that refuses to boot an old backend image against a newer database schema. --- AGENTS.md | 4 + CHANGELOG.md | 11 ++ backend/AGENTS.md | 20 ++ backend/app/config.py | 17 ++ backend/app/db/schema_guard.py | 142 ++++++++++++++ backend/app/main.py | 17 ++ backend/tests/db/test_schema_guard.py | 233 +++++++++++++++++++++++ docs/operations/PRODUCTION-DEPLOYMENT.md | 52 +++++ scripts/AGENTS.md | 6 + scripts/lib.sh | 26 +++ scripts/manage.d/backup.sh | 17 +- scripts/manage.d/db-migrate.sh | 102 ++++++++-- scripts/nukelabctl-completion.bash | 5 +- 13 files changed, 624 insertions(+), 28 deletions(-) create mode 100644 backend/app/db/schema_guard.py create mode 100644 backend/tests/db/test_schema_guard.py diff --git a/AGENTS.md b/AGENTS.md index e7c3b0bf..e3e9843a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,10 @@ Notes: `NUKELAB_IMAGE_TAG` so `nukelabctl up` / `update` pull tagged images from `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` instead of rebuilding from source. Unpinned deploys keep the source-build path. +- Rollback safety: `./nukelabctl db-migrate` takes an automatic pre-migration + snapshot, and the backend startup guard refuses to boot an old image on a + newer schema. To roll back, pin the previous release, run `./nukelabctl + update`, restore the `backups/pre-migrate-*` snapshot, and restart. - Cut a release with `scripts/bump-version.sh X.Y.Z` — it syncs `VERSION`, `frontend/package.json`, and `CHANGELOG.md` (the backend version is dynamic via `APP_VERSION`; `backend/app/version.py` stays `0.0.0-dev`), then prints diff --git a/CHANGELOG.md b/CHANGELOG.md index 38583794..e98ff424 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,17 @@ release version. `APP_VERSION` build arg to the backend and celery images, so stacks deployed with `nukelabctl` report the checkout version instead of `0.0.0-dev`. +- Automatic pre-migration backups: `./nukelabctl db-migrate` now takes a + `pg_dump` snapshot before running `alembic upgrade head`. The snapshot name + is `backups/pre-migrate---.dump`. + If the backup fails, the migration aborts; if the migration then fails, the + exact `./nukelabctl restore ` command is printed. Use `--no-backup` to + skip the snapshot. +- Startup schema-compatibility guard (`app/db/schema_guard.py`) refuses to boot + an old backend image against a newer database schema. Controlled by the + `DB_SCHEMA_GUARD` setting (`auto` refuse in production/warn elsewhere, + `enforce` always refuse, `off` disabled). If the database is unreachable the + guard logs a warning and does not block startup. ### Changed diff --git a/backend/AGENTS.md b/backend/AGENTS.md index d607f720..ad30f624 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -54,6 +54,26 @@ All files under `backend/` except generated artifacts (`.venv-dev`, `__pycache__ - Review the generated migration before committing; autogenerated scripts can miss renames and complex changes. - Test upgrade and downgrade locally: `alembic upgrade head && alembic downgrade -1`. - Migrations must be reversible and tested against the current schema. +- **Expand-contract for destructive changes:** split drops, renames, and removals across two releases so the previous release's code still runs on the new schema. For example, release N adds the replacement column and dual-writes; release N+1 removes the old column once no deployed image references it. + +### Schema-compatibility guard + +`app/db/schema_guard.py` protects against the pull-deploy rollback hazard where +an older backend image boots against a database that already ran newer Alembic +migrations. + +- `check_schema_compatibility(engine, script_dir_path)` reads the DB's + `alembic_version` revision(s), walks the local Alembic `ScriptDirectory`, and + reports whether every DB revision is known to the running image. +- `run_schema_guard(engine, script_dir_path, mode, app_env)` applies the + configured policy and raises `RuntimeError` when a rollback hazard is detected + in a refusing mode. +- The guard is controlled by the `DB_SCHEMA_GUARD` setting (`app/config.py`): + - `auto` (default): refuse to start in production, warn in other environments. + - `enforce`: always refuse. + - `off`: disabled. +- If the database is unreachable, the guard logs a warning and does not block + startup, so DB outages do not become a new failure mode. ### Background tasks diff --git a/backend/app/config.py b/backend/app/config.py index 9549d39e..c2f04227 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -141,6 +141,12 @@ class Settings(BaseSettings): # instead of relying on auto-create. auto_create_tables: bool = True + # Schema-compatibility guard. "auto" refuses to start in production when + # the DB Alembic revision is newer than the revisions known to this image, + # and warns in other environments. "enforce" always refuses. "off" disables + # the guard (still logs at debug level during startup). + db_schema_guard: str = "auto" + # Observability — Query Performance Monitoring observability_slow_query_threshold_ms: int = 100 observability_pg_stat_statements_enabled: bool = True @@ -321,6 +327,17 @@ def _empty_api_docs_to_none(cls, value: Any) -> Any: return None return value + @field_validator("db_schema_guard", mode="before") + @classmethod + def _validate_db_schema_guard(cls, value: Any) -> Any: + """Reject unsupported schema-guard modes.""" + allowed = {"off", "auto", "enforce"} + if value not in allowed: + raise ValueError( + f"DB_SCHEMA_GUARD must be one of {sorted(allowed)}, got {value!r}" + ) + return value + @model_validator(mode="after") def set_key_paths(self) -> "Settings": """Derive key paths from secrets_dir if not explicitly set.""" diff --git a/backend/app/db/schema_guard.py b/backend/app/db/schema_guard.py new file mode 100644 index 00000000..7dd0ebaa --- /dev/null +++ b/backend/app/db/schema_guard.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2023-2026 NukeHub Developers +# SPDX-License-Identifier: BSD-2-Clause + +"""Startup schema-compatibility guard. + +Detects when the database Alembic revision is newer than the revisions known to +the running backend image, which is the rollback hazard surfaced by pull-based +deploys. +""" + +import os +from dataclasses import dataclass + +from alembic.script import ScriptDirectory +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.core.logging import get_logger + +logger = get_logger(__name__) + +ALEMBIC_VERSION_QUERY = text("SELECT version_num FROM alembic_version") + + +@dataclass(frozen=True) +class SchemaCompatibilityResult: + """Result of a schema-compatibility check.""" + + ok: bool + unknown_revisions: tuple[str, ...] + db_revisions: tuple[str, ...] + known_revisions: frozenset[str] + db_unreachable: bool = False + + +async def check_schema_compatibility( + engine: AsyncEngine, + script_dir_path: str | os.PathLike[str], +) -> SchemaCompatibilityResult: + """Check whether the DB schema revision is known to this backend image. + + Returns a result indicating compatibility. A DB revision that is absent from + the local ScriptDirectory means the schema is newer than the code (rollback + hazard). No ``alembic_version`` table or an empty ``alembic_version`` is + treated as a fresh/unmanaged database and is considered compatible. + + If the database cannot be reached, the guard degrades gracefully: it logs a + warning and returns a result with ``db_unreachable=True`` and ``ok=True`` so + startup is not blocked by a transient DB outage. + """ + try: + async with engine.connect() as conn: + result = await conn.execute(ALEMBIC_VERSION_QUERY) + db_revisions = tuple(row[0] for row in result) + except Exception as exc: + logger.warning(f"Schema guard could not reach database: {exc}") + return SchemaCompatibilityResult( + ok=True, + unknown_revisions=(), + db_revisions=(), + known_revisions=frozenset(), + db_unreachable=True, + ) + + if not db_revisions: + # Fresh or unmanaged database. + return SchemaCompatibilityResult( + ok=True, + unknown_revisions=(), + db_revisions=(), + known_revisions=frozenset(), + ) + + try: + script_dir = ScriptDirectory(str(script_dir_path)) + known_revisions = frozenset(rev.revision for rev in script_dir.walk_revisions()) + except Exception as exc: + logger.warning(f"Schema guard could not load Alembic script directory: {exc}") + return SchemaCompatibilityResult( + ok=True, + unknown_revisions=(), + db_revisions=db_revisions, + known_revisions=frozenset(), + db_unreachable=False, + ) + + unknown_revisions = tuple(r for r in db_revisions if r not in known_revisions) + return SchemaCompatibilityResult( + ok=not unknown_revisions, + unknown_revisions=unknown_revisions, + db_revisions=db_revisions, + known_revisions=known_revisions, + ) + + +def _recovery_action() -> str: + return ( + "Restore a pre-migrate backup or deploy a newer image " + "matching the database schema revision." + ) + + +async def run_schema_guard( + engine: AsyncEngine, + script_dir_path: str | os.PathLike[str], + mode: str, + app_env: str, +) -> None: + """Run the guard and raise on a rollback hazard when configured to refuse. + + ``mode`` must be one of ``off``, ``auto``, or ``enforce``. ``app_env`` is + the resolved application environment (e.g. ``production`` or + ``development``). + """ + if mode == "off": + return + + result = await check_schema_compatibility(engine, script_dir_path) + if result.ok or not result.unknown_revisions: + return + + recovery_action = _recovery_action() + if mode == "enforce" or (mode == "auto" and app_env == "production"): + logger.error( + "Database schema is newer than this backend image; refusing to start", + extra={ + "unknown_db_revisions": list(result.unknown_revisions), + "recovery_action": recovery_action, + }, + ) + raise RuntimeError( + f"Database schema revision(s) {list(result.unknown_revisions)} " + f"are newer than this backend image. {recovery_action}" + ) + + logger.warning( + "Database schema is newer than this backend image", + extra={ + "unknown_db_revisions": list(result.unknown_revisions), + "recovery_action": recovery_action, + }, + ) diff --git a/backend/app/main.py b/backend/app/main.py index b441c037..d15408b7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -54,6 +54,23 @@ async def startup(): init_tracing() init_sentry() + # Schema-compatibility guard: refuse to boot an old image against a newer + # schema. DB outages must not become a new startup failure mode, so any + # unexpected exception is logged and swallowed. + try: + import os + + from app.db.schema_guard import run_schema_guard + + script_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "alembic" + ) + await run_schema_guard(engine, script_dir, settings.db_schema_guard, settings.app_env) + except RuntimeError: + raise + except Exception as exc: + logger.warning(f"Schema compatibility guard failed: {exc}") + # Create tables unless disabled (production should use Alembic migrations). # When enabled, use a Postgres advisory lock so multiple uvicorn workers # starting in parallel don't race to create the same tables/types. diff --git a/backend/tests/db/test_schema_guard.py b/backend/tests/db/test_schema_guard.py new file mode 100644 index 00000000..a7206e5b --- /dev/null +++ b/backend/tests/db/test_schema_guard.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: 2023-2026 NukeHub Developers +# SPDX-License-Identifier: BSD-2-Clause + +"""Tests for the schema-compatibility guard.""" + +from unittest import mock + +import pytest + + +class FakeScript: + def __init__(self, revision): + self.revision = revision + + +def _make_engine(rows=None, execute_side_effect=None): + """Return a mocked async engine yielding the given query rows.""" + mock_conn = mock.AsyncMock() + if execute_side_effect is not None: + mock_conn.execute.side_effect = execute_side_effect + else: + mock_conn.execute.return_value = rows or [] + mock_conn.__aenter__.return_value = mock_conn + mock_conn.__aexit__.return_value = False + + engine = mock.AsyncMock() + engine.connect = mock.MagicMock(return_value=mock_conn) + return engine + + +class TestCheckSchemaCompatibility: + """Tests for app.db.schema_guard.check_schema_compatibility.""" + + @pytest.mark.asyncio + async def test_known_revision_passes(self): + """A DB revision present in the local ScriptDirectory is compatible.""" + from app.db.schema_guard import check_schema_compatibility + + engine = _make_engine(rows=[("abc123",)]) + + with mock.patch( + "app.db.schema_guard.ScriptDirectory" + ) as mock_script_dir_cls: + mock_script_dir_cls.return_value.walk_revisions.return_value = [ + FakeScript("abc123") + ] + result = await check_schema_compatibility(engine, "/app/alembic") + + assert result.ok is True + assert result.unknown_revisions == () + assert result.db_revisions == ("abc123",) + assert "abc123" in result.known_revisions + + @pytest.mark.asyncio + async def test_unknown_revision_detected(self): + """A DB revision absent from ScriptDirectory is a rollback hazard.""" + from app.db.schema_guard import check_schema_compatibility + + engine = _make_engine(rows=[("newer_rev",)]) + + with mock.patch( + "app.db.schema_guard.ScriptDirectory" + ) as mock_script_dir_cls: + mock_script_dir_cls.return_value.walk_revisions.return_value = [ + FakeScript("abc123") + ] + result = await check_schema_compatibility(engine, "/app/alembic") + + assert result.ok is False + assert result.unknown_revisions == ("newer_rev",) + + @pytest.mark.asyncio + async def test_multiple_db_revisions_all_known(self): + """Multiple known DB revisions are compatible.""" + from app.db.schema_guard import check_schema_compatibility + + engine = _make_engine(rows=[("rev1",), ("rev2",)]) + + with mock.patch( + "app.db.schema_guard.ScriptDirectory" + ) as mock_script_dir_cls: + mock_script_dir_cls.return_value.walk_revisions.return_value = [ + FakeScript("rev1"), + FakeScript("rev2"), + ] + result = await check_schema_compatibility(engine, "/app/alembic") + + assert result.ok is True + assert result.db_revisions == ("rev1", "rev2") + + @pytest.mark.asyncio + async def test_no_alembic_version_table_is_fresh_db(self): + """Missing alembic_version table means fresh/unmanaged DB: compatible.""" + from app.db.schema_guard import check_schema_compatibility + + engine = _make_engine( + execute_side_effect=Exception( + "relation 'alembic_version' does not exist" + ) + ) + + result = await check_schema_compatibility(engine, "/app/alembic") + + assert result.ok is True + assert result.db_unreachable is True + assert result.db_revisions == () + + @pytest.mark.asyncio + async def test_empty_alembic_version_is_fresh_db(self): + """Empty alembic_version table is treated as fresh/unmanaged.""" + from app.db.schema_guard import check_schema_compatibility + + engine = _make_engine(rows=[]) + + result = await check_schema_compatibility(engine, "/app/alembic") + + assert result.ok is True + assert result.db_revisions == () + + @pytest.mark.asyncio + async def test_script_directory_failure_does_not_block_startup(self): + """Failure to load the Alembic script directory is logged, not fatal.""" + from app.db.schema_guard import check_schema_compatibility + + engine = _make_engine(rows=[("abc123",)]) + + with mock.patch( + "app.db.schema_guard.ScriptDirectory", + side_effect=Exception("corrupt env.py"), + ): + result = await check_schema_compatibility(engine, "/app/alembic") + + assert result.ok is True + assert result.db_revisions == ("abc123",) + + +class TestRunSchemaGuard: + """Tests for app.db.schema_guard.run_schema_guard decision logic.""" + + @pytest.mark.asyncio + async def test_known_revision_passes_all_modes(self): + """No-op when the DB revision is known.""" + from app.db.schema_guard import run_schema_guard + + engine = mock.AsyncMock() + with mock.patch( + "app.db.schema_guard.check_schema_compatibility", + return_value=mock.Mock(ok=True, unknown_revisions=()), + ): + await run_schema_guard(engine, "/app/alembic", "enforce", "production") + await run_schema_guard(engine, "/app/alembic", "auto", "production") + await run_schema_guard(engine, "/app/alembic", "off", "production") + + @pytest.mark.asyncio + async def test_unknown_revision_refuses_in_enforce_mode(self): + """enforce mode always refuses an unknown DB revision.""" + from app.db.schema_guard import run_schema_guard + + engine = mock.AsyncMock() + with mock.patch( + "app.db.schema_guard.check_schema_compatibility", + return_value=mock.Mock( + ok=False, unknown_revisions=("newer_rev",), db_revisions=("newer_rev",) + ), + ): + with pytest.raises(RuntimeError, match="newer_rev"): + await run_schema_guard(engine, "/app/alembic", "enforce", "development") + + @pytest.mark.asyncio + async def test_unknown_revision_refuses_in_production_auto(self): + """auto mode refuses an unknown DB revision in production.""" + from app.db.schema_guard import run_schema_guard + + engine = mock.AsyncMock() + with mock.patch( + "app.db.schema_guard.check_schema_compatibility", + return_value=mock.Mock( + ok=False, unknown_revisions=("newer_rev",), db_revisions=("newer_rev",) + ), + ): + with pytest.raises(RuntimeError, match="newer_rev"): + await run_schema_guard(engine, "/app/alembic", "auto", "production") + + @pytest.mark.asyncio + async def test_unknown_revision_warns_in_non_production_auto(self): + """auto mode warns in non-production environments.""" + from app.db.schema_guard import run_schema_guard + + engine = mock.AsyncMock() + with mock.patch( + "app.db.schema_guard.check_schema_compatibility", + return_value=mock.Mock( + ok=False, unknown_revisions=("newer_rev",), db_revisions=("newer_rev",) + ), + ), mock.patch("app.db.schema_guard.logger") as mock_logger: + await run_schema_guard(engine, "/app/alembic", "auto", "development") + + mock_logger.warning.assert_called_once() + assert "newer_rev" in str(mock_logger.warning.call_args) + + @pytest.mark.asyncio + async def test_off_mode_does_not_raise_or_warn(self): + """off mode skips the guard entirely.""" + from app.db.schema_guard import run_schema_guard + + engine = mock.AsyncMock() + with mock.patch( + "app.db.schema_guard.check_schema_compatibility" + ) as mock_check, mock.patch("app.db.schema_guard.logger") as mock_logger: + await run_schema_guard(engine, "/app/alembic", "off", "production") + + mock_check.assert_not_called() + mock_logger.warning.assert_not_called() + mock_logger.error.assert_not_called() + + +class TestSettingsValidation: + """Tests for the DB_SCHEMA_GUARD setting validation.""" + + @pytest.mark.parametrize("mode", ["off", "auto", "enforce"]) + def test_valid_db_schema_guard_values(self, mode): + """off/auto/enforce are accepted.""" + from app.config import Settings + + settings = Settings(DB_SCHEMA_GUARD=mode) + assert settings.db_schema_guard == mode + + def test_invalid_db_schema_guard_value_rejected(self): + """An unsupported value is rejected at settings construction time.""" + from app.config import Settings + + with pytest.raises(ValueError, match="DB_SCHEMA_GUARD"): + Settings(DB_SCHEMA_GUARD="warn-only") diff --git a/docs/operations/PRODUCTION-DEPLOYMENT.md b/docs/operations/PRODUCTION-DEPLOYMENT.md index 8047507c..527f83c2 100644 --- a/docs/operations/PRODUCTION-DEPLOYMENT.md +++ b/docs/operations/PRODUCTION-DEPLOYMENT.md @@ -79,6 +79,58 @@ When neither variable is pinned, deploys keep the source-build behavior. --- +## Rollback runbook + +Rolling back to a previous release is safe only when the database schema is +compatible with the older image. Because `./nukelabctl db-migrate` takes an +automatic pre-migration snapshot, you can restore the exact state from before +the failed upgrade. + +1. Pin the previous release in `.env`: + + ```env + NUKELAB_VERSION=2.0.0 + ``` + +2. Pull and restart the old images: + + ```bash + ./nukelabctl update + ``` + +3. Restore the pre-migration snapshot (taken automatically by `db-migrate`): + + ```bash + ./nukelabctl restore backups/pre-migrate-2.1.0--.dump + ``` + + Use `--yes` to skip the confirmation prompt in automation: + + ```bash + ./nukelabctl restore --yes backups/pre-migrate-2.1.0--.dump + ``` + +4. Restart the backend so it boots against the restored schema: + + ```bash + ./nukelabctl restart backend + ``` + +The backend startup guard refuses to boot an old image on a newer schema: if +you forget to restore the snapshot, the container logs an error naming the +unknown Alembic revision and exits. In that case, restore the snapshot and +restart again. + +**Avoiding rollback hazards:** split destructive migrations (column drops, +renames, table deletions) across two releases using expand-contract: + +- Release N: add the new column/table and dual-write, but keep the old one. +- Release N+1: remove the old column/table once no deployed image references it. + +This lets the previous release's code continue running on the newer schema. + +--- + ## Cgroup Controllers ### What You Need diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index e06a3f63..82d2c47c 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -34,6 +34,12 @@ All files under `scripts/`, plus the top-level `nukelabctl` dispatcher. - Environment build order matters: `manage.d/build.sh` builds `services/build-auth-sidecar.sh` before any `env base` build (base embeds the sidecar binary), then `conda-base`, then `workspace`/`dev`. `build-all.sh` mirrors that order. Keep the sidecar first when touching build orchestration. - When adding or changing `nukelabctl` commands, targets, or flags, update `scripts/nukelabctl-completion.bash` so bash tab-completion stays in sync. +- `db-migrate` takes an automatic `pg_dump` snapshot before running + `alembic upgrade head`. The snapshot is written to `backups/` with the name + `pre-migrate---.dump`. If the + backup fails, the migration aborts. If the migration then fails, the exact + `./nukelabctl restore ` command is printed. Use `--no-backup` to skip + the snapshot when you have already arranged your own protection. ## Verification diff --git a/scripts/lib.sh b/scripts/lib.sh index 4d08cd03..63c0171e 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1246,6 +1246,32 @@ _direct_database_url() { echo "postgresql+asyncpg://${DATABASE_USER:-nukelab}:${DATABASE_PASSWORD:-nukelab123}@${DATABASE_HOST:-postgres}:${DATABASE_PORT:-5432}/${DATABASE_NAME:-nukelab}" } +# Usage: _pg_dump_backup +# Run pg_dump for the configured database into . Returns 0 on +# success and removes partial/empty files on failure. stdout is redirected to +# the file; stderr remains visible so errors surface. +_pg_dump_backup() { + local output_file="$1" + local backup_dir + backup_dir=$(dirname "$output_file") + mkdir -p "$backup_dir" + + local _dump_exit=0 + $COMPOSE "${COMPOSE_ARGS[@]}" exec -T postgres \ + pg_dump -U "${DATABASE_USER:-nukelab}" "${DATABASE_NAME:-nukelab}" \ + > "$output_file" || _dump_exit=$? + + if [ "$_dump_exit" -ne 0 ]; then + rm -f "$output_file" + return 1 + fi + if [ ! -s "$output_file" ]; then + rm -f "$output_file" + return 1 + fi + return 0 +} + wait_for_backend() { # Always check the local Traefik-exposed backend, not APP_URL. APP_URL may # point to an external hostname (e.g., https://lab.nukehub.org) that isn't diff --git a/scripts/manage.d/backup.sh b/scripts/manage.d/backup.sh index 2f5f3fea..2f460a99 100755 --- a/scripts/manage.d/backup.sh +++ b/scripts/manage.d/backup.sh @@ -11,23 +11,10 @@ cmd_backup() { die "Postgres container is not running. Start the backend first:\n ./nukelabctl start backend" fi - mkdir -p "$backup_dir" step "Creating backup..." - # pg_dump's stdout IS the backup, so redirect it to the file directly. - # Routing through _run_quiet_unless_verbose would send stdout to /dev/null - # unless --verbose is set, producing an empty backup. stderr stays visible - # so pg_dump errors surface. - local _dump_exit=0 - $COMPOSE "${COMPOSE_ARGS[@]}" exec -T postgres pg_dump -U "${DATABASE_USER:-nukelab}" "${DATABASE_NAME:-nukelab}" > "$backup_file" || _dump_exit=$? - - if [ "$_dump_exit" -ne 0 ]; then - rm -f "$backup_file" - die "Backup failed: pg_dump exited with status $_dump_exit (partial file removed)" - fi - if [ ! -s "$backup_file" ]; then - rm -f "$backup_file" - die "Backup failed: pg_dump produced an empty backup (file removed)" + if ! _pg_dump_backup "$backup_file"; then + die "Backup failed: pg_dump exited with a non-zero status or produced an empty backup (partial file removed)" fi ok "Backup created: ${CYAN}$backup_file${RESET}" diff --git a/scripts/manage.d/db-migrate.sh b/scripts/manage.d/db-migrate.sh index e2fd970d..85193c23 100755 --- a/scripts/manage.d/db-migrate.sh +++ b/scripts/manage.d/db-migrate.sh @@ -2,33 +2,111 @@ # SPDX-FileCopyrightText: 2023-2026 NukeHub Developers # SPDX-License-Identifier: BSD-2-Clause +# Default values for db-migrate options. +DB_MIGRATE_NO_BACKUP=false + cmd_db_migrate() { step "Running database migrations..." - if is_backend_container_running; then - # Backend is running in containers, run migrations there. - # Force a direct Postgres URL even if DATABASE_URL points to PgBouncer; - # DDL must not go through the connection pooler. - local direct_url - direct_url=$(_direct_database_url) - if [[ "${DATABASE_HOST:-postgres}" == "pgbouncer" ]] || [[ "${DATABASE_PORT:-5432}" == "6432" ]]; then - info "Routing migration through direct Postgres connection" - fi - _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" exec -e "DATABASE_URL=$direct_url" backend alembic upgrade head - else + if ! is_backend_container_running; then die "Backend not running. Start it first:\n ./nukelabctl start backend" fi + local snapshot_file="" + local backup_created=false + local rev="none" + local ts + ts=$(date +%Y%m%d_%H%M%S) + + if ! $DB_MIGRATE_NO_BACKUP; then + # Query current Alembic revision before the upgrade. Use psql directly + # against postgres so this works even if the backend env/alembic CLI is + # in an unexpected state. + rev=$(_current_alembic_revision) + snapshot_file="$DIR/backups/pre-migrate-${NUKELAB_VERSION}-${rev}-${ts}.dump" + + step "Creating pre-migration snapshot ${snapshot_file}..." + if ! _pg_dump_backup "$snapshot_file"; then + die "Pre-migration backup failed; migration aborted to avoid an unprotected upgrade" + fi + backup_created=true + ok "Pre-migration snapshot created: ${CYAN}$snapshot_file${RESET}" + fi + + # Force a direct Postgres URL even if DATABASE_URL points to PgBouncer; + # DDL must not go through the connection pooler. + local direct_url + direct_url=$(_direct_database_url) + if [[ "${DATABASE_HOST:-postgres}" == "pgbouncer" ]] || [[ "${DATABASE_PORT:-5432}" == "6432" ]]; then + info "Routing migration through direct Postgres connection" + fi + + local _migrate_exit=0 + _run_quiet_unless_verbose $COMPOSE "${COMPOSE_ARGS[@]}" exec -e "DATABASE_URL=$direct_url" backend alembic upgrade head || _migrate_exit=$? + + if [ "$_migrate_exit" -ne 0 ]; then + if $backup_created; then + err "Migration failed. To restore the pre-migration snapshot, run:" + err " ./nukelabctl restore ${snapshot_file}" + fi + die "Migration failed: alembic upgrade head exited with status $_migrate_exit" + fi + ok "Migrations applied" } +# Return the current Alembic revision stored in the database, or "none" if the +# alembic_version table does not exist yet (fresh/unmanaged database). +_current_alembic_revision() { + local rev="" + local _exit=0 + rev=$($COMPOSE "${COMPOSE_ARGS[@]}" exec -T postgres \ + psql -U "${DATABASE_USER:-nukelab}" -d "${DATABASE_NAME:-nukelab}" \ + -v ON_ERROR_STOP=1 -t -A \ + -c "SELECT version_num FROM alembic_version LIMIT 1" 2> /dev/null | tr -d '[:space:]') || _exit=$? + if [ "$_exit" -ne 0 ] || [ -z "$rev" ]; then + echo "none" + else + echo "$rev" + fi +} + +parse_db_migrate_args() { + while [[ ${#EXTRA_ARGS[@]} -gt 0 ]]; do + case "${EXTRA_ARGS[0]}" in + --no-backup) + DB_MIGRATE_NO_BACKUP=true + EXTRA_ARGS=("${EXTRA_ARGS[@]:1}") + ;; + --help | -h) + help_db_migrate + exit 0 + ;; + --*) + die "Unknown option for db-migrate: ${EXTRA_ARGS[0]}" + ;; + *) + die "Unexpected argument for db-migrate: ${EXTRA_ARGS[0]}" + ;; + esac + done +} + help_db_migrate() { cat <<- EOF -${BOLD}Usage:${RESET} ./nukelabctl db-migrate +${BOLD}Usage:${RESET} ./nukelabctl db-migrate [options] Run Alembic database migrations inside the backend container. +A pg_dump snapshot is taken automatically before the upgrade. The filename +includes the platform version, the current Alembic revision, and a timestamp: +backups/pre-migrate---.dump + +${BOLD}Options:${RESET} + --no-backup Skip the automatic pre-migration snapshot + ${BOLD}Examples:${RESET} ./nukelabctl db-migrate + ./nukelabctl db-migrate --no-backup EOF } diff --git a/scripts/nukelabctl-completion.bash b/scripts/nukelabctl-completion.bash index fd9e602c..3b17e5fd 100644 --- a/scripts/nukelabctl-completion.bash +++ b/scripts/nukelabctl-completion.bash @@ -122,9 +122,12 @@ _manage_sh_complete() { update) COMPREPLY=($(compgen -W "--cache --build ${global_flags[*]}" -- "$cur")) ;; - pull | e2e | db-migrate | db-shell | backup | selftest | install-completion | help | security | init-user-auth-keys | rotate-user-auth-key | cleanup-user-auth-keys) + pull | e2e | db-shell | backup | selftest | install-completion | help | security | init-user-auth-keys | rotate-user-auth-key | cleanup-user-auth-keys) COMPREPLY=($(compgen -W "${global_flags[*]}" -- "$cur")) ;; + db-migrate) + COMPREPLY=($(compgen -W "--no-backup ${global_flags[*]}" -- "$cur")) + ;; *) COMPREPLY=($(compgen -W "${global_flags[*]}" -- "$cur")) ;; From 291904eae710ca085fe1e2f03dfafe89ff4f33d7 Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Fri, 21 Aug 2026 22:40:50 +0600 Subject: [PATCH 6/7] Fix Trivy image refs and skip empty SARIF uploads in CI --- .github/workflows/security.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index fea5f633..05b68983 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -108,7 +108,8 @@ jobs: uses: aquasecurity/trivy-action@v0.36.0 continue-on-error: true with: - image-ref: 'nukelab-backend:latest' + # compose.yml tags locally built images with the ghcr.io name. + image-ref: 'ghcr.io/nukehub-dev/nukelab-backend:latest' format: 'sarif' output: 'trivy-backend.sarif' severity: 'HIGH,CRITICAL' @@ -117,17 +118,20 @@ jobs: uses: aquasecurity/trivy-action@v0.36.0 continue-on-error: true with: - image-ref: 'nukelab-frontend:latest' + image-ref: 'ghcr.io/nukehub-dev/nukelab-frontend:latest' format: 'sarif' output: 'trivy-frontend.sarif' severity: 'HIGH,CRITICAL' - name: Merge SARIF files + id: merge run: | python3 -c " - import json, glob + import json, glob, os runs = [] for f in glob.glob('trivy-*.sarif'): + if f == 'trivy-merged.sarif': + continue runs.extend(json.load(open(f)).get('runs', [])) if not runs: merged = {'\$schema': 'https://json.schemastore.org/sarif-2.1.0.json', 'version': '2.1.0', 'runs': []} @@ -139,11 +143,16 @@ jobs: for run in runs: merged['runs'][0]['results'].extend(run.get('results', [])) json.dump(merged, open('trivy-merged.sarif', 'w'), indent=2) + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + fh.write('runs=%d\n' % len(merged['runs'])) " - name: Upload Trivy SARIF uses: github/codeql-action/upload-sarif@v4 - if: always() + # The code-scanning API rejects a SARIF payload with zero runs + # ("1 item required; only 0 were supplied") — skip the upload when + # both scans failed to produce a report instead of failing the job. + if: always() && steps.merge.outputs.runs != '0' with: sarif_file: trivy-merged.sarif From 76dc6cb76fe05ad7fd01a35d583b183b70d89aa0 Mon Sep 17 00:00:00 2001 From: Ahnaf Tahmid Chowdhury Date: Fri, 21 Aug 2026 23:01:39 +0600 Subject: [PATCH 7/7] Update schema guard test to use lowercase settings fields --- backend/tests/db/test_schema_guard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/tests/db/test_schema_guard.py b/backend/tests/db/test_schema_guard.py index a7206e5b..84e02a38 100644 --- a/backend/tests/db/test_schema_guard.py +++ b/backend/tests/db/test_schema_guard.py @@ -222,7 +222,7 @@ def test_valid_db_schema_guard_values(self, mode): """off/auto/enforce are accepted.""" from app.config import Settings - settings = Settings(DB_SCHEMA_GUARD=mode) + settings = Settings(db_schema_guard=mode) assert settings.db_schema_guard == mode def test_invalid_db_schema_guard_value_rejected(self): @@ -230,4 +230,4 @@ def test_invalid_db_schema_guard_value_rejected(self): from app.config import Settings with pytest.raises(ValueError, match="DB_SCHEMA_GUARD"): - Settings(DB_SCHEMA_GUARD="warn-only") + Settings(db_schema_guard="warn-only")