Update image registry and implement pull-based deploys

.env.example

@@ -20,6 +20,16 @@ APP_URL=http://localhost:8080         # Your application URL
 # FRONTEND_URL=http://localhost:5173  # Optional: Set when frontend runs separately (e.g., Vite dev server)
 APP_TIMEZONE=UTC
 
+# Version pinning / pull-based deploys
+# NUKELAB_VERSION is baked into locally built images as the APP_VERSION build
+# arg; it defaults to the VERSION file / git tag resolved by nukelabctl.
+# Setting NUKELAB_VERSION or NUKELAB_IMAGE_TAG switches nukelabctl to pull
+# pre-built images from ghcr.io/nukehub-dev instead of building from source.
+# The registry must be readable on the deploy host (public packages, or run
+# `podman login ghcr.io` / `docker login ghcr.io`).
+# NUKELAB_VERSION=2.1.0               # Pin platform version (also switches to pull-based deploy)
+# NUKELAB_IMAGE_TAG=2.1.0             # Exact registry tag; defaults to pinned version or `latest`
+
 # =============================================================================
 # SECURITY  ⚠️  CHANGE SECRETS FOR PRODUCTION
 # =============================================================================

.github/workflows/ci.yml

@@ -225,6 +225,9 @@ jobs:
           target: ${{ matrix.target }}
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
+          # Inject the resolved version into the backend image so the running
+          # API reports the exact image tag (APP_VERSION env, see backend/Dockerfile).
+          build-args: ${{ matrix.name == 'backend' && format('APP_VERSION={0}', steps.meta.outputs.version) || '' }}
           labels: |
             org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
             org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/security.yml

@@ -108,7 +108,8 @@ jobs:
         uses: aquasecurity/trivy-action@v0.36.0
         continue-on-error: true
         with:
-          image-ref: 'nukelab-backend:latest'
+          # compose.yml tags locally built images with the ghcr.io name.
+          image-ref: 'ghcr.io/nukehub-dev/nukelab-backend:latest'
           format: 'sarif'
           output: 'trivy-backend.sarif'
           severity: 'HIGH,CRITICAL'
@@ -117,17 +118,20 @@ jobs:
         uses: aquasecurity/trivy-action@v0.36.0
         continue-on-error: true
         with:
-          image-ref: 'nukelab-frontend:latest'
+          image-ref: 'ghcr.io/nukehub-dev/nukelab-frontend:latest'
           format: 'sarif'
           output: 'trivy-frontend.sarif'
           severity: 'HIGH,CRITICAL'
 
       - name: Merge SARIF files
+        id: merge
         run: |
           python3 -c "
-          import json, glob
+          import json, glob, os
           runs = []
           for f in glob.glob('trivy-*.sarif'):
+              if f == 'trivy-merged.sarif':
+                  continue
               runs.extend(json.load(open(f)).get('runs', []))
           if not runs:
               merged = {'\$schema': 'https://json.schemastore.org/sarif-2.1.0.json', 'version': '2.1.0', 'runs': []}
@@ -139,11 +143,16 @@ jobs:
               for run in runs:
                   merged['runs'][0]['results'].extend(run.get('results', []))
           json.dump(merged, open('trivy-merged.sarif', 'w'), indent=2)
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+              fh.write('runs=%d\n' % len(merged['runs']))
           "
 
       - name: Upload Trivy SARIF
         uses: github/codeql-action/upload-sarif@v4
-        if: always()
+        # The code-scanning API rejects a SARIF payload with zero runs
+        # ("1 item required; only 0 were supplied") — skip the upload when
+        # both scans failed to produce a report instead of failing the job.
+        if: always() && steps.merge.outputs.runs != '0'
         with:
           sarif_file: trivy-merged.sarif
 

AGENTS.md

@@ -124,6 +124,30 @@ Notes:
   path/to/file.test.ts` directly. See `frontend/AGENTS.md` for frontend
   conventions.
 
+## Releases
+
+- Version source of truth: git tags (`vX.Y.Z`). `scripts/ci-version.sh` turns
+  them into semver image tags plus `latest` in CI, and CI injects the resolved
+  version into the backend image via the `APP_VERSION` build arg (runtime
+  resolution: `settings.app_version`, fallback `backend/app/version.py`).
+  Local compose builds get the same treatment: `nukelabctl` exports
+  `NUKELAB_VERSION` (VERSION file / git describe) and `compose.yml` passes it
+  as the `APP_VERSION` build arg.
+- Production hosts can deploy by pinning `NUKELAB_VERSION` or
+  `NUKELAB_IMAGE_TAG` so `nukelabctl up` / `update` pull tagged images from
+  `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` instead of rebuilding
+  from source. Unpinned deploys keep the source-build path.
+- Rollback safety: `./nukelabctl db-migrate` takes an automatic pre-migration
+  snapshot, and the backend startup guard refuses to boot an old image on a
+  newer schema. To roll back, pin the previous release, run `./nukelabctl
+  update`, restore the `backups/pre-migrate-*` snapshot, and restart.
+- Cut a release with `scripts/bump-version.sh X.Y.Z` — it syncs `VERSION`,
+  `frontend/package.json`, and `CHANGELOG.md` (the backend version is dynamic
+  via `APP_VERSION`; `backend/app/version.py` stays `0.0.0-dev`), then prints
+  the commit/tag/push commands (it never runs them).
+- Record notable changes in the root `CHANGELOG.md` (Keep a Changelog format,
+  `[Unreleased]` section).
+
 ## Architecture pointer
 
 High-level layout; see the Child NAD Index below for domain-specific details.

CHANGELOG.md

@@ -0,0 +1,70 @@
+# Changelog
+
+All notable changes to the NukeLab platform are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Releases are cut with `scripts/bump-version.sh X.Y.Z`, which stamps the
+`[Unreleased]` section below and syncs the version across `VERSION`,
+`backend/app/version.py`, and `frontend/package.json`. Git tags (`vX.Y.Z`)
+are the release source of truth; CI builds container images tagged with the
+release version.
+
+## [Unreleased]
+
+### Added
+
+- Pull-based production deploys: pin `NUKELAB_VERSION` or `NUKELAB_IMAGE_TAG`
+  to switch `nukelabctl up` / `update` from source builds to pulling tagged
+  `ghcr.io/nukehub-dev/nukelab-backend` and `-frontend` images. The three
+  backend services share one backend image; `update --build` forces a source
+  rebuild even in pull mode. Unpinned deploys keep today's source-build
+  behavior.
+- Release versioning: git tags (`vX.Y.Z`) are the single source of truth.
+  `scripts/bump-version.sh` syncs the version across `VERSION`,
+  `backend/app/version.py`, `frontend/package.json`, and this changelog.
+- Dynamic version injection: CI image builds pass the resolved tag as the
+  `APP_VERSION` build arg (`backend/Dockerfile`), so running containers
+  report the exact image tag (`2.1.0`, `main`, `pr-42`). The API root,
+  `/health/status`, and the OpenTelemetry service version resolve via
+  `settings.app_version`; empty or unset `APP_VERSION` falls back to the
+  static literal in `app/version.py`.
+- Compose builds bake the platform version into locally built backend images:
+  `nukelabctl` exports `NUKELAB_VERSION` (from the `VERSION` file / git tag,
+  overridable via the env file) and `compose.yml` passes it as the
+  `APP_VERSION` build arg to the backend and celery images, so stacks
+  deployed with `nukelabctl` report the checkout version instead of
+  `0.0.0-dev`.
+- Automatic pre-migration backups: `./nukelabctl db-migrate` now takes a
+  `pg_dump` snapshot before running `alembic upgrade head`. The snapshot name
+  is `backups/pre-migrate-<NUKELAB_VERSION>-<current-revision>-<timestamp>.dump`.
+  If the backup fails, the migration aborts; if the migration then fails, the
+  exact `./nukelabctl restore <file>` command is printed. Use `--no-backup` to
+  skip the snapshot.
+- Startup schema-compatibility guard (`app/db/schema_guard.py`) refuses to boot
+  an old backend image against a newer database schema. Controlled by the
+  `DB_SCHEMA_GUARD` setting (`auto` refuse in production/warn elsewhere,
+  `enforce` always refuse, `off` disabled). If the database is unreachable the
+  guard logs a warning and does not block startup.
+
+### Changed
+
+- Backend version string is no longer hardcoded: `app/main.py`,
+  `app/api/health.py`, and the OpenTelemetry service version in
+  `app/config.py` all resolve through `settings.app_version` (env-first,
+  static fallback in `app/version.py`).
+- The static fallback in `app/version.py` is now the fixed sentinel
+  `0.0.0-dev` instead of a release-looking literal — a version that can
+  never be mistaken for a release, and which `bump-version.sh` no longer
+  touches (only `VERSION`, `frontend/package.json` + lockfile, and this
+  changelog are bumped at release time).
+- `frontend/package.json` version set from `0.0.0` placeholder to the
+  platform version.
+
+## [2.0.0]
+
+The 2.0 platform was delivered phase-by-phase without git tags. See
+[docs/plan/IMPLEMENTATION-PHASES.md](docs/plan/IMPLEMENTATION-PHASES.md)
+for the full delivery record and [docs/plan/ROADMAP.md](docs/plan/ROADMAP.md)
+for recent milestones.

VERSION

@@ -0,0 +1 @@
+v2.0.0

backend/AGENTS.md

@@ -12,6 +12,7 @@ All files under `backend/` except generated artifacts (`.venv-dev`, `__pycache__
 
 - Python 3.13; formatting and linting configured in `pyproject.toml`.
 - `app/main.py` is the ASGI entry point.
+- `app/version.py` holds the static fallback `__version__` — permanently `0.0.0-dev`, never bumped. The effective runtime version is `settings.app_version` (`app/config.py`): the `APP_VERSION` env var wins, with empty/unset falling back to the literal. `APP_VERSION` is injected as a Docker build arg both by CI (the image tag) and by local compose builds (`NUKELAB_VERSION` exported by `nukelabctl`). `main.py` and `app/api/health.py` read `settings.app_version` — never hardcode the version string.
 - `app/api/` owns route definitions; `app/services/` owns business logic; `app/models/` owns SQLAlchemy models; `app/db/` owns session/connection logic; `app/core/` owns cross-cutting utilities; `app/middleware/` owns ASGI middleware; `app/container/` owns container-runtime orchestration; `app/tasks.py` and `app/worker.py` own Celery.
 - `app/container/` is a driver layer: `driver.py` defines the `ContainerDriver` ABC + `ContainerDriverError` (plain-data returns only — no runtime objects escape), `docker_driver.py` is the Docker/Podman implementation, `factory.py` selects the driver via `CONTAINER_RUNTIME` (default `docker`), `client.py` is a compatibility shim for legacy imports/test seams, and `spawner.py` (server lifecycle) talks only to driver methods. To add a runtime (e.g. Kubernetes): implement `ContainerDriver` (synthesizing the documented return shapes, e.g. Docker-stats for `get_container_stats`) and register it in `factory.py`. Container health is driver-level config, not image metadata: `docker_driver.py` injects a uniform `Healthcheck` (`/usr/local/bin/nukelab-healthcheck.sh`) into every create config, because OCI images drop Dockerfile `HEALTHCHECK` and Kubernetes ignores it — a k8s driver must translate the same definition into pod liveness/startup probes and surface failures as the same `State.Health.Status` shape `HealthCheckService` consumes. `spawner.py` performs two readiness probes before marking a server `running`: (1) the container's own `/health` endpoint over the Docker network alias, and (2) the public server path through the internal Traefik load balancer (`TRAEFIK_INTERNAL_URL`, default `http://traefik:80`) with a `healthy` body check. This ensures the browser-facing route exists before the frontend is told to redirect.
 - Notifications: `app/services/notification_service.py` owns notification creation and delivery. Channels are `in_app`, `email`, `webhook`, and (when VAPID keys are configured) `push`. Push payloads are short previews only; dead subscriptions are removed on `404`/`410`. VAPID config and the `push_subscriptions` model live in `app/config.py` and `app/models/push_subscription.py`; pass `VAPID_*` env vars to both the `backend` and `celery-worker` containers.
@@ -53,6 +54,26 @@ All files under `backend/` except generated artifacts (`.venv-dev`, `__pycache__
 - Review the generated migration before committing; autogenerated scripts can miss renames and complex changes.
 - Test upgrade and downgrade locally: `alembic upgrade head && alembic downgrade -1`.
 - Migrations must be reversible and tested against the current schema.
+- **Expand-contract for destructive changes:** split drops, renames, and removals across two releases so the previous release's code still runs on the new schema. For example, release N adds the replacement column and dual-writes; release N+1 removes the old column once no deployed image references it.
+
+### Schema-compatibility guard
+
+`app/db/schema_guard.py` protects against the pull-deploy rollback hazard where
+an older backend image boots against a database that already ran newer Alembic
+migrations.
+
+- `check_schema_compatibility(engine, script_dir_path)` reads the DB's
+  `alembic_version` revision(s), walks the local Alembic `ScriptDirectory`, and
+  reports whether every DB revision is known to the running image.
+- `run_schema_guard(engine, script_dir_path, mode, app_env)` applies the
+  configured policy and raises `RuntimeError` when a rollback hazard is detected
+  in a refusing mode.
+- The guard is controlled by the `DB_SCHEMA_GUARD` setting (`app/config.py`):
+  - `auto` (default): refuse to start in production, warn in other environments.
+  - `enforce`: always refuse.
+  - `off`: disabled.
+- If the database is unreachable, the guard logs a warning and does not block
+  startup, so DB outages do not become a new failure mode.
 
 ### Background tasks
 

backend/Dockerfile

@@ -33,6 +33,12 @@ CMD ["python", "-m", "pytest"]
 # ── Production / runtime target (default) ───────────────────────────────────
 FROM base AS runtime
 
+# Platform version injected by CI (from scripts/ci-version.sh). Empty for
+# local builds; the app then falls back to the static version in
+# app/version.py (see app/config.py: app_version).
+ARG APP_VERSION=""
+ENV APP_VERSION=${APP_VERSION}
+
 # Expose port
 EXPOSE 8000
 

backend/app/api/health.py

@@ -190,7 +190,7 @@ async def platform_status():
     from app.services.oauth_service import oauth_service
 
     return {
-        "version": "2.0.0",
+        "version": settings.app_version,
         "features": {
             "auth_mode": settings.auth_mode,
             "oauth_enabled": oauth_service.is_configured

backend/app/config.py

@@ -8,6 +8,8 @@
 from pydantic import field_validator, model_validator
 from pydantic_settings import BaseSettings
 
+from app.version import __version__
+
 
 class Settings(BaseSettings):
     app_name: str = "NukeLab"
@@ -18,6 +20,11 @@ class Settings(BaseSettings):
     frontend_url: str = ""  # Defaults to public_url if not set
     app_timezone: str = "UTC"
 
+    # Platform version. Defaults to the checked-in fallback in app/version.py;
+    # CI image builds inject APP_VERSION (from scripts/ci-version.sh) so
+    # containers report the exact image tag (e.g. 2.1.0, main, pr-42).
+    app_version: str = __version__
+
     maintenance_mode: bool = False
     maintenance_message: str = "System under maintenance"
 
@@ -134,6 +141,12 @@ class Settings(BaseSettings):
     # instead of relying on auto-create.
     auto_create_tables: bool = True
 
+    # Schema-compatibility guard. "auto" refuses to start in production when
+    # the DB Alembic revision is newer than the revisions known to this image,
+    # and warns in other environments. "enforce" always refuses. "off" disables
+    # the guard (still logs at debug level during startup).
+    db_schema_guard: str = "auto"
+
     # Observability — Query Performance Monitoring
     observability_slow_query_threshold_ms: int = 100
     observability_pg_stat_statements_enabled: bool = True
@@ -240,7 +253,7 @@ def gpu_device_list(self) -> list[str]:
     otel_exporter_otlp_endpoint: str = "http://otel-collector:4317"
     otel_exporter_otlp_protocol: str = "grpc"  # grpc | http
     otel_service_name: str = "nukelab-backend"
-    otel_service_version: str = "2.0.0"
+    otel_service_version: str = ""  # Defaults to app_version; override via OTEL_SERVICE_VERSION
     otel_log_correlation: bool = True
     otel_sampler_ratio: float = 1.0
 
@@ -290,6 +303,14 @@ def gpu_device_list(self) -> list[str]:
     user_auth_denylist_fail_closed: bool = True
     user_auth_key_rotation_grace_seconds: int | None = None
 
+    @field_validator("app_version", mode="before")
+    @classmethod
+    def _empty_app_version_to_fallback(cls, value: Any) -> Any:
+        """Treat an empty APP_VERSION env value as "use the static fallback"."""
+        if value == "" or value is None:
+            return __version__
+        return value
+
     @field_validator("user_auth_key_rotation_grace_seconds", mode="before")
     @classmethod
     def _empty_rotation_grace_to_none(cls, value: Any) -> Any:
@@ -306,6 +327,17 @@ def _empty_api_docs_to_none(cls, value: Any) -> Any:
             return None
         return value
 
+    @field_validator("db_schema_guard", mode="before")
+    @classmethod
+    def _validate_db_schema_guard(cls, value: Any) -> Any:
+        """Reject unsupported schema-guard modes."""
+        allowed = {"off", "auto", "enforce"}
+        if value not in allowed:
+            raise ValueError(
+                f"DB_SCHEMA_GUARD must be one of {sorted(allowed)}, got {value!r}"
+            )
+        return value
+
     @model_validator(mode="after")
     def set_key_paths(self) -> "Settings":
         """Derive key paths from secrets_dir if not explicitly set."""
@@ -327,6 +359,13 @@ def set_key_paths(self) -> "Settings":
             )
         return self
 
+    @model_validator(mode="after")
+    def set_otel_service_version(self) -> "Settings":
+        """Default the OTEL service version to the resolved app version."""
+        if not self.otel_service_version:
+            self.otel_service_version = self.app_version
+        return self
+
     @model_validator(mode="after")
     def set_user_auth_rotation_grace(self) -> "Settings":
         """Default key rotation grace period to 2× access-token lifetime."""