Playbook rc1

.claude-plugin/marketplace.json

@@ -4,24 +4,24 @@
     "name": "NVIDIA Nemotron Team"
   },
   "metadata": {
-    "description": "NVIDIA Nemotron AI stack plugins — pipeline builder, model knowledge bases, and contributor tools"
+    "description": "NVIDIA Nemotron AI stack plugins"
   },
   "plugins": [
     {
-      "name": "nemotron",
-      "source": "./plugins/nemotron",
-      "description": "NVIDIA Nemotron AI stack — pipeline builder and model knowledge bases",
-      "version": "0.3.0",
+      "name": "nemotron-customize",
+      "source": "./skills/nemotron-customize",
+      "description": "Compose runnable Nemotron model-customization pipelines from repo steps.",
+      "version": "0.1.0",
       "category": "ml-pipelines",
-      "keywords": ["nvidia", "nemotron", "training", "sft", "rl", "megatron", "models"]
-    },
-    {
-      "name": "nemotron-dev",
-      "source": "./plugins/nemotron-dev",
-      "description": "Internal: contributor tools for Nemotron repo developers",
-      "version": "0.3.0",
-      "category": "developer-tools",
-      "keywords": ["nvidia", "nemotron", "internal", "contributing", "dev"]
+      "keywords": [
+        "nvidia",
+        "nemotron",
+        "training",
+        "sft",
+        "rl",
+        "megatron",
+        "customization"
+      ]
     }
   ]
 }

.gitignore

@@ -105,6 +105,7 @@ CLAUDE.md
 # Compiled config
 config.yaml
 main.py
+src/nemotron/steps/_bootstrap/runtime/
 
 # Documentation build
 docs/_build/

README.md

@@ -36,6 +36,36 @@
 
 ---
 
+## Use from Claude Code
+
+This repo ships a Claude Code plugin called **`nemotron-customize`** that turns the step catalog under [`src/nemotron/steps/`](./src/nemotron/steps/) into a guided, repo-native pipeline builder.
+
+Install once:
+
+```text
+/plugin marketplace add NVIDIA/Nemotron
+/plugin install nemotron-customize@nvidia-nemotron
+```
+
+Then, **start Claude Code from the repo root** and invoke the skill:
+
+```bash
+cd /path/to/Nemotron        # repo root: must contain pyproject.toml and src/nemotron/steps/
+claude
+```
+
+```text
+/nemotron-customize
+```
+
+The skill resolves all file paths against your current working directory, so it must be invoked from the Nemotron checkout root. Running it from a subdirectory will cause file reads to fail.
+
+The skill plans the step DAG, validates artifact wiring, and emits the YAML configs needed to run the requested pipeline. See [`skills/nemotron-customize/SKILL.md`](./skills/nemotron-customize/SKILL.md) for the full contract.
+
+> The marketplace installs **only** `nemotron-customize`. The other folders under [`skills/`](./skills/) (model knowledge bases, contributor add-`*` skills) stay on disk for repo browsing but are not loaded as plugins.
+
+---
+
 ## Repository Overview
 
 ```

deploy/nemotron-customizer/airgap/.gitignore

@@ -0,0 +1,7 @@
+# Generated by airgap runner.
+out/
+airgap-bundle/
+archives/
+__pycache__/
+*.lock.yaml
+*.tar

deploy/nemotron-customizer/airgap/Dockerfile.execution

@@ -0,0 +1,52 @@
+# Derivative execution image for Nemotron Customizer airgap.
+# Built from the real training/runtime image and only adds small missing
+# wrapper packages.
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG EXECUTION_REQUIREMENTS
+ARG REPO_OVERLAYS
+ARG REPO_OVERLAYS_DIR
+ARG PYTHON_BIN=python
+ARG PIP_NO_DEPS=true
+
+ENV HF_HUB_OFFLINE=1
+ENV TRANSFORMERS_OFFLINE=1
+ENV HF_DATASETS_OFFLINE=1
+ENV WANDB_MODE=offline
+
+COPY ${EXECUTION_REQUIREMENTS} /opt/nemotron-airgap/execution-requirements.txt
+COPY ${REPO_OVERLAYS} /opt/nemotron-airgap/repo-overlays.json
+COPY ${REPO_OVERLAYS_DIR}/ /opt/nemotron-airgap/repo-overlays/
+
+# Build-time installs keep --no-cache-dir so derivative image layers stay small.
+RUN if [ -s /opt/nemotron-airgap/execution-requirements.txt ]; then \
+      if [ "${PIP_NO_DEPS}" = "true" ]; then \
+        ${PYTHON_BIN} -m pip install --no-cache-dir --no-deps -r /opt/nemotron-airgap/execution-requirements.txt; \
+      else \
+        ${PYTHON_BIN} -m pip install --no-cache-dir -r /opt/nemotron-airgap/execution-requirements.txt; \
+      fi; \
+    fi && \
+    ${PYTHON_BIN} - <<'PY'
+import json
+import pathlib
+import shutil
+
+root = pathlib.Path("/opt/nemotron-airgap/repo-overlays")
+items = json.loads(pathlib.Path("/opt/nemotron-airgap/repo-overlays.json").read_text())
+for item in items:
+    repo = item["repo"]
+    source = item.get("source", repo)
+    target = pathlib.Path(item["target"])
+    src = root / source
+    if not src.exists():
+        raise SystemExit(f"missing baked repo overlay: {src}")
+    if target.exists() or target.is_symlink():
+        if target.is_dir() and not target.is_symlink():
+            shutil.rmtree(target)
+        else:
+            target.unlink()
+    target.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copytree(src, target)
+PY

deploy/nemotron-customizer/airgap/Dockerfile.execution.dockerignore

@@ -0,0 +1,14 @@
+**
+
+!deploy
+!deploy/nemotron-customizer
+!deploy/nemotron-customizer/airgap
+!deploy/nemotron-customizer/airgap/out
+!deploy/nemotron-customizer/airgap/out/execution-context
+!deploy/nemotron-customizer/airgap/out/execution-context/**
+!deploy/nemotron-customizer/airgap/out/repo-overlays
+!deploy/nemotron-customizer/airgap/out/repo-overlays/**
+
+**/.git
+**/__pycache__
+**/*.pyc

deploy/nemotron-customizer/airgap/Dockerfile.launcher

@@ -0,0 +1,30 @@
+# Launcher image for Nemotron Customizer airgap.
+# It contains the repo and a uv-synced environment. It does not run training.
+
+ARG BASE_IMAGE=python:3.12-slim
+FROM ${BASE_IMAGE}
+
+ARG UV_VERSION=0.11.1
+
+WORKDIR /workspace/Nemotron
+
+ENV UV_LINK_MODE=copy
+ENV UV_PYTHON_DOWNLOADS=never
+ENV HF_HUB_OFFLINE=1
+ENV TRANSFORMERS_OFFLINE=1
+ENV HF_DATASETS_OFFLINE=1
+ENV WANDB_MODE=offline
+ENV PYTHONPATH=/workspace/Nemotron/src
+ENV PATH=/workspace/Nemotron/.venv/bin:$PATH
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python -m pip install --no-cache-dir "uv==${UV_VERSION}"
+
+COPY . .
+
+RUN uv sync --frozen --no-dev
+
+CMD ["bash"]

deploy/nemotron-customizer/airgap/Dockerfile.launcher.dockerignore

@@ -0,0 +1,21 @@
+.git
+.venv
+.ruff_cache
+.pytest_cache
+**/__pycache__
+**/*.pyc
+
+/.nemo_run
+/outputs
+/output
+/logs
+/checkpoints
+/wandb
+/data
+/downloads
+
+deploy/nemotron-customizer/airgap/out
+deploy/nemotron-customizer/airgap/airgap-bundle
+deploy/nemotron-customizer/airgap/archives
+deploy/nemotron-customizer/airgap/*.tar
+deploy/nemotron-customizer/airgap/*.lock.yaml

deploy/nemotron-customizer/airgap/README.md

@@ -0,0 +1,135 @@
+# Nemotron Customizer Airgap
+
+This folder is scoped only to Nemotron Customizer steps under
+`src/nemotron/steps/`.
+
+The flow is intentionally small:
+
+1. Build one **launcher image** with this repo and `uv.lock`.
+2. Build one or more **execution images** by grouping selected workflow stages by base image.
+3. Save those images as tarballs for the airgapped side.
+4. Keep models, datasets, checkpoints, and customer files on persistent storage.
+
+Edit `airgap.yaml` first:
+
+- `workflow.stages`: the Nemotron Customizer steps the customer wants to run
+- `dependencies`: central step dependency map, for example SFT training needs SFT packing
+- `step_execution_images`: which execution image each step should use
+- `execution_images`: the base image, output tag, and known/import-probed Python requirements
+
+Only steps reached from `workflow.stages` are built. Steps are grouped by
+`base_image + repo_overlays`; each group gets one derivative image with the
+union of its small missing packages. If two selected step families share the
+same base image and repo overlays, the runner emits one combined execution image for
+both.
+
+Run from the repo root:
+
+```bash
+uv run python deploy/nemotron-customizer/airgap/runner.py \
+  --config deploy/nemotron-customizer/airgap/airgap.yaml
+```
+
+That prints the plan. To actually pull/build/save images on the connected
+machine:
+
+```bash
+uv run python deploy/nemotron-customizer/airgap/runner.py \
+  --config deploy/nemotron-customizer/airgap/airgap.yaml \
+  --execute
+```
+
+To run only a few stages:
+
+```bash
+uv run python deploy/nemotron-customizer/airgap/runner.py \
+  --config deploy/nemotron-customizer/airgap/airgap.yaml \
+  --stage validate \
+  --stage discover-execution-deps
+```
+
+To override the workflow without editing YAML, pass one or more selected
+Nemotron step targets. Dependencies are still expanded from `dependencies`.
+For example, SDG plus SFT also adds `data_prep/sft_packing` because SFT needs packed
+data:
+
+```bash
+uv run python deploy/nemotron-customizer/airgap/runner.py \
+  --config deploy/nemotron-customizer/airgap/airgap.yaml \
+  --target sdg/data_designer:tiny \
+  --target sft/megatron_bridge:tiny
+```
+
+Outputs are written under `deploy/nemotron-customizer/airgap/out/` by default:
+
+- `airgap-manifest.yaml`: what was validated and built
+- `airgap-build-state.yaml`: incomplete execute run state used for resume
+- `airgap-build-complete.yaml`: final execute run state after success
+- `requirements-<execution-group>.txt`: small missing packages per execution image
+- `repo-overlays-<execution-group>.json`: git auto-mounts discovered from selected step configs
+- `launcher-image.tar`
+- `execution-*.tar`
+- SHA256 checksums for saved image tarballs in `airgap-manifest.yaml`
+
+If an execute run fails midway, leave `airgap-build-state.yaml` in place and rerun
+the same command. Completed expensive actions are reused when their artifacts
+still exist. If you intentionally change the workflow or image plan before
+finishing, move or remove `airgap-build-state.yaml` first; the runner will not
+silently overwrite incomplete state from a different plan.
+
+Runtime dependency probes use Docker volumes named
+`nemotron-airgap-pip-cache-<platform>` to avoid downloading the same wheels on
+every probe loop. To reset them, run `docker volume ls | grep
+nemotron-airgap-pip-cache` and remove the relevant volume with
+`docker volume rm`.
+
+Large assets are not baked into images. The customer should stage them on
+executor-visible persistent storage and reference them through config overrides
+and `run.env.mounts`.
+
+During dependency discovery, the runner mounts the connected-machine checkout
+into each execution image only to probe imports. The final execution image deliberately
+does not bake this repo; the launcher image and the normal nemo-run/nemo-runspec
+code transport provide the repo to the remote job at submission time.
+
+Repo logistics stay outside `airgap.yaml`. If a selected step config contains
+`${auto_mount:git+...}`, the runner treats it as a connected-machine build input:
+it fetches that pinned repo and bakes it into the derivative execution image at the
+requested target path. Runtime jobs then use the baked image and do not clone
+from GitHub. Site-specific data/model mounts remain in env profiles or step
+overrides.
+
+If the connected machine is not the same architecture as the target cluster,
+set `platform: linux/amd64` on the `launcher_image` or execution image entry in
+`airgap.yaml`. If you need to minimize transfer size for several images that
+share layers, `docker save -o all-images.tar tag1 tag2 ...` can be used after
+the runner builds the images; a single tar deduplicates shared layers better
+than one tar per image.
+
+The Dockerfiles expect the chosen base images to have Python and `pip` available
+for bootstrapping small offline additions. The runtime defaults bake
+`HF_HUB_OFFLINE=1`, `TRANSFORMERS_OFFLINE=1`, `HF_DATASETS_OFFLINE=1`, and
+`WANDB_MODE=offline`; customers with an internal mirror can override those at
+submission time through their env profile or `run.env.env_vars`.
+
+For SFT Megatron-Bridge, build with the normal config so the runner can discover
+the pinned Megatron-LM and Megatron-Bridge auto-mounts:
+
+```yaml
+workflow:
+  stages:
+    - sft/megatron_bridge:tiny
+```
+
+When submitting inside the airgap, use the deploy overlay config so those git
+auto-mounts are cleared at runtime while persistent storage mounts from the env
+profile still apply. Use the image printed by the runner under
+`selected execution images`, or read it from `out/airgap-manifest.yaml` under
+`step_execution_images`.
+
+```bash
+uv run nemotron steps run sft/megatron_bridge \
+  -c deploy/nemotron-customizer/airgap/configs/sft_megatron_bridge_tiny.yaml \
+  -b <your-airgap-profile> \
+  run.env.container_image=<image-printed-for-sft/megatron_bridge>
+```