datarobot-oss · taras-pokornyy · Sep 4, 2026
@@ -69,3 +69,44 @@ jobs:
           DR_API_TOKEN: ${{ secrets.DR_API_TOKEN }}
         run: |
           task smoke-test-pre-release DR_API_TOKEN="$DR_API_TOKEN"
+
+  # A separate job, not more steps above: that job already runs the `dr
+  # start` Agentic Starter flow inside its own 30-minute cap, and a
+  # stop->stopped wait independently observed to take ~11 minutes
+  # (RAPTOR-19749) risks blowing that budget for an unrelated reason.
+  # Ubuntu only -- this surface's behavior isn't OS-sensitive the way
+  # `dr start`'s template tooling is, so there's no need to double the cost
+  # by also running it on macOS.
+  workload-smoke-test:
+    runs-on: ubuntu-latest
+    name: Run Workload/Artifact Smoke Tests
+    timeout-minutes: 45
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.version }}
+
+      - name: Install smoke-test dependencies
+        uses: datarobot-oss/cli/.github/actions/install-deps@main
+
+      - name: Set up Go and Task
+        uses: datarobot-oss/cli/.github/actions/setup@main
+
+      - name: Build and Install the DR CLI Binary
+        uses: datarobot-oss/cli/.github/actions/install-dr-bin@main
+
+      - name: Run Workload/Artifact Smoke Tests
+        env:
+          DR_API_TOKEN: ${{ secrets.DR_API_TOKEN }}
+        # Scenarios A/B/C/E + artifact-lifecycle. D (~20-30 min, two image
+        # builds) stays local/opt-in -- its cost is a bigger ask than this
+        # ticket's scope; revisit if/when the pre-release gate's total
+        # runtime budget can absorb it.
+        #
+        # lib.sh depends on jq throughout; it isn't installed explicitly by
+        # install-deps, only relied on as preinstalled on GitHub-hosted
+        # runner images. Low risk today, but worth knowing if this job ever
+        # starts failing at the very first `dr workload list` probe.
+        run: |
+          task smoke-test-workload -- a b c e artifact
@@ -11,7 +11,7 @@ This document tracks what is covered by smoke tests across all platforms. Use it
 | `run_plugin_update_smoke_test.sh` | Unix | `task smoke-test` |
 | `run_self_update_smoke_test.sh` | Unix (macOS for brew tests) | manual / CI |
 | `run_pre_release_smoke_test.sh` | Unix (Linux / macOS) | `task smoke-test-pre-release` (gates release promotion) |
-| `run_workload_smoke_test.sh` | Unix | `task smoke-test-workload` (local only; CI wiring pending team review) |
+| `run_workload_smoke_test.sh` | Unix | `task smoke-test-workload` (local, full set incl. opt-in D/E) + pre-release CI runs scenarios A/B/C/E/artifact on every release smoke gate; D stays local/opt-in (~20-30 min build cost) |
 
 ---
 
@@ -70,11 +70,12 @@ Legend: ✅ Covered · ⚠️ Partial · ❌ Not covered · ⏭️ Intentionally
 | brew install → `dr self update` uses brew path | ✅ (macOS) | ❌ | Skipped on Linux |
 | Template min-version satisfied → update is no-op | ✅ | ❌ | Stretch test |
 | Template min-version satisfied → `dr self update -f` upgrades | ✅ | ❌ | Stretch test |
-| **Workload / Artifact** (`run_workload_smoke_test.sh`) — `task smoke-test-workload` | | | Local only; CI wiring pending team review. Uses the CLI's existing `drconfig.yaml` auth; set `DATAROBOT_API_TOKEN` / `DATAROBOT_ENDPOINT` only to override. Sets `DATAROBOT_CLI_FEATURE_WORKLOAD=true`. |
+| **Workload / Artifact** (`run_workload_smoke_test.sh`) — `task smoke-test-workload` | | | Pre-release CI runs A/B/C/E/artifact (see Test Scripts table); D stays local/opt-in. Uses the CLI's existing `drconfig.yaml` auth; set `DATAROBOT_API_TOKEN` / `DATAROBOT_ENDPOINT` only to override. Sets `DATAROBOT_CLI_FEATURE_WORKLOAD=true`. |
 | A: whoami create → bind → up round trip + validation probes | ✅ | ❌ | Guards `up` validates at load time before server mutation |
 | B: account sweep — dry-run bind every workload on the account | ✅ | ❌ | Mirrors Woj's 4/4 binding-failure repro |
 | C: re-bind preserves live tuning; FileExists guard; delete-rebind restore | ✅ | ❌ | Known bug RAPTOR-19697: memory renders as 512MB (asserted) |
 | D: built-workload rebuild round trip (ErrImagePull regression) | ✅ (opt-in) | ❌ | ~20-30 min, two image builds; `task smoke-test-workload-full` or `d` arg |
+| E: `up --dry-run` idempotency; `stop` → `up --yes` reconcile; `delete --yes` clears binding | ✅ | ❌ | ~15-20 min, dominated by an observed ~11 min stop→stopped wait; RAPTOR-19749 |
 | Artifact: create → get → list → code init/sync/versions → del | ✅ | ❌ | CLI-side focus (manifest, .drignore, exit codes); complements `workload-api/tests/acceptance`. `dr artifact lock` not automated: a build-config draft needs a completed build to lock, and a locked artifact can't be deleted/unlocked via the CLI (would leak state) — covered manually / via Scenario D |
 
 ---
@@ -86,6 +87,7 @@ Legend: ✅ Covered · ⚠️ Partial · ❌ Not covered · ⏭️ Intentionally
 - `datarobot` alias is not verified.
 - Plugin auto-update and self-update flows are not covered.
 - Shell detection only covers PowerShell (not cmd.exe in the main suite; covered by a standalone `.bat` script).
+- `dr workload` / `dr artifact` are entirely untested on Windows — the whole suite is Unix-only (RAPTOR-19749).
 
 ### Unix
 - `dr help run` is not checked (only `dr help`).

@@ -13,7 +13,10 @@
 #
 # Defaults to the fast set: a b c artifact. Scenario D (~20-30 min, two image
 # builds) is opt-in — pass `d` as a scenario, or set
-# WORKLOAD_SMOKE_INCLUDE_D=1 to add it to the default set.
+# WORKLOAD_SMOKE_INCLUDE_D=1 to add it to the default set. Scenario E
+# (~15-20 min, dominated by a stop->stopped wait) is likewise opt-in — pass
+# `e` explicitly; the pre-release CI job does. Neither is silently added to
+# a bare local run, since both push well past the "fast" set's usual runtime.
 #
 # Exit code is non-zero if any scenario failed.
 
@@ -43,6 +46,7 @@ scenario_script() {
         b|B)        printf '%s/RAPTOR-19533-B-sweep.sh'      "$WL_DIR" ;;
         c|C)        printf '%s/RAPTOR-19533-C-rebind.sh'     "$WL_DIR" ;;
         d|D)        printf '%s/RAPTOR-19533-D-built.sh'      "$WL_DIR" ;;
+        e|E)        printf '%s/RAPTOR-19749-E-dryrun-stop-reconcile.sh' "$WL_DIR" ;;
         artifact)   printf '%s/artifact-lifecycle.sh'       "$WL_DIR" ;;
         *) echo "❌ unknown scenario: $1" >&2; return 1 ;;
     esac

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Ticket: RAPTOR-19749
+# Scenario E — dry-run idempotency, stop/up reconcile, delete clears binding.
+#
+# The whole `dr workload` surface had no smoke coverage before this and
+# Scenario A/D. This scenario closes the three items Scenario A/D leave open:
+# `up --dry-run` reports "Already up to date" on a stable tree, a stopped
+# workload is started and reconciled by a plain `up` in one command (not
+# recreated), and `delete --yes` both removes the workload and clears the
+# `workloadId:` binding it left in .datarobot.yaml.
+#
+# Uses an image-based whoami workload (like Scenario A), not a Dockerfile
+# build (like Scenario D): items under test here are up/stop/delete
+# mechanics, orthogonal to image vs. build origin, and stacking a second
+# server-side build on top of this scenario's own stop-wait would burn
+# 20-30 extra minutes for no added coverage.
+#
+# ~15-20 min, dominated by the stop-wait (see E.3): stop->stopped has been
+# observed to take up to ~11 minutes on staging. Source: RAPTOR-19749.
+
+# shellcheck shell=bash
+set -euo pipefail
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+wl::init_env
+wl::register_cleanup
+
+wl::start_timer "E: dry-run idempotency, stop/up reconcile, delete binding"
+
+# --- E.1 Create + bind (fast path: image-based, no build) -------------------
+work="$WL_SCRATCH/project"
+mkdir -p "$work"
+cat > "$work/workload.yaml" <<EOF
+name: aj-re-${RUN}
+artifact:
+  name: whoami-artifact
+  type: service
+  spec:
+    containerGroups:
+      - name: default
+        containers:
+          - name: whoami
+            imageUri: containous/whoami:latest
+            port: 8080
+            primary: true
+            entrypoint: ["/whoami", "--port", "8080"]
+            readinessProbe:
+              path: "/"
+              port: 8080
+              initialDelaySeconds: 5
+runtime:
+  containerGroups:
+    - name: default
+      replicaCount: 1
+      containers:
+        - name: whoami
+          resourceAllocation:
+            cpu: 1
+            memory: "512MB"
+EOF
+
+wl::dr_capture workload create --spec-file "$work/workload.yaml" --output-format json
+wl::assert_cmd_ok "$WL_RC" "$WL_OUT" "$WL_ERR" "workload create"
+WID="$(printf '%s' "$WL_OUT" | jq -r '.id')"
+wl::register_workload "$WID"
+wl::pass "created draft workload $WID"
+
+wl::wait_for_status "$WID" running 600 >/dev/null
+
+cd "$work"
+wl::dr_capture workload config --yes --workload-id "$WID"
+wl::assert_cmd_ok "$WL_RC" "$WL_OUT" "$WL_ERR" "workload config --workload-id"
+[[ -f .datarobot.yaml ]] || wl::fail "config did not write .datarobot.yaml"
+wl::pass "bound to workload $WID"
+
+# --- E.2 Item 3: `up --dry-run` on an unchanged tree is idempotent ----------
+# Twice, to rule out a one-shot fluke (e.g. a plan that only looks empty on
+# the very first read after config wrote the file).
+for attempt in 1 2; do
+    wl::dr_capture workload up --dry-run
+    wl::assert_cmd_ok "$WL_RC" "$WL_OUT" "$WL_ERR" "workload up --dry-run (attempt $attempt)"
+    printf '%s\n%s' "$WL_OUT" "$WL_ERR" | grep -q 'Already up to date' \
+        || wl::fail "up --dry-run (attempt $attempt) did not report 'Already up to date'"
+done
+wl::pass "up --dry-run reports 'Already up to date' on a stable tree (x2)"
+
+# --- E.3 Item 4: stop, then a plain `up` reconciles in one run --------------
+wl::dr_capture workload stop "$WID"
+wl::assert_cmd_ok "$WL_RC" "$WL_OUT" "$WL_ERR" "workload stop"
+wl::pass "stop accepted"
+
+# 900s (15 min) floor, not the 300s helper default: a stop->stopped
+# transition has been observed to take ~11 minutes on staging with the
+# container already gone and updatedAt never advancing past createdAt
+# (RAPTOR-19749). Do not tighten this without re-measuring.
+wl::wait_for_status "$WID" stopped 900 >/dev/null
+wl::pass "workload reached stopped"
+
+wl::dr_capture workload up --yes --output-format json
+wl::assert_cmd_ok "$WL_RC" "$WL_OUT" "$WL_ERR" "workload up --yes (reconcile from stopped)"
+GOT_WID="$(printf '%s' "$WL_OUT" | jq -r '.workloadId')"
+[[ "$GOT_WID" == "$WID" ]] \
+    || wl::fail "up after stop created/targeted a different workload: got $GOT_WID, want $WID"
+wl::pass "up reconciled the stopped workload in one run (no recreate)"
+
+wl::wait_for_status "$WID" running 600 >/dev/null
+
+# --- E.4 Item 6: --output-format json stdout is valid JSON ------------------
+status="$(wl::dr_jq '.status' workload status "$WID")"
+[[ "$status" == "running" ]] || wl::fail "workload status --output-format json: expected running, got $status"
+wl::pass "workload status --output-format json parses and reports running"
+
+# --- E.5 Item 5: `delete --yes` clears the binding --------------------------
+wl::dr_capture workload delete "$WID" --yes
+wl::assert_cmd_ok "$WL_RC" "$WL_OUT" "$WL_ERR" "workload delete --yes"
+printf '%s' "$WL_OUT" | grep -q "Deleted workload: $WID" \
+    || wl::fail "delete did not report 'Deleted workload: $WID' (got: $WL_OUT)"
+printf '%s' "$WL_ERR" | grep -q 'Removed workloadId from' \
+    || wl::fail "delete did not report clearing the binding (stderr: $WL_ERR)"
+wl::assert_absent .datarobot.yaml '^workloadId:' "workloadId binding cleared from .datarobot.yaml"
+wl::pass "delete removed the workload and cleared the manifest binding"
+
+wl::stop_timer
+echo "✅ Scenario E passed"
@@ -11,6 +11,7 @@ scenario script (grep with `rg '^# Ticket:' smoke_test_scripts/workload`).
 | B — account sweep | `RAPTOR-19533-B-sweep.sh` | RAPTOR-19533 | binding works against every workload on the account |
 | C — re-bind tuned | `RAPTOR-19533-C-rebind.sh` | RAPTOR-19533 | re-bind preserves live tuning; FileExists guard; delete-rebind restore |
 | D — built-workload rebuild | `RAPTOR-19533-D-built.sh` | RAPTOR-19533 | no ErrImagePull after re-bind; platform rebuilds from imageBuildConfig |
+| E — dry-run idempotency, stop/up reconcile, delete binding | `RAPTOR-19749-E-dryrun-stop-reconcile.sh` | RAPTOR-19749 | `up --dry-run` reports "Already up to date"; `stop` then `up --yes` reconciles the same workload; `delete --yes` clears the `workloadId:` binding |
 | Artifact lifecycle | `artifact-lifecycle.sh` | none — basic acceptance | `dr artifact` create/get/list/code sync/versions/del CLI-side state |
 
 ## Adding a scenario for a new ticket