From c978f80f2612c2b2b95229d6bf3fd3d5a4c96fea Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 03:46:02 +0000 Subject: [PATCH 1/7] feat(docker): ship the CLIs the server shells out to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runtime stage was gcr.io/distroless/static-debian12, which holds exactly one file. But the server execs `devpod` for every workspace operation, `docker` for the SSH proxy, SFTP, container-user lookup and devcontainer prebuild, and `git` for the files tab. The published image booted, migrated, served the SPA, and then failed on the first session create with an exec-not-found. Replace the runtime base with debian:bookworm-slim carrying git plus pinned Docker CLI (client only — the daemon is the host's) and DevPod, fetched in a separate stage so curl and the tarball never reach the runtime layer. Distroless is not kept as a second variant: there is no working use case for an image that cannot run the product's core feature, and publishing both invites deploying the wrong one. HOME is set explicitly because it is load-bearing. Both DevPod state trees resolve through os.UserHomeDir() — the CLI's workspace records that supply the reconciler's container label, and the agent's cloned content that the files tab reads — as does the SSH host key. DevPod's pin is duplicated from .devcontainer/tool-versions.env rather than shared: .dockerignore excludes .devcontainer from the build context, so the file is not readable at build time. Image goes from ~40MB to ~103MB. Verified: all three binaries resolve, container runs as UID 65532 with a writable HOME, migrations apply, the SPA and version endpoint serve, and a missing hashed asset still 404s rather than falling back to the SPA shell. Co-Authored-By: Claude Opus 5 (1M context) --- Dockerfile | 82 ++++- ...7-30-vm-deploy-and-upgrade-requirements.md | 207 +++++++++++++ ...-001-feat-vm-deploy-topology-spike-plan.md | 286 ++++++++++++++++++ 3 files changed, 568 insertions(+), 7 deletions(-) create mode 100644 docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md create mode 100644 docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md diff --git a/Dockerfile b/Dockerfile index b581f4a..3721440 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,11 +27,79 @@ RUN cd server && CGO_ENABLED=0 GOOS=linux go build \ -ldflags="-s -w -X main.Version=${VERSION}" \ -o /out/deuce . -# Stage 2: minimal runtime. distroless/static-debian12:nonroot is correct for -# CGO_ENABLED=0 static Go binaries: no glibc, no shell, no package manager. -# Runs as UID 65532. -FROM gcr.io/distroless/static-debian12:nonroot -COPY --from=backend /out/deuce /deuce -EXPOSE 8080 +# Stage 2: fetch the CLI binaries the server shells out to. +# +# Deuce is not a self-contained binary at runtime: it drives `devpod` for every +# workspace operation, `docker` for the SSH proxy / SFTP / container-user lookup +# / devcontainer prebuild, and `git` for the files tab. A distroless runtime +# (the previous base) boots, migrates, and serves the SPA, then fails on the +# first session create with an exec-not-found. Both CLIs are fetched here so +# curl and the tarball never reach the runtime stage. +# +# Version pins are duplicated from .devcontainer/tool-versions.env rather than +# read from it: .dockerignore excludes .devcontainer from the build context, so +# the file is not readable here. Keep DEVPOD_VERSION in step with that file when +# bumping either one — a drift means the devcontainer and the shipped image +# drive DevPod differently. +FROM debian:bookworm-slim AS tools +ARG DEVPOD_VERSION=v0.6.15 +ARG DOCKER_CLI_VERSION=29.7.0 +# TARGETARCH is supplied by buildx. The release workflow builds linux/amd64 +# only today; parameterizing here means adding linux/arm64 is a workflow +# change rather than a Dockerfile change. +ARG TARGETARCH +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* +RUN set -eux; \ + case "${TARGETARCH}" in \ + amd64) docker_arch="x86_64" ;; \ + arm64) docker_arch="aarch64" ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl --fail --silent --show-error --location \ + --output /tmp/docker.tgz \ + "https://download.docker.com/linux/static/stable/${docker_arch}/docker-${DOCKER_CLI_VERSION}.tgz"; \ + # The tarball carries the full engine; extract only the client. dockerd, + # containerd and runc have no business in this image — the daemon is the + # host's. + tar --extract --file /tmp/docker.tgz --strip-components=1 --directory /usr/local/bin docker/docker; \ + curl --fail --silent --show-error --location \ + --output /usr/local/bin/devpod \ + "https://github.com/loft-sh/devpod/releases/download/${DEVPOD_VERSION}/devpod-linux-${TARGETARCH}"; \ + chmod 0755 /usr/local/bin/docker /usr/local/bin/devpod; \ + /usr/local/bin/docker --version; \ + /usr/local/bin/devpod version + +# Stage 3: runtime. +# +# HOME is load-bearing, not cosmetic. Both DevPod state trees resolve through +# os.UserHomeDir(): the CLI's workspace records (which supply the container +# label the reconciler matches on) and the agent's cloned workspace content +# (which the files tab reads and runs git against). The SSH host key defaults +# under it too. A deployment mounts one host directory here so all three +# survive container replacement — and, for the socket-mounted topology, so the +# path string resolves to the same directory for both Deuce and the host +# daemon. See docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md. +FROM debian:bookworm-slim +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + && rm -rf /var/lib/apt/lists/* +# UID 65532 matches the distroless `nonroot` user this image replaced, so any +# host directory already owned for the old image keeps working. +RUN groupadd --gid 65532 deuce \ + && useradd --uid 65532 --gid 65532 --home-dir /var/lib/deuce --create-home deuce +COPY --from=tools /usr/local/bin/docker /usr/local/bin/docker +COPY --from=tools /usr/local/bin/devpod /usr/local/bin/devpod +COPY --from=backend /out/deuce /usr/local/bin/deuce +ENV HOME=/var/lib/deuce +WORKDIR /var/lib/deuce +# 8080 HTTP (API + WS + embedded SPA), 2222 embedded SSH proxy for +# "Open in VS Code" (DEUCE_SSH_LISTEN_ADDR default). +EXPOSE 8080 2222 USER 65532:65532 -ENTRYPOINT ["/deuce"] +ENTRYPOINT ["/usr/local/bin/deuce"] diff --git a/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md b/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md new file mode 100644 index 0000000..c521746 --- /dev/null +++ b/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md @@ -0,0 +1,207 @@ +--- +date: 2026-07-30 +topic: vm-deploy-and-upgrade +--- + +# Self-Hosted VM Deploy and Upgrade Path + +## Summary + +Ship Deuce's VM deployment as repo artifacts — a production compose file, an env template, and deploy docs — that anyone can run and that the maintainers run as user #1. Upgrading is pull, restart, restart sessions. The container topology is settled by a timeboxed spike rather than picked up front, because the constraint most likely to break it fails silently. + +--- + +## Problem Frame + +Deuce has no deployment. The build-and-publish half of the original dogfood plan shipped — a semver tag produces `ghcr.io/forgeutah/deuce:X.Y.Z` and a GitHub Release — but the deploy half never landed, and no deploy workflow has ever existed in the repo's history. Nobody has run Deuce on a VM. The only validated configuration is a local devcontainer. + +That gap costs twice. The team can't dogfood a product whose entire thesis is *multiple people in one shared room*, so the premise stays theoretical. And the README's promise to end users — "one-command Docker compose for end users (not just dev)" — has no artifact behind it, so anyone who wants to try Deuce has to reconstruct a dev environment. + +The published image compounds it. `Dockerfile` produces a `gcr.io/distroless/static-debian12` runtime containing exactly one file, `/deuce`. But the server shells out to three binaries that aren't in it: `devpod` for every workspace operation (`server/internal/workspace/manager.go`), `docker` for the SSH proxy, SFTP, container-user lookup, and prebuild (`server/internal/sshproxy/docker.go`, `server/internal/workspace/prebuild.go`), and `git` for the files tab (`server/internal/handler/files.go`). `docker run` against the published image boots, migrates, and serves the SPA — then fails on the first session create. The artifact that looks deployable isn't. + +Underneath all of it sits a constraint that shapes every option. Deuce does not read workspace files through DevPod. It reads them directly off its own filesystem at `~/.devpod/agent/contexts//workspaces//content/` and runs `git` there with `cmd.Dir`, a deliberate design recorded in `docs/solutions/architecture-patterns/devpod-docker-workspace-bind-mount-2026-05-13.md`. Deuce and the Docker daemon it drives must therefore resolve the same path string to the same directory. The local devcontainer satisfies this with a genuine nested daemon — `.devcontainer/docker-compose.yml` runs `privileged: true` with the docker-in-docker feature, and its comment names the reason: to mount workspace paths into child containers "without host/container path mismatches." + +--- + +## Key Decisions + +- **Self-host is the shape; the maintainers are user #1.** The deploy path ships in-repo as artifacts an adopter runs, and the project's own instance uses those same artifacts rather than a private pipeline. A deploy path its authors don't exercise is how self-host docs rot, and this closes the standing README roadmap item instead of adding a parallel one. + +- **A spike decides the topology, not this document.** Three shapes are viable (below). None has been run on a VM, and the way the wrong one fails — an empty files tab while the agent edits a different directory than the team is looking at — is quiet enough that discovering it in production is expensive. The spike is the first unit of work, with a written decision rule. + +- **Workspaces stopping during an upgrade is acceptable.** Sessions live in Postgres and survive; workspace containers may be stopped and restarted by hand afterward. This is an explicit product call, and it is what lets the upgrade stay a container restart instead of an orchestration problem. + +- **The reconciler already implements the upgrade's session behavior.** `server/internal/reconcile/reconciler.go` polls container state every ~10s and writes truth into the DB: container absent but on-disk DevPod metadata present becomes `stopped`; no on-disk state at all becomes `missing`. Post-upgrade session recovery needs no new product code. It does mean the deploy's persistence choices are load-bearing — losing DevPod's on-disk state downgrades every session from `stopped` to `missing`. + +- **Tailscale Serve is the documented default, not one option among three.** Proxy mode already supports it, the tailnet is the trust boundary so there is no shared secret to manage, and it gives an adopter a safe configuration on day one rather than a decision they are not equipped to make. forge-proxy and exe.dev remain documented alternatives. + +- **The prebuild cache-key fix is part of this effort, not a separate one.** It is the only requirement here that changes product code rather than deploy artifacts, and it is tempting to split out. It stays because it is what makes the upgrade story true: without it, upgrading Deuce leaves cached repos starting sessions from previously baked agent tooling, and an adopter debugging that from the outside has almost no chance. The deploy and the fix fail together, so they ship together. + +- **Distroless is given up deliberately.** Whichever topology wins, the runtime needs real binaries on `PATH`. The image grows from ~40MB and loses the no-shell posture. That was the right default for an artifact that only served an SPA; it is the wrong default for a process whose job is driving other processes. + +### Candidate topologies + +The spike chooses among these. The distinguishing question is where the filesystem-namespace boundary falls relative to Deuce and the daemon it drives. + +```mermaid +flowchart TB + subgraph A["A — host-native"] + A1["deuce (systemd)"] --- A2["host dockerd"] + A2 --- A3["one host filesystem"] + A1 --- A3 + end + subgraph B["B — DinD container"] + B1["deuce + nested dockerd
(privileged)"] --- B2["container filesystem"] + end + subgraph C["C — socket + path parity"] + C1["deuce container"] --- C2["host dockerd via socket"] + C1 --- C3["shared path,
identical string both sides"] + C2 --- C3 + end +``` + +- **A — host-native.** Deuce runs as a systemd unit on the VM; Docker, DevPod, and git are host packages. Namespaces coincide by construction, nothing is privileged, and an upgrade restarts a small process while workspace containers keep running untouched. Costs: assumes a systemd/Debian-ish VM, and needs a raw binary artifact the release workflow deliberately does not publish today. +- **B — DinD container.** Mirrors the devcontainer: a privileged Deuce container running its own daemon. It is the only configuration anyone has seen work, and it is fully self-contained. Costs: privileged; a much larger image; overlayfs-on-overlayfs with a volume over the daemon's storage; and every upgrade necessarily stops every workspace, because restarting the container kills the nested daemon. +- **C — socket mount with path parity.** An unprivileged Deuce container with the host socket mounted and DevPod state bind-mounted at an *identical absolute path* inside and out, so the same string resolves to the same directory on both sides. Workspace containers are host siblings, so an upgrade leaves them running. Costs: parity is a discipline that can be broken silently, and the container's docker gid must match the host's. + +The naive variant of C — socket mounted without path parity — is the shape most people reach for first and must not ship. DevPod writes content to a path inside the container while sibling containers bind-mount that same path from the host. Two different directories, one string. + +--- + +## Key Flows + +- F1. First-time self-host install + - **Trigger:** An operator (adopter or maintainer) wants Deuce running on a fresh VM. + - **Actors:** Operator; the VM; Tailscale. + - **Steps:** + 1. Operator provisions a VM with Docker available and joins it to their tailnet. + 2. Operator copies `deploy/` from the repo (or a release), copies the env template, and fills in the handful of required values. + 3. Operator brings the stack up with a single documented command. + 4. Deuce runs migrations in-process, refuses to start if they fail, then binds and serves. + 5. Operator exposes it via Tailscale Serve and reaches it at the tailnet hostname; the first sign-in provisions their user from proxy identity headers. + - **Outcome:** A working Deuce reachable by their team, authenticated, with no secret to rotate. + - **Covered by:** R1, R2, R3, R11, R12, R13, R14, R17 + +- F2. Upgrade to a new version + - **Trigger:** A new Deuce release is published and the operator wants it. + - **Actors:** Operator; the reconciler. + - **Steps:** + 1. Operator stops running workspace containers (or accepts that the upgrade stops them). + 2. Operator points the deployment at the new image tag and restarts. + 3. Migrations run in-process before the listener binds; a failure exits non-zero rather than serving a partially-migrated schema. + 4. On boot, the reconciler observes each session's workspace and writes `stopped` where on-disk DevPod state survived. + 5. Operator (or any team member) restarts the sessions they want back, using the existing controls. + - **Outcome:** New version serving; sessions intact and restartable; no session shows `missing`. + - **Covered by:** R7, R8, R9, R10, R15, R16 + +--- + +## Requirements + +**Deploy artifacts** + +- R1. A production deployment lives in the repo under `deploy/` and is what both adopters and maintainers use. It is not a private pipeline mirrored by separate docs. +- R2. The deployment stands up Deuce plus Postgres, with database data on a named volume that survives container replacement. +- R3. An env template ships alongside it, listing every variable an operator must set and defaulting the rest to safe values. It is distinct from the dev-oriented `.env.example` at the repo root. +- R4. The deployment names an explicit image tag rather than a floating one, so an upgrade is a deliberate edit and a rollback is the reverse edit. + +**Runtime topology** + +- R5. Deuce and the Docker daemon it drives resolve DevPod's content paths to the same directory. Whichever topology is chosen must satisfy this, and the deploy artifacts must make it hard to break by accident. +- R6. The runtime image carries `devpod`, `docker`, and `git` on `PATH`, pinned to known versions. (Not applicable if the spike selects the host-native topology, where these are host packages and the docs pin them instead.) + +**Upgrade and state persistence** + +- R7. An upgrade is a documented sequence an operator can perform without reading source: point at the new tag, restart, restart sessions. +- R8. State that must survive an upgrade is persisted explicitly, not incidentally: Postgres data, DevPod agent state and workspace content, the SSH host key, and — when configured — the devcontainer prebuild cache and the VS Code server cache. +- R9. After an upgrade, sessions whose workspace content survived report `stopped`, not `missing`, and are restartable through the existing controls. +- R10. Migrations run before the listener binds and a failure prevents serving. (Already true in `server/main.go`; stated so the deployment does not undermine it, e.g. by racing multiple app containers.) + +**Security defaults** + +- R11. The shipped configuration uses proxy auth mode with Tailscale Serve headers, not dev mode. +- R12. A bind-address setting is introduced so an operator can bind loopback only. The server currently binds all interfaces unconditionally, which makes CLAUDE.md's existing "dev mode is localhost-only" guidance impossible to follow. +- R13. The server refuses to start in dev auth mode when bound to a non-loopback address. Existing local and devcontainer setups must keep working unchanged. +- R14. Deploy docs state plainly that dev mode grants any reachable client the ability to act as any user, and that exposing a dev-mode instance is the single most damaging misconfiguration available. + +**Release artifacts** + +- R15. The devcontainer prebuild cache key incorporates the inputs to Deuce's own baked layer, not only DevPod's hash of the devcontainer definition. Today `bakedTag()` in `server/internal/workspace/prebuild.go` reuses DevPod's definition hash and skips the bake when that tag exists, so upgrading Deuce leaves cached repos starting sessions from the previously baked agent tooling. +- R16. Upgrading Deuce causes affected workspaces to be rebuilt from the new baked layer on their next start, without an operator manually clearing images. +- R17. The release publishes `linux/arm64` alongside `linux/amd64`. Common low-cost self-host VMs are ARM, and the binary is statically linked and cross-compiles. + +**Documentation** + +- R18. Deploy docs cover install, upgrade, rollback, the required VM prerequisites, and what to do when a session comes back `missing`. +- R19. The README's stale claim that the devcontainer "mounts the host Docker socket" is corrected — no such mount exists in the repo; the devcontainer runs a nested daemon. + +--- + +## Acceptance Examples + +- AE1. **Covers R5.** Given a deployment on a fresh VM, when a session is created and its workspace reaches ready, then the files tab lists the repository's real contents and `git status` reflects the same working tree the agent sees. An empty or partial tree indicates the path-parity constraint is violated and the topology is wrong. +- AE2. **Covers R9, R8.** Given a running instance with two active sessions, when the operator upgrades to a new image tag and restarts, then within roughly one reconciler interval both sessions report `stopped`, and restarting either brings back the same workspace content rather than a fresh clone. +- AE3. **Covers R13.** Given `DEUCE_AUTH_MODE=dev` and a bind address that is not loopback, when the server starts, then it exits non-zero with a message naming both the auth mode and the bind address. Given the same dev mode bound to loopback, or the existing devcontainer configuration, then it starts normally. +- AE4. **Covers R15, R16.** Given a repo with a cached baked image and an unchanged `devcontainer.json`, when Deuce is upgraded to a version that bakes a different Pi version, then the next session start rebuilds the baked layer and the workspace runs the new Pi. +- AE5. **Covers R10.** Given an upgrade whose migration fails, when the new container starts, then it exits non-zero and never serves, leaving the operator with a failed start rather than a partially-migrated running instance. +- AE6. **Covers R17.** Given an ARM VM, when the operator follows the install docs unchanged, then the image pulls and runs without an architecture error or emulation. + +--- + +## Success Criteria + +- An operator who has never seen the codebase gets from a fresh VM to a reachable, authenticated Deuce by following the deploy doc alone, without reading Go source or asking a maintainer. +- The maintainers' own instance runs the artifacts in `deploy/`. If the adopter path breaks, the maintainers feel it. +- An upgrade takes a couple of minutes of operator attention: change a tag, restart, restart the sessions that matter. +- The spike produces a written decision naming the chosen topology and the evidence that settled it, so the next person to question the shape reads a page instead of re-running the experiment. + +--- + +## Scope Boundaries + +- No backups of any kind. Carried forward from the earlier dogfood decision and still accepted; revisit when the database holds something anyone would mourn. +- No zero-downtime, blue-green, or rolling deploys. A short interruption during restart is fine. +- No multi-node, Kubernetes, or managed-Postgres topology. One VM with an on-box database. +- No per-PR preview environments. +- No unattended or auto-upgrade. The operator decides when to move. +- No custom domain or TLS management beyond what the fronting proxy provides. +- No Terraform or other IaC for the VM itself. Provisioned by hand; only the app stack is described in-repo. +- No continuous deployment on merge to `main`. Deployment consumes published release tags. + +--- + +## Dependencies / Assumptions + +- The path-parity approach (topology C) is unproven here. It is a known pattern, but nothing in this repo has exercised it, and DevPod may record absolute paths in its own state that behave differently than expected. This is the spike's central risk. +- DevPod's on-disk layout under `~/.devpod/agent/contexts//workspaces/` is treated as stable enough to persist across upgrades. The existing content-directory env override is the escape hatch if a DevPod release moves it. +- Pi runs inside workspace containers, so restarting Deuce drops the JSONL channel to any in-flight agent run. Agent session continuity across server restarts is unbuilt and unchecked on the README roadmap; in-flight agent work is lost on upgrade regardless of topology. +- Adopters can join a VM to a tailnet. Reasonable for the target audience, and the alternatives are documented, but it is a real prerequisite the install flow depends on. +- The SSH proxy path requires the Deuce process to reach the Docker daemon that owns the workspace containers. This holds under all three candidate topologies but constrains any future split of Deuce from its daemon. +- Whether an operator can be expected to set the host's Docker group id in their env, or whether the deployment should discover it, is unresolved and depends on the topology chosen. + +--- + +## Outstanding Questions + +### Resolve Before Planning + +- [Affects R5, R6, R7] The topology spike. Stand up candidate C on a real VM and confirm, at minimum: the files tab shows real content (AE1), the terminal attaches, the SSH proxy's `docker exec` reaches the container as the right user, and an upgrade leaves workspaces running. Decision rule: if C passes, take it; if parity proves fragile, fall back to A rather than B, since A is the same architecture with the container removed. B remains the backstop if both fail, at the cost of privileged mode and workspaces stopping on every upgrade. +### Deferred to Planning + +- [Affects R3, R4] Whether the deployment pins a tag in the compose file, in the env file, or both, and how rollback is documented against that choice. +- [Affects R8] Exact persistence surface — which paths become volumes or bind mounts — since it follows directly from the topology the spike selects. +- [Affects R12, R13] Naming and default of the bind-address setting, and whether the dev-mode guard keys on the resolved bind address, an explicit opt-out, or both. +- [Affects R6] How `devpod` and `docker` CLI versions are pinned in the image, and how that pin is kept current. +- [Affects R17] Whether arm64 is built natively or via emulation in the release workflow, and the build-time cost of each. +- [Affects R18] Whether deploy docs live in `README.md`, a dedicated `docs/deploying.md`, or a `deploy/README.md` next to the artifacts. + +--- + +## Sources / Research + +- `docs/solutions/architecture-patterns/devpod-docker-workspace-bind-mount-2026-05-13.md` — establishes host-filesystem reads as the workspace data plane, which is what makes path parity load-bearing rather than incidental. +- `docs/brainstorms/2026-05-23-exe-dev-dogfood-deploy-requirements.md` and `docs/plans/2026-05-23-001-feat-exe-dev-dogfood-deploy-plan.md` — the earlier deploy effort, still `status: active`. Its build-and-publish slice shipped as `docs/plans/2026-05-26-001-feat-tag-triggered-release-plan.md`; its deploy-side units never landed and are superseded by this document. +- `.devcontainer/docker-compose.yml` — the only validated runtime configuration, and the source of the path-mismatch rationale. +- `server/internal/reconcile/reconciler.go` — the `stopped` versus `missing` distinction that defines what a successful upgrade preserves. +- `server/internal/workspace/prebuild.go` — `bakedTag()` and the existence check that together produce the stale-agent-tooling bug. +- `.github/workflows/release.yml` — current publish surface: amd64 only, image only, no raw binary. diff --git a/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md b/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md new file mode 100644 index 0000000..20868e4 --- /dev/null +++ b/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md @@ -0,0 +1,286 @@ +--- +title: VM deploy topology spike — validate socket-mount with path parity +type: feat +status: active +date: 2026-07-31 +origin: docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md +--- + +# VM Deploy Topology Spike — Validate Socket-Mount With Path Parity + +## Summary + +Build the socket-mount-with-path-parity deployment candidate to a genuinely testable state — a runtime image carrying the binaries the server shells out to, and a deployment wired so Deuce and the host Docker daemon resolve DevPod paths identically — then run it on a real VM against the origin's checks and record the decision. A pass leaves most of a working deployment behind; a fail stops at the recorded decision rather than pivoting to the fallback in the same pass. + +--- + +## Problem Frame + +Nothing about Deuce's deployment has been run. The published image is `gcr.io/distroless/static-debian12` holding one file, so it boots, migrates, serves the SPA, and then fails on the first session create because `devpod`, `docker`, and `git` are not present. The only validated runtime configuration is the local devcontainer, which uses a genuine nested daemon. + +The constraint that makes topology choice consequential is that Deuce reads workspace files off its own filesystem rather than through DevPod (see origin). Research confirms this reaches further than the files tab: `readWorkspaceUID` in `server/internal/workspace/manager.go` parses `$HOME/.devpod/contexts/default/workspaces//workspace.json` to get the container label the reconciler matches on, and `workspaceContentPath` in `server/internal/handler/files.go` resolves `$HOME/.devpod/agent/contexts/default/workspaces//content`. Both hang off `os.UserHomeDir()`. So the whole question — files tab, git status, reconciler truth, SSH proxy container lookup — reduces to whether Deuce's `$HOME` and the host daemon's view of that same path string are the same directory. + +That is testable, and it is worth testing before committing, because the failure mode is quiet. A misconfigured deployment shows an empty file tree and sessions that read as `missing` while the agent works normally in a directory nobody is looking at. + +--- + +## Requirements + +Origin requirements this spike advances. The origin's remaining requirements are deferred (see Scope Boundaries). + +**Validated by this spike** + +- R1. Confirm Deuce and the Docker daemon it drives resolve DevPod content paths to the same directory (origin R5). +- R2. Produce a runtime image carrying `devpod`, `docker`, and `git` on `PATH`, pinned to known versions (origin R6). +- R3. Produce a deployment definition standing up Deuce plus Postgres with database data on a named volume, naming an explicit image tag (origin R2, R4). +- R4. Confirm an upgrade leaves sessions restartable — surviving sessions report `stopped`, not `missing` (origin R9). +- R5. Confirm the persistence surface actually covers what must survive an upgrade: database data, DevPod agent state and workspace content, and the SSH host key (origin R8). + +**Produced as a decision artifact** + +- R6. A written decision naming the chosen topology and the evidence that settled it, so the shape does not get re-litigated from scratch (origin success criteria). + +--- + +## Key Technical Decisions + +- **Replace the distroless runtime stage rather than adding a second image.** There is no working use case for the current published artifact — it cannot run the product's core feature. Maintaining both variants would mean continuing to publish an image that looks deployable and isn't, and inviting someone to deploy the wrong one. The Go build stage is unchanged; only the runtime base and its contents change. + +- **Path parity is achieved through `HOME`, not through per-directory mounts.** Both DevPod state trees and the SSH host key derive from `os.UserHomeDir()`. Binding a single host directory at an identical absolute path inside the container and pointing `HOME` at it gives parity for all of them at once, and leaves one obvious thing to get right instead of four. The existing `DEVPOD_AGENT_CONTENT_DIR` override stays available as an escape hatch if a DevPod release moves its layout. + +- **Pin the CLI versions the way the devcontainer already does.** `.devcontainer/tool-versions.env` pins `DEVPOD_VERSION`; the runtime image should draw from the same pinning discipline rather than tracking latest, so a rebuild is not a silent DevPod upgrade. The devcontainer installs DevPod as a downloaded release binary, which is a pattern to mirror rather than invent around. + +- **The spike runs in dev auth mode on a private VM.** What is under test is topology, not exposure. Introducing proxy auth, a bind-address setting, and the dev-mode startup guard at the same time would mean debugging two unrelated classes of failure at once. This is a deliberate, temporary posture — the follow-up plan hardens it before anything is documented for adopters. + +- **First validation pass runs with the devcontainer prebuild cache disabled.** The cache has a known staleness defect (the bake is skipped when the definition hash is unchanged, regardless of whether Deuce's baked layer inputs moved). Leaving it on during the upgrade check would confuse a topology result with a caching result. Cache-on is a second pass once the topology reads clean. + +- **A failing spike stops at the recorded decision.** If parity proves fragile, the fallback is the host-native topology — the same architecture with the container removed — but building it belongs in its own plan with its own units. Pivoting mid-spike would produce a half-tested version of both. + +--- + +## High-Level Technical Design + +### What path parity means concretely + +The candidate topology's whole claim is that one path string resolves to one directory on both sides of the container boundary. The failing variant is identical except that the state directory is container-local. + +```mermaid +flowchart TB + subgraph pass["Candidate C — parity holds"] + P1["deuce container
HOME = /var/lib/deuce"] + P2["host path /var/lib/deuce
bound at the same string"] + P3["host dockerd
via mounted socket"] + P4["workspace container
bind-mounts /var/lib/deuce/..."] + P1 -->|"devpod clones content"| P2 + P1 -->|"asks for a container"| P3 + P3 -->|"resolves on the host"| P2 + P2 --> P4 + end + subgraph fail["Naive variant — parity broken"] + F1["deuce container
HOME = container-local"] + F2["container filesystem"] + F3["host dockerd"] + F4["host filesystem
same string, empty dir"] + F1 -->|"devpod clones content"| F2 + F1 -->|"asks for a container"| F3 + F3 -->|"resolves on the host"| F4 + end +``` + +The failing variant produces no error. DevPod succeeds, the container starts, and the bind mount silently resolves to an empty host directory — which is why this gets tested before it gets committed to. + +### Decision rule + +The spike's output is a routing decision, not a preference. + +```mermaid +flowchart TB + A["Run the checks against candidate C"] --> B{"File tree, git status,
terminal, SSH exec
all correct?"} + B -->|no| F["Record failure evidence"] + B -->|yes| C{"Upgrade leaves workspaces
running and sessions
restartable?"} + C -->|no| F + C -->|yes| D{"Second pass with
prebuild cache on
still correct?"} + D -->|no| E["Record: C viable,
cache fix is a prerequisite"] + D -->|yes| G["Adopt C — follow-up plan
hardens and documents it"] + F --> H["Fall back to host-native.
New plan, not this one."] +``` + +--- + +## Implementation Units + +### U1. Runtime image carrying the tooling the server shells out to + +**Goal:** Produce an image that can actually run a workspace operation, replacing a runtime stage that cannot. + +**Requirements:** R2 + +**Dependencies:** none + +**Files:** +- `Dockerfile` — replace the runtime stage; leave the Go build stage as-is +- `.dockerignore` — revisit only if the new stage needs context it currently excludes +- `.devcontainer/tool-versions.env` — read for the DevPod pin; extend only if the pin needs to be shared rather than duplicated + +**Approach:** Swap `gcr.io/distroless/static-debian12:nonroot` for a slim Debian base. Install `git` and CA certificates from the package manager; install the Docker CLI (client only — no daemon, no containerd) and DevPod as pinned release binaries, mirroring how `.devcontainer/post-create.sh` already provisions DevPod. Run as a fixed non-root UID whose home directory is the path parity will later bind, so the image does not assume it owns its own `$HOME` contents. Keep the `VERSION` ldflag wiring and the exposed ports unchanged. + +**Patterns to follow:** `.devcontainer/post-create.sh` for the pinned-release-binary install shape and its arch detection; `.devcontainer/Dockerfile` for the package set a Debian base actually needs. + +**Test scenarios:** +- All three binaries (`devpod`, `docker`, `git`) resolve on `PATH` inside a container started from the image. +- The container starts as the intended non-root UID, not root. +- The SPA is served at the root path and the version endpoint reports the injected build version rather than `dev`. +- A request for a missing hashed asset still returns a 404 rather than the SPA shell, confirming the embedded-frontend behavior survived the base change. + +**Verification:** A container from the image serves the SPA and reports its version, and each of the three binaries is present and executable. The image builds through the existing release path without changes to the frontend or Go build stages. + +--- + +### U2. Deployment definition wired for path parity + +**Goal:** Express the candidate topology as a runnable deployment, with parity as the property that is hard to get wrong rather than easy to get wrong. + +**Requirements:** R1, R3, R5 + +**Dependencies:** U1 + +**Files:** +- `deploy/docker-compose.yml` — new +- `deploy/.env.example` — new; distinct from the dev-oriented `.env.example` at the repo root + +**Approach:** Two services. Deuce runs from an explicitly pinned image tag, with the host Docker socket mounted, a single host state directory bound at an identical absolute path inside the container, and `HOME` pointed at that path so both DevPod trees and the SSH host key land inside it. Membership in the host's Docker group is supplied through configuration rather than assumed. HTTP and the SSH proxy port are published. Postgres runs alongside with its data on a named volume. The env template carries the values an operator must set and defaults the rest, with dev auth mode flagged in a comment as temporary spike posture rather than presented as a normal setting. + +**Patterns to follow:** `docker-compose.yml` at the repo root for the Postgres service shape and credentials; `.env.example` for variable naming and the comment style that explains why a setting exists rather than restating it. + +**Test scenarios:** +- Bringing the stack up on a clean host runs migrations before the listener binds, and a deliberately broken migration prevents serving rather than producing a partially-migrated running instance. +- After a session is created, the DevPod state directory is visible on the host at the same absolute path the container writes to. +- Restarting only the Deuce service leaves the Postgres volume and the host state directory intact. +- Bringing the stack up twice in a row is idempotent — the second run does not re-clone or re-key. + +**Verification:** The stack starts on a clean host, serves the SPA, and the host state directory is populated at the parity path. `Test expectation: none` does not apply — these are observable integration behaviors, exercised in U4 rather than by automated tests, because the property under test is a deployment topology. + +--- + +### U3. Provision the spike VM and bring the stack up + +**Goal:** Get a real VM running the candidate, and capture what it actually took, since the prerequisite list is currently guesswork. + +**Requirements:** R1 + +**Dependencies:** U1, U2 + +**Files:** none — this unit's output is recorded findings that feed U5, not committed code. + +**Approach:** Provision a VM, install Docker, place the deployment and its env file, resolve the host Docker group id, create the state directory with the ownership the container's UID expects, and bring the stack up. Record every step that was not obvious and every place the deployment needed adjusting — that record is the raw material the follow-up documentation plan consumes. + +Two prerequisites are easy to discover the hard way. The image built in U1 has to reach the VM: pushing a prerelease tag exercises the real release path and publishes to the registry without claiming the floating latest tag, which is the closer analogue to how an adopter would get it. And session creation clones a repository inside the container, so the VM needs credentials for whatever repo the checks use — see the private-repo clone-auth solutions doc for the failure shape when it doesn't. + +**Execution note:** Expect ownership and group-id friction on first run. Resolve it by adjusting the deployment rather than by hand-patching the VM, so the fix lands in the artifact instead of in a VM nobody else can see. + +**Test scenarios:** `Test expectation: none — provisioning unit with no committed behavior. Validated by U4.` + +**Verification:** The stack is reachable on the VM and a session can be created. Prerequisites and friction points are written down. + +--- + +### U4. Run the validation checks + +**Goal:** Determine whether the candidate topology actually holds, against the checks the origin named. + +**Requirements:** R1, R4, R5 + +**Dependencies:** U3 + +**Files:** none — this unit's output is recorded evidence. + +**Approach:** Create a session against a real repository and work through each check in turn, recording the observed result rather than a pass/fail impression. Then perform an upgrade and re-observe. Then repeat the session-start and upgrade checks with the prebuild cache enabled, treating a divergence there as a caching result rather than a topology result. + +**Test scenarios:** +- Covers the origin's files-tab acceptance case. The file tree lists the repository's real contents and git status reflects the same working tree the agent sees. An empty or partial tree means parity is broken and the topology is wrong. +- The terminal attaches to the workspace container and runs an interactive shell. +- Opening in VS Code over the SSH proxy lands in the workspace as the devcontainer's `remoteUser`, not as root — the ownership symptom to watch for is git refusing the workspace as dubious. +- Covers the origin's upgrade acceptance case. After changing the image tag and restarting, workspace containers are still running, sessions report `stopped` where on-disk state survived rather than `missing`, and restarting a session returns the same workspace content instead of a fresh clone. +- With the prebuild cache enabled, a session starts from the baked image; after upgrading Deuce, whether the baked layer is rebuilt is recorded as evidence for the separately-planned cache-key fix rather than treated as a spike failure. + +**Verification:** Every check has a recorded observed result. Any failure carries enough detail — what was expected, what appeared, where the paths diverged — to route the decision in U5 without re-running the spike. + +--- + +### U5. Record the decision and route the follow-up + +**Goal:** Turn the spike's evidence into a durable decision, so the next person to question the topology reads a page instead of re-running the experiment. + +**Requirements:** R6 + +**Dependencies:** U4 + +**Files:** +- `docs/solutions/architecture-patterns/.md` — new, following the existing frontmatter and section conventions in that directory +- `docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md` — resolve the topology item under Resolve Before Planning +- `docs/plans/2026-05-23-001-feat-exe-dev-dogfood-deploy-plan.md` — mark superseded; it is still `status: active` and its deploy-side units are now obsolete + +**Approach:** Write the decision as a solutions doc rather than plan prose, because the audience is a future reader hitting the same question, not an implementer executing this plan. Name the topology chosen, the evidence, and — importantly — the symptom that would indicate parity has been broken later, since that is the knowledge most likely to be needed and least likely to be re-derived. If the candidate failed, the doc records why and the follow-up is a host-native plan rather than a documentation plan. + +**Patterns to follow:** `docs/solutions/architecture-patterns/devpod-docker-workspace-bind-mount-2026-05-13.md` — the closest analogue in subject and shape, and the doc this one extends. + +**Test scenarios:** `Test expectation: none — documentation unit.` + +**Verification:** The decision doc states the outcome and its evidence. The origin's blocking item is resolved. The superseded plan no longer reads as active work. + +--- + +## Scope Boundaries + +- No auth hardening. Dev mode on a private VM is the spike posture; proxy mode, the Tailscale default, the bind-address setting, and the dev-mode startup guard are all follow-up work. +- No deploy documentation for adopters. U3 records prerequisites as raw material; turning that into a deploy guide is the follow-up plan's job. +- No arm64 image. The spike runs on whatever the spike VM is. +- No prebuild cache-key fix. Its behavior is observed and recorded here; fixing it is separate work the origin already scopes. +- No backups, zero-downtime deploys, multi-node topology, or continuous deployment — carried forward from the origin. +- No building the host-native or nested-daemon topologies. They are the documented fallbacks; constructing one is a new plan. + +### Deferred to Follow-Up Work + +- Deploy artifacts hardening and documentation, gated on this spike passing: proxy auth defaults, the bind-address setting and dev-mode guard, the adopter-facing deploy guide, and rollback instructions. +- The prebuild cache-key fix, so that upgrading Deuce rebuilds the baked agent-tooling layer. +- arm64 in the release workflow. +- Correcting the README's claim that the devcontainer mounts the host Docker socket — it runs a nested daemon. + +--- + +## Risks & Dependencies + +- **Parity may hold for content but break for something subtler.** DevPod may record absolute paths in its own state that behave differently than the clone path. The checks in U4 are chosen to surface this — the reconciler's `stopped`-versus-`missing` distinction is a second, independent probe of the same property, since it reads a different tree under the same `$HOME`. +- **Docker group id is host-specific.** The container's access to the mounted socket depends on a gid that varies between hosts. This is the most likely first-run failure and the most likely thing to get hand-patched on the VM instead of fixed in the artifact. +- **File ownership across the boundary.** DevPod writes into the shared directory as the container's UID; anything on the host touching those files sees that UID. Ownership mismatches here can look like parity failures without being one. +- **In-flight agent runs do not survive the upgrade.** Pi runs inside workspace containers driven over a per-session channel, and agent session continuity across server restarts is unbuilt. This is expected, not a spike failure, and should not be mistaken for one during U4. +- **A spike VM is a real cost surface.** It needs to exist for the duration and be reachable from wherever the VS Code check runs. + +--- + +## Open Questions + +### Deferred to Implementation + +- Which slim Debian base and whether the Docker CLI comes from Docker's apt repository or a pinned static binary. Both work; the choice follows from what keeps the image small and the pin honest. +- Whether the DevPod version pin is shared with `.devcontainer/tool-versions.env` or duplicated in the image. Sharing is cleaner but couples the release image to a devcontainer file; decide when the Dockerfile is in front of you. +- The exact parity path. It needs to be absolute, stable, and unlikely to collide with anything else on a host, but nothing in the design depends on the specific string. +- Whether the deployment resolves the host Docker group id automatically or requires the operator to supply it. Automatic is friendlier; explicit is more predictable. Decide after seeing the first-run friction in U3. +- Whether Postgres readiness needs explicit ordering in the deployment, or whether the existing database-wait in the server's startup path already covers it. +- Whether the spike image reaches the VM via a published prerelease tag or a local build on the VM. Prerelease is recommended in U3 because it exercises the real path, but a local build is faster to iterate on if the first passes need several image rebuilds. + +--- + +## Sources & Research + +- `docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md` — origin. Carries the constraint, the three candidate topologies, the decision rule, and the requirements this spike advances. +- `server/internal/handler/files.go` — `workspaceContentPath` resolves content under `os.UserHomeDir()` with a `DEVPOD_AGENT_CONTENT_DIR` override. +- `server/internal/workspace/manager.go` — `readWorkspaceUID` parses DevPod's CLI-side workspace record under the same home; `ContainerName` and `BulkContainerStatus` locate containers by the `dev.containers.id` label through the Docker CLI. +- `server/internal/reconcile/reconciler.go` — the `stopped`-versus-`missing` derivation that makes the upgrade check meaningful. +- `server/internal/sshproxy/docker.go` — the exec shapes the VS Code check exercises, including the `--user` flag that makes the `remoteUser` symptom visible. +- `server/internal/web/web.go` — embedded SPA serving with the assets-404 behavior that U1 must not regress. +- `.devcontainer/post-create.sh` and `.devcontainer/tool-versions.env` — the existing pinned-binary install pattern for DevPod. +- `.devcontainer/docker-compose.yml` — the nested-daemon configuration that currently works, and the source of the path-mismatch rationale. +- `docs/solutions/architecture-patterns/devpod-docker-workspace-bind-mount-2026-05-13.md` — establishes host-filesystem reads as the workspace data plane. +- `docs/solutions/integration-issues/devpod-private-repo-clone-auth.md` — the clone-auth failure shape the spike VM will hit if repository credentials aren't in place before the checks. +- `.github/workflows/release.yml` — semver-tag publish path, including prerelease handling, which is how the spike image reaches the VM. From 8b052428b067886ae75e0d4c93c8c03be2b0c57e Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 03:51:14 +0000 Subject: [PATCH 2/7] feat(deploy): add the spike deployment wired for path parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deuce and the Docker daemon it drives must agree on what a path string means. Deuce reads workspace files off its own filesystem and runs git against them, and it locates workspace containers by a label it reads from DevPod's on-disk records — both resolve through $HOME. So the state directory is mounted at the same absolute path on both sides and HOME points at it. Breaking that parity produces no error: the bind mount silently resolves to an empty host directory while the agent works normally somewhere nobody is looking. Required variables use compose's `:?` form rather than defaults. Every one of them (pinned image tag, state dir, docker gid, db password) is something that misbehaves quietly when guessed, so the stack refuses to start instead of starting wrong. Ports bind loopback by default. Dev auth mode admits any reachable client as any user, and that mode is deliberately set here for the spike, so the default must not be broadly published. The env template documents a verified loader behavior worth knowing: a set-but-empty variable falls back to its built-in default rather than meaning "empty". `DEUCE_WS_ALLOWED_ORIGINS=` silently becomes the localhost dev default. There is no way to express "empty" through the env file at all. Verified locally against the devcontainer's nested daemon: required-var enforcement fails loudly, the stack serves, the container's $HOME writes appear on the host at the same path with matching ownership, and a down/up cycle preserves both the SSH host key and the database. Co-Authored-By: Claude Opus 5 (1M context) --- deploy/.env.example | 90 +++++++++++++++++++++++++++++++++++++++ deploy/docker-compose.yml | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 deploy/.env.example create mode 100644 deploy/docker-compose.yml diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..8781b65 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,90 @@ +# Deuce deployment configuration — SPIKE CANDIDATE. +# +# Copy to .env in this directory: cp .env.example .env +# This is the deployment env file. It is NOT the same as .env.example at the +# repo root, which configures a developer laptop. +# +# ── A GOTCHA THAT WILL COST YOU AN HOUR ──────────────────────────────────── +# Leaving a variable empty here does NOT mean "unset" or "off". The config +# loader falls back to the built-in default when a variable is set-but-empty, +# exactly as if you had never written the line. Verified against a running +# container, not inferred. +# +# So `DEUCE_WS_ALLOWED_ORIGINS=` does not mean "no origins allowed" — it means +# "localhost:4000,localhost:8080", which is a dev default you do not want on a +# server. To change a setting, give it a real value. To get default behavior, +# delete the line. There is no way to express "empty" through this file. +# ─────────────────────────────────────────────────────────────────────────── + +# ── Required. The compose file refuses to start without these. ───────────── + +# Released image tag to run. Never a floating tag — an upgrade should be a +# deliberate edit here and a rollback the reverse edit. Note the tag has no +# leading "v": the git tag v0.2.0 publishes the image tag 0.2.0. +DEUCE_IMAGE_TAG= + +# Absolute host path holding everything that must survive a container +# replacement: DevPod's workspace records and cloned workspace content, and the +# SSH host key. It is mounted at this same absolute path INSIDE the container, +# and HOME is set to it. Both sides must match — see the compose file header +# for what breaks when they don't, and how quietly it breaks. +# +# Create it owned by the container's UID before first start: +# sudo mkdir -p /var/lib/deuce && sudo chown 65532:65532 /var/lib/deuce +DEUCE_STATE_DIR=/var/lib/deuce + +# Numeric group id of the host's docker group, so the container can reach the +# mounted socket. Host-specific: +# getent group docker | cut -d: -f3 +DOCKER_GID= + +# Database password. Generate one: openssl rand -hex 32 +POSTGRES_PASSWORD= + +# ── Exposure ─────────────────────────────────────────────────────────────── +# Defaults bind loopback only. Read the auth warning below before widening. + +DEUCE_HTTP_BIND=127.0.0.1 +DEUCE_HTTP_PORT=8080 + +# The SSH proxy backing "Open in VS Code". Reaching it from a VS Code client +# means widening this bind and letting the tailnet be the boundary. +DEUCE_SSH_BIND=127.0.0.1 +DEUCE_SSH_PORT=2222 + +# ── Auth ─────────────────────────────────────────────────────────────────── +# +# TEMPORARY SPIKE POSTURE. `dev` mode admits ANY request as a single fixed +# user — anyone who can reach the port is fully authenticated as that user. +# It is set here only because the spike is testing deployment topology, not +# exposure, on a private VM. It must not survive into a real deployment. +# +# The intended default is proxy mode behind Tailscale Serve, which is planned +# work and not yet wired up here. Until it is, keep the binds on loopback. +DEUCE_AUTH_MODE=dev +DEUCE_USER_ID=10000000-0000-0000-0000-000000000001 + +# Hostname the deployment is reached at. Used for WebSocket origin checks and +# baked into the vscode:// URIs the UI hands out, so a wrong value produces +# "Open in VS Code" links that point somewhere unreachable. +DEUCE_WS_ALLOWED_ORIGINS=localhost:8080 +DEUCE_PUBLIC_HOSTNAME=localhost + +# ── Workspaces ───────────────────────────────────────────────────────────── + +DEVPOD_PROVIDER=docker + +# GitHub PAT for repo discovery in the session-create flow. Sessions clone +# their repository inside the container, so this also has to carry enough +# access for whatever repos you point the deployment at. +GITHUB_TOKEN= + +# Required for the agent to actually run inside a workspace. +ANTHROPIC_API_KEY= + +# Devcontainer prebuild cache. Deliberately left off for the first spike pass: +# the cache key is derived only from the devcontainer definition hash, so a +# Deuce upgrade that bakes newer agent tooling reuses the stale image and the +# upgrade check would report a caching result as if it were a topology result. +# Enable it only for the second pass, with that behavior in mind. +# DEUCE_PREBUILD_REPOSITORY=deuce-prebuild diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..c9a08d7 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,82 @@ +# Deuce single-VM deployment — SPIKE CANDIDATE, not yet validated. +# +# This is the "socket mount with path parity" topology under test in +# docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md. Do not treat +# it as a supported deployment until that spike records a decision. +# +# THE ONE THING THAT MATTERS HERE +# +# Deuce does not read workspace files through DevPod. It reads them off its own +# filesystem and runs git against them, and it locates workspace containers by a +# label it reads from DevPod's own on-disk records. Both of those trees resolve +# through $HOME. So Deuce and the host Docker daemon must agree on what a path +# string means: DEUCE_STATE_DIR is mounted at the SAME ABSOLUTE PATH on both +# sides, and HOME points at it. +# +# Break that parity and nothing errors. DevPod succeeds, the container starts, +# and the bind mount silently resolves to an empty host directory — the files +# tab shows nothing while the agent works normally somewhere nobody is looking. +# If you change the mount, change both sides together. +# +# Copy .env.example to .env in this directory before bringing the stack up. +# The required-variable checks below fail loudly rather than defaulting, because +# every one of them is something that silently misbehaves when guessed. + +services: + deuce: + # Pinned tag, never a floating one: an upgrade should be a deliberate edit + # and a rollback the reverse edit. + image: ghcr.io/forgeutah/deuce:${DEUCE_IMAGE_TAG:?set DEUCE_IMAGE_TAG to a released tag, e.g. 0.2.0 — floating tags make rollback ambiguous} + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + environment: + # Overrides anything in .env. HOME is what makes the mount below + # meaningful — see the header. + HOME: ${DEUCE_STATE_DIR:?set DEUCE_STATE_DIR to an absolute host path, e.g. /var/lib/deuce} + DATABASE_URL: postgres://deuce:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@postgres:5432/deuce?sslmode=disable + volumes: + # The host daemon Deuce drives. Workspace containers are created as + # siblings of this container, not as children. + - /var/run/docker.sock:/var/run/docker.sock + # Path parity. Both sides are the same string on purpose. + - ${DEUCE_STATE_DIR:?set DEUCE_STATE_DIR}:${DEUCE_STATE_DIR:?set DEUCE_STATE_DIR} + # The container runs as UID 65532 and needs the host's docker group to reach + # the socket. This gid is host-specific — find it with: + # getent group docker | cut -d: -f3 + group_add: + - "${DOCKER_GID:?set DOCKER_GID to the host docker group id — see the comment above}" + ports: + # Loopback by default. In dev auth mode ANY reachable client is admitted + # as any user, so this must not be published broadly until proxy auth is + # configured. Front it with `tailscale serve`, which proxies from + # localhost, or tunnel to it. + - "${DEUCE_HTTP_BIND:-127.0.0.1}:${DEUCE_HTTP_PORT:-8080}:8080" + # The SSH proxy for "Open in VS Code". Must be reachable by the VS Code + # client to exercise that path, which means widening this bind and + # relying on the tailnet as the boundary. + - "${DEUCE_SSH_BIND:-127.0.0.1}:${DEUCE_SSH_PORT:-2222}:2222" + + postgres: + image: postgres:17 + restart: unless-stopped + environment: + POSTGRES_DB: deuce + POSTGRES_USER: deuce + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + # Gates deuce's start. Migrations run in-process before the HTTP listener + # binds and exit non-zero on failure, so a half-migrated schema never + # serves — but the app still has to find a live database first. + test: ["CMD-SHELL", "pg_isready -U deuce -d deuce"] + interval: 5s + timeout: 5s + retries: 10 + +volumes: + pgdata: From d34d8ad127d6cee419325789784d9dfc652d63df Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 04:21:30 +0000 Subject: [PATCH 3/7] fix(docker): let Deuce run git against workspaces it doesn't own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found on the spike VM. DevPod clones workspace content as the devcontainer's remoteUser — uid 1000 on most images — while the server process runs as 65532. Git refuses to operate across that mismatch and exits with "detected dubious ownership". The failure is silent in the worst way. Listing a directory needs no ownership match, so the files tab still renders the full tree; only the per-file git status disappears. Nothing errors and nothing logs, so the tab looks fine while quietly showing no modification state at all. This never surfaced in the devcontainer because Deuce runs there as vscode (uid 1000), which happens to match what DevPod writes as. It appears the moment the two uids differ, which is every containerized deployment. Matching the uids is not available as a fix: remoteUser varies per devcontainer image, one deployment serves many repos at once, and the value isn't known until the workspace is built. Declaring the trees safe describes the actual situation — this process's job is reading repositories owned by other uids, which is not what git's ownership check was built to guard. Verified on the VM: before the change the files API returned the tree with no gitStatus on any entry; after it, a modified file reports M and untracked files report U. Co-Authored-By: Claude Opus 5 (1M context) --- Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3721440..03f10d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,6 +93,20 @@ RUN apt-get update \ # host directory already owned for the old image keeps working. RUN groupadd --gid 65532 deuce \ && useradd --uid 65532 --gid 65532 --home-dir /var/lib/deuce --create-home deuce +# Deuce runs git against workspace trees it does not own. DevPod clones content +# as the devcontainer's remoteUser (uid 1000 on most images), while this process +# runs as 65532, and git refuses to operate across that mismatch — it reports +# "detected dubious ownership" and exits non-zero. The files tab degrades +# quietly when that happens: the tree still lists, because walking a directory +# needs no ownership match, but every file loses its git status. +# +# Matching the uids is not available as a fix. remoteUser varies per +# devcontainer image, one deployment serves many repos at once, and the value +# is not known until after the workspace is built. Declaring the trees safe is +# the accurate description of the situation: this container's entire job is +# reading repositories owned by other uids, which is the case git's ownership +# check was never meant to cover. +RUN git config --system --add safe.directory '*' COPY --from=tools /usr/local/bin/docker /usr/local/bin/docker COPY --from=tools /usr/local/bin/devpod /usr/local/bin/devpod COPY --from=backend /out/deuce /usr/local/bin/deuce From 360406d49f6bb3b7e5a2ed103d78d43b106ef0b6 Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 04:39:50 +0000 Subject: [PATCH 4/7] docs(deploy): record the topology decision and what the spike found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts the socket-mounted container with path parity. It was the shape with the quiet failure mode, which is why it was validated before being committed to: parity held, writes crossed the boundary both ways, and rolling the image left workspace containers running untouched across three restarts while sessions reported stopped rather than missing. The solutions doc leads with the two degradation symptoms rather than the architecture, because that is the knowledge most likely to be needed later and least likely to be re-derived. An empty file tree means path parity broke; a full tree with no git status means the file-ownership declaration is missing. Neither logs an error. Also records three defects the spike surfaced that the origin did not anticipate: The prebuild cache is non-functional against DevPod v0.6.15. `devpod build --repository R --skip-push` reports success and exits zero, but tags the image :latest with no devpod- prefix to parse AND leaves no image on the daemon. Every session silently falls back to a from-scratch build with an over-SSH tooling install. This inverts the origin's requirement: the cache-key staleness bug it identified is unreachable, because a cache that never populates cannot go stale. A fresh deployment has no user row and no team membership, and the consequences cascade into unreadable sessions, empty member lists, and an SSH proxy that rejects every key. The config loader cannot express "off". A set-but-empty variable falls back to its built-in default, so DEUCE_SSH_LISTEN_ADDR= does not disable the SSH proxy the way the deployment checklist instructs. The superseded exe.dev plan is marked as such — its publish half shipped separately and its deploy half is replaced. Co-Authored-By: Claude Opus 5 (1M context) --- ...7-30-vm-deploy-and-upgrade-requirements.md | 16 +++- ...23-001-feat-exe-dev-dogfood-deploy-plan.md | 11 ++- ...-as-a-container-sharing-the-host-daemon.md | 94 +++++++++++++++++++ 3 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md diff --git a/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md b/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md index c521746..fef9160 100644 --- a/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md +++ b/docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md @@ -126,8 +126,8 @@ The naive variant of C — socket mounted without path parity — is the shape m **Release artifacts** -- R15. The devcontainer prebuild cache key incorporates the inputs to Deuce's own baked layer, not only DevPod's hash of the devcontainer definition. Today `bakedTag()` in `server/internal/workspace/prebuild.go` reuses DevPod's definition hash and skips the bake when that tag exists, so upgrading Deuce leaves cached repos starting sessions from the previously baked agent tooling. -- R16. Upgrading Deuce causes affected workspaces to be rebuilt from the new baked layer on their next start, without an operator manually clearing images. +- R15. The devcontainer prebuild cache produces a usable image at all. The spike found it non-functional against DevPod v0.6.15: `devpod build --repository R --skip-push` prints `Successfully build image R:latest` and exits zero, but the tag carries no `devpod-` prefix for `bakedTag()` to parse **and no image is left on the daemon**. Every session silently falls back to a from-scratch build plus an over-SSH tooling install, logging one WARN. Verified with two repositories, one with a `devcontainer.json` and one without. +- R16. Once the cache populates, its key incorporates the inputs to Deuce's own baked layer rather than only DevPod's definition hash, so upgrading Deuce rebuilds the baked tooling instead of reusing a stale image. This was the originally-identified defect; the spike showed it is currently unreachable, since a cache that never populates cannot go stale. It remains real and becomes live the moment R15 is fixed. - R17. The release publishes `linux/arm64` alongside `linux/amd64`. Common low-cost self-host VMs are ARM, and the binary is statically linked and cross-compiles. **Documentation** @@ -135,6 +135,13 @@ The naive variant of C — socket mounted without path parity — is the shape m - R18. Deploy docs cover install, upgrade, rollback, the required VM prerequisites, and what to do when a session comes back `missing`. - R19. The README's stale claim that the devcontainer "mounts the host Docker socket" is corrected — no such mount exists in the repo; the devcontainer runs a nested daemon. +**First boot** + +Both of these were found by standing the deployment up on a real VM. Neither appears on a developer laptop, where the database has accumulated state over time. + +- R20. A fresh deployment is usable by its first user without hand-editing the database. On first boot the `users` table is empty and no `team_members` rows exist, and the consequences cascade: every read returns `FORBIDDEN — not a team member`, session listing returns empty despite seeded sessions, newly created sessions get `members: []` because the creator row doesn't exist to be added, and the SSH proxy then rejects every key because it authorizes on session membership. The spike got past this by inserting a user and a membership by hand. +- R21. Configuration expresses "off" in a way that works. The config loader falls back to a field's built-in default when a variable is set-but-empty, so an env file cannot express an empty value at all. `DEUCE_SSH_LISTEN_ADDR=` does not disable the SSH proxy — verified across unset, empty, and explicit values, the listener came up on `:2222` in the first two cases — yet the SSH deployment checklist instructs operators to disable it exactly that way. `DEUCE_WS_ALLOWED_ORIGINS=` silently becomes the localhost dev default, which is the same trap on a security-relevant setting. + --- ## Acceptance Examples @@ -183,9 +190,10 @@ The naive variant of C — socket mounted without path parity — is the shape m ## Outstanding Questions -### Resolve Before Planning +### Resolved + +- The topology spike is complete and candidate C — the socket-mounted container with path parity — is adopted. Every check passed on an Ubuntu 24.04 VM: parity held, writes crossed the boundary in both directions, upgrades left workspace containers running untouched, the reconciler reported `stopped` rather than `missing`, and the SSH proxy landed in the container as the devcontainer's `remoteUser`. See `docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md` for the evidence and for the two degradation symptoms that indicate parity or file ownership has been broken later. -- [Affects R5, R6, R7] The topology spike. Stand up candidate C on a real VM and confirm, at minimum: the files tab shows real content (AE1), the terminal attaches, the SSH proxy's `docker exec` reaches the container as the right user, and an upgrade leaves workspaces running. Decision rule: if C passes, take it; if parity proves fragile, fall back to A rather than B, since A is the same architecture with the container removed. B remains the backstop if both fail, at the cost of privileged mode and workspaces stopping on every upgrade. ### Deferred to Planning - [Affects R3, R4] Whether the deployment pins a tag in the compose file, in the env file, or both, and how rollback is documented against that choice. diff --git a/docs/plans/2026-05-23-001-feat-exe-dev-dogfood-deploy-plan.md b/docs/plans/2026-05-23-001-feat-exe-dev-dogfood-deploy-plan.md index e9ec42d..fa5e0d1 100644 --- a/docs/plans/2026-05-23-001-feat-exe-dev-dogfood-deploy-plan.md +++ b/docs/plans/2026-05-23-001-feat-exe-dev-dogfood-deploy-plan.md @@ -1,12 +1,21 @@ --- title: Deploy Deuce to exe.dev VM via GitHub Actions type: feat -status: active +status: superseded date: 2026-05-23 deepened: 2026-05-23 origin: docs/brainstorms/2026-05-23-exe-dev-dogfood-deploy-requirements.md +superseded_by: docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md --- +> **Superseded.** The build-and-publish half of this plan shipped as +> `docs/plans/2026-05-26-001-feat-tag-triggered-release-plan.md`. The deploy half +> never landed and is replaced by +> `docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md`, which +> targets a self-hostable deployment rather than a single dogfood VM. The +> container topology that plan left open was settled by +> `docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md`. + # Deploy Deuce to exe.dev VM via GitHub Actions ## Summary diff --git a/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md b/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md new file mode 100644 index 0000000..4fef10a --- /dev/null +++ b/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md @@ -0,0 +1,94 @@ +--- +title: Deploy Deuce as a container sharing the host Docker daemon, with path parity +date: 2026-07-31 +category: architecture-patterns +module: server +problem_type: architecture_pattern +component: deployment +applies_when: + - "Deploying Deuce to a VM or any host outside a developer laptop" + - "Changing how the runtime image is built or which user the server process runs as" + - "The files tab renders a tree but every file is missing its git status" + - "Sessions report `missing` after a restart when their workspaces should have survived" +related_components: + - workspace + - sshproxy + - development_workflow +tags: + - deployment + - devpod + - docker-provider + - path-parity + - upgrade + - bind-mount +--- + +# Deploy Deuce as a container sharing the host Docker daemon, with path parity + +## Context + +Deuce orchestrates containers, which makes "where does Deuce itself run" a real architectural question rather than a packaging detail. Three shapes were considered: run it directly on the host under systemd; run it in a privileged container with its own nested Docker daemon (what the devcontainer does); or run it in an ordinary container that drives the *host's* daemon through a mounted socket. + +The socket-mounted shape looks obviously best on paper — no privileged container, no nested storage, and workspace containers are siblings of Deuce rather than children, so replacing Deuce doesn't disturb them. The reason it needed validating first is that it has a failure mode which produces no error at all. + +Deuce does not read workspace files through DevPod. It reads them off its own filesystem and runs `git` against them (`server/internal/handler/files.go`), and it finds workspace containers via a label it reads from DevPod's own on-disk records (`server/internal/workspace/manager.go`). Both resolve through `os.UserHomeDir()`. So Deuce and the daemon it drives must agree on what a path *string* means. If they disagree, DevPod still succeeds, the container still starts, and the bind mount silently resolves to an empty host directory — the team sees an empty file tree while the agent works normally somewhere nobody is looking. + +## Guidance + +Run Deuce in an unprivileged container with the host Docker socket mounted, and bind the state directory **at the same absolute path inside and outside**, with `HOME` pointing at it: + +```yaml +volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${DEUCE_STATE_DIR}:${DEUCE_STATE_DIR} # same string on purpose +environment: + HOME: ${DEUCE_STATE_DIR} +group_add: + - "${DOCKER_GID}" # host-specific; getent group docker +``` + +One mount covers everything that must survive a container replacement, because all of it hangs off `HOME`: DevPod's CLI-side workspace records, DevPod's agent-side cloned content, and the SSH host key. Do not mount these individually — the single-`HOME` mount is what makes parity checkable by inspection instead of by audit. + +Three non-negotiables: + +1. **The runtime image must carry `devpod`, the Docker CLI, and `git`.** Deuce is not self-contained; it shells out to all three. A distroless image boots, migrates, serves the SPA, and then fails on the first session create. + +2. **Declare workspace trees safe for git.** DevPod clones as the devcontainer's `remoteUser` (uid 1000 on most images) while the server runs as its own uid. Git refuses across that mismatch, and it fails *quietly* — the tree still lists, because walking a directory needs no ownership match, so only the per-file status vanishes. Matching uids is not available as a fix: `remoteUser` varies per devcontainer image, one deployment serves many repos, and the value isn't known until the workspace is built. `git config --system --add safe.directory '*'` in the image is the accurate description of the situation. + +3. **Pin an explicit image tag, never a floating one.** Upgrade becomes a deliberate edit and rollback is the reverse edit. + +### The symptom to recognize later + +This is the knowledge most likely to be needed and least likely to be re-derived. If someone changes the mount, the state directory, or the user the process runs as, the deployment does not break loudly — it degrades in one of two specific ways: + +- **Empty or partial file tree, sessions reporting `missing` after a restart** → path parity is broken. Deuce and the daemon are resolving the same string to different directories. Check that the bind mount's source and destination are identical and that `HOME` matches. +- **Full file tree, but no file has a git status** → the ownership declaration is missing. Run `git status` inside the container against a workspace content path; "detected dubious ownership" confirms it. + +Neither logs an error. Both look like "the files tab is a bit broken." + +## Why This Matters + +The alternative shapes each cost something concrete. + +A nested-daemon container works — it's what the devcontainer does, and it sidesteps parity entirely because there is only one filesystem. But restarting it kills the nested daemon, so **every upgrade stops every workspace**, and it needs `privileged: true` plus overlayfs-on-overlayfs storage. + +Host-native under systemd also works and has no parity risk, but it gives up containerized packaging and needs a raw binary artifact the release pipeline doesn't publish. + +The socket-mounted shape keeps compose-simple packaging *and* leaves workspaces running across upgrades, which is the property that matters most in practice: upgrading Deuce should not disturb work in progress. + +## Verification + +Confirmed on an Ubuntu 24.04 VM (Docker 29.1.3) against a live deployment: + +- The workspace container's bind mount source was the exact host path Deuce reads from. +- Files written *inside* the workspace container appeared in Deuce's file listing immediately, and modifications reported the correct git status. +- Rolling the Deuce image to a new tag left the workspace container running untouched — `Up 13 minutes` before and after — across three separate restarts. +- Stopping a workspace container produced `stopped`, not `missing`; restarting returned `ready` with content preserved and no re-clone. +- The SSH proxy authenticated a registered key and landed in the container as `vscode` (uid 1000), not root, with git working. + +## Related + +- [devpod-docker-workspace-bind-mount-2026-05-13.md](devpod-docker-workspace-bind-mount-2026-05-13.md) — establishes host-filesystem reads as the workspace data plane, which is what makes path parity load-bearing rather than incidental. +- [embedded-ssh-proxy-for-vscode-remote.md](embedded-ssh-proxy-for-vscode-remote.md) — the `--user` handling this topology depends on. +- [docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md](../../brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md) — the deploy requirements this decision unblocks. +- [docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md](../../plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md) — the spike that produced it. From 9794980f4220994000e257d2444fa39ceded8375 Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 04:44:36 +0000 Subject: [PATCH 5/7] fix(files): stop untrusted repo config executing in the server process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git executes core.fsmonitor as a command and honours it from the repository's own .git/config. Workspace repositories are not trusted — anyone with terminal access to a session, or the agent itself, can write into .git/config. Git's defence against exactly this is the dubious-ownership check, which fires here because DevPod clones as the devcontainer's remoteUser while the server runs as its own uid. The previous commit suppressed that check so the files tab could report git status at all, which also removed the protection it was providing. On a deployment that mounts the Docker socket, the result is a path from workspace container to server process to host root. Pin core.fsmonitor off on the command line, where it takes precedence over repository config. Verified by hand first: a planted value runs under a plain `git status` and does not run with the override, and status output is unchanged either way. The regression test plants the config the way an attacker would and asserts the command never ran. Confirmed it fails when the override is removed rather than passing regardless. Co-Authored-By: Claude Opus 5 (1M context) --- ...-as-a-container-sharing-the-host-daemon.md | 6 +- server/internal/handler/files.go | 26 +++- server/internal/handler/files_test.go | 114 ++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 server/internal/handler/files_test.go diff --git a/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md b/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md index 4fef10a..f20273d 100644 --- a/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md +++ b/docs/solutions/architecture-patterns/deploy-deuce-as-a-container-sharing-the-host-daemon.md @@ -53,7 +53,11 @@ Three non-negotiables: 1. **The runtime image must carry `devpod`, the Docker CLI, and `git`.** Deuce is not self-contained; it shells out to all three. A distroless image boots, migrates, serves the SPA, and then fails on the first session create. -2. **Declare workspace trees safe for git.** DevPod clones as the devcontainer's `remoteUser` (uid 1000 on most images) while the server runs as its own uid. Git refuses across that mismatch, and it fails *quietly* — the tree still lists, because walking a directory needs no ownership match, so only the per-file status vanishes. Matching uids is not available as a fix: `remoteUser` varies per devcontainer image, one deployment serves many repos, and the value isn't known until the workspace is built. `git config --system --add safe.directory '*'` in the image is the accurate description of the situation. +2. **Declare workspace trees safe for git — and compensate for what that turns off.** DevPod clones as the devcontainer's `remoteUser` (uid 1000 on most images) while the server runs as its own uid. Git refuses across that mismatch, and it fails *quietly* — the tree still lists, because walking a directory needs no ownership match, so only the per-file status vanishes. Matching uids is not available as a fix: `remoteUser` varies per devcontainer image, one deployment serves many repos, and the value isn't known until the workspace is built. So the image sets `git config --system --add safe.directory '*'`. + + That check was not merely pedantic, and turning it off has a cost that must be paid back. Git executes `core.fsmonitor` as a command and honours it from a repository's *own* `.git/config` — and workspace repositories are not trusted, since anyone with terminal access to a session, or the agent itself, can write there. With the ownership check suppressed, a planted value would run in the server process, which on this topology can reach the host Docker daemon: a path from workspace container to host root. Every git invocation against workspace content therefore pins the setting off on the command line, where it takes precedence over repository config (`server/internal/handler/files.go`), covered by a regression test that plants the config and asserts it never runs. + + The general rule when adding git invocations here: the command line is the only barrier between untrusted repository config and the server process. Anything git will execute from config must be pinned there. 3. **Pin an explicit image tag, never a floating one.** Upgrade becomes a deliberate edit and rollback is the reverse edit. diff --git a/server/internal/handler/files.go b/server/internal/handler/files.go index 236d1dd..5cb2691 100644 --- a/server/internal/handler/files.go +++ b/server/internal/handler/files.go @@ -387,12 +387,36 @@ func discoverRepoRoots(ctx context.Context, rootPath string) ([]string, error) { return roots, nil } +// gitStatusArgs is the argv for the workspace git-status probe. +// +// core.fsmonitor is pinned off because the repositories this runs against are +// not trusted. Git treats fsmonitor as a command to execute, and it honours it +// from the repository's own .git/config — so a value planted inside a workspace +// would run here, in the server process, rather than in the workspace container +// where whoever planted it already had execution. On a deployment that mounts +// the Docker socket that is a path from workspace to host root. +// +// Git's own defence against this is the dubious-ownership check, which fires +// because DevPod clones as the devcontainer's remoteUser while this process +// runs as its own uid. Deuce has to suppress that check to read workspaces at +// all (see the safe.directory line in the Dockerfile), which means suppressing +// the protection it was providing. Pinning the setting on the command line +// takes precedence over repository config and restores it. +// +// Confine additions here to flags that are inert or safe by construction; a +// command-line -c is the only layer standing between untrusted repository +// config and this process. +var gitStatusArgs = []string{ + "-c", "core.fsmonitor=false", + "status", "--porcelain=v1", "--untracked-files=normal", +} + // loadGitStatus runs `git status --porcelain=v1` in the given repo root and // records workspace-relative paths in statusByPath. Errors from one repo do // not abort the whole walk. func loadGitStatus(ctx context.Context, rootPath, repoRoot string, statusByPath map[string]string) error { absRepoPath := filepath.Join(rootPath, repoRoot) - cmd := exec.CommandContext(ctx, "git", "status", "--porcelain=v1", "--untracked-files=normal") + cmd := exec.CommandContext(ctx, "git", gitStatusArgs...) cmd.Dir = absRepoPath out, err := cmd.Output() diff --git a/server/internal/handler/files_test.go b/server/internal/handler/files_test.go new file mode 100644 index 0000000..7278f1d --- /dev/null +++ b/server/internal/handler/files_test.go @@ -0,0 +1,114 @@ +package handler + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +// gitInit builds a throwaway repository with one staged file and one untracked +// file, so loadGitStatus has something to report either way. +func gitInit(t *testing.T, dir string) { + t.Helper() + for _, args := range [][]string{ + {"init", "--quiet"}, + {"config", "user.email", "test@example.com"}, + {"config", "user.name", "Test"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + if err := os.WriteFile(filepath.Join(dir, "staged.txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "untracked.txt"), []byte("y"), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("git", "add", "staged.txt") + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git add: %v: %s", err, out) + } +} + +// TestLoadGitStatus_IgnoresRepoPlantedFsmonitor is a security regression test. +// +// Git executes core.fsmonitor as a command and honours it from the +// repository's own .git/config. Workspace repositories are not trusted: anyone +// with terminal access to a session, or the agent itself, can write into +// .git/config. Because the server suppresses git's dubious-ownership check in +// order to read workspaces at all, that planted value would otherwise run here +// — in the server process, which on a socket-mounted deployment can reach the +// host Docker daemon. +// +// The test plants the config the way an attacker would and asserts the command +// never ran. It fails if the -c override is dropped from the git invocation. +func TestLoadGitStatus_IgnoresRepoPlantedFsmonitor(t *testing.T) { + t.Parallel() + + repo := t.TempDir() + gitInit(t, repo) + + // Sentinel lives outside the repo so a stray `git clean` can't mask it. + sentinel := filepath.Join(t.TempDir(), "fsmonitor-ran") + cmd := exec.Command("git", "config", "core.fsmonitor", + "sh -c 'printf executed > "+sentinel+"; echo'") + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("planting fsmonitor config: %v: %s", err, out) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + statusByPath := map[string]string{} + if err := loadGitStatus(ctx, repo, "", statusByPath); err != nil { + t.Fatalf("loadGitStatus: %v", err) + } + + if _, err := os.Stat(sentinel); err == nil { + t.Fatal("core.fsmonitor from repository config was executed by the server process; " + + "the -c core.fsmonitor=false override has been lost from gitStatusArgs") + } else if !os.IsNotExist(err) { + t.Fatalf("checking sentinel: %v", err) + } + + // The override must not cost us the actual feature. + if got := statusByPath["staged.txt"]; got == "" { + t.Errorf("expected a status for staged.txt, got none (statuses: %v)", statusByPath) + } + if got := statusByPath["untracked.txt"]; got == "" { + t.Errorf("expected a status for untracked.txt, got none (statuses: %v)", statusByPath) + } +} + +// TestLoadGitStatus_ReportsStatusesUnderRepoRoot covers the sub-repo path, +// where keys are prefixed with the repo root relative to the workspace. +func TestLoadGitStatus_ReportsStatusesUnderRepoRoot(t *testing.T) { + t.Parallel() + + workspace := t.TempDir() + sub := filepath.Join(workspace, "packages", "api") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + gitInit(t, sub) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + statusByPath := map[string]string{} + if err := loadGitStatus(ctx, workspace, "packages/api", statusByPath); err != nil { + t.Fatalf("loadGitStatus: %v", err) + } + + if _, ok := statusByPath["packages/api/staged.txt"]; !ok { + t.Errorf("expected key prefixed with the repo root, got: %v", statusByPath) + } +} From c26f079eed179ac2431646a551da542d1f5fd340 Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 04:45:41 +0000 Subject: [PATCH 6/7] docs(plans): mark the topology spike plan completed Co-Authored-By: Claude Opus 5 (1M context) --- docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md b/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md index 20868e4..73de99b 100644 --- a/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md +++ b/docs/plans/2026-07-31-001-feat-vm-deploy-topology-spike-plan.md @@ -1,7 +1,7 @@ --- title: VM deploy topology spike — validate socket-mount with path parity type: feat -status: active +status: completed date: 2026-07-31 origin: docs/brainstorms/2026-07-30-vm-deploy-and-upgrade-requirements.md --- From 2b9e444704d517ff71d5ee905ccffc330416b68c Mon Sep 17 00:00:00 2001 From: Clint Berry Date: Fri, 31 Jul 2026 04:57:28 +0000 Subject: [PATCH 7/7] fix(sshproxy): run client exec commands under bash, not dash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported from a real client: connecting to a session printed `/bin/sh: 29: Syntax error: "(" unexpected` and dropped the connection before anything ran. The exec path handed client-supplied commands to /bin/sh, which is dash on Debian-derived images. Clients write commands for a login shell, because that is what a real sshd gives them — VS Code Remote-SSH's bootstrap and the shell-integration payloads terminals inject on connect are multi-line bash scripts. Dash rejects the bash-only syntax in them at parse time, so the whole command dies. The reporter's script was 29 lines; a one-line `cat <(echo hi)` reproduces it at line 1. This was on track to break Open-in-VS-Code, which is the reason the proxy exists. The interactive shell modes have always used /bin/bash, so this aligns the exec path with them rather than introducing a new dependency, and bash is already required of devcontainers used with this proxy because VS Code's own install probe needs it. Verified against the live deployment: the failing syntax now runs, multi-line scripts with functions and [[ ]] work, exec still lands as the devcontainer's remoteUser, and the interactive shell is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- server/internal/sshproxy/docker.go | 20 ++++++++++++++-- server/internal/sshproxy/session_test.go | 29 ++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/server/internal/sshproxy/docker.go b/server/internal/sshproxy/docker.go index c5424b9..77be6a7 100644 --- a/server/internal/sshproxy/docker.go +++ b/server/internal/sshproxy/docker.go @@ -15,6 +15,22 @@ import ( // allocation-free. const defaultDockerBin = "docker" +// execShell interprets client-supplied exec commands inside the container. +// +// Bash, not sh. A real sshd hands the command to the account's login shell, +// and clients write to that expectation: VS Code Remote-SSH's bootstrap, and +// the shell-integration payloads terminals like iTerm2 and Warp inject on +// connect, are multi-line bash scripts. On Debian-derived images /bin/sh is +// dash, which rejects process substitution and the rest of the bash-only +// syntax those scripts use — the whole command dies on a syntax error before +// anything runs. +// +// bash is already a hard requirement for devcontainers used with this proxy +// (VS Code's own install probe needs it), and the interactive shell modes +// below have always used it, so this only aligns the exec path with the shell +// path. +const execShell = "/bin/bash" + // Env-var allowlist. Forwarded onto cmd.Env as-is when the request name // matches. Everything else is silently dropped — see filterEnv. This is // the only path by which a hostile session-member key can influence the @@ -194,10 +210,10 @@ func dockerArgs(container, command string, mode execMode, user string) []string case execModeNonPTYShell: return append(pre, "-i", container, "/bin/bash", "-l") case execModePTYExec: - return append(pre, "-it", container, "/bin/sh", "-c", command) + return append(pre, "-it", container, execShell, "-c", command) case execModeSFTP: return append(pre, "-i", container, "/usr/lib/openssh/sftp-server", "-e") default: // execModeNonPTY - return append(pre, "-i", container, "/bin/sh", "-c", command) + return append(pre, "-i", container, execShell, "-c", command) } } diff --git a/server/internal/sshproxy/session_test.go b/server/internal/sshproxy/session_test.go index 53d668b..3f7221c 100644 --- a/server/internal/sshproxy/session_test.go +++ b/server/internal/sshproxy/session_test.go @@ -24,9 +24,14 @@ import ( // Pure unit tests: command builders + env filter. // ---------------------------------------------------------------------- +// TestDockerArgs_NonPTY pins bash as the exec interpreter. Clients send +// commands written for a login shell — VS Code Remote-SSH's bootstrap and the +// shell-integration payloads terminals inject on connect are multi-line bash +// scripts — and /bin/sh is dash on Debian-derived images, which dies on +// bash-only syntax before running anything. func TestDockerArgs_NonPTY(t *testing.T) { got := dockerArgs("alice", "echo hi", execModeNonPTY, "") - want := []string{"exec", "-i", "alice", "/bin/sh", "-c", "echo hi"} + want := []string{"exec", "-i", "alice", "/bin/bash", "-c", "echo hi"} if !reflect.DeepEqual(got, want) { t.Errorf("dockerArgs(non-pty):\n got: %#v\nwant: %#v", got, want) } @@ -54,12 +59,32 @@ func TestDockerArgs_NonPTYShell(t *testing.T) { func TestDockerArgs_PTYExec(t *testing.T) { got := dockerArgs("alice", "ls /", execModePTYExec, "") - want := []string{"exec", "-it", "alice", "/bin/sh", "-c", "ls /"} + want := []string{"exec", "-it", "alice", "/bin/bash", "-c", "ls /"} if !reflect.DeepEqual(got, want) { t.Errorf("dockerArgs(pty-exec):\n got: %#v\nwant: %#v", got, want) } } +// TestDockerArgs_ExecUsesBashNotDash is the regression guard for the failure +// this replaced: a client sent a multi-line bash script and dash rejected it +// with `Syntax error: "(" unexpected`, closing the connection before the +// command ran. Both exec modes must reach bash. +func TestDockerArgs_ExecUsesBashNotDash(t *testing.T) { + for _, mode := range []execMode{execModeNonPTY, execModePTYExec} { + args := dockerArgs("alice", "cat <(echo bash-only)", mode, "") + var interp string + for i, a := range args { + if a == "-c" && i > 0 { + interp = args[i-1] + break + } + } + if interp != "/bin/bash" { + t.Errorf("mode %v: exec interpreter = %q, want /bin/bash (dash cannot parse bash-only syntax)", mode, interp) + } + } +} + func TestDockerArgs_SFTP(t *testing.T) { got := dockerArgs("alice", "", execModeSFTP, "") want := []string{"exec", "-i", "alice", "/usr/lib/openssh/sftp-server", "-e"}