diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1ba274..fa80481 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,10 @@ on: # Least privilege by default (audit T61): the workflow as a whole is # read-only; only the publishing job escalates to contents: write. +concurrency: + group: teploy-release-distribution + cancel-in-progress: false + permissions: contents: read @@ -34,4 +38,34 @@ jobs: args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + - name: Check out the Homebrew tap + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: useteploy/homebrew-tap + token: ${{ secrets.HOMEBREW_TAP_TOKEN }} + path: distribution/homebrew-tap + - name: Check out the Scoop bucket + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: useteploy/scoop-bucket + token: ${{ secrets.HOMEBREW_TAP_TOKEN }} + path: distribution/scoop-bucket + - name: Render distribution manifests from release checksums + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python3 scripts/render-distribution.py "$RELEASE_TAG" dist/checksums.txt distribution/homebrew-tap distribution/scoop-bucket + - name: Publish distribution manifests + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + for repo in distribution/homebrew-tap distribution/scoop-bucket; do + git -C "$repo" config user.name github-actions[bot] + git -C "$repo" config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ -f "$repo/teploy.rb" ]; then git -C "$repo" add teploy.rb; else git -C "$repo" add teploy.json; fi + if ! git -C "$repo" diff --cached --quiet; then + git -C "$repo" commit -m "Update teploy to $RELEASE_TAG" + git -C "$repo" pull --rebase + git -C "$repo" push + fi + done diff --git a/.goreleaser.yml b/.goreleaser.yml index 3c0a7a3..ff6f3e6 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -3,6 +3,9 @@ version: 2 builds: - main: ./cmd/teploy binary: teploy + flags: + - -trimpath + - -buildvcs=false env: # Without this, the linked-against-glibc Linux binary fails to run # at all inside any musl-based container (Alpine) with a bare "no @@ -36,38 +39,5 @@ checksum: changelog: sort: asc -# NOTE (2026-05-29): goreleaser 2.16 flags `brews:` as deprecated but does -# not yet ship a 1:1 replacement (homebrew_casks rejects `install:`; no -# homebrew_formulae key exists). Leaving as-is until goreleaser publishes a -# clean migration target; release path still works (deprecation, not error). -brews: - # Pin the formula name to `teploy` (the binary + what users install). Without - # this, goreleaser derives it from the project/repo name — which became - # `teploy-cli` after the repo rename and split the formula (a stale - # `teploy-cli.rb` at 0.1.5 alongside the real `teploy.rb`). Keep it `teploy`. - - name: teploy - repository: - owner: useteploy - name: homebrew-tap - token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" - homepage: https://github.com/useteploy/teploy-cli - description: Zero-downtime Docker deploys to any server via SSH - license: MIT - install: | - bin.install "teploy" - # Carried into the generated tap formula so `brew test teploy` (and the - # tap's CI) verifies the installed binary reports the formula's own - # version — the generated formulas previously shipped with no test - # block at all (audit useteploy__homebrew-tap-01). - test: | - assert_match version.to_s, shell_output("#{bin}/teploy version") - -scoops: - - name: teploy - repository: - owner: useteploy - name: scoop-bucket - token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" - homepage: https://github.com/useteploy/teploy-cli - description: Zero-downtime Docker deploys to any server via SSH - license: MIT +# Distribution manifests are rendered from verified archive checksums by +# scripts/render-distribution.py and published by the release workflow. diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 34861e5..9c31140 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -2602,3 +2602,92 @@ Validation: full Go tests and vet; source-selection/rsync argument tests; private-directory permission tests; real repository context checks for Ship, Dash and Observe. This change does not remove historical uploaded copies or rotate potentially exposed credentials; the doctor reports those separately. + + +## 2026-10-07 campaign reconciliation + +Implemented the current CLI/distribution remediation set: CLI-01..09, +LIF-01..15, ADM-01..14, ENG-01..09, RCLI-01..07, CFG-01..03, +STA-01..03, DIST-01..04 and the CLI portion of TSEM-03. This includes +reviewed source snapshots and under-lock admission, durable duplicate webhook +admission/torn-tail repair, DR authority and key validation, compensated +lifecycle/migration, explicit literal environments, bounded secret-safe backup +verification, framed preview authority, and validated binary publication. +Historical checksum transcription was reconciled by rebuilding the exact +six-platform source revision; published v0.1.37 archives were independently +verified. Final integration status is recorded outside this public repository. + +CC05 now has a supported additive pre-start `volume_ownership` map (see +`docs/volume-ownership.md`); existing volume maps keep their meanings. The +contract provisions only an empty managed root, preserves matching populated +roots, refuses populated mismatches and never recursively changes data. + +Plan record schema 2 requires re-planning schema-1 records. Templates emit +`env_literal` and require a matching CLI revision. Provider enrollment-window +wording preserves deliberate external ACL ownership; it does not claim active +device access expires. Private audit material is not part of this repository. + +Real consumer dependencies remain explicit: Nucleus must support +`shell --command-stdin --json` for private KV transport; older images refuse. +Observe can deduplicate stable `source_event_id` metadata for delivery retries. +Live integration and new release publication are separate from source fixes. + +### Source correction follow-up + +Reviewed-input mode binding, native/private-stdin DR restoration, unique owned +DR validation, retryable preview destruction, authoritative lifecycle selection +and fleet publication-outcome propagation have additional source corrections. +Managed volume admission now precedes preparation effects and uses no-follow +directory descriptors with explicit trusted-administrator ancestor admission. +SSH policy installation quotes the complete script and uses private invocation +inputs. The corresponding regression sources are present; compilation and +runtime acceptance must be completed for these new bytes before closure. + +### Trigger convergence and compare-destroy — landing review (2026-10-07, r5) + +The round4 mandatory engineering OPEN is implemented and locally validated. +`deploy`, `preview deploy` and `preview destroy` accept `--trigger-stdin +--json`: one bounded (32 KiB) duplicate-key-refusing schema-1 request whose +canonical domain-separated key binds target/source/repository/exact ref/full +commit/private execution binding/action; the target's enrolled identity is +proved from the target, never from a caller alias. Durable per-operation +receipts live target-resident under `/deployments//.operations` (0600, +O_NOFOLLOW, fsync+rename, completed receipts immutable), are written before +any preparation under the SAME renewable app lease, and replay reconciles +pending or unknown debt against authoritative state, immutable release/image +and binding before new preparation. Results are truthful +`not_started|not_committed|committed|unknown` with reconciliation status, and +the final JSON reaches stdout even on nonzero Cobra exits. Authenticated +webhook deliveries without a usable immutable commit are refused before +durable admission (C02); explicit tip mode can no longer be inferred from an +authenticated malformed event. + +Preview records now carry a durable ownership ID, a monotonic per-app +generation and an updated-at token. Destruction compares the admitted +identity against authority re-read under the same lease — including between +prune's listing and the destroy — and refuses renewed previews before any +route, container or record effect; teardown proves route and container +absence (immutable IDs) before unlinking authority, tolerates proven-absent +retries, and never converts unknown inventory into absence. Legacy-era +records compare their all-empty identity triple exactly and remain prunable. + +`teploy version --json` advertises `trigger-convergence-v1` and +`preview-compare-destroy-v1` (contracts corpus rev 9). Dash and other +consumers must keep cross-trigger consumption inactive until their own +pinned contract acceptance runs against an exact released artifact — the +capability token is the CLI-side handshake, not a consumer pass. + +Executed locally: build, vet (plus the `integration` tag type-check), full +unit and race suites (27/27 packages, no races), gofmt on the round's paths, +and the Linux/Windows/Darwin compile matrix including the static +`CGO_ENABLED=0` Linux binaries the release config builds. Two completion +defects were fixed beyond the interrupted lane's bytes: DR cutover engine +environment now honors `env_literal` database passwords (previously restored +with an empty password), and legacy preview records no longer serialize a +zero `updated_at`. Still open and not claimed: live-DB/SSH integration +suites (env-gated), real Linux/SSH/Docker/Caddy acceptance, native Windows +and release publication, ADM-11 upstream acceptance, migrating the resident +autodeploy deploy path onto the trigger adapter, and scheduler/manual +tip-mode admission plumbing. Private evidence and exact receipts live +outside this repository. + diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index cf342d1..36e1d80 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,6 +10,8 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| +| 9 | campaign based on 9939eaf, trigger-fix-r5 (uncommitted source patch; exact source digest in private report) | 2 | Additive. version-handshake advertises `trigger-convergence-v1` and `preview-compare-destroy-v1`: the reserved `--trigger-stdin` interface is now implemented (bounded schema1 stdin request, target-resident durable operation receipts reconciled under the app lease before any preparation, truthful committed/not_committed/unknown results with final JSON on nonzero Cobra paths) and preview destruction compares admitted ownership/generation against authority re-read under the same lease (renewal between listing and destroy is refused without effects). preview-state records/list rows may carry optional `ownership_id`, `generation`, `updated_at`, `operation_key`, `execution_binding_digest`, `source_revision`, `image_digest` (all absent on legacy records). Regenerated fixture: version-handshake mi2.json (two new capability tokens). Consumer activation (Dash) remains gated on its own pinned contract acceptance. No MI bump. | +| 8 | campaign based on 9939eaf (uncommitted source patch; exact source digest in private report) | 2 | Plan record schema 2; literal template environment; release/attempt identifier grammar parity; 128-bit preview ID with legacy lookup; additive managed `volume_ownership` for pre-start UID/GID/mode provisioning. | | 7 | main (L4 cli-defects: ID-created container image resolution) | 2 | Additive. The container object in app-list-envelope and server-status-envelope gains optional `image_id` (full sha256) and `image_tags` (string array), emitted only for containers created by image ID (A52 creates web/worker containers from the immutable ID). For those, `image` now carries the first repo tag instead of the bare 12-hex short ID docker ps reports - restoring the pre-A52 meaning (Ship wave-9: ID-created containers never matched the artifact tag). Name-created containers and existing fixtures are unchanged (no fixture regenerated: the corpus fixtures use name-form images). No MI bump. | | 6 | main (tailnet preview mode, DELEGATED_DECISIONS §10) | 2 | Additive. preview-state schema gains optional record/list-row fields on both eras (`domain`, `url`, `base_domain`, `http_only`, `allow_ips`) with the invariant url scheme = `http://` iff `http_only` (else `https://`); two valid fixtures GENERATED from the real `preview list --json` row encoder (`previewListRows`, `contracts_golden_test.go`): canonical-list-row (default mode, no exposure keys, https url) and canonical-list-row-tailnet (base_domain + http_only + allow_ips, http url), each wrapped with the artifact's `era`/`app` classification keys (the wire row carries neither). The hand-authored identity fixtures (canonical, legacy, ambiguous) are unchanged. version-handshake gains the `preview-exposure` capability token (additive). No MI bump. | | 5 | main (X02 S2 tail: server-status fixtures + schema correction) | 2 | server-status-envelope fixtures landed (was "pending live capture"): valid x2 (full healthy observation, partial-caddy-unavailable — the class a target without a caddy container produces) + legacy pre-MI (machine_interface absent, the 42243e2-era shape). Encoder-derived: generated from the REAL `collectServerStatus` via a mock SSH executor (`contracts_golden_test.go`, TEPLOY_UPDATE_CONTRACTS) — synthetic values, real encoder and parse stages; the wire shape was verified against a live `server status --json` run before pinning. Defect fixed in the same commit: the schema had copied the appStatus root since its S2 draft (its own defect-fix commit 08cfb1b said so) and never described the actual serverStatusDTO wire format (server/host/uptime/load/memory/disks/docker/caddy) — rewritten to the real root with strict required-key coverage of the DTO's no-omitempty fields. Additive to consumers (a schema that matched nothing before now matches the wire); no MI bump. | @@ -70,3 +72,7 @@ treats id-vanished as ambiguous-legacy requiring explicit re-binding, never auto-re-mint). Consumers MUST NOT treat a missing `machine_interface` field as MI 0 — it means "pre-MI producer", the legacy decode path. + +## 2026-10-07 contract update + +PlanRecord schema 2 requires re-planning records from schema 1. The private config binding includes execution commands, hook contents and routing values; environment values retain the established exclusion policy. Static plans bind the selected source and exact plan output exclusion. Release and attempt identifier schemas now match runtime release IDs (1–128 grammar-valid characters); attempts append a dot and 16 hex nonce. Templates emit literal `env_literal` maps for rendered environment values, preserving user passwords through deployment. diff --git a/contracts/fixtures/attempt-name/invalid/examples.json b/contracts/fixtures/attempt-name/invalid/examples.json index 56b862b..8f2436b 100644 --- a/contracts/fixtures/attempt-name/invalid/examples.json +++ b/contracts/fixtures/attempt-name/invalid/examples.json @@ -1,6 +1,5 @@ [ "deadb17ecafef00d", - "ABC1234.deadb17ecafef00d", "abc1234.DeadB17eCafef00d", "abc1234.deadb17ecafef00", "../escape.attempt0000000" diff --git a/contracts/fixtures/attempt-name/valid/examples.json b/contracts/fixtures/attempt-name/valid/examples.json index 2b164d2..1e97fdf 100644 --- a/contracts/fixtures/attempt-name/valid/examples.json +++ b/contracts/fixtures/attempt-name/valid/examples.json @@ -1,4 +1,7 @@ [ "abc1234.deadb17ecafef00d", - "9f31c02.0123456789abcdef" + "9f31c02.0123456789abcdef", + "ABC1234.deadb17ecafef00d", + "v1.2.3.deadb17ecafef00d", + "release_1.deadb17ecafef00d" ] diff --git a/contracts/fixtures/plan-record/invalid/tampered-id.json b/contracts/fixtures/plan-record/invalid/tampered-id.json index bfce6ba..5f15142 100644 --- a/contracts/fixtures/plan-record/invalid/tampered-id.json +++ b/contracts/fixtures/plan-record/invalid/tampered-id.json @@ -1,6 +1,6 @@ { - "schema_version": 1, - "plan_id": "e6e098f1e0322e3f", + "schema_version": 2, + "plan_id": "1eefc411f1a95ead", "app": "myapp", "server": "srv.example.com", "server_name": "prod", diff --git a/contracts/fixtures/plan-record/valid/build.json b/contracts/fixtures/plan-record/valid/build.json index 1eed2b8..5fd4900 100644 --- a/contracts/fixtures/plan-record/valid/build.json +++ b/contracts/fixtures/plan-record/valid/build.json @@ -1,6 +1,6 @@ { - "schema_version": 1, - "plan_id": "f5f4997ca665419e", + "schema_version": 2, + "plan_id": "3d6f4a1a3f9dd67d", "app": "myapp", "server": "srv.example.com", "user": "root", diff --git a/contracts/fixtures/plan-record/valid/prebuilt-digest.json b/contracts/fixtures/plan-record/valid/prebuilt-digest.json index d4d3731..0065375 100644 --- a/contracts/fixtures/plan-record/valid/prebuilt-digest.json +++ b/contracts/fixtures/plan-record/valid/prebuilt-digest.json @@ -1,6 +1,6 @@ { - "schema_version": 1, - "plan_id": "e6e098f1e0322e3f", + "schema_version": 2, + "plan_id": "1eefc411f1a95ead", "app": "myapp", "server": "srv.example.com", "server_name": "prod", diff --git a/contracts/fixtures/preview-state/valid/canonical-list-row-tailnet.json b/contracts/fixtures/preview-state/valid/canonical-list-row-tailnet.json index c988476..2712de8 100644 --- a/contracts/fixtures/preview-state/valid/canonical-list-row-tailnet.json +++ b/contracts/fixtures/preview-state/valid/canonical-list-row-tailnet.json @@ -11,7 +11,7 @@ "era": "canonical", "expires_at": "2026-09-27T12:00:00Z", "http_only": true, - "id": "myapp-p-08e81639", + "id": "myapp-p-08e8163960152ff19d4fc9c432d3270b", "image": "myapp-build-abc1234", "port": 49200, "repo": "github.com/example/myapp", diff --git a/contracts/fixtures/preview-state/valid/canonical-list-row.json b/contracts/fixtures/preview-state/valid/canonical-list-row.json index d4b546a..51a54cd 100644 --- a/contracts/fixtures/preview-state/valid/canonical-list-row.json +++ b/contracts/fixtures/preview-state/valid/canonical-list-row.json @@ -6,7 +6,7 @@ "domain": "preview-feature-login-08e81639.myapp.com", "era": "canonical", "expires_at": "2026-09-27T12:00:00Z", - "id": "myapp-p-08e81639", + "id": "myapp-p-08e8163960152ff19d4fc9c432d3270b", "image": "myapp-build-abc1234", "port": 49200, "repo": "github.com/example/myapp", diff --git a/contracts/fixtures/version-handshake/valid/mi2.json b/contracts/fixtures/version-handshake/valid/mi2.json index 4292447..13c68e3 100644 --- a/contracts/fixtures/version-handshake/valid/mi2.json +++ b/contracts/fixtures/version-handshake/valid/mi2.json @@ -10,6 +10,7 @@ "plan-apply", "preview-blue-green", "preview-canonical-id", + "preview-compare-destroy-v1", "preview-exposure", "provenance-records", "readiness-receipts", @@ -17,7 +18,8 @@ "server-rename", "server-status-machine", "server-update", - "template-var-stdin" + "template-var-stdin", + "trigger-convergence-v1" ], "machine_interface": 2, "version": "v0.0.0-contracts" diff --git a/contracts/schema/attempt-name.schema.json b/contracts/schema/attempt-name.schema.json index 30af49c..b1f0cd5 100644 --- a/contracts/schema/attempt-name.schema.json +++ b/contracts/schema/attempt-name.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://teploy.github.io/contracts/schema/attempt-name.schema.json", - "title": "deploy attempt directory name grammar: .<16hex> (F08)", + "title": "deploy attempt directory name grammar: .<16hex>", "type": "string", - "pattern": "^[a-f0-9]{7}\\.[a-f0-9]{16}$" + "pattern": "^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}\\.[a-f0-9]{16}$" } diff --git a/contracts/schema/plan-record.schema.json b/contracts/schema/plan-record.schema.json index e581bb7..f1894cc 100644 --- a/contracts/schema/plan-record.schema.json +++ b/contracts/schema/plan-record.schema.json @@ -3,78 +3,232 @@ "$id": "https://teploy.github.io/contracts/schema/plan-record.schema.json", "title": "C05 plan record (teploy plan --out; cli/planrecord.go PlanRecord)", "type": "object", - "required": ["schema_version", "plan_id", "app", "server", "target_version", "version_known", "config_digest", "image", "target_state", "effects"], + "required": [ + "schema_version", + "plan_id", + "app", + "server", + "target_version", + "version_known", + "config_digest", + "image", + "target_state", + "effects" + ], "properties": { - "schema_version": {"type": "integer", "minimum": 1}, - "plan_id": {"type": "string", "pattern": "^[a-f0-9]{16}$"}, - "app": {"type": "string", "minLength": 1}, - "server": {"type": "string", "minLength": 1}, - "user": {"type": "string"}, - "server_name": {"type": "string"}, - "destination": {"type": "string"}, - "target_version": {"type": "string"}, - "version_known": {"type": "boolean"}, - "version_explicit": {"type": "boolean"}, - "config_digest": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "schema_version": { + "type": "integer", + "const": 2 + }, + "plan_id": { + "type": "string", + "pattern": "^[a-f0-9]{16}$" + }, + "app": { + "type": "string", + "minLength": 1 + }, + "server": { + "type": "string", + "minLength": 1 + }, + "user": { + "type": "string" + }, + "server_name": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "target_version": { + "type": "string" + }, + "version_known": { + "type": "boolean" + }, + "version_explicit": { + "type": "boolean" + }, + "config_digest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, "image": { "type": "object", - "required": ["resolution"], + "required": [ + "resolution" + ], "properties": { - "ref": {"type": "string"}, - "needs_build": {"type": "boolean"}, - "resolution": {"type": "string", "enum": ["resolved-by-digest", "resolved-by-image-id", "unresolved-mutable-tag", "unresolved-awaiting-build"]}, - "digest": {"type": "string"}, - "context_path": {"type": "string"}, - "context_fingerprint": {"type": "string"}, - "dockerfile": {"type": "string"}, - "dockerfile_sha256": {"type": "string"}, - "platform": {"type": "string"} + "ref": { + "type": "string" + }, + "needs_build": { + "type": "boolean" + }, + "resolution": { + "type": "string", + "enum": [ + "resolved-by-digest", + "resolved-by-image-id", + "unresolved-mutable-tag", + "unresolved-awaiting-build" + ] + }, + "digest": { + "type": "string" + }, + "context_path": { + "type": "string" + }, + "context_fingerprint": { + "type": "string" + }, + "dockerfile": { + "type": "string" + }, + "dockerfile_sha256": { + "type": "string" + }, + "platform": { + "type": "string" + } } }, "target_state": { "type": "object", - "required": ["deployed", "generation"], + "required": [ + "deployed", + "generation" + ], "properties": { - "deployed": {"type": "boolean"}, - "generation": {"type": "integer", "minimum": 0}, - "current_hash": {"type": "string"}, - "manifest_sha256": {"type": "string"} + "deployed": { + "type": "boolean" + }, + "generation": { + "type": "integer", + "minimum": 0 + }, + "current_hash": { + "type": "string" + }, + "manifest_sha256": { + "type": "string" + } } }, "effects": { "type": "object", - "required": ["containers"], + "required": [ + "containers" + ], "properties": { - "containers": {"type": "array", "items": {"$ref": "#/$defs/planChange"}}, - "routing": {"type": "array", "items": {"$ref": "#/$defs/planEffect"}}, - "env": {"type": "array", "items": {"$ref": "#/$defs/planEffect"}}, - "storage": {"type": "array", "items": {"$ref": "#/$defs/planEffect"}}, - "resources": {"type": "array", "items": {"$ref": "#/$defs/planEffect"}}, - "accessories": {"type": "array", "items": {"$ref": "#/$defs/planEffect"}} + "containers": { + "type": "array", + "items": { + "$ref": "#/$defs/planChange" + } + }, + "routing": { + "type": "array", + "items": { + "$ref": "#/$defs/planEffect" + } + }, + "env": { + "type": "array", + "items": { + "$ref": "#/$defs/planEffect" + } + }, + "storage": { + "type": "array", + "items": { + "$ref": "#/$defs/planEffect" + } + }, + "resources": { + "type": "array", + "items": { + "$ref": "#/$defs/planEffect" + } + }, + "accessories": { + "type": "array", + "items": { + "$ref": "#/$defs/planEffect" + } + } + } + }, + "unresolved": { + "type": "array", + "items": { + "type": "string" } }, - "unresolved": {"type": "array", "items": {"type": "string"}}, - "written_at": {"type": "string", "format": "date-time"} + "written_at": { + "type": "string", + "format": "date-time" + }, + "input_exclusions": { + "type": "array", + "items": { + "type": "string" + } + } }, "$defs": { "planChange": { "type": "object", - "required": ["action", "name"], + "required": [ + "action", + "name" + ], "properties": { - "action": {"type": "string", "enum": ["create", "stop", "unchanged"]}, - "name": {"type": "string"}, - "detail": {"type": "string"} + "action": { + "type": "string", + "enum": [ + "create", + "stop", + "unchanged" + ] + }, + "name": { + "type": "string" + }, + "detail": { + "type": "string" + } } }, "planEffect": { "type": "object", - "required": ["action", "name"], + "required": [ + "action", + "name" + ], "properties": { - "action": {"type": "string", "enum": ["add", "remove", "change"]}, - "name": {"type": "string"}, - "from": {"type": "string"}, - "to": {"type": "string"}, - "detail": {"type": "string"} + "action": { + "type": "string", + "enum": [ + "add", + "remove", + "change" + ] + }, + "name": { + "type": "string" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + }, + "detail": { + "type": "string" + } } } } diff --git a/contracts/schema/release-record.schema.json b/contracts/schema/release-record.schema.json index 1d1b686..2dffc7d 100644 --- a/contracts/schema/release-record.schema.json +++ b/contracts/schema/release-record.schema.json @@ -3,20 +3,92 @@ "$id": "https://teploy.github.io/contracts/schema/release-record.schema.json", "title": "releasemeta Record (F14; releasemeta.go Record)", "type": "object", - "required": ["schema_version", "app", "hash", "created_at", "deployment_type", "ingress_mode"], + "required": [ + "schema_version", + "app", + "hash", + "created_at", + "deployment_type", + "ingress_mode" + ], "properties": { - "schema_version": {"type": "integer", "minimum": 1}, - "app": {"type": "string", "minLength": 1}, - "hash": {"type": "string", "pattern": "^[a-f0-9]{7}(\\.[a-f0-9]{16})?$"}, - "created_at": {"type": "string", "format": "date-time"}, - "backfilled": {"type": "boolean"}, - "generation": {"type": "integer", "minimum": 0}, - "deployment_type": {"type": "string", "enum": ["container", "static"]}, - "ingress_mode": {"type": "string"}, - "domain": {"type": "string"}, - "image_ref": {"type": "string"}, - "image_digest": {"type": "string"}, - "manifest_sha256": {"type": "string"}, - "provenance": {"type": ["object", "null"]} + "schema_version": { + "type": "integer", + "minimum": 1 + }, + "app": { + "type": "string", + "minLength": 1 + }, + "hash": { + "type": "string", + "pattern": "^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "backfilled": { + "type": "boolean" + }, + "generation": { + "type": "integer", + "minimum": 0 + }, + "deployment_type": { + "type": "string", + "enum": [ + "container", + "static" + ] + }, + "ingress_mode": { + "type": "string" + }, + "domain": { + "type": "string" + }, + "image_ref": { + "type": "string" + }, + "image_digest": { + "type": "string" + }, + "manifest_sha256": { + "type": "string" + }, + "provenance": { + "type": [ + "object", + "null" + ] + }, + "volume_ownership": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "uid", + "gid", + "mode" + ], + "properties": { + "uid": { + "type": "integer", + "minimum": 0, + "maximum": 4294967294 + }, + "gid": { + "type": "integer", + "minimum": 0, + "maximum": 4294967294 + }, + "mode": { + "type": "string", + "pattern": "^0[0-7]{2}[0145]$" + } + } + } + } } } diff --git a/docs/build-context.md b/docs/build-context.md index 942049b..c55fa3e 100644 --- a/docs/build-context.md +++ b/docs/build-context.md @@ -31,3 +31,27 @@ static sources normally point at generated output directories. `teploy doctor` reports older build directories that are traversable by other users and potentially secret-bearing files under build/static release paths. The check lists paths only and does not delete files or change permissions. + +### Checked static output and source maps + +A Dockerfile that copies locally generated, Gitignored `dist/` output needs an +explicit `.teployignore` allowlist. This selection includes ignored descendants +while retaining mandatory secret exclusions: + +``` +!/dist/ +/dist/**/*.map +``` + +Source sync transfers exactly the admitted file list, creating its parent +directories. Nested ignored parents can be selected with an anchored path such as +`!/nested/parent/dist/`. Excludes win over allowlists, including source maps in any +depth. Patterns containing a `..` path segment and literal allowlists through +symlinked ancestors are refused rather than traversing outside the source root. + +Reviewed plans use a separate mode-bearing input fingerprint, including the +context root, regular-file modes and directory modes (including implied +parents). A chmod-only change requires re-planning. Provenance retains its +existing mode-free fingerprint. Apply preserves reviewed modes inside a private +parent directory, then validates and builds/transfers that same frozen tree. +Previously saved build plans must be regenerated for this reviewed identity. diff --git a/docs/failure-and-recovery.md b/docs/failure-and-recovery.md index 6f1409d..4d9618e 100644 --- a/docs/failure-and-recovery.md +++ b/docs/failure-and-recovery.md @@ -158,3 +158,39 @@ read them before trusting a volume snapshot of a writing database. For the topology-level version (N+1, state off-box, the dead-server runbook), see [resilience.md](resilience.md). + +### Reviewed plan compatibility + +Saved plans use record schema 2. Re-plan older records: their execution binding +omitted fields now covered by the private config digest. The public display +manifest remains redacted; environment values retain the established exclusion +policy. Apply revalidates authority under the app lock, then builds or serves a +private copy of the admitted source rather than a changing caller directory. +Static serving output is fingerprinted even when Gitignored. The plan output +file and parents it creates are excluded from its own fingerprint. Reviewed +static source must be inside the project; absolute source symlinks are refused. +Direct deploy keeps its existing source-path support. + +Rendered template environments use `env_literal`, beside ordinary `env`: +variables that become passwords such as `generate`, `auto`, `$NAME` or +`vault:literal#field` remain literal bytes. Original template generation +sentinels are resolved before variable substitution. New template manifests +require a CLI revision supporting `env_literal`; do not feed them to an older +release that lacks this field. + +DR restore receipts now use schema 2 and include a private invocation identity. +Each restore retains its own staging directory; another restore never wipes +it. Cutover selects the latest completed receipt for the requested bundle, +then binds to that invocation's own receipt and manifest. Older schema-1 +receipts require a fresh isolated restore. Scratch validation uses unique names +and cleans only the immutable IDs created by that invocation after checking +ownership; unknown create outcomes require inspection, not removal by name. + +Fleet publication errors retain committed or unknown workload authority and +withhold automatic load-balancer changes and fleet convergence rollback until +reconciliation. Lifecycle stop/start/restart requires a current release record +and selects only its web/workers plus accessories in the current applied +manifest. Start activates those configured accessories before app processes, +including accessories previously stopped by an operator; removed accessories, +previews, historical releases and unknown processes are excluded. A legacy +release without a record must be redeployed before using lifecycle commands. diff --git a/docs/volume-ownership.md b/docs/volume-ownership.md new file mode 100644 index 0000000..a74aee3 --- /dev/null +++ b/docs/volume-ownership.md @@ -0,0 +1,61 @@ +# Managed volume ownership before startup + +Existing `volumes` maps keep their meaning. A logical name resolves to +`/deployments//volumes/`; absolute host bind keys stay operator managed. +Add an explicit initialization contract for an image running as a non-root user: + +```yaml +volumes: + observe-state: /var/lib/observe +volume_ownership: + observe-state: + uid: 10001 + gid: 10001 + mode: "0700" +``` + +The ownership key must match a named entry in `volumes`. UID and GID are required +numeric IDs from 0 through 4294967294. Mode is four octal digits without special +bits; world-writable modes are refused. Host paths cannot be ownership keys. + +The engine provisions the directory before any app container starts, while holding +the deployment fence. Root or passwordless sudo is required on the server. Every +parent and the target must be real directories, not symlinks. An empty directory +can receive the requested owner/group/mode; an already matching directory is an +idempotent success, including when populated. A populated mismatch refuses before +startup and requires a deliberate operator migration. Initialization never performs +recursive chown/chmod and never changes existing files or subdirectories. + +Accessories support the same map beside their own `volumes`: + +```yaml +accessories: + database: + image: postgres:17 + volumes: + data: /var/lib/postgresql/data + volume_ownership: + data: {uid: 999, gid: 999, mode: "0700"} +``` + +Accessory roots resolve below `/deployments//accessories//`. +Explicit entries bypass the historical image-user ownership inference for that +volume. Verify the image's actual numeric UID/GID rather than copying this example +across images. Ownership is part of the reviewed plan binding and release metadata. +A pre-deploy hook runs inside a started web container, so it cannot replace this +pre-start contract. + +Provisioning uses descriptor-relative, no-follow directory access. Ownership +and mode changes apply to the admitted directory descriptor, never a later +replacement of its pathname. Ancestors must belong to root or the SSH +administrator and have no group/world write permission. Root and the SSH +administrator (including that account's processes) are trusted; the app lease +does not serialize unrelated privileged administrators. Unsafe ancestors +refuse with migration guidance. The Linux target needs an existing Python 3 +interpreter and root or passwordless sudo; the CLI does not install it. + +All explicit app and accessory roots are checked before accessory or agent +startup and before volume migration stops writers. Running accessories also +undergo ownership admission. Explicitly owned roots bypass legacy recursive +image ownership repair. Configure these ancestor permissions deliberately when +upgrading an existing deployment tree. diff --git a/internal/accessories/accessories.go b/internal/accessories/accessories.go index 51c8902..98f2371 100644 --- a/internal/accessories/accessories.go +++ b/internal/accessories/accessories.go @@ -16,6 +16,7 @@ import ( "github.com/useteploy/teploy/internal/docker" "github.com/useteploy/teploy/internal/secret" "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" ) const deploymentsDir = "/deployments" @@ -44,13 +45,50 @@ func ContainerName(app, name string) string { // EnsureRunning checks if an accessory is running and starts it if not. // Returns env vars to inject into the app (e.g., DATABASE_URL for postgres). func (m *Manager) EnsureRunning(ctx context.Context, app, name string, cfg config.AccessoryConfig) (map[string]string, error) { + if err := config.ValidateVolumeOwnership(cfg.Volumes, cfg.VolumeOwnership); err != nil { + return nil, err + } + if fenced, ok := m.exec.(interface{ UnderFence() bool }); !ok || !fenced.UnderFence() { + if err := m.docker.EnsureManagedDirectory(ctx, app, "", ""); err != nil { + return nil, err + } + lk, err := state.AcquireLockFenced(ctx, m.exec, app) + if err != nil { + return nil, err + } + lk.StartRenewal(m.exec) + defer state.ReleaseLockFenced(m.exec, lk, app) + return NewManager(&state.FencedExecutor{Executor: m.exec, Lock: lk}, m.out).EnsureRunning(ctx, app, name, cfg) + } + for _, volume := range sortedKeys(cfg.VolumeOwnership) { + if err := m.docker.ValidateManagedVolume(ctx, app, name, volume, cfg.VolumeOwnership[volume]); err != nil { + return nil, err + } + } + for _, volume := range sortedKeys(cfg.VolumeOwnership) { + if err := m.docker.ProvisionManagedVolume(ctx, app, name, volume, cfg.VolumeOwnership[volume]); err != nil { + return nil, err + } + } containerName := ContainerName(app, name) // Always resolve env (needed for connection strings even if already running). - env, err := m.resolveEnv(ctx, app, name, cfg.Env) + directives := make(map[string]string, len(cfg.Env)) + for k, v := range cfg.Env { + if _, literal := cfg.EnvLiteral[k]; !literal { + directives[k] = v + } + } + env, err := m.resolveEnv(ctx, app, name, directives) if err != nil { return nil, fmt.Errorf("resolving env vars: %w", err) } + if env == nil { + env = map[string]string{} + } + for k, v := range cfg.EnvLiteral { + env[k] = v + } // Check if already running. status, err := m.exec.Run(ctx, fmt.Sprintf( @@ -81,7 +119,7 @@ func (m *Manager) EnsureRunning(ctx context.Context, app, name string, cfg confi // Ensure directory structure. accDir := fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, app, name) - if _, err := m.exec.Run(ctx, fmt.Sprintf("mkdir -p %s", accDir)); err != nil { + if err := m.docker.EnsureManagedDirectory(ctx, app, name, ""); err != nil { return nil, fmt.Errorf("creating accessory directory: %w", err) } @@ -99,12 +137,16 @@ func (m *Manager) EnsureRunning(ctx context.Context, app, name string, cfg confi // the engine. That is the upgrade failure reconcileDataOwnership already // covers, hit on a fresh install by every deploy — `template install` // included. - for _, hostPath := range sortedKeys(volumes) { - if _, err := m.exec.Run(ctx, "mkdir -p "+hostPath); err != nil { - return nil, fmt.Errorf("creating accessory volume directory %s: %w", hostPath, err) + for _, volume := range sortedKeys(cfg.Volumes) { + if _, owned := cfg.VolumeOwnership[volume]; owned { + continue + } + if err := m.docker.EnsureManagedDirectory(ctx, app, name, volume); err != nil { + return nil, err } } - if err := m.reconcileDataOwnership(ctx, app, name, cfg); err != nil { + + if err := m.reconcileDataOwnership(ctx, app, name, withoutOwnedVolumes(cfg)); err != nil { return nil, err } @@ -454,6 +496,28 @@ func (m *Manager) Logs(ctx context.Context, app, name string, lines int) error { // Upgrade stops the old container, pulls the new image, and starts a new one with same config. func (m *Manager) Upgrade(ctx context.Context, app, name, newImage string, cfg config.AccessoryConfig) error { + if fenced, ok := m.exec.(interface{ UnderFence() bool }); !ok || !fenced.UnderFence() { + if err := m.docker.EnsureManagedDirectory(ctx, app, "", ""); err != nil { + return err + } + lk, err := state.AcquireLockFenced(ctx, m.exec, app) + if err != nil { + return err + } + lk.StartRenewal(m.exec) + defer state.ReleaseLockFenced(m.exec, lk, app) + return NewManager(&state.FencedExecutor{Executor: m.exec, Lock: lk}, m.out).Upgrade(ctx, app, name, newImage, cfg) + } + for _, volume := range sortedKeys(cfg.VolumeOwnership) { + if err := m.docker.ValidateManagedVolume(ctx, app, name, volume, cfg.VolumeOwnership[volume]); err != nil { + return err + } + } + for _, volume := range sortedKeys(cfg.VolumeOwnership) { + if err := m.docker.ProvisionManagedVolume(ctx, app, name, volume, cfg.VolumeOwnership[volume]); err != nil { + return err + } + } containerName := ContainerName(app, name) fmt.Fprintf(m.out, "Pulling %s...\n", newImage) @@ -529,6 +593,9 @@ func (m *Manager) reconcileDataOwnership(ctx context.Context, app, name string, accDir := fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, app, name) for _, volName := range sortedKeys(cfg.Volumes) { + if _, explicit := cfg.VolumeOwnership[volName]; explicit { + continue + } dir := fmt.Sprintf("%s/%s", accDir, volName) dirOwner, ownErr := m.exec.Run(ctx, fmt.Sprintf("stat -c '%%u' %s 2>/dev/null", ssh.ShellQuote(dir))) if ownErr != nil || strings.TrimSpace(dirOwner) == "" || strings.TrimSpace(dirOwner) == uid { @@ -575,8 +642,14 @@ func connectionEnvVars(app, name, image string, port int, env map[string]string) vars["DATABASE_URL"] = dbURL("postgres", user, password, alias, port, db) case isImageType(image, "mysql"), isImageType(image, "mariadb"): - password := env["MYSQL_ROOT_PASSWORD"] - db := env["MYSQL_DATABASE"] + password := env["MARIADB_ROOT_PASSWORD"] + if password == "" { + password = env["MYSQL_ROOT_PASSWORD"] + } + db := env["MARIADB_DATABASE"] + if db == "" { + db = env["MYSQL_DATABASE"] + } if db == "" { db = app } @@ -635,7 +708,7 @@ func generatePassword() (string, error) { return hex.EncodeToString(b), nil } -func sortedKeys(m map[string]string) []string { +func sortedKeys[V any](m map[string]V) []string { keys := make([]string, 0, len(m)) for k := range m { keys = append(keys, k) @@ -643,3 +716,14 @@ func sortedKeys(m map[string]string) []string { sort.Strings(keys) return keys } + +func withoutOwnedVolumes(cfg config.AccessoryConfig) config.AccessoryConfig { + copyCfg := cfg + copyCfg.Volumes = make(map[string]string) + for name, target := range cfg.Volumes { + if _, explicit := cfg.VolumeOwnership[name]; !explicit { + copyCfg.Volumes[name] = target + } + } + return copyCfg +} diff --git a/internal/accessories/accessories_command_test.go b/internal/accessories/accessories_command_test.go index 7768953..6b4faa4 100644 --- a/internal/accessories/accessories_command_test.go +++ b/internal/accessories/accessories_command_test.go @@ -13,14 +13,14 @@ import ( // The command: override must land as trailing args after the image — // MinIO/ntfy style images need an explicit verb (`server /data`, `serve`). func TestEnsureRunningAppendsCommand(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := ssh.NewMockExecutor("h", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker inspect", Output: "exited"}, // "exited" = the container exists but is stopped, so EnsureRunning // removes it before recreating (docker run would hit a name conflict). ssh.MockCommand{Match: "docker rm -f", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc"}, - ) + )...) m := NewManager(mock, io.Discard) _, err := m.EnsureRunning(context.Background(), "myapp", "minio", config.AccessoryConfig{ Image: "minio/minio:latest", @@ -43,14 +43,14 @@ func TestEnsureRunningAppendsCommand(t *testing.T) { // Without command:, the docker run line must end at the image (no regression). func TestEnsureRunningNoCommandUnchanged(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := ssh.NewMockExecutor("h", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker inspect", Output: "exited"}, // "exited" = the container exists but is stopped, so EnsureRunning // removes it before recreating (docker run would hit a name conflict). ssh.MockCommand{Match: "docker rm -f", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc"}, - ) + )...) m := NewManager(mock, io.Discard) _, err := m.EnsureRunning(context.Background(), "myapp", "db", config.AccessoryConfig{ Image: "postgres:16", diff --git a/internal/accessories/accessories_test.go b/internal/accessories/accessories_test.go index 26aa7b8..8ed7e02 100644 --- a/internal/accessories/accessories_test.go +++ b/internal/accessories/accessories_test.go @@ -11,8 +11,20 @@ import ( "github.com/useteploy/teploy/internal/ssh" ) +// admissionStubs registers the remote surface every unfenced EnsureRunning +// and Upgrade now walks before any accessory effect: the descriptor-based +// managed-directory admission (R2-06) and the fenced app lock's mkdir +// (F16). The lock info upload and the guarded release are modeled by the +// mock's file state, so only these two registrations are test plumbing. +func admissionStubs(app string) []ssh.MockCommand { + return []ssh.MockCommand{ + ssh.MockCommand{Match: "teploy_volume_actor", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/" + app + "/.lock", Output: ""}, + } +} + func TestEnsureRunning_New(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), // No stored credentials. ssh.MockCommand{Match: "if [ ! -e ", Output: "absent"}, // Not running. @@ -21,7 +33,7 @@ func TestEnsureRunning_New(t *testing.T) { ssh.MockCommand{Match: "mkdir -p /deployments/myapp/accessories/postgres", Output: ""}, // Start container. ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -129,7 +141,7 @@ func TestEnsureRunning_New(t *testing.T) { } func TestEnsureRunning_ReconcilesFreshVolumeOwnership(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "if [ ! -e ", Output: "absent"}, ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p /deployments/myapp/accessories/nucleus", Output: ""}, @@ -138,7 +150,7 @@ func TestEnsureRunning_ReconcilesFreshVolumeOwnership(t *testing.T) { ssh.MockCommand{Match: "stat -c '%u' '/deployments/myapp/accessories/nucleus/data'", Output: "1000\n"}, ssh.MockCommand{Match: "docker run --rm --user 0 --entrypoint chown", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -152,11 +164,13 @@ func TestEnsureRunning_ReconcilesFreshVolumeOwnership(t *testing.T) { t.Fatalf("EnsureRunning: %v", err) } - var mkdir, chown, run int = -1, -1, -1 + // The volume directory is created by the descriptor-based provision + // (R2-06), which must still precede the ownership reconcile. + var provision, chown, run int = -1, -1, -1 for i, call := range mock.Calls { switch { - case call == "mkdir -p /deployments/myapp/accessories/nucleus/data": - mkdir = i + case strings.HasPrefix(call, "teploy_volume_actor=") && strings.Contains(call, "'/deployments/myapp/accessories/nucleus/data' provision"): + provision = i case strings.HasPrefix(call, "docker run --rm --user 0 --entrypoint chown"): chown = i if !strings.Contains(call, "'/deployments/myapp/accessories/nucleus/data:/teploy-data'") || !strings.Contains(call, "-R 10001:10001 /teploy-data") { @@ -166,11 +180,11 @@ func TestEnsureRunning_ReconcilesFreshVolumeOwnership(t *testing.T) { run = i } } - if mkdir < 0 || chown < 0 || run < 0 { - t.Fatalf("expected mkdir, chown and run calls, got: %v", mock.Calls) + if provision < 0 || chown < 0 || run < 0 { + t.Fatalf("expected volume provision, chown and run calls, got: %v", mock.Calls) } - if !(mkdir < chown && chown < run) { - t.Errorf("expected volume mkdir, then chown, then the accessory start; got order mkdir=%d chown=%d run=%d", mkdir, chown, run) + if !(provision < chown && chown < run) { + t.Errorf("expected volume provision, then chown, then the accessory start; got order provision=%d chown=%d run=%d", provision, chown, run) } if !strings.Contains(buf.String(), "is owned by uid 1000 — reconciling") { t.Errorf("expected reconcile message, got: %s", buf.String()) @@ -178,13 +192,13 @@ func TestEnsureRunning_ReconcilesFreshVolumeOwnership(t *testing.T) { } func TestEnsureRunning_AlreadyRunning(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), // Stored credentials exist (needed for connection string). ssh.MockCommand{Match: "if [ ! -e ", Output: "present"}, ssh.MockCommand{Match: "cat -- ", Output: "POSTGRES_PASSWORD=existingpass123\n"}, // Already running. ssh.MockCommand{Match: "docker inspect", Output: "running"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -219,7 +233,7 @@ func TestEnsureRunning_AlreadyRunning(t *testing.T) { } func TestEnsureRunning_StoredCredentials(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), // Stored credentials exist. ssh.MockCommand{Match: "if [ ! -e ", Output: "present"}, ssh.MockCommand{Match: "cat -- ", Output: "POSTGRES_PASSWORD=storedpass456\n"}, @@ -229,7 +243,7 @@ func TestEnsureRunning_StoredCredentials(t *testing.T) { ssh.MockCommand{Match: "mkdir -p", Output: ""}, // Start container. ssh.MockCommand{Match: "docker run", Output: "def456"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -259,14 +273,14 @@ func TestEnsureRunning_StoredCredentials(t *testing.T) { } func TestEnsureRunning_NoEnv(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), // Not running. ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, // Create directory. ssh.MockCommand{Match: "mkdir -p", Output: ""}, // Start container. ssh.MockCommand{Match: "docker run", Output: "ghi789"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -389,7 +403,7 @@ func TestLogs(t *testing.T) { } func TestUpgrade(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), // Pull new image. ssh.MockCommand{Match: "docker pull", Output: ""}, // Stop old container. @@ -404,7 +418,7 @@ func TestUpgrade(t *testing.T) { ssh.MockCommand{Match: "mkdir -p", Output: ""}, // EnsureRunning: start new container. ssh.MockCommand{Match: "docker run", Output: "new123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -423,9 +437,15 @@ func TestUpgrade(t *testing.T) { t.Fatalf("Upgrade: %v", err) } - // Verify pull was called with new image. - if !strings.Contains(mock.Calls[0], "docker pull 'postgres:17'") { - t.Errorf("expected pull of postgres:17, got: %s", mock.Calls[0]) + // Verify pull was called with new image (after the admission surface). + var pullCmd string + for _, call := range mock.Calls { + if strings.HasPrefix(call, "docker pull") { + pullCmd = call + } + } + if !strings.Contains(pullCmd, "docker pull 'postgres:17'") { + t.Errorf("expected pull of postgres:17, got: %s", pullCmd) } // Verify new container uses new image. @@ -627,7 +647,7 @@ func TestIsImageType(t *testing.T) { } func TestEnsureRunning_SecretReference(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), // No stored credentials (first test -f), then the secret EXISTS // (second test -f) and decrypts. ssh.MockCommand{Match: "if [ ! -e ", Output: "absent", Once: true}, @@ -637,7 +657,7 @@ func TestEnsureRunning_SecretReference(t *testing.T) { ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p /deployments/myapp/accessories/nucleus", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -688,11 +708,11 @@ func TestEnsureRunning_SecretReference(t *testing.T) { } func TestEnsureRunning_SecretReferenceMissing(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "if [ ! -e ", Output: "absent"}, // Secret file does not exist. ssh.MockCommand{Match: "if [ ! -e /deployments/myapp/secrets/NUCLEUS_PASSWORD.age", Output: "absent"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -716,9 +736,9 @@ func TestEnsureRunning_SecretReferenceMissing(t *testing.T) { } func TestEnsureRunning_SecretReferenceEmptyKey(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "if [ ! -e ", Output: "absent"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) _, err := mgr.EnsureRunning(context.Background(), "myapp", "nucleus", config.AccessoryConfig{ @@ -734,11 +754,11 @@ func TestEnsureRunning_SecretReferenceEmptyKey(t *testing.T) { // containers down with it — an engine's own advisory memory budget does not // prevent that. These cover the limits actually reaching docker run. func TestEnsureRunning_ResourceLimits(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -766,11 +786,11 @@ func TestEnsureRunning_ResourceLimits(t *testing.T) { } func TestEnsureRunning_NoLimitsWhenUnset(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -790,11 +810,11 @@ func TestEnsureRunning_NoLimitsWhenUnset(t *testing.T) { // already up does nothing until it is recreated. Silence there would leave the // operator believing in a cap that does not exist. func TestEnsureRunning_WarnsWhenRunningContainerLacksLimit(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker inspect -f '{{.State.Status}}'", Output: "running"}, // HostConfig.Memory / NanoCpus: both unset on the running container. ssh.MockCommand{Match: "docker inspect -f '{{.HostConfig.Memory}}", Output: "0 0"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -814,10 +834,10 @@ func TestEnsureRunning_WarnsWhenRunningContainerLacksLimit(t *testing.T) { } func TestEnsureRunning_NoWarningWhenLimitAlreadyApplied(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker inspect -f '{{.State.Status}}'", Output: "running"}, ssh.MockCommand{Match: "docker inspect -f '{{.HostConfig.Memory}}", Output: "8589934592 0"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -836,7 +856,7 @@ func TestEnsureRunning_NoWarningWhenLimitAlreadyApplied(t *testing.T) { // root to 10001) leaves the data dir owned by the old user, and the upgraded // container crash-loops on a permission error that names nothing useful. func TestUpgrade_ReconcilesDataOwnershipOnUIDChange(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("ship"), ssh.MockCommand{Match: "docker pull", Output: ""}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker rm", Output: ""}, @@ -847,7 +867,7 @@ func TestUpgrade_ReconcilesDataOwnershipOnUIDChange(t *testing.T) { ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -874,7 +894,7 @@ func TestUpgrade_ReconcilesDataOwnershipOnUIDChange(t *testing.T) { } func TestUpgrade_LeavesOwnershipAloneWhenUIDMatches(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("ship"), ssh.MockCommand{Match: "docker pull", Output: ""}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker rm", Output: ""}, @@ -883,7 +903,7 @@ func TestUpgrade_LeavesOwnershipAloneWhenUIDMatches(t *testing.T) { ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -901,7 +921,7 @@ func TestUpgrade_LeavesOwnershipAloneWhenUIDMatches(t *testing.T) { // A root-running image must not trigger a chown — that would be teploy // rewriting ownership on every upgrade of an ordinary image. func TestUpgrade_SkipsReconcileForRootImage(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("myapp"), ssh.MockCommand{Match: "docker pull", Output: ""}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker rm", Output: ""}, @@ -909,7 +929,7 @@ func TestUpgrade_SkipsReconcileForRootImage(t *testing.T) { ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -929,12 +949,12 @@ func TestUpgrade_SkipsReconcileForRootImage(t *testing.T) { // surfaced a raw daemon error naming no accessory. Recreate it instead — the // data is on a bind mount, so the container is disposable. func TestEnsureRunning_RecreatesStoppedContainer(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("ship"), ssh.MockCommand{Match: "docker inspect -f '{{.State.Status}}'", Output: "exited"}, ssh.MockCommand{Match: "docker rm -f", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -964,10 +984,10 @@ func TestEnsureRunning_RecreatesStoppedContainer(t *testing.T) { // A running accessory must never be torn down as a side effect of a deploy. func TestEnsureRunning_NeverRemovesARunningContainer(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := ssh.NewMockExecutor("1.2.3.4", append(admissionStubs("ship"), ssh.MockCommand{Match: "docker inspect -f '{{.State.Status}}'", Output: "running"}, ssh.MockCommand{Match: "docker inspect -f '{{.HostConfig.Memory}}", Output: "0 0"}, - ) + )...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -982,3 +1002,24 @@ func TestEnsureRunning_NeverRemovesARunningContainer(t *testing.T) { } } } + +func TestAccessoryLiteralDirectiveValuesNeverResolve(t *testing.T) { + mock := ssh.NewMockExecutor("test", append(admissionStubs("demo"), + ssh.MockCommand{Match: "docker inspect", Err: fmt.Errorf("not found")}, + ssh.MockCommand{Match: "mkdir -p", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: ""})...) + mgr := NewManager(mock, &bytes.Buffer{}) + _, err := mgr.EnsureRunning(context.Background(), "demo", "db", config.AccessoryConfig{Image: "redis:7", Env: map[string]string{"AUTO": "auto", "SECRET": "secret:retired"}, EnvLiteral: map[string]string{"AUTO": "auto", "SECRET": "secret:retired"}}) + if err != nil { + t.Fatal(err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "credentials") || strings.Contains(call, "age -d") { + t.Fatalf("literal values triggered secret resolution: %s", call) + } + } + contents := string(mock.Files["/deployments/demo/accessories/db/container.env"]) + if !strings.Contains(contents, "AUTO=auto\n") || !strings.Contains(contents, "SECRET=secret:retired\n") { + t.Fatalf("literal values changed: %q", contents) + } +} diff --git a/internal/accessories/round3_regression_test.go b/internal/accessories/round3_regression_test.go new file mode 100644 index 0000000..2790539 --- /dev/null +++ b/internal/accessories/round3_regression_test.go @@ -0,0 +1,34 @@ +package accessories + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +type round3AccessoryExecutor struct{ *ssh.MockExecutor } + +func (e *round3AccessoryExecutor) UnderFence() bool { return true } +func (e *round3AccessoryExecutor) Run(ctx context.Context, command string) (string, error) { + if strings.Contains(command, "' validate ") { + return "", errors.New("populated mismatch") + } + return e.MockExecutor.Run(ctx, command) +} +func TestRound3RunningAccessoryMustAdmitOwnershipBeforeAnyEffect(t *testing.T) { + uid, gid := uint32(10001), uint32(10001) + mock := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "docker inspect", Output: "running"}) + executor := &round3AccessoryExecutor{mock} + cfg := config.AccessoryConfig{Image: "postgres:16", Volumes: map[string]string{"data": "/var/lib/postgresql/data"}, VolumeOwnership: map[string]config.VolumeOwnership{"data": {UID: &uid, GID: &gid, Mode: "0700"}}} + if _, err := NewManager(executor, io.Discard).EnsureRunning(context.Background(), "demo", "db", cfg); err == nil { + t.Fatal("running fast path bypassed ownership") + } + if len(mock.Calls) != 0 { + t.Fatalf("effect/inspection before ownership admission: %v", mock.Calls) + } +} diff --git a/internal/accessories/round5_regression_test.go b/internal/accessories/round5_regression_test.go new file mode 100644 index 0000000..8be1b48 --- /dev/null +++ b/internal/accessories/round5_regression_test.go @@ -0,0 +1,92 @@ +package accessories + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func TestRound5LogsIsPureStreamingWrapper(t *testing.T) { + mock := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "docker logs", Output: "actual-log\n"}) + var out bytes.Buffer + if err := NewManager(mock, &out).Logs(context.Background(), "demo", "db", 20); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "actual-log") { + t.Fatal("log stream lost") + } + for _, cmd := range mock.Calls { + if !strings.Contains(cmd, "docker logs") { + t.Fatalf("logs provisioned/started: %s", cmd) + } + } +} + +type round5UpgradeExecutor struct { + *ssh.MockExecutor + running bool + refuse bool +} + +func (e *round5UpgradeExecutor) Run(ctx context.Context, cmd string) (string, error) { + if e.refuse && strings.Contains(cmd, "' validate ") { + return "", errors.New("populated ownership mismatch") + } + if strings.Contains(cmd, "docker inspect") && strings.Contains(cmd, "State.Status") { + if e.running { + return "running", nil + } + return "", errors.New("absent") + } + if strings.Contains(cmd, "docker rm") { + e.running = false + } + return e.MockExecutor.Run(ctx, cmd) +} +func TestRound5UnfencedUpgradeUsesRequestedImageBeforeOwnershipEffects(t *testing.T) { + for _, refuse := range []bool{false, true} { + t.Run(map[bool]string{false: "new-image", true: "ownership-refusal"}[refuse], func(t *testing.T) { + mock := ssh.NewMockExecutor("host", append(admissionStubs("demo"), + ssh.MockCommand{Match: "docker pull", Output: ""}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker rm", Output: ""}, + ssh.MockCommand{Match: "docker run", Output: strings.Repeat("a", 64)}, + ssh.MockCommand{Match: "mkdir -p", Output: ""})...) + e := &round5UpgradeExecutor{MockExecutor: mock, running: true, refuse: refuse} + cfg := config.AccessoryConfig{Image: "image:A"} + if refuse { + uid, gid := uint32(10001), uint32(10001) + cfg.VolumeOwnership = map[string]config.VolumeOwnership{"data": {UID: &uid, GID: &gid, Mode: "0700"}} + cfg.Volumes = map[string]string{"data": "/data"} + } + err := NewManager(e, io.Discard).Upgrade(context.Background(), "demo", "db", "image:B", cfg) + if refuse && err == nil { + t.Fatal("ownership refusal bypassed") + } + if !refuse && err != nil { + t.Fatal(err) + } + pull, create := false, false + for _, cmd := range mock.Calls { + if strings.Contains(cmd, "docker pull 'image:B'") { + pull = true + } + if strings.Contains(cmd, "docker run") && strings.Contains(cmd, "'image:B'") { + create = true + } + if refuse && (strings.Contains(cmd, "docker stop") || strings.Contains(cmd, "docker rm") || strings.Contains(cmd, "docker run")) { + t.Fatalf("mutation before ownership proof: %s", cmd) + } + } + if !refuse && (!pull || !create) { + t.Fatalf("requested B not upgraded: %v", mock.Calls) + } + }) + } +} diff --git a/internal/autodeploy/autodeploy.go b/internal/autodeploy/autodeploy.go index 2d20f39..1fe7c28 100644 --- a/internal/autodeploy/autodeploy.go +++ b/internal/autodeploy/autodeploy.go @@ -476,66 +476,20 @@ func (m *Manager) Remove(ctx context.Context, app string) error { return nil } -// generateScheduledRedeployScript returns the bash script that the cron job -// invokes. It runs entirely on the server with no teploy-binary dependency: -// 1. Find the running container for this app's "web" process by labels. -// 2. Pull the image tag the container was started with. -// 3. Compare the running image digest with the just-pulled one. If they -// match, exit silently — the container is already current. -// 4. Otherwise, capture the current container's env, volumes, ports, -// labels, network, and restart policy via docker inspect, then -// stop+rm and recreate it with the same name and the new image. -// -// We keep the same container name on purpose: Caddy's reverse_proxy upstream -// resolves containers by their network alias / DNS name, and re-using the -// engine entry. The script only performs the CHEAP part — find the web -// container, pull its image tag, compare digests, and exit 0 when nothing -// changed. When the digest DID move it invokes the on-server teploy binary's -// `autodeploy redeploy`, which runs the exact same fenced, health-gated, -// release-recorded deploy code as `teploy deploy` and the webhook listener -// (C02: one execution path for every trigger). The old version of this -// script reconstructed the container from docker inspect and did its own -// stop/rm/run — no lock, no health gate, no release record, no rollback, -// and a stop-to-start downtime window. +// generateScheduledRedeployScript delegates every scheduled trigger to the +// full engine, which resolves the original configured image reference and +// handles source builds. Container Config.Image is an immutable local ID, +// not a registry reference and cannot be used as a pull preflight. func generateScheduledRedeployScript(app, branch, binaryPath string) string { return fmt.Sprintf(`#!/bin/bash -# Scheduled redeploy for %[1]s (branch %[2]s) — digest pre-check, then the full deploy engine. set -e - APP=%[4]q BRANCH=%[5]q LOG="/deployments/$APP/scheduled-redeploy.log" - ts() { date -u +%%Y-%%m-%%dT%%H:%%M:%%SZ; } - CONTAINER=$(docker ps --filter "label=teploy.app=$APP" --filter "label=teploy.process=web" --format '{{.Names}}' | head -n 1) -if [ -z "$CONTAINER" ]; then - echo "$(ts) [skip] no running container for $APP/web" >> "$LOG" - exit 0 -fi - -IMAGE=$(docker inspect --format='{{.Config.Image}}' "$CONTAINER") -if [ -z "$IMAGE" ]; then - echo "$(ts) [error] could not read image for $CONTAINER" >> "$LOG" - exit 1 -fi - -CURRENT_DIGEST=$(docker inspect --format='{{.Image}}' "$CONTAINER") - -# Pull the same tag — gets the latest digest if the upstream was updated. -if ! docker pull "$IMAGE" >> "$LOG" 2>&1; then - echo "$(ts) [error] docker pull failed for $IMAGE" >> "$LOG" - exit 1 -fi - -NEW_DIGEST=$(docker image inspect --format='{{.Id}}' "$IMAGE") - -if [ "$CURRENT_DIGEST" = "$NEW_DIGEST" ]; then - # No-op silent exit. Don't spam the log on every cron tick. - exit 0 -fi - -echo "$(ts) [redeploy] new digest for $IMAGE — running the deploy engine" >> "$LOG" +if [ -z "$CONTAINER" ]; then exit 0; fi +echo "$(ts) [redeploy] running the deploy engine" >> "$LOG" %[3]q autodeploy redeploy --app "$APP" --branch "$BRANCH" >> "$LOG" 2>&1 `, app, branch, binaryPath, app, branch) } diff --git a/internal/autodeploy/autodeploy_test.go b/internal/autodeploy/autodeploy_test.go index 26ada65..121c8ff 100644 --- a/internal/autodeploy/autodeploy_test.go +++ b/internal/autodeploy/autodeploy_test.go @@ -460,18 +460,13 @@ func TestSchedule(t *testing.T) { for _, want := range []string{ `APP="myapp"`, `BRANCH="main"`, - "docker pull", - "docker inspect", - "teploy.app=$APP", - "CURRENT_DIGEST", - "NEW_DIGEST", `"/deployments/.bin/teploy" autodeploy redeploy --app "$APP" --branch "$BRANCH"`, } { if !strings.Contains(string(script), want) { t.Errorf("scheduled-redeploy.sh missing %q", want) } } - for _, forbidden := range []string{"docker run", "docker stop", "docker rm"} { + for _, forbidden := range []string{"docker run", "docker stop", "docker rm", "docker pull", ".Config.Image"} { if strings.Contains(string(script), forbidden) { t.Errorf("scheduled-redeploy.sh must not reconstruct containers itself (%q present) — the redeploy goes through the engine", forbidden) } @@ -557,10 +552,6 @@ func TestGenerateScheduledRedeployScript(t *testing.T) { for _, want := range []string{ `APP="myapp"`, `BRANCH="release"`, - "docker pull", - "docker inspect", - "CURRENT_DIGEST", - "NEW_DIGEST", `"/deployments/.bin/teploy" autodeploy redeploy --app "$APP" --branch "$BRANCH"`, "$(ts) [redeploy]", } { @@ -570,7 +561,7 @@ func TestGenerateScheduledRedeployScript(t *testing.T) { } // C02: the script must never reconstruct the container itself — the // redeploy runs through the engine (locks, health gate, records). - for _, forbidden := range []string{"docker run", "docker stop", "docker rm"} { + for _, forbidden := range []string{"docker run", "docker stop", "docker rm", "docker pull", ".Config.Image"} { if strings.Contains(script, forbidden) { t.Errorf("scheduled redeploy script must not contain %q — container reconstruction bypasses the engine", forbidden) } diff --git a/internal/autodeploy/campaign_test.go b/internal/autodeploy/campaign_test.go new file mode 100644 index 0000000..f903027 --- /dev/null +++ b/internal/autodeploy/campaign_test.go @@ -0,0 +1,73 @@ +package autodeploy + +import ( + "github.com/useteploy/teploy/internal/ssh" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCampaignLedgerTornTailRepairAndExclusiveWriter(t *testing.T) { + path := filepath.Join(t.TempDir(), "ledger") + good := `{"kind":"admitted","id":"one"}` + "\n" + if err := os.WriteFile(path, []byte(good+`{"kind":`), 0600); err != nil { + t.Fatal(err) + } + l, err := OpenLedger(path) + if err != nil { + t.Fatal(err) + } + if second, err := OpenLedger(path); err == nil { + second.Close() + t.Fatal("concurrent writer accepted") + } + if err := l.Append(AdmissionRecord{Kind: AdmissionKindAdmitted, ID: "two"}); err != nil { + t.Fatal(err) + } + l.Close() + l, err = OpenLedger(path) + if err != nil { + t.Fatal(err) + } + defer l.Close() + b, _ := os.ReadFile(path) + records, err := ParseLedger(b) + if err != nil || len(records) != 2 || records[1].ID != "two" { + t.Fatalf("replay: %+v %v", records, err) + } +} +func TestCampaignLedgerInteriorCorruptionPreserved(t *testing.T) { + path := filepath.Join(t.TempDir(), "ledger") + bad := []byte("invalid\n") + os.WriteFile(path, bad, 0600) + if l, err := OpenLedger(path); err == nil { + l.Close() + t.Fatal("corruption admitted") + } + got, _ := os.ReadFile(path) + if string(got) != string(bad) { + t.Fatal("corrupt history modified") + } +} + +func TestCampaignScheduledScriptInvokesEngineForImmutableContainer(t *testing.T) { + dir := t.TempDir() + capture := filepath.Join(dir, "invocation") + docker := filepath.Join(dir, "docker") + os.WriteFile(docker, []byte("#!/bin/sh\n[ \"$1\" = ps ] || exit 90\nprintf 'demo-web-v1\\n'\n"), 0700) + engine := filepath.Join(dir, "engine") + os.WriteFile(engine, []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > "+ssh.ShellQuote(capture)+"\n"), 0700) + script := generateScheduledRedeployScript("demo", "main", engine) + script = strings.ReplaceAll(script, `LOG="/deployments/$APP/scheduled-redeploy.log"`, `LOG="`+filepath.Join(dir, "log")+`"`) + cmd := exec.Command("bash", "-c", script) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("script %v %s", err, out) + } + b, err := os.ReadFile(capture) + if err != nil || string(b) != "autodeploy\nredeploy\n--app\ndemo\n--branch\nmain\n" { + t.Fatalf("engine invocation %s %v", b, err) + } +} diff --git a/internal/autodeploy/ledger.go b/internal/autodeploy/ledger.go index 073e8c5..3afcf03 100644 --- a/internal/autodeploy/ledger.go +++ b/internal/autodeploy/ledger.go @@ -1,12 +1,14 @@ package autodeploy import ( + "bytes" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" + "io" "os" "sort" "strings" @@ -96,17 +98,36 @@ type LedgerAppender interface { // O_APPEND|O_CREATE (0600 — same privacy class as the dedup file) and // fsyncs after every appended line. type FileLedger struct { - path string - mu chan struct{} // binary semaphore; keeps Append a mutex-free test target - f *os.File + path string + mu chan struct{} // binary semaphore; keeps Append a mutex-free test target + f *os.File + failed error } // OpenLedger opens (or creates) the ledger at path for durable appends. func OpenLedger(path string) (*FileLedger, error) { - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600) if err != nil { return nil, fmt.Errorf("opening webhook admission ledger %s: %w", path, err) } + if err = lockLedgerFile(f); err != nil { + f.Close() + return nil, fmt.Errorf("locking admission ledger: %w", err) + } + data, err := io.ReadAll(f) + if err == nil { + _, err = ParseLedger(data) + } + if err == nil && len(data) > 0 && data[len(data)-1] != '\n' { + err = f.Truncate(int64(bytes.LastIndexByte(data, '\n') + 1)) + if err == nil { + err = f.Sync() + } + } + if err != nil { + f.Close() + return nil, fmt.Errorf("repairing admission ledger: %w", err) + } return &FileLedger{path: path, mu: make(chan struct{}, 1), f: f}, nil } @@ -120,10 +141,18 @@ func (l *FileLedger) Append(rec AdmissionRecord) error { line = append(line, '\n') l.mu <- struct{}{} + if l.failed != nil { + err := l.failed + <-l.mu + return err + } _, werr := l.f.Write(line) if werr == nil { werr = l.f.Sync() } + if werr != nil { + l.failed = werr + } <-l.mu if werr != nil { return fmt.Errorf("appending to webhook admission ledger %s: %w", l.path, werr) diff --git a/internal/autodeploy/ledger_lock_unix.go b/internal/autodeploy/ledger_lock_unix.go new file mode 100644 index 0000000..f787733 --- /dev/null +++ b/internal/autodeploy/ledger_lock_unix.go @@ -0,0 +1,10 @@ +//go:build !windows + +package autodeploy + +import ( + "golang.org/x/sys/unix" + "os" +) + +func lockLedgerFile(f *os.File) error { return unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) } diff --git a/internal/autodeploy/ledger_lock_windows.go b/internal/autodeploy/ledger_lock_windows.go new file mode 100644 index 0000000..c6902c4 --- /dev/null +++ b/internal/autodeploy/ledger_lock_windows.go @@ -0,0 +1,12 @@ +//go:build windows + +package autodeploy + +import ( + "golang.org/x/sys/windows" + "os" +) + +func lockLedgerFile(f *os.File) error { + return windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped)) +} diff --git a/internal/autodeploy/paths.go b/internal/autodeploy/paths.go index 261adb8..20ef353 100644 --- a/internal/autodeploy/paths.go +++ b/internal/autodeploy/paths.go @@ -65,7 +65,8 @@ func PushEvent(body []byte, branch string) (ok bool) { // Empty when the payload carries no usable commit: a non-push shape (the // caller has already filtered with PushEvent), a branch deletion, the // all-zero deletion marker, or a malformed hash. An empty result means -// "deploy the tip and say so", never "pin to garbage". +// "refuse authenticated-event deployment". Explicit scheduler/manual tip +// mode is admitted separately and must not infer authentication from this. func PushCommit(body []byte) string { var p pushPayload if err := json.Unmarshal(body, &p); err != nil { diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 8ac2432..b8d52ab 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -644,20 +644,21 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st // failure path DELIBERATELY keeps tmpdir for inspection, so the // credential file must be removed by name there: keep the SQL, // never the secret (C08). - execEnv := "" + client := "mysql" + if isDBType(image, "mariadb") { + client = "mariadb" + } + sqlPath := tmpdir + "/restore.sql" + receiver := "docker exec -i " + qContainer + " " + client + " -u root " + ssh.ShellQuote(db) + " < " + ssh.ShellQuote(sqlPath) if pwd := mysqlRootPassword(env); pwd != "" { envFile := tmpdir + "/mysql.env" if err := c.exec.Upload(ctx, strings.NewReader("MYSQL_PWD="+pwd+"\n"), envFile, "0600"); err != nil { - return keepTmp(fmt.Errorf("staging the mysql credential file for %s: %w", name, err)) + return keepTmp(err) } - execEnv = " --env-file " + ssh.ShellQuote(envFile) + receiver = "{ cat " + ssh.ShellQuote(envFile) + " && cat " + ssh.ShellQuote(sqlPath) + "; } | docker exec -i " + qContainer + " sh -c " + ssh.ShellQuote(mysqlPasswordPrelude+"exec "+client+" -u root "+ssh.ShellQuote(db)) } - // Same pipeline-to-redirect shape as postgres (mysql itself exits - // nonzero on SQL errors when reading a script, but gunzip's failure - // must not be masked either). - sqlPath := tmpdir + "/restore.sql" - restoreCmd = fmt.Sprintf("gunzip -c %s > %s && docker exec -i%s %s mysql -u root %s < %s", - ssh.ShellQuote(restorePath), ssh.ShellQuote(sqlPath), execEnv, qContainer, ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) + restoreCmd = "gunzip -c " + ssh.ShellQuote(restorePath) + " > " + ssh.ShellQuote(sqlPath) + " && " + receiver + // The kept-on-failure scratch dir must never keep the credential. innerKeep := keepTmp keepTmp = func(err error) error { @@ -867,28 +868,27 @@ func planAccessoryDump(app, name, image string, env map[string]string, workDir s } case isDBType(image, "mysql"), isDBType(image, "mariadb"): engine := "mysql" + client := "mysqldump" if isDBType(image, "mariadb") { engine = "mariadb" + client = "mariadb-dump" } db := mysqlDB(app, env) // Root password rides a 0600 env-file consumed by --env-file, never // argv (mysqldump argv is visible in `ps` inside the container, and // -e MYSQL_PWD puts the secret in the docker CLI's own argv on the // host). Absent = current behavior (passwordless root). - execEnv := "" var cred *planCredential + command := "docker exec " + qContainer + " " + client + " -u root " + ssh.ShellQuote(db) if pwd := mysqlRootPassword(env); pwd != "" { envFile := workDir + "/mysql.env" cred = &planCredential{path: envFile, content: "MYSQL_PWD=" + pwd + "\n"} - execEnv = " --env-file " + ssh.ShellQuote(envFile) - } - return accessoryDumpPlan{ - engine: engine, method: "mysqldump", consistency: consistencyEngineDump, ext: ".sql.gz", - artifactPath: dumpPath, - cmd: fmt.Sprintf("docker exec%s %s mysqldump -u root %s > %s && gzip -c %s > %s", - execEnv, qContainer, ssh.ShellQuote(db), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpTmp), ssh.ShellQuote(dumpPath)), - credential: cred, + inner := mysqlPasswordPrelude + "exec " + client + " -u root " + ssh.ShellQuote(db) + command = "docker exec -i " + qContainer + " sh -c " + ssh.ShellQuote(inner) + " < " + ssh.ShellQuote(envFile) } + return accessoryDumpPlan{engine: engine, method: client, consistency: consistencyEngineDump, ext: ".sql.gz", artifactPath: dumpPath, credential: cred, + cmd: command + " > " + ssh.ShellQuote(dumpTmp) + " && gzip -c " + ssh.ShellQuote(dumpTmp) + " > " + ssh.ShellQuote(dumpPath)} + case isDBType(image, "mongo"): return accessoryDumpPlan{ engine: "mongo", method: "mongodump", consistency: consistencyEngineDump, ext: ".archive.gz", @@ -962,15 +962,28 @@ func postgresRestoreCmd(container, user, db, dumpPath, sqlPath string) string { ssh.ShellQuote(dumpPath), ssh.ShellQuote(sqlPath), ssh.ShellQuote(container), ssh.ShellQuote(user), ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) } -// mysqlRestoreCmd mirrors postgresRestoreCmd for mysql/mariadb; the root -// password rides MYSQL_PWD container env, never argv (audit F22). -func mysqlRestoreCmd(container, db, pwd, dumpPath, sqlPath string) string { - execEnv := "" - if pwd != "" { - execEnv = " -e MYSQL_PWD=" + ssh.ShellQuote(pwd) +// restoreMySQL decompresses successfully before streaming a credential header and SQL. +// The password is confined to SSH stdin and the client environment, never command text. +func (c *Client) restoreMySQL(ctx context.Context, engine, container, db, pwd, dumpPath, sqlPath string) error { + if strings.ContainsAny(pwd, "\r\n") { + return fmt.Errorf("database credential contains a line break") } - return fmt.Sprintf("gunzip -c %s > %s && docker exec -i%s %s mysql -u root %s < %s", - ssh.ShellQuote(dumpPath), ssh.ShellQuote(sqlPath), execEnv, ssh.ShellQuote(container), ssh.ShellQuote(db), ssh.ShellQuote(sqlPath)) + if _, err := c.exec.Run(ctx, "umask 077; gunzip -c "+ssh.ShellQuote(dumpPath)+" > "+ssh.ShellQuote(sqlPath)); err != nil { + return err + } + client, _ := mysqlTools(engine) + receiver := "{ cat; cat " + ssh.ShellQuote(sqlPath) + "; } | docker exec -i " + ssh.ShellQuote(container) + " sh -c " + ssh.ShellQuote(mysqlPasswordPrelude+"exec "+client+" -u root "+ssh.ShellQuote(db)) + return c.exec.RunInput(ctx, receiver, strings.NewReader("MYSQL_PWD="+pwd+"\n")) +} +func mysqlTools(engine string) (string, string) { + if engine == "mariadb" { + return "mariadb", "mariadb-admin" + } + return "mysql", "mysqladmin" +} +func mysqlReadyProbe(engine string) string { + _, admin := mysqlTools(engine) + return "sh -c " + ssh.ShellQuote("MYSQL_PWD=${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-${MYSQL_PASSWORD:-}}}; export MYSQL_PWD; exec "+admin+" ping -u root --silent") } // mongoRestoreCmd streams a mongodump archive into mongorestore. @@ -1076,7 +1089,10 @@ func postgresDBAndUser(app string, env map[string]string) (db, user string) { // user, matching connectionEnvVars' own hardcoded assumption there — no // equivalent MYSQL_USER override exists to resolve. func mysqlDB(app string, env map[string]string) string { - db := env["MYSQL_DATABASE"] + db := env["MARIADB_DATABASE"] + if db == "" { + db = env["MYSQL_DATABASE"] + } if db == "" { db = app } @@ -1090,8 +1106,13 @@ func mysqlDB(app string, env map[string]string) string { // the host's `ps` output. Empty means no password configured; callers // keep the bare command. func mysqlRootPassword(env map[string]string) string { + if pwd := env["MARIADB_ROOT_PASSWORD"]; pwd != "" { + return pwd + } if pwd := env["MYSQL_ROOT_PASSWORD"]; pwd != "" { return pwd } return env["MYSQL_PASSWORD"] } + +const mysqlPasswordPrelude = `IFS= read -r teploy_credential || exit 1; MYSQL_PWD=${teploy_credential#MYSQL_PWD=}; export MYSQL_PWD; ` diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index fae7da5..3655cc0 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -303,8 +303,8 @@ func TestAccessoryBackup_MySQL_UsesRootPasswordEnv(t *testing.T) { if dumpCmd == "" { t.Fatal("expected a mysqldump command") } - if !strings.Contains(dumpCmd, "docker exec --env-file '/tmp/teploy-backup.abc123/mysql.env' 'myapp-mysql' mysqldump -u root 'myapp'") { - t.Errorf("password must ride via docker exec --env-file, got: %s", dumpCmd) + if !strings.Contains(dumpCmd, "docker exec -i 'myapp-mysql' sh -c") { + t.Errorf("password must ride via private stdin transport, got: %s", dumpCmd) } if got := string(mock.Files["/tmp/teploy-backup.abc123/mysql.env"]); got != "MYSQL_PWD=sekret\n" { t.Errorf("credential file content = %q", got) @@ -335,7 +335,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { } found := false for _, call := range mock.Calls { - if strings.Contains(call, "--env-file") { + if strings.Contains(call, "MYSQL_PWD=${teploy_credential#MYSQL_PWD=}") { found = true } if strings.Contains(call, "fall") { @@ -433,8 +433,8 @@ func TestAccessoryRestore_MySQL_UsesRootPasswordEnv(t *testing.T) { if restoreCmd == "" { t.Fatal("expected a mysql restore command") } - if !strings.Contains(restoreCmd, "docker exec -i --env-file '/tmp/teploy-restore.abc123/mysql.env' 'myapp-mysql' mysql -u root 'myapp'") { - t.Errorf("password must ride via docker exec --env-file, got: %s", restoreCmd) + if !strings.Contains(restoreCmd, "docker exec -i 'myapp-mysql' sh -c") { + t.Errorf("password must ride via private stdin transport, got: %s", restoreCmd) } if got := string(mock.Files["/tmp/teploy-restore.abc123/mysql.env"]); got != "MYSQL_PWD=sekret\n" { t.Errorf("credential file content = %q", got) diff --git a/internal/backup/bundle.go b/internal/backup/bundle.go index 6d69a0b..ff4faa3 100644 --- a/internal/backup/bundle.go +++ b/internal/backup/bundle.go @@ -3,12 +3,14 @@ package backup import ( "context" "encoding/json" + "errors" "fmt" "sort" "strings" "time" "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/releasemeta" "github.com/useteploy/teploy/internal/secret" "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" @@ -266,7 +268,7 @@ func (s S3BundleStore) ListIDs(ctx context.Context, exec ssh.Executor, app strin var ids []string for _, line := range strings.Split(strings.TrimSpace(out), "\n") { fields := strings.Fields(line) - if len(fields) < 4 || !strings.HasSuffix(fields[len(fields)-1], "/") { + if len(fields) != 2 || fields[0] != "PRE" || !strings.HasSuffix(fields[1], "/") { continue } id := strings.TrimSuffix(fields[len(fields)-1], "/") @@ -359,6 +361,9 @@ func ParseBundleManifest(data []byte) (*BundleManifest, error) { if err := ValidateDate(m.ID); err != nil { return nil, fmt.Errorf("bundle manifest id: %w", err) } + if err := validateBundlePaths(&m); err != nil { + return nil, err + } return &m, nil } @@ -433,7 +438,7 @@ func guessVolumeEngine(containerPath string) string { // CreateBundle assembles a complete DR bundle on the server and publishes it // through the store. The manifest is uploaded LAST so a partial upload can // never be mistaken for a bundle. -func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store BundleStore) (*BundleManifest, error) { +func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store BundleStore) (result *BundleManifest, retErr error) { if !safeName.MatchString(opts.App) { return nil, fmt.Errorf("invalid app name %q", opts.App) } @@ -506,7 +511,7 @@ func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store Bun var stoppedContainers []string if opts.StopApp { names, err := c.exec.Run(ctx, fmt.Sprintf( - "docker ps --filter label=teploy.app=%s --filter label=teploy.process=web --format '{{.Names}}'", + "docker ps --filter label=teploy.app=%s --filter label=teploy.process --format '{{.Names}}'", ssh.ShellQuote(opts.App))) if err != nil { cleanup() @@ -517,16 +522,22 @@ func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store Bun continue } fmt.Fprintf(c.out, "Stopping %s for quiesced snapshot...\n", n) + stoppedContainers = append(stoppedContainers, n) if _, err := c.exec.Run(ctx, "docker stop "+ssh.ShellQuote(n)); err != nil { // Restart anything already stopped before aborting. - c.restartContainers(context.WithoutCancel(ctx), stoppedContainers) + restartErr := c.restartContainers(context.WithoutCancel(ctx), stoppedContainers) cleanup() - return nil, fmt.Errorf("stopping %s for quiesced snapshot: %w", n, err) + return nil, errors.Join(fmt.Errorf("stopping %s for quiesced snapshot: %w", n, err), restartErr) } - stoppedContainers = append(stoppedContainers, n) } } - defer c.restartContainers(context.WithoutCancel(ctx), stoppedContainers) + defer func() { + recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() + if restartErr := c.restartContainers(recoveryCtx, stoppedContainers); restartErr != nil { + retErr = errors.Join(retErr, restartErr) + } + }() // Secret material — only on explicit operator selection, with the // manifest stating exactly what traveled. @@ -840,16 +851,20 @@ func (c *Client) accessoryImageEnv(ctx context.Context, app, name string, cfg co return cfg.Image, cfg.Env, nil } -func (c *Client) restartContainers(ctx context.Context, names []string) { +func (c *Client) restartContainers(ctx context.Context, names []string) error { + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + var failures []error for _, n := range names { if n == "" { continue } fmt.Fprintf(c.out, "Restarting %s...\n", n) if _, err := c.exec.Run(ctx, "docker start "+ssh.ShellQuote(n)); err != nil { - fmt.Fprintf(c.out, "WARNING: could not restart %s after bundling: %v\n", n, err) + failures = append(failures, fmt.Errorf("restarting %s after bundling: %w", n, err)) } } + return errors.Join(failures...) } // wsCopy copies a server file into the bundle workspace under member, @@ -957,3 +972,84 @@ func (c *Client) collectReleaseRecords(ctx context.Context, app string) ([]json. } return records, nil } + +func validateBundlePaths(m *BundleManifest) error { + if err := config.ValidateName(m.App); err != nil { + return err + } + seen := map[string]bool{} + for _, snap := range m.Snapshots { + if !safeName.MatchString(snap.Name) { + return fmt.Errorf("invalid snapshot name %q", snap.Name) + } + prefix := "accessories/" + if snap.Role == "app-volume" { + prefix = "volumes/" + } else if snap.Role != "accessory" { + return fmt.Errorf("invalid snapshot role %q", snap.Role) + } + valid := false + for _, ext := range []string{".tar.gz", ".sql.gz", ".archive.gz", ".rdb.gz"} { + if snap.Artifact == prefix+snap.Name+ext { + valid = true + } + } + if !valid || seen[snap.Artifact] { + return fmt.Errorf("invalid or duplicate snapshot artifact %q", snap.Artifact) + } + seen[snap.Artifact] = true + } + for _, key := range m.Secrets.Keys { + if err := secret.ValidateKey(key); err != nil { + return err + } + } + for _, member := range m.Secrets.Included { + valid := member == "env/.env" + if strings.HasPrefix(member, "secrets/") && strings.HasSuffix(member, ".age") { + key := strings.TrimSuffix(strings.TrimPrefix(member, "secrets/"), ".age") + valid = secret.ValidateKey(key) == nil + } + if strings.HasPrefix(member, "env/credentials/") { + valid = safeName.MatchString(strings.TrimPrefix(member, "env/credentials/")) + } + if !valid || seen[member] { + return fmt.Errorf("invalid or duplicate secret member %q", member) + } + seen[member] = true + } + if len(m.State) > 0 { + var st state.AppState + if err := json.Unmarshal(m.State, &st); err != nil { + return fmt.Errorf("invalid bundled state: %w", err) + } + for _, hash := range []string{st.CurrentHash, st.PreviousHash} { + if hash != "" { + if err := releasemeta.ValidateHash(hash); err != nil { + return err + } + } + } + if st.PreviousRelease != nil && st.PreviousRelease.Hash != "" { + if err := releasemeta.ValidateHash(st.PreviousRelease.Hash); err != nil { + return err + } + } + } + for _, raw := range m.ReleaseRecords { + var rec struct { + App string `json:"app"` + Hash string `json:"hash"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return fmt.Errorf("invalid bundled release record: %w", err) + } + if rec.App != "" && rec.App != m.App { + return fmt.Errorf("bundled release belongs to a different app") + } + if err := releasemeta.ValidateHash(rec.Hash); err != nil { + return err + } + } + return nil +} diff --git a/internal/backup/bundle_campaign_test.go b/internal/backup/bundle_campaign_test.go new file mode 100644 index 0000000..a4da80f --- /dev/null +++ b/internal/backup/bundle_campaign_test.go @@ -0,0 +1,136 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/useteploy/teploy/internal/ssh" + "strings" + "testing" +) + +func TestBundleManifestRejectsUnsafeMembersBeforeStaging(t *testing.T) { + mutations := []func(*BundleManifest){ + func(m *BundleManifest) { m.App = "../demo" }, + func(m *BundleManifest) { m.Snapshots[0].Name = "../escape" }, + func(m *BundleManifest) { m.Snapshots[0].Artifact = "../../escape.sql.gz" }, + func(m *BundleManifest) { m.Secrets.Included = []string{"env/credentials/../../escape"} }, + func(m *BundleManifest) { + m.ReleaseRecords = []json.RawMessage{json.RawMessage(`{"hash":"../../escape"}`)} + }, + } + for i, mutation := range mutations { + manifest := drTestManifest() + mutation(manifest) + data, _ := json.Marshal(manifest) + if _, err := ParseBundleManifest(data); err == nil { + t.Fatalf("unsafe mutation %d accepted", i) + } + } +} + +func TestS3BundleListRecognizesPrefixRows(t *testing.T) { + mock := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "aws", Output: " PRE 20260923-101500-0123456789abcdef/\n2026-09-23 10:15:00 12 manifest.json\n"}) + ids, err := (S3BundleStore{S3: S3Config{Bucket: "dummy"}}).ListIDs(context.Background(), mock, "myapp") + if err != nil { + t.Fatal(err) + } + if len(ids) != 1 || ids[0] != drTestBundleID { + t.Fatalf("prefix missing: %v", ids) + } +} + +func TestDRCutoverPublishesFreshTargetGeneration(t *testing.T) { + manifest := drTestManifest() + mock := drCutoverMock(drOKReceipt(), manifest) + mock.Files["/deployments/myapp/state.json"] = []byte(`{"schema_version":2,"generation":9,"current_hash":"v9"}`) + mock.Files["/deployments/myapp/.generation"] = []byte("12\n") + receipt, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err != nil { + t.Fatal(err) + } + if !receipt.OK { + t.Fatal("completed receipt not marked successful") + } + var got struct { + Generation uint64 `json:"generation"` + } + json.Unmarshal(mock.Files["/deployments/myapp/state.json"], &got) + if got.Generation != 13 { + t.Fatalf("historical generation reinstated: %d", got.Generation) + } + if strings.TrimSpace(string(mock.Files["/deployments/myapp/.generation"])) != "13" { + t.Fatal("sidecar disagrees") + } +} + +func TestDRCutoverPreservesLiveAgeKeyAndRefusesBeforeStop(t *testing.T) { + manifest := drTestManifest() + manifest.Secrets = SecretsRecord{Mode: "encrypted", AgeKeyIncluded: true, Included: []string{"secrets/API_KEY.age"}} + mock := drCutoverMock(drOKReceipt(), manifest, ssh.MockCommand{Match: "age -d -i '/deployments/.age-key'", Err: fmt.Errorf("wrong target key")}) + mock.Files["/deployments/.age-key"] = []byte("dummy-key-B") + _, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err == nil { + t.Fatal("undecryptable live key accepted") + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker stop") { + t.Fatalf("stopped before key proof: %s", call) + } + } + if string(mock.Files["/deployments/.age-key"]) != "dummy-key-B" { + t.Fatal("global key overwritten") + } +} + +func TestBundleStopsWorkersAndReportsRestartFailure(t *testing.T) { + mock := drBaseMock(ssh.MockCommand{Match: "docker ps --filter label=teploy.app='myapp' --filter label=teploy.process ", Output: "myapp-web\nmyapp-worker\n"}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker start 'myapp-worker'", Err: fmt.Errorf("restart refused")}, ssh.MockCommand{Match: "docker start", Output: ""}) + _, err := NewClient(mock, &bytes.Buffer{}).CreateBundle(context.Background(), BundleOptions{App: "myapp", Config: drTestConfig(), StopApp: true}, DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "myapp-worker") { + t.Fatalf("restart error hidden: %v", err) + } + found := false + for _, call := range mock.Calls { + if call == "docker stop 'myapp-worker'" { + found = true + } + } + if !found { + t.Fatal("worker kept writing during snapshot") + } +} + +func TestCutoverRefusesAlteredValidatedManifest(t *testing.T) { + manifest := drTestManifest() + mock := drCutoverMock(drOKReceipt(), manifest) + manifest.Snapshots[0].Image = "postgres:17" + changed, _ := json.Marshal(manifest) + mock.Files[DRStagingPath("myapp", manifest.ID)+"/"+strings.Repeat("b", 32)+"/manifest.json"] = changed + _, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) + if err == nil || !strings.Contains(err.Error(), "does not bind") { + t.Fatalf("changed staged manifest accepted: %v", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker stop") { + t.Fatal("effect before receipt binding refusal") + } + } +} + +func TestBundleStopErrorStillAttemptsRecoveryForUncertainTarget(t *testing.T) { + mock := drBaseMock(ssh.MockCommand{Match: "docker ps --filter label=teploy.app='myapp' --filter label=teploy.process ", Output: "myapp-web\nmyapp-worker\n"}, ssh.MockCommand{Match: "docker stop 'myapp-worker'", Err: fmt.Errorf("transport died after stop")}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker start", Output: ""}) + _, err := NewClient(mock, &bytes.Buffer{}).CreateBundle(context.Background(), BundleOptions{App: "myapp", Config: drTestConfig(), StopApp: true}, DirBundleStore{Root: "/var/drstore"}) + if err == nil { + t.Fatal("stop error hidden") + } + found := false + for _, call := range mock.Calls { + if call == "docker start 'myapp-worker'" { + found = true + } + } + if !found { + t.Fatal("uncertain stop outcome left without recovery") + } +} diff --git a/internal/backup/bundle_restore.go b/internal/backup/bundle_restore.go index 26afd01..f2f687d 100644 --- a/internal/backup/bundle_restore.go +++ b/internal/backup/bundle_restore.go @@ -2,6 +2,9 @@ package backup import ( "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -12,6 +15,7 @@ import ( "github.com/useteploy/teploy/internal/accessories" "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" "github.com/useteploy/teploy/internal/secret" "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" @@ -30,7 +34,7 @@ import ( const DRStagingRoot = "/var/tmp/teploy-dr" // RestoreReceiptSchemaVersion versions the restore receipt. -const RestoreReceiptSchemaVersion = 1 +const RestoreReceiptSchemaVersion = 2 // DRStagingPath is the deterministic staging directory for one bundle of // one app — deterministic so a later `teploy dr cutover` finds it without @@ -58,22 +62,25 @@ type DRCheckResult struct { // start to validated staging (download + extract + validation) — the // recovery-time floor before cutover. type RestoreReceipt struct { - SchemaVersion int `json:"schema_version"` - BundleID string `json:"bundle_id"` - App string `json:"app"` - StartedAt time.Time `json:"started_at"` - ValidatedAt time.Time `json:"validated_at"` - RPOSeconds int64 `json:"rpo_seconds"` - RTOSeconds int64 `json:"rto_seconds"` - StagingPath string `json:"staging_path"` - Checks []DRCheckResult `json:"checks"` - OK bool `json:"ok"` - NextStep string `json:"next_step"` + InvocationID string `json:"invocation_id"` + ManifestSHA256 string `json:"manifest_sha256"` + SchemaVersion int `json:"schema_version"` + BundleID string `json:"bundle_id"` + App string `json:"app"` + StartedAt time.Time `json:"started_at"` + ValidatedAt time.Time `json:"validated_at"` + RPOSeconds int64 `json:"rpo_seconds"` + RTOSeconds int64 `json:"rto_seconds"` + StagingPath string `json:"staging_path"` + Checks []DRCheckResult `json:"checks"` + OK bool `json:"ok"` + NextStep string `json:"next_step"` } // CutoverReceipt records what the explicit cutover changed and where the // pre-cutover originals were preserved. type CutoverReceipt struct { + OK bool `json:"ok"` SchemaVersion int `json:"schema_version"` BundleID string `json:"bundle_id"` App string `json:"app"` @@ -115,18 +122,18 @@ func (c *Client) RestoreBundleIsolated(ctx context.Context, opts BundleRestoreOp return nil, err } - staging := DRStagingPath(opts.App, manifest.ID) - if !drStagingRE.MatchString(staging) { - return nil, fmt.Errorf("internal error: staging path %q fails the safety pattern", staging) + invocation, err := drInvocationID() + if err != nil { + return nil, err } - - // ---- Staging (isolated: /var/tmp/teploy-dr/..., never /deployments). - // A re-restore of the same bundle wipes only this bundle's staging tree. - if _, err := c.exec.Run(ctx, "rm -rf "+ssh.ShellQuote(staging)); err != nil { - return nil, fmt.Errorf("clearing previous staging: %w", err) + base := DRStagingPath(opts.App, manifest.ID) + if !drStagingRE.MatchString(base) { + return nil, fmt.Errorf("invalid DR staging base") } - if _, err := c.exec.Run(ctx, "umask 077; mkdir -p "+ssh.ShellQuote(staging)); err != nil { - return nil, fmt.Errorf("creating staging dir: %w", err) + staging := base + "/" + invocation + // Exclusively create a private invocation tree; never wipe an incumbent. + if _, err := c.exec.Run(ctx, "umask 077; mkdir -p "+ssh.ShellQuote(base)+" && mkdir -m 0700 -- "+ssh.ShellQuote(staging)); err != nil { + return nil, fmt.Errorf("creating private DR invocation: %w", err) } // ---- Download every artifact named by the manifest (+ secret members). @@ -151,8 +158,12 @@ func (c *Client) RestoreBundleIsolated(ctx context.Context, opts BundleRestoreOp } // Keep the manifest itself in staging: cutover re-reads it without the // store, and the receipt records which bundle produced this tree. - if manifestBytes, err := json.MarshalIndent(manifest, "", " "); err == nil { - _ = c.exec.Upload(context.WithoutCancel(ctx), strings.NewReader(string(manifestBytes)+"\n"), staging+"/manifest.json", "0600") + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, err + } + if err := c.exec.Upload(ctx, strings.NewReader(string(manifestBytes)+"\n"), staging+"/manifest.json", "0600"); err != nil { + return nil, fmt.Errorf("persisting staged manifest: %w", err) } // ---- Integrity: every gzip artifact must prove itself BEFORE any @@ -174,12 +185,14 @@ func (c *Client) RestoreBundleIsolated(ctx context.Context, opts BundleRestoreOp } receipt := &RestoreReceipt{ - SchemaVersion: RestoreReceiptSchemaVersion, - BundleID: manifest.ID, - App: opts.App, - StartedAt: started.UTC(), - StagingPath: staging, - NextStep: fmt.Sprintf("teploy dr cutover %s", manifest.ID), + InvocationID: invocation, + ManifestSHA256: bundleManifestDigest(manifest), + SchemaVersion: RestoreReceiptSchemaVersion, + BundleID: manifest.ID, + App: opts.App, + StartedAt: started.UTC(), + StagingPath: staging, + NextStep: fmt.Sprintf("teploy dr cutover %s", manifest.ID), } // ---- Extract app volumes + generic accessory tars into staging. @@ -224,7 +237,19 @@ func (c *Client) RestoreBundleIsolated(ctx context.Context, opts BundleRestoreOp } else { fmt.Fprintf(c.out, "Isolated restore FAILED validation — staging kept at %s for inspection; nothing live was touched\n", staging) } - writeReceipt(ctx, c.exec, staging+"/receipt.json", receipt) + if err := writeReceipt(ctx, c.exec, staging+"/receipt.json", receipt); err != nil { + return receipt, err + } + // Publish only a completed receipt as the latest selection. Cutover reads + // that selection once and follows its immutable invocation, never another + // validation's mutable staging. Older invocations remain inspectable. + data, err := json.Marshal(receipt) + if err != nil { + return receipt, err + } + if err := ssh.UploadAtomic(ctx, c.exec, strings.NewReader(string(data)+"\n"), base+"/receipt.json", "0600"); err != nil { + return receipt, err + } return receipt, nil } @@ -289,47 +314,49 @@ func (c *Client) decryptProof(ctx context.Context, manifest *BundleManifest, sta if manifest.Secrets.Mode != "encrypted" || len(manifest.Secrets.Included) == 0 { return nil } - var member string - for _, inc := range manifest.Secrets.Included { - if strings.HasSuffix(inc, ".age") { - member = inc - break - } - } - if member == "" { - return nil // only .env/credentials included; no ciphertext to prove - } keyPath := deploymentsDir + "/.age-key" - if manifest.Secrets.AgeKeyIncluded { + hasKey, err := c.remoteExists(ctx, keyPath) + if err != nil { + return fmt.Errorf("checking surviving target age key: %w", err) + } + if !hasKey { + if !manifest.Secrets.AgeKeyIncluded { + return fmt.Errorf("target age key missing; restore an encrypted bundle with its age key or supply matching target credentials") + } keyPath = staging + "/age-key" } - // age -d reads the file; output goes to the shell's trash — this is a - // decryptability proof, not an extraction. - cmd := fmt.Sprintf("age -d -i %s %s >/dev/null 2>&1", ssh.ShellQuote(keyPath), ssh.ShellQuote(staging+"/"+member)) - if _, err := c.exec.Run(ctx, cmd); err != nil { - if manifest.Secrets.AgeKeyIncluded { - return fmt.Errorf("the bundled age key cannot decrypt the bundled ciphertext %s — the bundle is internally inconsistent; nothing has been touched", member) + for _, member := range manifest.Secrets.Included { + if !strings.HasPrefix(member, "secrets/") || !strings.HasSuffix(member, ".age") { + continue + } + cmd := fmt.Sprintf("age -d -i %s %s >/dev/null 2>&1", ssh.ShellQuote(keyPath), ssh.ShellQuote(staging+"/"+member)) + if _, err := c.exec.Run(ctx, cmd); err != nil { + return fmt.Errorf("the age key that will remain live cannot decrypt bundled ciphertext %s; supply secrets encrypted for the target key or restore references only; target key was preserved", member) } - return fmt.Errorf("this target's age key cannot decrypt the bundle's secret material (ciphertext %s) — the bundle was encrypted on %s; supply that host's /deployments/.age-key on this target, or restore a references-only bundle and set the secrets by hand; nothing has been touched", member, manifest.Server) } + return nil } // validateEngineSnapshot boots a scratch engine, restores the dump into it, // and checks the restored data is real. The scratch container is always torn // down. The LIVE accessory (if any) is never contacted. -func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleManifest, snap SnapshotRecord, staging string) DRCheckResult { - check := DRCheckResult{ +func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleManifest, snap SnapshotRecord, staging string) (check DRCheckResult) { + check = DRCheckResult{ Name: fmt.Sprintf("accessory:%s (%s)", snap.Name, snap.Engine), Kind: "data", } - scratch := manifest.App + "-" + snap.Name + "-drcheck" - teardown := func() { - c.exec.Run(context.WithoutCancel(ctx), "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") + invocation, err := drInvocationID() + if err != nil { + return checkFailed(check, err) } - // Remove stale scratch from an interrupted earlier run. - c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") - defer teardown() + scratch := "teploy-dr-engine-" + invocation + scratchID := "" + defer func() { + if err := c.cleanupDRScratch(ctx, scratchID, invocation); err != nil { + check = checkFailed(check, err) + } + }() dump := staging + "/" + snap.Artifact image := snap.Image @@ -346,20 +373,21 @@ func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleMan if user == "" { user = "postgres" } - if _, err := c.exec.Run(ctx, fmt.Sprintf( - "docker run -d --name %s -e POSTGRES_DB=%s -e POSTGRES_USER=%s -e POSTGRES_HOST_AUTH_METHOD=trust %s >/dev/null", - ssh.ShellQuote(scratch), ssh.ShellQuote(db), ssh.ShellQuote(user), ssh.ShellQuote(image))); err != nil { + scratchID, err = c.createDRScratch(ctx, scratch, invocation, fmt.Sprintf( + "docker run -d --name %s -e POSTGRES_DB=%s -e POSTGRES_USER=%s -e POSTGRES_HOST_AUTH_METHOD=trust %s", + ssh.ShellQuote(scratch), ssh.ShellQuote(db), ssh.ShellQuote(user), ssh.ShellQuote(image)), true) + if err != nil { return checkFailed(check, fmt.Errorf("starting scratch postgres: %w", err)) } - if err := c.waitReady(ctx, scratch, fmt.Sprintf("pg_isready -U %s", ssh.ShellQuote(user)), 30); err != nil { + if err := c.waitReady(ctx, scratchID, fmt.Sprintf("pg_isready -U %s", ssh.ShellQuote(user)), 30); err != nil { return checkFailed(check, err) } - if _, err := c.exec.Run(ctx, postgresRestoreCmd(scratch, user, db, dump, staging+"/"+snap.Name+".sql")); err != nil { + if _, err := c.exec.Run(ctx, postgresRestoreCmd(scratchID, user, db, dump, staging+"/"+snap.Name+".sql")); err != nil { return checkFailed(check, fmt.Errorf("restoring dump into scratch: %w", err)) } out, err := c.exec.Run(ctx, fmt.Sprintf( "docker exec %s psql -tA -U %s %s -c \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'\"", - ssh.ShellQuote(scratch), ssh.ShellQuote(user), ssh.ShellQuote(db))) + ssh.ShellQuote(scratchID), ssh.ShellQuote(user), ssh.ShellQuote(db))) if err != nil { return checkFailed(check, fmt.Errorf("verify query: %w", err)) } @@ -375,19 +403,27 @@ func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleMan case "mysql", "mariadb": db := snap.EngineParams["db"] - if _, err := c.exec.Run(ctx, fmt.Sprintf( - "docker run -d --name %s -e MYSQL_DATABASE=%s -e MYSQL_ALLOW_EMPTY_PASSWORD=yes %s >/dev/null", - ssh.ShellQuote(scratch), ssh.ShellQuote(db), ssh.ShellQuote(image))); err != nil { + client, _ := mysqlTools(snap.Engine) + envPrefix := "MYSQL" + emptyPassword := "MYSQL_ALLOW_EMPTY_PASSWORD" + if snap.Engine == "mariadb" { + envPrefix = "MARIADB" + emptyPassword = "MARIADB_ALLOW_EMPTY_ROOT_PASSWORD" + } + scratchID, err = c.createDRScratch(ctx, scratch, invocation, fmt.Sprintf( + "docker run -d --name %s -e "+envPrefix+"_DATABASE=%s -e "+emptyPassword+"=yes %s", + ssh.ShellQuote(scratch), ssh.ShellQuote(db), ssh.ShellQuote(image)), true) + if err != nil { return checkFailed(check, fmt.Errorf("starting scratch mysql: %w", err)) } - if err := c.waitReady(ctx, scratch, "mysqladmin ping -u root --silent", 40); err != nil { + if err := c.waitReady(ctx, scratchID, mysqlReadyProbe(snap.Engine), 40); err != nil { return checkFailed(check, err) } - if _, err := c.exec.Run(ctx, mysqlRestoreCmd(scratch, db, "", dump, staging+"/"+snap.Name+".sql")); err != nil { + if err := c.restoreMySQL(ctx, snap.Engine, scratchID, db, "", dump, staging+"/"+snap.Name+".sql"); err != nil { return checkFailed(check, fmt.Errorf("restoring dump into scratch: %w", err)) } - out, err := c.exec.Run(ctx, fmt.Sprintf("docker exec %s sh -c %s", ssh.ShellQuote(scratch), - ssh.ShellQuote(fmt.Sprintf("mysql -u root -N -e \"SHOW TABLES\" %s | wc -l", db)))) + out, err := c.exec.Run(ctx, fmt.Sprintf("docker exec %s sh -c %s", ssh.ShellQuote(scratchID), + ssh.ShellQuote(fmt.Sprintf(client+" -u root -N -e \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE()\" %s", ssh.ShellQuote(db))))) if err != nil { return checkFailed(check, fmt.Errorf("verify query: %w", err)) } @@ -402,20 +438,21 @@ func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleMan return check case "mongo": - if _, err := c.exec.Run(ctx, fmt.Sprintf("docker run -d --name %s %s >/dev/null", - ssh.ShellQuote(scratch), ssh.ShellQuote(image))); err != nil { + scratchID, err = c.createDRScratch(ctx, scratch, invocation, fmt.Sprintf("docker run -d --name %s %s", + ssh.ShellQuote(scratch), ssh.ShellQuote(image)), true) + if err != nil { return checkFailed(check, fmt.Errorf("starting scratch mongo: %w", err)) } probe := "sh -c 'mongosh --quiet --eval 1 || mongo --quiet --eval 1'" - if err := c.waitReady(ctx, scratch, probe, 30); err != nil { + if err := c.waitReady(ctx, scratchID, probe, 30); err != nil { return checkFailed(check, err) } - if _, err := c.exec.Run(ctx, mongoRestoreCmd(scratch, dump)); err != nil { + if _, err := c.exec.Run(ctx, mongoRestoreCmd(scratchID, dump)); err != nil { return checkFailed(check, fmt.Errorf("restoring dump into scratch: %w", err)) } out, err := c.exec.Run(ctx, fmt.Sprintf( "docker exec %s sh -c 'mongosh --quiet --eval \"db.getMongo().getDBNames().length\" || mongo --quiet --eval \"db.getMongo().getDBNames().length\"'", - ssh.ShellQuote(scratch))) + ssh.ShellQuote(scratchID))) if err != nil { return checkFailed(check, fmt.Errorf("verify query: %w", err)) } @@ -428,16 +465,21 @@ func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleMan // start it: redis boots FROM the backup, and a corrupt rdb fails the // readiness probe (same shape as verify.go). rdb := staging + "/" + snap.Name + ".rdb" - if _, err := c.exec.Run(ctx, fmt.Sprintf( - "gunzip -c %s > %s && docker create --name %s %s >/dev/null && docker cp %s %s:/data/dump.rdb && docker start %s >/dev/null", - ssh.ShellQuote(dump), ssh.ShellQuote(rdb), ssh.ShellQuote(scratch), ssh.ShellQuote(image), - ssh.ShellQuote(rdb), ssh.ShellQuote(scratch), ssh.ShellQuote(scratch))); err != nil { - return checkFailed(check, fmt.Errorf("seeding scratch redis: %w", err)) + if _, err := c.exec.Run(ctx, "umask 077; gunzip -c "+ssh.ShellQuote(dump)+" > "+ssh.ShellQuote(rdb)); err != nil { + return checkFailed(check, err) } - if err := c.waitReady(ctx, scratch, "redis-cli ping", 20); err != nil { + scratchID, err = c.createDRScratch(ctx, scratch, invocation, "docker create --name "+ssh.ShellQuote(scratch)+" "+ssh.ShellQuote(image), false) + if err != nil { return checkFailed(check, err) } - out, err := c.exec.Run(ctx, fmt.Sprintf("docker exec %s redis-cli dbsize", ssh.ShellQuote(scratch))) + if _, err := c.exec.Run(ctx, "docker cp "+ssh.ShellQuote(rdb)+" "+ssh.ShellQuote(scratchID+":/data/dump.rdb")+" && docker start "+ssh.ShellQuote(scratchID)); err != nil { + return checkFailed(check, err) + } + + if err := c.waitReady(ctx, scratchID, "redis-cli ping", 20); err != nil { + return checkFailed(check, err) + } + out, err := c.exec.Run(ctx, fmt.Sprintf("docker exec %s redis-cli dbsize", ssh.ShellQuote(scratchID))) if err != nil { return checkFailed(check, fmt.Errorf("verify query: %w", err)) } @@ -456,8 +498,8 @@ func (c *Client) validateEngineSnapshot(ctx context.Context, manifest *BundleMan // bundled env (when the operator included it) and reports whether it reaches // running state. It is isolated: no network alias, no published port, no // Caddy involvement — it can never receive production traffic. -func (c *Client) appCheck(ctx context.Context, manifest *BundleManifest, opts BundleRestoreOptions, staging string) DRCheckResult { - check := DRCheckResult{Name: "app", Kind: "app"} +func (c *Client) appCheck(ctx context.Context, manifest *BundleManifest, opts BundleRestoreOptions, staging string) (check DRCheckResult) { + check = DRCheckResult{Name: "app", Kind: "app"} var appState state.AppState if err := json.Unmarshal(manifest.State, &appState); err != nil { @@ -474,9 +516,17 @@ func (c *Client) appCheck(ctx context.Context, manifest *BundleManifest, opts Bu check.Detail = "bundle carries no deployed image reference — application check skipped (data checks still ran)" return check } - scratch := manifest.App + "-dr-appcheck" - c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") - defer c.exec.Run(context.WithoutCancel(ctx), "docker rm -f "+ssh.ShellQuote(scratch)+" >/dev/null 2>&1 || true") + invocation, err := drInvocationID() + if err != nil { + return checkFailed(check, err) + } + scratch := "teploy-dr-app-" + invocation + scratchID := "" + defer func() { + if err := c.cleanupDRScratch(ctx, scratchID, invocation); err != nil { + check = checkFailed(check, err) + } + }() args := []string{"docker", "run", "-d", "--name", ssh.ShellQuote(scratch), "--label", "teploy.app=" + manifest.App, "--label", "teploy.role=dr-appcheck"} @@ -495,7 +545,8 @@ func (c *Client) appCheck(ctx context.Context, manifest *BundleManifest, opts Bu args = append(args, ssh.ShellQuote(w)) } - if _, err := c.exec.Run(ctx, strings.Join(args, " ")); err != nil { + scratchID, err = c.createDRScratch(ctx, scratch, invocation, strings.Join(args, " "), true) + if err != nil { // An unavailable image is a SKIP with the reason, not a silent // pass: the operator sees exactly what was not proven. if strings.Contains(err.Error(), "Unable to find image") || strings.Contains(err.Error(), "pull access denied") || strings.Contains(err.Error(), "not found") { @@ -508,7 +559,7 @@ func (c *Client) appCheck(ctx context.Context, manifest *BundleManifest, opts Bu // Bounded wait for "running": docker ps filtering by exact name. waitCmd := fmt.Sprintf( "for i in $(seq 1 15); do s=$(docker inspect -f '{{.State.Status}}' %s 2>/dev/null) && [ \"$s\" = running ] && exit 0; sleep 2; done; docker logs --tail 20 %s >&2; exit 1", - ssh.ShellQuote(scratch), ssh.ShellQuote(scratch)) + ssh.ShellQuote(scratchID), ssh.ShellQuote(scratchID)) if _, err := c.exec.Run(ctx, waitCmd); err != nil { return checkFailed(check, fmt.Errorf("app container never reached running state: %w", err)) } @@ -560,6 +611,25 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( if err := json.Unmarshal(receiptData, &receipt); err != nil { return nil, fmt.Errorf("parsing the staged restore receipt: %w", err) } + if receipt.SchemaVersion != RestoreReceiptSchemaVersion || !drInvocationRE.MatchString(receipt.InvocationID) || receipt.StagingPath != staging+"/"+receipt.InvocationID { + return nil, fmt.Errorf("restore receipt lacks private invocation identity; re-restore before cutover") + } + staging = receipt.StagingPath + immutableReceipt, present, err := state.ReadRemoteFile(ctx, c.exec, staging+"/receipt.json") + if err != nil || !present { + return nil, fmt.Errorf("private invocation receipt unavailable: %v", err) + } + var selected RestoreReceipt + if err := json.Unmarshal(immutableReceipt, &selected); err != nil { + return nil, err + } + { // compare canonical records, including checks + left, _ := json.Marshal(receipt) + right, _ := json.Marshal(selected) + if string(left) != string(right) { + return nil, fmt.Errorf("selected invocation receipt changed; re-restore") + } + } if !receipt.OK { return nil, fmt.Errorf("the staged restore for bundle %s FAILED validation — cutover refused; inspect %s or re-run teploy dr restore", opts.ID, staging) } @@ -572,6 +642,12 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( if err != nil { return nil, err } + if receipt.BundleID != opts.ID || receipt.App != opts.App || receipt.ManifestSHA256 == "" || receipt.ManifestSHA256 != bundleManifestDigest(manifest) { + return nil, fmt.Errorf("validated receipt does not bind this staged manifest; run teploy dr restore again before cutover") + } + if manifest.ID != opts.ID { + return nil, fmt.Errorf("staged bundle id does not match requested cutover") + } if manifest.App != opts.App { return nil, fmt.Errorf("staged bundle belongs to app %q, not %q", manifest.App, opts.App) } @@ -608,6 +684,40 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( return nil, fmt.Errorf("acquiring the app lock for cutover: %w", err) } defer state.ReleaseLockFenced(c.exec, lk, opts.App) + lk.StartRenewal(c.exec) + current, err := state.Read(ctx, c.exec, opts.App) + if err != nil { + return nil, err + } + baseGeneration, err := state.ReadCommittedGeneration(ctx, c.exec, opts.App) + if err != nil { + return nil, err + } + if current != nil && current.Generation > baseGeneration { + baseGeneration = current.Generation + } + if baseGeneration == ^uint64(0) { + return nil, fmt.Errorf("target generation exhausted") + } + dk := docker.NewClient(&state.FencedExecutor{Executor: c.exec, Lock: lk}) + for name, permission := range opts.Config.VolumeOwnership { + if err := dk.ValidateManagedVolume(ctx, opts.App, "", name, permission); err != nil { + return nil, err + } + } + for accessory, cfg := range opts.Config.Accessories { + for name, permission := range cfg.VolumeOwnership { + if err := dk.ValidateManagedVolume(ctx, opts.App, accessory, name, permission); err != nil { + return nil, err + } + } + } + if err := c.ensureBundleAgeKey(ctx, manifest, staging); err != nil { + return nil, err + } + if err := c.decryptProof(ctx, manifest, staging); err != nil { + return nil, err + } // Stop live containers (web + accessories). They are STOPPED, not // removed — an aborted cutover restarts them. @@ -622,11 +732,11 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( continue } fmt.Fprintf(c.out, "Stopping %s...\n", n) + stopped = append(stopped, n) if _, err := c.exec.Run(ctx, "docker stop "+ssh.ShellQuote(n)); err != nil { - c.restartContainers(context.WithoutCancel(ctx), stopped) - return nil, fmt.Errorf("stopping %s for cutover (it was left as-is): %w", n, err) + restartErr := c.restartContainers(context.WithoutCancel(ctx), stopped) + return nil, errors.Join(fmt.Errorf("stopping %s for cutover: %w; recovery restart attempted", n, err), restartErr) } - stopped = append(stopped, n) } out.Stopped = stopped @@ -635,9 +745,12 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( // state is not left dark; staged data and recovery dirs are KEPT // and named. If a promotion already happened for some path, the // receipt lists it and the error says exactly where originals are. - c.restartContainers(context.WithoutCancel(ctx), stopped) - writeReceipt(context.WithoutCancel(ctx), c.exec, staging+"/cutover-receipt.json", out) - return out, fmt.Errorf("%w — staged data kept at %s, pre-cutover originals kept in %v; stopped containers were restarted", err, staging, out.RecoveryDirs) + if state.PreservePublishedState(err) { + return out, fmt.Errorf("cutover state publication needs reconciliation; workload preserved: %w", err) + } + restartErr := c.restartContainers(context.WithoutCancel(ctx), stopped) + receiptErr := writeReceipt(context.WithoutCancel(ctx), c.exec, staging+"/cutover-receipt.json", out) + return out, fmt.Errorf("%w — staged data kept at %s, originals kept in %v; recovery restart attempted", errors.Join(err, restartErr, receiptErr), staging, out.RecoveryDirs) } // Engine-dump accessories first: their data lands through the engine, @@ -702,7 +815,7 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( } // Start the accessory from config (fresh dirs), then pipe the dump. - mgr := accessories.NewManager(c.exec, c.out) + mgr := accessories.NewManager(&state.FencedExecutor{Executor: c.exec, Lock: lk}, c.out) env, err := mgr.EnsureRunning(ctx, opts.App, snap.Name, accCfg) if err != nil { return fail(fmt.Errorf("starting accessory %s for cutover: %w", snap.Name, err)) @@ -720,10 +833,10 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( } case "mysql", "mariadb": db := mysqlDB(opts.App, engineEnv(snap, env, accCfg)) - if err := c.waitReady(ctx, container, "mysqladmin ping -u root --silent", 40); err != nil { + if err := c.waitReady(ctx, container, mysqlReadyProbe(snap.Engine), 40); err != nil { restoreErr = err } else { - _, restoreErr = c.exec.Run(ctx, mysqlRestoreCmd(container, db, mysqlRootPassword(engineEnv(snap, env, accCfg)), dump, staging+"/"+snap.Name+".cutover.sql")) + restoreErr = c.restoreMySQL(ctx, snap.Engine, container, db, mysqlRootPassword(engineEnv(snap, env, accCfg)), dump, staging+"/"+snap.Name+".cutover.sql") } case "mongo": _, restoreErr = c.exec.Run(ctx, mongoRestoreCmd(container, dump)) @@ -776,12 +889,15 @@ func (c *Client) CutoverBundle(ctx context.Context, opts BundleRestoreOptions) ( // State, release records, and secret material land LAST — after all // data promotions succeeded, so a failed promotion never leaves the // target claiming a generation it does not have. - if err := c.installBundleState(ctx, manifest, staging); err != nil { + if err := c.installBundleState(ctx, manifest, staging, lk, baseGeneration); err != nil { return fail(fmt.Errorf("installing restored state: %w", err)) } out.CompletedAt = timeNow(opts.Now).UTC() - writeReceipt(ctx, c.exec, staging+"/cutover-receipt.json", out) + out.OK = true + if err := writeReceipt(ctx, c.exec, staging+"/cutover-receipt.json", out); err != nil { + return out, fmt.Errorf("cutover completed but receipt could not be persisted: %w", err) + } fmt.Fprintf(c.out, "Cutover complete. Pre-cutover originals kept in %v\n", out.RecoveryDirs) fmt.Fprintf(c.out, "Next: %s\n", out.NextStep) return out, nil @@ -832,6 +948,13 @@ func engineEnv(snap SnapshotRecord, liveEnv map[string]string, cfg config.Access for k, v := range liveEnv { merged[k] = v // resolved values (auto passwords, DATABASE_URL) win } + // env_literal values are the operator's exact bytes: they win over both, + // exactly as the accessory container itself applies them (a MariaDB root + // password with directive-looking bytes can only be declared literally, + // and restoring its dump without it would authenticate as nobody). + for k, v := range cfg.EnvLiteral { + merged[k] = v + } if p := snap.EngineParams; p != nil { if db := p["db"]; db != "" && merged["POSTGRES_DB"] == "" && merged["MYSQL_DATABASE"] == "" { switch snap.Engine { @@ -847,14 +970,11 @@ func engineEnv(snap SnapshotRecord, liveEnv map[string]string, cfg config.Access // installBundleState writes the bundle's state.json, release records and // secret material into /deployments/ — atomically per file, 0600. -func (c *Client) installBundleState(ctx context.Context, manifest *BundleManifest, staging string) error { +func (c *Client) installBundleState(ctx context.Context, manifest *BundleManifest, staging string, lk *state.Lock, baseGeneration uint64) error { appDir := fmt.Sprintf("%s/%s", deploymentsDir, manifest.App) if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(appDir+"/meta")+" "+ssh.ShellQuote(appDir+"/secrets")); err != nil { return fmt.Errorf("preparing state dirs: %w", err) } - if err := c.exec.Upload(ctx, strings.NewReader(string(manifest.State)+"\n"), appDir+"/state.json", "0600"); err != nil { - return fmt.Errorf("writing restored state.json: %w", err) - } for _, rec := range manifest.ReleaseRecords { var r struct { Hash string `json:"hash"` @@ -897,13 +1017,24 @@ func (c *Client) installBundleState(ctx context.Context, manifest *BundleManifes return fmt.Errorf("checking the target age key: %w", err) } if !hasKey { - if _, err := c.exec.Run(ctx, fmt.Sprintf("cp -p %s %s && chmod 600 %s", - ssh.ShellQuote(staging+"/age-key"), ssh.ShellQuote(deploymentsDir+"/.age-key"), ssh.ShellQuote(deploymentsDir+"/.age-key"))); err != nil { + if _, err := c.exec.Run(ctx, fmt.Sprintf("umask 077; tmp=$(mktemp /deployments/.age-key.XXXXXXXX) || exit 1; trap 'rm -f -- \"$tmp\"' EXIT; cp %s \"$tmp\" && chmod 600 \"$tmp\" && ln \"$tmp\" %s", + ssh.ShellQuote(staging+"/age-key"), ssh.ShellQuote(deploymentsDir+"/.age-key"))); err != nil { return fmt.Errorf("installing the bundled age key: %w", err) } } } } + var restored state.AppState + if err := json.Unmarshal(manifest.State, &restored); err != nil { + return err + } + restored.Generation = baseGeneration + 1 + restored.OperationID = "dr-cutover-" + manifest.ID + restored.UpdatedAt = time.Now().UTC() + if err := state.WriteFencedGeneration(ctx, c.exec, manifest.App, &restored, lk, baseGeneration); err != nil { + return fmt.Errorf("publishing restored state as target generation %d: %w", restored.Generation, err) + } + return nil } @@ -911,16 +1042,17 @@ func (c *Client) failReceipt(ctx context.Context, receipt *RestoreReceipt, err e receipt.Checks = append(receipt.Checks, DRCheckResult{Name: "restore", Kind: "data", Status: "fail", Detail: err.Error()}) receipt.ValidatedAt = time.Now().UTC() receipt.OK = false - writeReceipt(context.WithoutCancel(ctx), c.exec, receipt.StagingPath+"/receipt.json", receipt) - return err + return errors.Join(err, writeReceipt(context.WithoutCancel(ctx), c.exec, receipt.StagingPath+"/receipt.json", receipt)) } -func writeReceipt(ctx context.Context, exec ssh.Executor, path string, v any) { +func writeReceipt(ctx context.Context, exec ssh.Executor, path string, v any) error { data, err := json.MarshalIndent(v, "", " ") if err != nil { - return + return err } - _ = exec.Upload(context.WithoutCancel(ctx), strings.NewReader(string(data)+"\n"), path, "0600") + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return exec.Upload(ctx, strings.NewReader(string(data)+"\n"), path, "0600") } func allPassed(checks []DRCheckResult) bool { @@ -955,3 +1087,75 @@ func timeNow(f func() time.Time) time.Time { } return time.Now() } + +// Publish a missing bundled key before stopping workloads, without replacing +// an existing global key used by another app. A concurrent publisher causes a +// refusal before cutover; retry then proves ciphertext with the surviving key. +func (c *Client) ensureBundleAgeKey(ctx context.Context, manifest *BundleManifest, staging string) error { + if manifest.Secrets.Mode != "encrypted" || !manifest.Secrets.AgeKeyIncluded { + return nil + } + hasKey, err := c.remoteExists(ctx, deploymentsDir+"/.age-key") + if err != nil { + return err + } + if hasKey { + return nil + } + cmd := fmt.Sprintf("umask 077; tmp=$(mktemp /deployments/.age-key.XXXXXXXX) || exit 1; trap 'rm -f -- \"$tmp\"' EXIT; cp %s \"$tmp\" && chmod 600 \"$tmp\" && ln \"$tmp\" %s", ssh.ShellQuote(staging+"/age-key"), ssh.ShellQuote(deploymentsDir+"/.age-key")) + if _, err := c.exec.Run(ctx, cmd); err != nil { + return fmt.Errorf("publishing bundled age key without overwriting an existing key: %w", err) + } + return nil +} + +func bundleManifestDigest(manifest *BundleManifest) string { + data, err := json.Marshal(manifest) + if err != nil { + return "" + } + sum := sha256.Sum256(data) + return fmt.Sprintf("%x", sum[:]) +} + +var drInvocationRE = regexp.MustCompile(`^[0-9a-f]{32}$`) +var drContainerIDRE = regexp.MustCompile(`^[0-9a-f]{64}$`) + +func drInvocationID() (string, error) { + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return "", err + } + return hex.EncodeToString(nonce[:]), nil +} +func (c *Client) createDRScratch(ctx context.Context, name, owner, command string, start bool) (string, error) { + command = strings.Replace(command, "docker run -d", "docker create", 1) + command = strings.TrimSuffix(command, " >/dev/null") + nameFlag := "--name " + ssh.ShellQuote(name) + command = strings.Replace(command, nameFlag, nameFlag+" --label "+ssh.ShellQuote("teploy.dr-owner="+owner)+" --label teploy.role=dr-check", 1) + output, err := c.exec.Run(ctx, command) + if err != nil { + return "", fmt.Errorf("scratch create outcome unknown (name %s, owner %s); no incumbent removed: %w", name, owner, err) + } + id := strings.TrimSpace(output) + if !drContainerIDRE.MatchString(id) { + return "", fmt.Errorf("scratch create returned no immutable ID (%s); inspect owner %s", name, owner) + } + if start { + _, err = c.exec.Run(ctx, "docker start "+ssh.ShellQuote(id)) + } + return id, err +} +func (c *Client) cleanupDRScratch(ctx context.Context, id, owner string) error { + if id == "" { + return nil + } + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + actual, err := c.exec.Run(cleanup, "docker inspect -f '{{index .Config.Labels \"teploy.dr-owner\"}}' "+ssh.ShellQuote(id)) + if err != nil || strings.TrimSpace(actual) != owner { + return fmt.Errorf("scratch ownership unproven for %s; cleanup withheld: %v", id, err) + } + _, err = c.exec.Run(cleanup, "docker rm -f "+ssh.ShellQuote(id)) + return err +} diff --git a/internal/backup/bundle_restore_test.go b/internal/backup/bundle_restore_test.go index 72c9014..56359f4 100644 --- a/internal/backup/bundle_restore_test.go +++ b/internal/backup/bundle_restore_test.go @@ -49,10 +49,15 @@ func drRestoreMock(manifest *BundleManifest, extra ...ssh.MockCommand) *ssh.Mock {Match: "gzip -t", Output: ""}, {Match: "tar -xzf", Output: ""}, {Match: "docker rm -f", Output: ""}, - {Match: "docker run -d", Output: "cid123\n"}, + {Match: "docker create", Output: strings.Repeat("a", 64)}, + {Match: "docker start", Output: ""}, + {Match: "docker inspect -f '{{index .Config.Labels", Output: ""}, {Match: "for i in $(seq", Output: ""}, + {Match: "umask 077; gunzip -c", Output: ""}, {Match: "gunzip -c", Output: ""}, - {Match: "docker exec 'myapp-db-drcheck' psql -tA", Output: "3\n"}, + {Match: "docker exec '" + strings.Repeat("a", 64) + "' psql -tA", Output: "3\n"}, + // r2-03 private stdin import: SQL + credential header piped to exec -i. + {Match: "{ cat; cat ", Output: ""}, }...)...) // The store's manifest, readable through the framed auto-answer. mock.Files["/var/drstore/myapp/dr/"+manifest.ID+"/manifest.json"] = manifestJSON @@ -128,6 +133,7 @@ func TestRestoreBundleIsolated_CorruptBundleFailsBeforeEngines(t *testing.T) { func TestRestoreBundleIsolated_ReceiptRPORTO(t *testing.T) { manifest := drTestManifest() mock := drRestoreMock(manifest) + executor := &round3DRExecutor{MockExecutor: mock, owners: map[string]string{}} start := time.Date(2026, 9, 23, 11, 0, 0, 0, time.UTC) validated := start.Add(90 * time.Second) calls := 0 @@ -139,7 +145,7 @@ func TestRestoreBundleIsolated_ReceiptRPORTO(t *testing.T) { return validated } - client := NewClient(mock, &bytes.Buffer{}) + client := NewClient(executor, &bytes.Buffer{}) receipt, err := client.RestoreBundleIsolated(context.Background(), BundleRestoreOptions{ App: "myapp", ID: manifest.ID, Config: drTestConfig(), Now: now, }, DirBundleStore{Root: "/var/drstore"}) @@ -156,7 +162,7 @@ func TestRestoreBundleIsolated_ReceiptRPORTO(t *testing.T) { if receipt.RTOSeconds != 90 { t.Errorf("RTO = %d, want 90", receipt.RTOSeconds) } - if receipt.StagingPath != DRStagingPath("myapp", manifest.ID) { + if receipt.StagingPath != DRStagingPath("myapp", manifest.ID)+"/"+receipt.InvocationID { t.Errorf("staging path: %s", receipt.StagingPath) } var dataCheck, appCheck bool @@ -173,8 +179,8 @@ func TestRestoreBundleIsolated_ReceiptRPORTO(t *testing.T) { } // Scratch teardown happened. var sawTeardown bool - for _, call := range mock.Calls { - if strings.Contains(call, "docker rm -f 'myapp-db-drcheck'") { + for _, call := range executor.commands { + if strings.HasPrefix(call, "docker rm -f '") { sawTeardown = true } if strings.Contains(call, "/deployments/myapp") { @@ -215,12 +221,20 @@ func TestRestoreBundleIsolated_EncryptedWithoutKeyFailsBeforeEngines(t *testing. // ---- Cutover ---- func drCutoverMock(receipt *RestoreReceipt, manifest *BundleManifest, extra ...ssh.MockCommand) *ssh.MockExecutor { - staging := DRStagingPath("myapp", manifest.ID) + base := DRStagingPath("myapp", manifest.ID) + staging := base + "/" + strings.Repeat("b", 32) + receipt.InvocationID = strings.Repeat("b", 32) + receipt.StagingPath = staging + if receipt.ManifestSHA256 == "" { + receipt.ManifestSHA256 = bundleManifestDigest(manifest) + } receiptJSON, _ := json.Marshal(receipt) manifestJSON, _ := json.Marshal(manifest) // extras are PREPENDED so injected failures outrank the base successes. mock := ssh.NewMockExecutor("dr-host", append(extra, []ssh.MockCommand{ {Match: "mkdir /deployments/myapp/.lock", Output: ""}, + // Accessory admission (R2-06) runs before any cutover effect. + {Match: "teploy_volume_actor", Output: ""}, {Match: "docker ps --filter label=teploy.app='myapp'", Output: "myapp-web\nmyapp-db\n"}, {Match: "docker stop", Output: ""}, {Match: "docker start", Output: ""}, @@ -232,7 +246,10 @@ func drCutoverMock(receipt *RestoreReceipt, manifest *BundleManifest, extra ...s {Match: "docker image inspect", Output: "\n"}, {Match: "docker run --detach", Output: ""}, {Match: "for i in $(seq", Output: ""}, + {Match: "umask 077; gunzip -c", Output: ""}, {Match: "gunzip -c", Output: ""}, + // r2-03 private stdin import: SQL + credential header piped to exec -i. + {Match: "{ cat; cat ", Output: ""}, {Match: "mkdir -p '/deployments/myapp/volumes/data'", Output: ""}, {Match: "mktemp -d '/deployments", Output: "/deployments/myapp/recovery123\n"}, {Match: "find ", Output: ""}, @@ -243,6 +260,7 @@ func drCutoverMock(receipt *RestoreReceipt, manifest *BundleManifest, extra ...s {Match: "mkdir -p '/deployments/myapp/meta' '/deployments/myapp/secrets'", Output: ""}, {Match: "mkdir -p /deployments/myapp/meta", Output: ""}, }...)...) + mock.Files[base+"/receipt.json"] = receiptJSON mock.Files[staging+"/receipt.json"] = receiptJSON mock.Files[staging+"/manifest.json"] = manifestJSON return mock @@ -385,7 +403,7 @@ func TestCutover_HappyPath(t *testing.T) { t.Errorf("promoted paths: %v", receipt.Promoted) } stateBytes, ok := mock.Files["/deployments/myapp/state.json"] - if !ok || !strings.Contains(string(stateBytes), `"generation":4`) { + if !ok || !strings.Contains(string(stateBytes), `"generation":1`) { t.Errorf("restored state.json not installed: %q", string(stateBytes)) } if _, ok := mock.Files["/deployments/myapp/meta/rel-77.json"]; !ok { @@ -413,7 +431,7 @@ func TestCutover_NonEmptyAccessoryDirWithoutVolumeData(t *testing.T) { t.Fatalf("CutoverBundle: %v", err) } for _, call := range mock.Calls { - if strings.HasPrefix(call, "mv -f ") || strings.Contains(call, "docker run --rm --user 0") { + if (strings.HasPrefix(call, "mv -f ") && !strings.Contains(call, "state.json.tmp-")) || strings.Contains(call, "docker run --rm --user 0") { t.Errorf("nothing existed to move aside, yet a move ran: %s", call) } if strings.Contains(call, "pre-cutover.XXXXXX") { diff --git a/internal/backup/bundle_test.go b/internal/backup/bundle_test.go index 48d6443..6a5aa88 100644 --- a/internal/backup/bundle_test.go +++ b/internal/backup/bundle_test.go @@ -181,7 +181,7 @@ func TestCreateBundle_VolumePatternHeuristic(t *testing.T) { // snapshot is then labeled quiesced with detection=teploy-stop. func TestCreateBundle_StopAppQuiescesVolumes(t *testing.T) { mock := drBaseMock( - ssh.MockCommand{Match: "docker ps --filter label=teploy.app='myapp' --filter label=teploy.process=web", Output: "myapp-web\n"}, + ssh.MockCommand{Match: "docker ps --filter label=teploy.app='myapp' --filter label=teploy.process", Output: "myapp-web\n"}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker start", Output: ""}, ) diff --git a/internal/backup/dr_integration_test.go b/internal/backup/dr_integration_test.go index 179f18e..a7ff695 100644 --- a/internal/backup/dr_integration_test.go +++ b/internal/backup/dr_integration_test.go @@ -342,8 +342,8 @@ func TestDRIntegration_CutoverCopyFailurePreservesOriginals(t *testing.T) { // Inject the copy failure: the staged tree becomes unreadable, so // promoteStaged's `cp -a` fails AFTER the originals were moved aside — // exactly the mid-restore partial-copy window. - run("chmod 000 " + ssh.ShellQuote(DRStagingPath(drFixtureApp, manifest.ID)+"/volumes/data")) - defer run("chmod -R 700 " + ssh.ShellQuote(DRStagingPath(drFixtureApp, manifest.ID)) + " >/dev/null 2>&1 || true") + run("chmod 000 " + ssh.ShellQuote(receipt.StagingPath+"/volumes/data")) + defer run("chmod -R 700 " + ssh.ShellQuote(receipt.StagingPath) + " >/dev/null 2>&1 || true") _, err = client.CutoverBundle(ctx, BundleRestoreOptions{App: drFixtureApp, ID: manifest.ID, Config: cfg}) if err == nil || !strings.Contains(err.Error(), "promoting data at cutover") { diff --git a/internal/backup/round3_regression_test.go b/internal/backup/round3_regression_test.go new file mode 100644 index 0000000..795e297 --- /dev/null +++ b/internal/backup/round3_regression_test.go @@ -0,0 +1,152 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" + "sync" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +// Owns only containers it created. Legal incumbent accessory names stay in +// the same world and any attempt to remove them is a fixture failure. +type round3DRExecutor struct { + *ssh.MockExecutor + mu sync.Mutex + commands []string + owners map[string]string +} + +func (e *round3DRExecutor) Run(ctx context.Context, command string) (string, error) { + e.mu.Lock() + defer e.mu.Unlock() + e.commands = append(e.commands, command) + if strings.HasPrefix(command, "docker create ") { + owner := regexp.MustCompile(`teploy.dr-owner=([0-9a-f]{32})`).FindStringSubmatch(command) + if len(owner) != 2 { + return "", errors.New("scratch lacks owner") + } + id := fmt.Sprintf("%064x", len(e.owners)+1) + e.owners[id] = owner[1] + return id, nil + } + if strings.HasPrefix(command, "docker inspect -f '{{index .Config.Labels") { + for id, owner := range e.owners { + if strings.HasSuffix(command, "'"+id+"'") { + return owner, nil + } + } + return "", errors.New("unknown scratch ID") + } + if strings.HasPrefix(command, "docker rm -f ") { + for id := range e.owners { + if strings.HasSuffix(command, "'"+id+"'") { + return "", nil + } + } + return "", fmt.Errorf("attempted incumbent/unknown cleanup: %s", command) + } + if strings.Contains(command, "SELECT COUNT(*)") { + return "3", nil + } + return e.MockExecutor.Run(ctx, command) +} +func TestRound3DRNativeMariaDBAndPasswordStdinAtCutover(t *testing.T) { + manifest := drTestManifest() + manifest.Snapshots = manifest.Snapshots[:1] + manifest.Snapshots[0].Engine = "mariadb" + manifest.Snapshots[0].Method = "mariadb-dump" + manifest.Snapshots[0].Image = "mariadb:11" + cfg := drTestConfig() + cfg.Volumes = nil + cfg.Accessories = map[string]config.AccessoryConfig{"db": {Image: "mariadb:11", EnvLiteral: map[string]string{"MARIADB_ROOT_PASSWORD": "DUMMY-r3-secret", "MARIADB_DATABASE": "appdb"}, Volumes: map[string]string{"data": "/var/lib/mysql"}}} + mock := drCutoverMock(drOKReceipt(), manifest) + receipt, err := NewClient(mock, &bytes.Buffer{}).CutoverBundle(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: cfg}) + if err != nil || !receipt.OK { + t.Fatalf("cutover: %v receipt=%+v calls=%v", err, receipt, mock.Calls) + } + nativeAdmin, nativeImport := false, false + for _, command := range mock.Calls { + if strings.Contains(command, "DUMMY-r3-secret") { + t.Fatalf("password in argv: %s", command) + } + if strings.Contains(command, "exec mariadb-admin ") { + nativeAdmin = true + } + if strings.Contains(command, "exec mariadb -u root") { + nativeImport = true + } + if strings.Contains(command, "exec mysql ") || strings.Contains(command, "mysqladmin ping") { + t.Fatal("MySQL alias required") + } + } + if !nativeAdmin || !nativeImport || !strings.Contains(strings.Join(mock.Inputs, ""), "MYSQL_PWD=DUMMY-r3-secret\n") { + t.Fatalf("native/private transport missing: %v inputs=%v", mock.Calls, mock.Inputs) + } +} +func TestRound3DRValidationScratchNeverDeletesIncumbents(t *testing.T) { + mock := drRestoreMock(drTestManifest()) + executor := &round3DRExecutor{MockExecutor: mock, owners: map[string]string{}} + client := NewClient(executor, &bytes.Buffer{}) + manifest := drTestManifest() + manifest.App = "demo" + manifest.AppRun.Image = "demo:v1" + manifest.State = json.RawMessage(`{"schema_version":2,"current_hash":"v1","image_ref":"demo:v1"}`) + snap := manifest.Snapshots[0] + snap.Engine = "mariadb" + snap.Image = "mariadb:11" + var wg sync.WaitGroup + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + result := client.validateEngineSnapshot(context.Background(), manifest, snap, "/var/tmp/private-r3") + if result.Status != "pass" { + t.Errorf("validation: %+v", result) + } + }() + } + wg.Wait() + result := client.appCheck(context.Background(), manifest, BundleRestoreOptions{}, "/var/tmp/private-r3") + if result.Status != "pass" { + t.Fatal(result) + } + for _, command := range executor.commands { + if strings.Contains(command, "demo-db-drcheck") || strings.Contains(command, "demo-dr-appcheck") { + t.Fatalf("incumbent touched: %s", command) + } + } + if len(executor.owners) != 3 { + t.Fatalf("overlapping validators reused scratch: %+v", executor.owners) + } +} +func TestRound3RestoreWorkspaceNeverWipesPriorInvocation(t *testing.T) { + manifest := drTestManifest() + mock := drRestoreMock(manifest) + // Engine/app checks use owned IDs; both full restores retain distinct trees. + executor := &round3DRExecutor{MockExecutor: mock, owners: map[string]string{}} + client := NewClient(executor, &bytes.Buffer{}) + one, err := client.RestoreBundleIsolated(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatal(err) + } + two, err := client.RestoreBundleIsolated(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}, DirBundleStore{Root: "/var/drstore"}) + if err != nil { + t.Fatal(err) + } + if one.InvocationID == two.InvocationID || one.StagingPath == two.StagingPath { + t.Fatal("workspace reused") + } + for _, command := range executor.commands { + if strings.HasPrefix(command, "rm -rf") { + t.Fatalf("prior invocation wiped: %s", command) + } + } +} diff --git a/internal/backup/verify.go b/internal/backup/verify.go index 7d29ed6..cd7bad5 100644 --- a/internal/backup/verify.go +++ b/internal/backup/verify.go @@ -2,8 +2,12 @@ package backup import ( "context" + "crypto/rand" + "encoding/hex" + "errors" "fmt" "os" + "path/filepath" "sort" "strconv" "strings" @@ -49,7 +53,7 @@ func (c *Client) InspectAccessory(ctx context.Context, app, name string) (image } env = make(map[string]string) for _, line := range strings.Split(rawEnv, "\n") { - line = strings.TrimSpace(line) + line = strings.TrimSuffix(line, "\r") if line == "" { continue } @@ -113,7 +117,7 @@ func verifyTmpDir() string { return "/var/tmp" } -func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3Config) (*VerifyResult, error) { +func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3Config) (retResult *VerifyResult, retErr error) { if err := c.ensureAWSCLI(ctx); err != nil { return nil, err } @@ -138,32 +142,35 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 App: app, Accessory: name, Image: image, Date: date, SizeBytes: size, } - scratch := app + "-" + name + "-verify" - // /var/tmp, not /tmp. Verifying a backup means writing the whole dataset - // to disk twice (the archive, then the extract), and /tmp is a tmpfs on - // many distributions — RAM-sized. A 28 GB Nucleus accessory against a - // 15 GB tmpfs failed here with "tar: Wrote only 2560 of 10240 bytes", - // which reads like a corrupt archive and is not: the archive was fine and - // the target was full. /var/tmp is disk-backed by convention and survives - // the cleanup semantics we want. TEPLOY_VERIFY_TMPDIR overrides it for - // hosts that keep their space somewhere else. - tmpDir := verifyTmpDir() - tmpBase := fmt.Sprintf("%s/teploy-verify-%s-%s", tmpDir, app, name) - - // Always clean up, pass or fail — a leftover scratch container must never - // survive a verification run. Covers every per-type artifact: the - // downloaded dump (tmpBase.sql.gz/.rdb.gz/.archive.gz/.tar.gz) and the - // extract dir (tmpBase.d) — a 597 MB .tar.gz was found orphaned in /tmp - // on a prod box because only tmpBase and tmpBase.d were removed. + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return nil, err + } + owner := hex.EncodeToString(nonce[:]) + scratch := "teploy-backup-verify-" + owner + workspace, err := c.exec.Run(ctx, "umask 077; mktemp -d "+ssh.ShellQuote(verifyTmpDir()+"/teploy-verify.XXXXXXXX")) + if err != nil { + return nil, fmt.Errorf("creating private verification workspace: %w", err) + } + workspace = strings.TrimSpace(workspace) + if !strings.HasPrefix(workspace, verifyTmpDir()+"/teploy-verify.") || strings.ContainsAny(workspace, "\n\r") || filepath.Clean(workspace) != workspace { + return nil, fmt.Errorf("invalid verification workspace") + } + tmpBase := workspace + "/backup" defer func() { - _, _ = c.exec.Run(context.WithoutCancel(ctx), fmt.Sprintf( - "docker rm -f %s >/dev/null 2>&1; rm -rf %s %s.d %s.sql.gz %s.rdb.gz %s.archive.gz %s.tar.gz", - ssh.ShellQuote(scratch), ssh.ShellQuote(tmpBase), ssh.ShellQuote(tmpBase), - ssh.ShellQuote(tmpBase), ssh.ShellQuote(tmpBase), ssh.ShellQuote(tmpBase), - ssh.ShellQuote(tmpBase))) + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, cleanupErr := c.exec.Run(cleanupCtx, fmt.Sprintf("if [ \"$(docker inspect -f '{{index .Config.Labels \"teploy.verify.owner\"}}' %s 2>/dev/null)\" = %s ]; then docker rm -f %s >/dev/null; fi; rm -rf -- %s", ssh.ShellQuote(scratch), ssh.ShellQuote(owner), ssh.ShellQuote(scratch), ssh.ShellQuote(workspace))) + if cleanupErr != nil { + retErr = errors.Join(retErr, fmt.Errorf("cleaning verification scratch: %w", cleanupErr)) + if retResult != nil { + retResult.OK = false + retResult.Detail = "scratch cleanup failed" + } + } }() - // Remove any stale scratch from a previous interrupted run. - _, _ = c.exec.Run(ctx, fmt.Sprintf("docker rm -f %s >/dev/null 2>&1 || true", ssh.ShellQuote(scratch))) + // The unique container name and owner label keep cleanup invocation-scoped. + env["TEPLOY_VERIFY_OWNER"] = owner switch { case isDBType(image, "postgres"): @@ -198,21 +205,30 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 return res, nil } -// envFlags renders `-e K=V` docker-run flags for the env keys relevant to a -// DB image, cloning the source accessory's configuration so the scratch -// container authenticates exactly like the one the backup was taken from. -func envFlags(env map[string]string, prefixes ...string) string { - var parts []string - for k, v := range env { +// verifyEnv stages database environment in the private workspace; values never enter argv. +func (c *Client) verifyEnv(ctx context.Context, env map[string]string, tmpBase string, prefixes ...string) (string, error) { + var keys []string + for k := range env { for _, p := range prefixes { if strings.HasPrefix(k, p) { - parts = append(parts, "-e "+ssh.ShellQuote(k+"="+v)) + keys = append(keys, k) break } } } - sort.Strings(parts) - return strings.Join(parts, " ") + sort.Strings(keys) + var data strings.Builder + for _, k := range keys { + if strings.ContainsAny(env[k], "\r\n\x00") { + return "", fmt.Errorf("invalid scratch environment value") + } + fmt.Fprintf(&data, "%s=%s\n", k, env[k]) + } + path := tmpBase + ".env" + if err := c.exec.Upload(ctx, strings.NewReader(data.String()), path, "0600"); err != nil { + return "", err + } + return "--label teploy.verify.owner=" + ssh.ShellQuote(env["TEPLOY_VERIFY_OWNER"]) + " --env-file " + ssh.ShellQuote(path), nil } func (c *Client) download(ctx context.Context, s3Key, dest string, s3 S3Config) error { @@ -245,7 +261,10 @@ func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, nam return err } - flags := envFlags(env, "POSTGRES_") + flags, err := c.verifyEnv(ctx, env, tmpBase, "POSTGRES_") + if err != nil { + return err + } if _, ok := env["POSTGRES_PASSWORD"]; !ok { // Source ran without auth config (trust); scratch images require one. flags += " -e POSTGRES_HOST_AUTH_METHOD=trust" @@ -261,8 +280,8 @@ func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, nam fmt.Fprintf(c.out, "Restoring into scratch...\n") if _, err := c.exec.Run(ctx, fmt.Sprintf( - "gunzip -c %s | docker exec -i %s psql -q -v ON_ERROR_STOP=1 -U %s %s >/dev/null", - ssh.ShellQuote(dump), ssh.ShellQuote(scratch), ssh.ShellQuote(user), ssh.ShellQuote(db))); err != nil { + "gunzip -c %s > %s.sql && docker exec -i %s psql -q -v ON_ERROR_STOP=1 -U %s %s < %s.sql >/dev/null", + ssh.ShellQuote(dump), ssh.ShellQuote(tmpBase), ssh.ShellQuote(scratch), ssh.ShellQuote(user), ssh.ShellQuote(db), ssh.ShellQuote(tmpBase))); err != nil { return fmt.Errorf("restore into scratch failed: %w", err) } @@ -289,7 +308,10 @@ func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, return err } - flags := envFlags(env, "MYSQL_", "MARIADB_") + flags, err := c.verifyEnv(ctx, env, tmpBase, "MYSQL_", "MARIADB_") + if err != nil { + return err + } if _, ok := env["MYSQL_ROOT_PASSWORD"]; !ok { if _, ok2 := env["MARIADB_ROOT_PASSWORD"]; !ok2 { flags += " -e MYSQL_ALLOW_EMPTY_PASSWORD=yes" @@ -300,25 +322,24 @@ func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, "docker run -d --name %s %s %s >/dev/null", ssh.ShellQuote(scratch), flags, ssh.ShellQuote(image))); err != nil { return fmt.Errorf("starting scratch container: %w", err) } - // mysqladmin ping via socket mirrors the backup's `mysqldump -u root` - // (socket auth, no password on argv). - if err := c.waitReady(ctx, scratch, "mysqladmin ping -u root --silent", 40); err != nil { + client, admin := "mysql", "mysqladmin" + if isDBType(image, "mariadb") { + client, admin = "mariadb", "mariadb-admin" + } + prelude := `MYSQL_PWD=${MYSQL_ROOT_PASSWORD:-${MARIADB_ROOT_PASSWORD:-}}; export MYSQL_PWD; ` + if err := c.waitReady(ctx, scratch, "sh -c "+ssh.ShellQuote(prelude+"exec "+admin+" ping -u root --silent"), 40); err != nil { return err } - - fmt.Fprintf(c.out, "Restoring into scratch...\n") - if _, err := c.exec.Run(ctx, fmt.Sprintf( - "gunzip -c %s | docker exec -i %s mysql -u root %s", - ssh.ShellQuote(dump), ssh.ShellQuote(scratch), ssh.ShellQuote(db))); err != nil { + restore := prelude + "exec " + client + " -u root " + ssh.ShellQuote(db) + if _, err := c.exec.Run(ctx, "gunzip -c "+ssh.ShellQuote(dump)+" > "+ssh.ShellQuote(tmpBase+".sql")+" && docker exec -i "+ssh.ShellQuote(scratch)+" sh -c "+ssh.ShellQuote(restore)+" < "+ssh.ShellQuote(tmpBase+".sql")); err != nil { return fmt.Errorf("restore into scratch failed: %w", err) } - - out, err := c.exec.Run(ctx, fmt.Sprintf( - "docker exec %s sh -c %s", ssh.ShellQuote(scratch), - ssh.ShellQuote(fmt.Sprintf("mysql -u root -N -e \"SHOW TABLES\" %s | wc -l", db)))) + query := prelude + "exec " + client + " -u root -N -e " + ssh.ShellQuote("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '"+strings.ReplaceAll(db, "'", "''")+"'") + out, err := c.exec.Run(ctx, "docker exec "+ssh.ShellQuote(scratch)+" sh -c "+ssh.ShellQuote(query)) if err != nil { return fmt.Errorf("verify query failed: %w", err) } + n, _ := strconv.Atoi(strings.TrimSpace(out)) res.Metric = fmt.Sprintf("tables=%d", n) if n == 0 { @@ -339,9 +360,9 @@ func (c *Client) verifyRedis(ctx context.Context, res *VerifyResult, app, name, // Create stopped, seed the rdb, then start — so redis boots FROM the // backup (a corrupt rdb fails the readiness probe). cmd := fmt.Sprintf( - "gunzip -c %s > %s.rdb && docker create --name %s %s >/dev/null && docker cp %s.rdb %s:/data/dump.rdb && docker start %s >/dev/null", + "gunzip -c %s > %s.rdb && docker create --name %s --label teploy.verify.owner=%s %s >/dev/null && docker cp %s.rdb %s:/data/dump.rdb && docker start %s >/dev/null", ssh.ShellQuote(dump), ssh.ShellQuote(tmpBase), - ssh.ShellQuote(scratch), ssh.ShellQuote(image), + ssh.ShellQuote(scratch), ssh.ShellQuote(env["TEPLOY_VERIFY_OWNER"]), ssh.ShellQuote(image), ssh.ShellQuote(tmpBase), ssh.ShellQuote(scratch), ssh.ShellQuote(scratch)) if _, err := c.exec.Run(ctx, cmd); err != nil { return fmt.Errorf("seeding scratch redis: %w", err) @@ -365,10 +386,14 @@ func (c *Client) verifyMongo(ctx context.Context, res *VerifyResult, app, name, return err } + flags, err := c.verifyEnv(ctx, env, tmpBase, "MONGO_") + if err != nil { + return err + } fmt.Fprintf(c.out, "Starting scratch %s...\n", scratch) if _, err := c.exec.Run(ctx, fmt.Sprintf( "docker run -d --name %s %s %s >/dev/null", - ssh.ShellQuote(scratch), envFlags(env, "MONGO_"), ssh.ShellQuote(image))); err != nil { + ssh.ShellQuote(scratch), flags, ssh.ShellQuote(image))); err != nil { return fmt.Errorf("starting scratch container: %w", err) } // mongosh on 5.x+, legacy mongo shell before that. @@ -414,10 +439,14 @@ func (c *Client) verifyNucleus(ctx context.Context, res *VerifyResult, app, name return fmt.Errorf("archive extract failed (or no nucleus-data dir inside): %w", err) } + flags, err := c.verifyEnv(ctx, env, tmpBase, "NUCLEUS_") + if err != nil { + return err + } fmt.Fprintf(c.out, "Booting scratch %s on the restored data dir...\n", scratch) if _, err := c.exec.Run(ctx, fmt.Sprintf( "docker run -d --name %s %s -v %s/nucleus-data:/data %s >/dev/null", - ssh.ShellQuote(scratch), envFlags(env, "NUCLEUS_"), ssh.ShellQuote(extractDir), ssh.ShellQuote(image))); err != nil { + ssh.ShellQuote(scratch), flags, ssh.ShellQuote(extractDir), ssh.ShellQuote(image))); err != nil { return fmt.Errorf("starting scratch container: %w", err) } diff --git a/internal/backup/verify_test.go b/internal/backup/verify_test.go index e5a5c16..3c035b6 100644 --- a/internal/backup/verify_test.go +++ b/internal/backup/verify_test.go @@ -12,6 +12,7 @@ import ( func TestLatestBackupDate_PicksNewest(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "" + "2026-07-08 04:00:01 1000 20260708-040000.sql.gz\n" + @@ -33,6 +34,7 @@ func TestLatestBackupDate_PicksNewest(t *testing.T) { func TestLatestBackupDate_NoBackups(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "\n"}, ) client := NewClient(mock, &bytes.Buffer{}) @@ -48,6 +50,7 @@ func TestLatestBackupDate_NoBackups(t *testing.T) { // real accessory), the verify query runs, and teardown removes the scratch. func TestVerifyBackup_Postgres(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:16\n"}, ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "POSTGRES_PASSWORD=sekret\nPOSTGRES_DB=appdb\nPATH=/usr/bin\n"}, @@ -57,7 +60,7 @@ func TestVerifyBackup_Postgres(t *testing.T) { ssh.MockCommand{Match: "for i in $(seq", Output: ""}, ssh.MockCommand{Match: "gunzip -c", Output: ""}, ssh.MockCommand{Match: "docker exec", Output: "42\n"}, - ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "if [", Output: ""}, ) client := NewClient(mock, &bytes.Buffer{}) res, err := client.VerifyBackup(context.Background(), "myapp", "db", "", S3Config{Bucket: "b", Region: "us-east-1"}) @@ -75,10 +78,10 @@ func TestVerifyBackup_Postgres(t *testing.T) { for _, call := range mock.Calls { if strings.Contains(call, "docker run -d") { sawScratchRun = true - if !strings.Contains(call, "myapp-db-verify") { + if !strings.Contains(call, "teploy-backup-verify-") { t.Errorf("scratch run must use the -verify container name: %s", call) } - if !strings.Contains(call, "POSTGRES_PASSWORD=sekret") { + if strings.Contains(call, "sekret") || !strings.Contains(string(mock.Files["/var/tmp/teploy-verify.test/backup.env"]), "POSTGRES_PASSWORD=sekret") { t.Errorf("scratch must clone source POSTGRES_* env: %s", call) } if strings.Contains(call, "PATH=") { @@ -87,14 +90,14 @@ func TestVerifyBackup_Postgres(t *testing.T) { } if strings.Contains(call, "gunzip -c") && strings.Contains(call, "psql") { sawRestore = true - if !strings.Contains(call, "myapp-db-verify") { + if !strings.Contains(call, "teploy-backup-verify-") { t.Errorf("restore must target the scratch container, got: %s", call) } if !strings.Contains(call, "'appdb'") { t.Errorf("restore must target POSTGRES_DB from inspected env: %s", call) } } - if strings.Contains(call, "docker rm -f") && strings.Contains(call, "myapp-db-verify") { + if strings.Contains(call, "docker rm -f") && strings.Contains(call, "teploy-backup-verify-") { sawTeardown = true } } @@ -107,6 +110,7 @@ func TestVerifyBackup_Postgres(t *testing.T) { // yields ok=false (not an operational error) and STILL removes the scratch. func TestVerifyBackup_FailureIsResultAndTearsDown(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:16\n"}, ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "POSTGRES_PASSWORD=x\n"}, @@ -114,7 +118,7 @@ func TestVerifyBackup_FailureIsResultAndTearsDown(t *testing.T) { ssh.MockCommand{Match: "docker run -d", Output: ""}, ssh.MockCommand{Match: "for i in $(seq", Output: ""}, ssh.MockCommand{Match: "gunzip -c", Err: fmt.Errorf("psql: ERROR: syntax error")}, - ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "if [", Output: ""}, ) client := NewClient(mock, &bytes.Buffer{}) res, err := client.VerifyBackup(context.Background(), "myapp", "db", "20260710-040000", S3Config{Bucket: "b", Region: "us-east-1"}) @@ -129,7 +133,7 @@ func TestVerifyBackup_FailureIsResultAndTearsDown(t *testing.T) { } teardown := false for _, call := range mock.Calls { - if strings.Contains(call, "docker rm -f") && strings.Contains(call, "myapp-db-verify") { + if strings.Contains(call, "docker rm -f") && strings.Contains(call, "teploy-backup-verify-") { teardown = true } } @@ -142,6 +146,7 @@ func TestVerifyBackup_FailureIsResultAndTearsDown(t *testing.T) { // nucleus accessories (generic tar + scratch engine on the restored data dir). func TestVerifyBackup_NucleusBootVerify(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "ghcr.io/neutron-build/nucleus:latest\n"}, ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "NUCLEUS_ALLOW_NO_AUTH=1\n"}, @@ -150,7 +155,7 @@ func TestVerifyBackup_NucleusBootVerify(t *testing.T) { ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "docker run -d", Output: ""}, ssh.MockCommand{Match: "for i in $(seq", Output: "Restored 57 table\n"}, - ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "if [", Output: ""}, ) client := NewClient(mock, &bytes.Buffer{}) res, err := client.VerifyBackup(context.Background(), "observe", "nucleus", "", S3Config{Bucket: "b", Region: "us-east-1"}) @@ -164,7 +169,7 @@ func TestVerifyBackup_NucleusBootVerify(t *testing.T) { t.Errorf("metric should parse the boot-restore line, got %q", res.Metric) } for _, call := range mock.Calls { - if strings.Contains(call, "docker run -d") && !strings.Contains(call, "NUCLEUS_ALLOW_NO_AUTH") { + if strings.Contains(call, "docker run -d") && !strings.Contains(string(mock.Files["/var/tmp/teploy-verify.test/backup.env"]), "NUCLEUS_ALLOW_NO_AUTH") { t.Errorf("scratch nucleus must clone NUCLEUS_* env: %s", call) } } @@ -174,12 +179,13 @@ func TestVerifyBackup_NucleusBootVerify(t *testing.T) { // integrity check, and an empty archive fails. func TestVerifyBackup_VolumeArchive(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "some/custom-thing:1\n"}, ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "done\n"}, ssh.MockCommand{Match: "mkdir -p", Output: "12\n"}, - ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "if [", Output: ""}, ) client := NewClient(mock, &bytes.Buffer{}) res, err := client.VerifyBackup(context.Background(), "myapp", "thing", "20260710-040000", S3Config{Bucket: "b", Region: "us-east-1"}) diff --git a/internal/build/campaign_test.go b/internal/build/campaign_test.go new file mode 100644 index 0000000..e2e393e --- /dev/null +++ b/internal/build/campaign_test.go @@ -0,0 +1,31 @@ +package build + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestCampaignGitDiscoveryFailsClosed(t *testing.T) { + repo := t.TempDir() + cmd := exec.Command("git", "init", "-q", repo) + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + cfg := filepath.Join(t.TempDir(), "config") + os.WriteFile(cfg, []byte("[malformed"), 0600) + t.Setenv("GIT_CONFIG_GLOBAL", cfg) + if git, err := isGitWorkTree(repo); err == nil || git { + t.Fatalf("git=%v error=%v", git, err) + } + if git, err := isGitWorkTree(t.TempDir()); err == nil || git { + t.Fatalf("unknown status accepted: git=%v error=%v", git, err) + } +} +func TestCampaignNonGitSourceAccepted(t *testing.T) { + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + if git, err := isGitWorkTree(t.TempDir()); err != nil || git { + t.Fatalf("git=%v error=%v", git, err) + } +} diff --git a/internal/build/fingerprint_test.go b/internal/build/fingerprint_test.go index b783590..2b86dd3 100644 --- a/internal/build/fingerprint_test.go +++ b/internal/build/fingerprint_test.go @@ -102,7 +102,6 @@ func TestContextFingerprint_HonorsExcludePatterns(t *testing.T) { writeTree(t, b, core) writeTree(t, b, map[string]string{ "node_modules/pkg/index.js": "junk", - ".git/config": "junk", ".env": "SECRET=1", ".env.local": "SECRET=2", }) diff --git a/internal/build/ignore.go b/internal/build/ignore.go index b9709dd..5dac5f6 100644 --- a/internal/build/ignore.go +++ b/internal/build/ignore.go @@ -128,6 +128,8 @@ type rule struct { } func compileRule(p string) (rule, error) { + for _,part:=range strings.Split(p,"/") {if part==".." {return rule{},fmt.Errorf("pattern must not traverse the source root")}} + if strings.ContainsRune(p,0) {return rule{},fmt.Errorf("pattern contains NUL")} var r rule if strings.HasSuffix(p, "/") { r.dirOnly = true diff --git a/internal/build/site_consumer_test.go b/internal/build/site_consumer_test.go new file mode 100644 index 0000000..57a4cfc --- /dev/null +++ b/internal/build/site_consumer_test.go @@ -0,0 +1,69 @@ +package build + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestSiteConsumerIgnoredDistOverrideAndMaps(t *testing.T) { + policy, err := os.ReadFile("testdata/site-teployignore") + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + gitRepo(t, root, map[string]string{".gitignore": "/dist/\n", ".teployignore": string(policy), "Dockerfile": "FROM caddy:2\nCOPY dist/ /srv/\n"}) + writeTree(t, root, map[string]string{"dist/index.html": "site", "dist/assets/site.js": "console.log('site')", "dist/site.js.map": "map", "dist/assets/deep/site.js.map": "map", "dist/.env": "protected secret", "dist/ignored/nested/asset.css": "body{}"}) + src := resolve(t, root) + assertEntries(t, src, []string{"dist/index.html", "dist/assets/site.js", "dist/ignored/nested/asset.css"}, []string{"dist/site.js.map", "dist/assets/deep/site.js.map", "dist/.env", ".teployignore"}) + if !src.Contains("dist/assets") || !src.GitAware { + t.Fatal("parent directory traversal missing") + } + if _, err := exec.LookPath("rsync"); err != nil { + t.Skip("rsync unavailable") + } + destination := t.TempDir() + transfer := exec.Command("rsync", "-az", "--from0", "--files-from=-", root+"/", destination+"/") + transfer.Stdin = bytes.NewReader(src.FileList()) + if out, err := transfer.CombinedOutput(); err != nil { + t.Fatalf("consumer sync: %v %s", err, out) + } + for _, name := range []string{"dist/index.html", "dist/assets/site.js", "dist/ignored/nested/asset.css"} { + if _, err := os.Stat(filepath.Join(destination, name)); err != nil { + t.Fatalf("missing transferred parent/tree: %s %v", name, err) + } + } + for _, name := range []string{"dist/site.js.map", "dist/assets/deep/site.js.map", "dist/.env"} { + if _, err := os.Stat(filepath.Join(destination, name)); !os.IsNotExist(err) { + t.Fatalf("excluded child transferred: %s", name) + } + } +} + +func TestAllowlistedIgnoredParentsAndTraversalRefusal(t *testing.T) { + root := t.TempDir() + gitRepo(t, root, map[string]string{".gitignore": "/nested/\n", ".teployignore": "!/nested/parent/dist/\n/nested/parent/dist/**/*.map\n"}) + writeTree(t, root, map[string]string{"nested/parent/dist/assets/app.js": "script", "nested/parent/dist/assets/app.js.map": "map"}) + assertEntries(t, resolve(t, root), []string{"nested/parent/dist/assets/app.js"}, []string{"nested/parent/dist/assets/app.js.map"}) + for _, pattern := range []string{"!/../outside/\n", "!/nested/../../outside/\n"} { + if err := os.WriteFile(filepath.Join(root, ".teployignore"), []byte(pattern), 0600); err != nil { + t.Fatal(err) + } + if _, err := ResolveSource(root); err == nil { + t.Fatal("parent traversal accepted") + } + } + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Fatal(err) + } + writeTree(t, outside, map[string]string{"dist/secret": "not source"}) + if err := os.WriteFile(filepath.Join(root, ".teployignore"), []byte("!/linked/dist/\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := ResolveSource(root); err == nil { + t.Fatal("symlinked ancestor traversed") + } +} diff --git a/internal/build/source.go b/internal/build/source.go index 61bc75a..18c8375 100644 --- a/internal/build/source.go +++ b/internal/build/source.go @@ -172,7 +172,26 @@ func isGitWorkTree(dir string) (bool, error) { } out, err := exec.Command("git", "-C", dir, "rev-parse", "--is-inside-work-tree").Output() if err != nil { - return false, nil + abs, absErr := filepath.Abs(dir) + if absErr != nil { + return false, absErr + } + for d := abs; ; d = filepath.Dir(d) { + if _, statErr := os.Lstat(filepath.Join(d, ".git")); statErr == nil { + return false, fmt.Errorf("git work-tree discovery failed for %s: %w", dir, err) + } else if !errors.Is(statErr, fs.ErrNotExist) { + return false, statErr + } + if filepath.Dir(d) == d { + break + } + } + var exit *exec.ExitError + // Git distinguishes a genuinely absent repository in its diagnostic. + if errors.As(err, &exit) && strings.Contains(string(exit.Stderr), "not a git repository") { + return false, nil + } + return false, fmt.Errorf("cannot establish whether %s is a git work tree: %w", dir, err) } return strings.TrimSpace(string(out)) == "true", nil } @@ -232,6 +251,17 @@ func addAllowlisted(root string, rules *Rules, set map[string]bool) error { starts = literals } for _, s := range starts { + // A literal start must not reach through a symlinked ancestor to an + // outside tree. Symlink entries themselves retain normal selection. + for parent := path.Dir(s); parent != "." && parent != ""; parent = path.Dir(parent) { + info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(parent))) + if err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("allowlisted path %s has a symlinked ancestor", s) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } // walkAdmitted judges every entry's ancestors too, so a literal // under a protected or excluded directory still yields nothing. found, err := walkAdmitted(root, s, rules, keep) @@ -290,7 +320,12 @@ func (s *Source) FileList() []byte { // Permission bits are ignored (umask stability across machines). The // fingerprint describes what the build host RECEIVED — a provenance // identity, not a security boundary. -func (s *Source) Fingerprint(sub string) (string, error) { +func (s *Source) Fingerprint(sub string) (string, error) { return s.fingerprint(sub, false) } + +// ReviewedFingerprint binds permission modes as well as bytes, including implied parents. +// Existing mode-free provenance remains v2; reviewed authorization is v3. +func (s *Source) ReviewedFingerprint(sub string) (string, error) { return s.fingerprint(sub, true) } +func (s *Source) fingerprint(sub string, reviewed bool) (string, error) { sub = strings.Trim(path.Clean("/"+filepath.ToSlash(sub)), "/") type entry struct { rel string @@ -300,6 +335,9 @@ func (s *Source) Fingerprint(sub string) (string, error) { target string } byRel := map[string]entry{} + if reviewed { + byRel[""] = entry{rel: "", kind: 'd'} + } addDirs := func(rel string) { for i := 0; i < len(rel); i++ { if rel[i] == '/' { @@ -354,7 +392,11 @@ func (s *Source) Fingerprint(sub string) (string, error) { h := sha256.New() // v2: the entry set is the L14 selection (gitignore-aware), not v1's // walk-minus-excludes — a new version so the two are never compared. - h.Write([]byte("teploy-context-v2\x00")) + if reviewed { + h.Write([]byte("teploy-reviewed-context-v3\x00")) + } else { + h.Write([]byte("teploy-context-v2\x00")) + } var num [8]byte binary.BigEndian.PutUint64(num[:], uint64(len(entries))) h.Write(num[:]) @@ -367,6 +409,15 @@ func (s *Source) Fingerprint(sub string) (string, error) { for _, e := range entries { h.Write([]byte{e.kind}) writeStr(e.rel) + if reviewed && e.kind != 'l' { + full := filepath.Join(s.Root, filepath.FromSlash(sub), filepath.FromSlash(e.rel)) + info, err := os.Lstat(full) + if err != nil { + return "", err + } + binary.BigEndian.PutUint64(len8[:], uint64(info.Mode().Perm())) + h.Write(len8[:]) + } switch e.kind { case 'f': binary.BigEndian.PutUint64(len8[:], uint64(e.size)) diff --git a/internal/build/source_test.go b/internal/build/source_test.go index e48223b..4bcb43d 100644 --- a/internal/build/source_test.go +++ b/internal/build/source_test.go @@ -152,12 +152,11 @@ func TestResolveSource_OutsideGit(t *testing.T) { "teploy.yml": "app: x", "teploy.home.yml": "password: hunter2", ".env.local": "SECRET=1", - ".git/config": "not a real repo", }) if err := os.MkdirAll(filepath.Join(dir, "empty/dir"), 0o755); err != nil { t.Fatal(err) } - // A bare .git directory without git's own layout is not a work tree. + // This fixture is genuinely outside a repository; corrupt .git directories now refuse. src := resolve(t, dir) if src.GitAware { t.Skip("temp dir resolved as a git work tree (unexpected environment)") diff --git a/internal/build/testdata/site-teployignore b/internal/build/testdata/site-teployignore new file mode 100644 index 0000000..cad21ba --- /dev/null +++ b/internal/build/testdata/site-teployignore @@ -0,0 +1,5 @@ +# Caddy's Dockerfile copies the locally generated static site. Teploy's +# gitignore-aware source sync must explicitly include this build output. +!/dist/ +# Keep source maps out of the published static directory. +/dist/**/*.map diff --git a/internal/caddy/caddy.go b/internal/caddy/caddy.go index 6a53f95..0a847fa 100644 --- a/internal/caddy/caddy.go +++ b/internal/caddy/caddy.go @@ -439,7 +439,9 @@ func (c *Client) SetMaintenance(ctx context.Context, app, domain string) error { begin := fmt.Sprintf(markerBeginFmt, app) end := fmt.Sprintf(markerEndFmt, app) var pol SitePolicy + var maintenanceGeneration uint64 if cur := extractCaddyfileBlock(prev, begin, end); cur != "" { + maintenanceGeneration, _ = RegionGeneration(cur) extracted, err := ExtractPolicy(cur) if err != nil { return "", fmt.Errorf("reading %s's TLS/access policy for maintenance (route left unchanged): %w", app, err) @@ -464,7 +466,11 @@ func (c *Client) SetMaintenance(ctx context.Context, app, domain string) error { } } } - updated, err := renderUpdated(prev, app, hosts, maintenanceBlock(hosts, pol)) + block := maintenanceBlock(hosts, pol) + if maintenanceGeneration > 0 { + block = fmt.Sprintf(generationStampFmt, maintenanceGeneration) + "\n" + block + } + updated, err := renderUpdated(prev, app, hosts, block) if err != nil { return "", err } diff --git a/internal/caddy/campaign_test.go b/internal/caddy/campaign_test.go new file mode 100644 index 0000000..3b3266e --- /dev/null +++ b/internal/caddy/campaign_test.go @@ -0,0 +1,46 @@ +package caddy + +import ( + "context" + "strings" + "testing" +) + +func TestCampaignStampedPolicyAndWebhook(t *testing.T) { + block := "# TEPLOY GENERATION 7\nexample.test {\n tls internal\n reverse_proxy app:3000\n}\n" + policy, err := ExtractPolicy(block) + if err != nil || !strings.Contains(policy.TLS, "internal") { + t.Fatalf("policy %+v %v", policy, err) + } + cfg := &webhookRouteConfig{} + got, err := injectWebhookFragment(block, "app", cfg) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(got, "# TEPLOY GENERATION 7\nexample.test {\n") { + t.Fatalf("metadata lost: %s", got) + } + if _, err = ExtractPolicy(got); err != nil { + t.Fatal(err) + } +} + +func TestCampaignStampedMaintenanceOnOff(t *testing.T) { + prior := "# TEPLOY BEGIN demo\n# TEPLOY GENERATION 7\nexample.test {\n tls internal\n reverse_proxy app:3000\n}\n# TEPLOY END demo\n" + m, _ := caddyMutateMocks(prior) + c := NewClient(m) + if err := c.SetMaintenance(context.Background(), "demo", "example.test"); err != nil { + t.Fatal(err) + } + b := string(m.Files[caddyfilePath]) + if !strings.Contains(b, "# TEPLOY GENERATION 7") { + t.Fatalf("generation lost %s", b) + } + if err := c.RemoveMaintenance(context.Background(), "demo"); err != nil { + t.Fatal(err) + } + b = string(m.Files[caddyfilePath]) + if !strings.Contains(b, "# TEPLOY GENERATION 7") || !strings.Contains(b, "reverse_proxy app:3000") { + t.Fatalf("restore lost identity %s", b) + } +} diff --git a/internal/caddy/routes.go b/internal/caddy/routes.go index be63cae..a3fcbd2 100644 --- a/internal/caddy/routes.go +++ b/internal/caddy/routes.go @@ -426,17 +426,32 @@ type SitePolicy struct { // directives, with nested spans (basic_auth's user list, forward_auth's // options) kept verbatim. Anything unparseable is an error — a maintenance // block built from a misread policy is worse than one built from none. +func siteOpeningLine(lines []string) (int, error) { + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + if strings.HasSuffix(trimmed, "{") && braceNet(line) > 0 { + return i, nil + } + return 0, fmt.Errorf("not a site block: %q", line) + } + return 0, fmt.Errorf("site block has no opening line") +} + func ExtractPolicy(block string) (SitePolicy, error) { var pol SitePolicy lines := logicalLines(strings.Split(block, "\n")) if len(lines) == 0 { return pol, nil } - depth := braceNet(lines[0]) // the address line's opening brace - if depth <= 0 { - return pol, fmt.Errorf("not a site block: %q", lines[0]) + opening, err := siteOpeningLine(lines) + if err != nil { + return pol, err } - for i := 1; i < len(lines); i++ { + depth := braceNet(lines[opening]) + for i := opening + 1; i < len(lines); i++ { raw := lines[i] code, err := codeLine(raw) if err != nil { diff --git a/internal/caddy/webhook.go b/internal/caddy/webhook.go index 184fbda..6f4c9ae 100644 --- a/internal/caddy/webhook.go +++ b/internal/caddy/webhook.go @@ -129,11 +129,14 @@ func stripWebhookFragment(block, app string) string { // TEPLOY markers (extractCaddyfileBlock's contract). func injectWebhookFragment(block, app string, cfg *webhookRouteConfig) (string, error) { lines := strings.Split(block, "\n") - if len(lines) < 2 || !strings.HasSuffix(strings.TrimSpace(lines[0]), "{") { - return "", fmt.Errorf("cannot place the webhook route: %s's site block has no recognizable opening line", app) + opening, err := siteOpeningLine(lines) + if err != nil { + return "", fmt.Errorf("cannot place webhook route for %s: %w", app, err) } fragment := webhookFragment(app, cfg) - out := append([]string{lines[0], strings.TrimRight(fragment, "\n")}, lines[1:]...) + out := append([]string{}, lines[:opening+1]...) + out = append(out, strings.TrimRight(fragment, "\n")) + out = append(out, lines[opening+1:]...) return strings.Join(out, "\n"), nil } diff --git a/internal/cli/apply.go b/internal/cli/apply.go index 84782f5..770933f 100644 --- a/internal/cli/apply.go +++ b/internal/cli/apply.go @@ -16,9 +16,11 @@ import ( "io" "os" "os/signal" + "path/filepath" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" ) @@ -71,6 +73,22 @@ func runApply(flags *Flags, planPath string, skipDNSCheck, migrateVolumes bool) return err } + if rec.Image.NeedsBuild { + snapshot, err := freezePlanInputs(rec.InputExclusions, appCfg) + if err != nil { + return refuseApply(fmt.Errorf("freezing reviewed inputs: %w", err)) + } + defer os.RemoveAll(filepath.Dir(snapshot)) + original, err := os.Getwd() + if err != nil { + return err + } + if err := os.Chdir(snapshot); err != nil { + return err + } + defer os.Chdir(original) + } + // Connect to the plan's target. serverName := rec.ServerName if serverName == "" { @@ -92,9 +110,17 @@ func runApply(flags *Flags, planPath string, skipDNSCheck, migrateVolumes bool) // identity. This catches edits the version cannot see (a dirty-tree // change between plan and apply). var curFingerprint, curDockerfileSHA string - if rec.Image.NeedsBuild { + if rec.Image.NeedsBuild && appCfg.IsStatic() { + curFingerprint, err = staticPlanFingerprint(rec.InputExclusions, appCfg) + if err != nil { + return err + } + } else if rec.Image.NeedsBuild { prov := resolveDeployProvenance(ctx, executor, io.Discard, appCfg, ".", image, rec.TargetVersion, curDigest, true) - curFingerprint = prov.ContextFingerprint + curFingerprint, err = staticPlanFingerprint(rec.InputExclusions, appCfg) + if err != nil { + return err + } curDockerfileSHA = prov.DockerfileSHA256 } @@ -118,7 +144,7 @@ func runApply(flags *Flags, planPath string, skipDNSCheck, migrateVolumes bool) // ONE execution path: the same deployAppConfig a direct `teploy // deploy` runs, with the plan's recorded image/version/overlay. The // plan id rides along and is stamped into the provenance receipt. - err = deployAppConfig(flags, appCfg, serverName, image, rec.TargetVersion, skipDNSCheck, migrateVolumes, rec.PlanID) + err = deployAppConfig(flags, appCfg, serverName, image, rec.TargetVersion, skipDNSCheck, migrateVolumes, rec.PlanID, rec) if err != nil { return err } @@ -154,7 +180,7 @@ func applyResolveCurrent(ctx context.Context, flags *Flags, rec *PlanRecord) (ap } image = rec.Image.Ref - _, curDigest, err = config.NormalizeAndDigest(appCfg, image) + curDigest, err = config.ExecutionBindingDigest(appCfg, image) if err != nil { return nil, "", "", "", fmt.Errorf("normalizing current config: %w", err) } @@ -187,3 +213,31 @@ func refuseApply(err error) error { } return refuseAdmission(err) } + +func revalidateLockedPlan(ctx context.Context, exec ssh.Executor, cfg *config.AppConfig, image, version string, rec *PlanRecord) error { + current, err := state.Read(ctx, exec, cfg.App) + if err != nil { + return err + } + digest, err := config.ExecutionBindingDigest(cfg, image) + if err != nil { + return err + } + facts := planCurrentFacts{App: cfg.App, Server: exec.Host(), Version: version, ConfigDigest: digest, State: current} + if rec.Image.NeedsBuild { + if cfg.IsStatic() { + facts.ContextFingerprint, err = staticPlanFingerprint(rec.InputExclusions, cfg) + if err != nil { + return err + } + } else { + prov := resolveDeployProvenance(ctx, exec, io.Discard, cfg, ".", image, version, digest, true) + facts.ContextFingerprint, err = staticPlanFingerprint(rec.InputExclusions, cfg) + if err != nil { + return err + } + facts.DockerfileSHA256 = prov.DockerfileSHA256 + } + } + return verifyPlanBinding(rec, facts) +} diff --git a/internal/cli/apply_test.go b/internal/cli/apply_test.go index 3f66e78..18af968 100644 --- a/internal/cli/apply_test.go +++ b/internal/cli/apply_test.go @@ -47,7 +47,7 @@ func planFromConfig(t *testing.T, dir, image, destination string) *PlanRecord { if err != nil { t.Fatalf("LoadApp: %v", err) } - _, digest, err := config.NormalizeAndDigest(appCfg, image) + digest, err := config.ExecutionBindingDigest(appCfg, image) if err != nil { t.Fatal(err) } @@ -256,6 +256,8 @@ func TestApplyStampReceiptThroughEngine(t *testing.T) { appliedManifest, _, _ := config.NormalizeAndDigest(appCfg, image) mock := ssh.NewMockExecutor("1.2.3.4", + // managed-directory admission (R2-06) precedes the engine + ssh.MockCommand{Match: "teploy_volume_actor", Output: ""}, // provenance mkdir + upload (UploadAtomic: UPLOAD + mv handled by mock) ssh.MockCommand{Match: "mkdir -p /deployments/myapp/meta/att", Output: ""}, // .env absent diff --git a/internal/cli/autodeploy.go b/internal/cli/autodeploy.go index b052808..5b4fbde 100644 --- a/internal/cli/autodeploy.go +++ b/internal/cli/autodeploy.go @@ -103,7 +103,7 @@ func runAutoDeploySetup(flags *Flags, branch, secret string) error { // pipeline as `teploy update` (see binarydist.go). const teployBinaryPath = "/deployments/.bin/teploy" fmt.Println("Installing teploy binary on server...") - binVersion, err := deployTeployBinaryToServer(ctx, executor, teployBinaryPath) + binVersion, err := deployTeployBinaryToServer(ctx, executor, teployBinaryPath, "autodeploy serve") if err != nil { return fmt.Errorf("installing teploy binary: %w", err) } @@ -320,12 +320,9 @@ func runAutoDeploySchedule(flags *Flags, schedule, branch string) error { // than reconstructing the container itself (C02), so the server needs // a teploy binary that speaks `autodeploy redeploy`. const teployBinaryPath = "/deployments/.bin/teploy" - if _, err := deployTeployBinaryToServer(ctx, executor, teployBinaryPath); err != nil { + if _, err := deployTeployBinaryToServer(ctx, executor, teployBinaryPath, "autodeploy redeploy"); err != nil { return fmt.Errorf("installing the teploy binary on the server: %w", err) } - if _, err := executor.Run(ctx, fmt.Sprintf("%s autodeploy redeploy --help >/dev/null 2>&1", ssh.ShellQuote(teployBinaryPath))); err != nil { - return fmt.Errorf("the server's teploy binary does not support 'autodeploy redeploy' (the scheduled redeploy now runs the full engine through it); release a teploy version that includes it, then re-run this command: %w", err) - } mgr := autodeploy.NewManager(executor, os.Stdout) return mgr.Schedule(ctx, appCfg.App, schedule, branch, teployBinaryPath) diff --git a/internal/cli/autodeploy_admission_test.go b/internal/cli/autodeploy_admission_test.go index d23cd3e..21f44e7 100644 --- a/internal/cli/autodeploy_admission_test.go +++ b/internal/cli/autodeploy_admission_test.go @@ -94,7 +94,7 @@ func TestAdmission_AckWaitsForFsync(t *testing.T) { logf: func(string, ...any) {}, }) - body := `{"ref":"refs/heads/main","after":"abc"}` + body := `{"ref":"refs/heads/main","after":"a9993e364706816aba3e25717850c26c9cd0d89d"}` req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) req.Header.Set("X-Hub-Signature-256", githubSign("s3cret", []byte(body))) w := newRecordingWriter() @@ -155,7 +155,7 @@ func TestAdmission_PersistenceFailureRefused(t *testing.T) { logf: func(string, ...any) {}, }) - rec := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"abc"}`) + rec := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"a9993e364706816aba3e25717850c26c9cd0d89d"}`) if rec.Code != http.StatusServiceUnavailable { t.Errorf("status = %d, want 503 (never ack what isn't durable)", rec.Code) @@ -177,7 +177,7 @@ func TestAdmission_PersistenceFailureRefused(t *testing.T) { ledger.mu.Lock() ledger.fail = nil ledger.mu.Unlock() - rec2 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"abc"}`) + rec2 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"a9993e364706816aba3e25717850c26c9cd0d89d"}`) if rec2.Code != http.StatusOK { t.Errorf("retry after persistence failure: status = %d, want 200 admitted", rec2.Code) } @@ -198,7 +198,7 @@ func TestAdmission_Supersede(t *testing.T) { handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) // W starts running (blocks in the deploy). - if rec := postSigned(t, handler, "s3cret", "w", `{"ref":"refs/heads/main","after":"w"}`); rec.Code != http.StatusOK { + if rec := postSigned(t, handler, "s3cret", "w", `{"ref":"refs/heads/main","after":"aff024fe4ab0fece4091de044c58c9ae4233383a"}`); rec.Code != http.StatusOK { t.Fatalf("running delivery status = %d", rec.Code) } else if !strings.Contains(rec.Body.String(), `"running"`) { t.Errorf("first delivery disposition = %q, want running", rec.Body.String()) @@ -206,14 +206,14 @@ func TestAdmission_Supersede(t *testing.T) { run.waitCall(t) // W is inside the deploy now // A queues behind it. - if rec := postSigned(t, handler, "s3cret", "a", `{"ref":"refs/heads/main","after":"a"}`); rec.Code != http.StatusOK { + if rec := postSigned(t, handler, "s3cret", "a", `{"ref":"refs/heads/main","after":"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}`); rec.Code != http.StatusOK { t.Fatalf("queued delivery status = %d", rec.Code) } else if !strings.Contains(rec.Body.String(), `"queued"`) { t.Errorf("second delivery disposition = %q, want queued", rec.Body.String()) } // B supersedes A. - if rec := postSigned(t, handler, "s3cret", "b", `{"ref":"refs/heads/main","after":"b"}`); rec.Code != http.StatusOK { + if rec := postSigned(t, handler, "s3cret", "b", `{"ref":"refs/heads/main","after":"e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98"}`); rec.Code != http.StatusOK { t.Fatalf("superseding delivery status = %d", rec.Code) } else if !strings.Contains(rec.Body.String(), `"superseded"`) { t.Errorf("third delivery disposition = %q, want superseded", rec.Body.String()) @@ -262,7 +262,7 @@ func TestAdmission_NoGoroutinePileup(t *testing.T) { running, queued, superseded := 0, 0, 0 for i := 0; i < 10; i++ { - body := `{"ref":"refs/heads/main","after":"c` + string(rune('0'+i)) + `"}` + body := `{"ref":"refs/heads/main","after":"` + strings.Repeat("c", 39) + string(rune('0'+i)) + `"}` rec := postSigned(t, handler, "s3cret", "", body) if rec.Code != http.StatusOK { t.Fatalf("delivery %d status = %d, want 200", i, rec.Code) @@ -454,9 +454,9 @@ func TestAdmission_SameDigestWhileQueuedIsDuplicate(t *testing.T) { run := newCountingRun().blocking(block) handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) - postSigned(t, handler, "s3cret", "w", `{"ref":"refs/heads/main","after":"w"}`) + postSigned(t, handler, "s3cret", "w", `{"ref":"refs/heads/main","after":"aff024fe4ab0fece4091de044c58c9ae4233383a"}`) run.waitCall(t) - body := `{"ref":"refs/heads/main","after":"q"}` + body := `{"ref":"refs/heads/main","after":"22ea1c649c82946aa6e479e1ffd321e4a318b1b0"}` if rec := postSigned(t, handler, "s3cret", "q1", body); !strings.Contains(rec.Body.String(), `"queued"`) { t.Fatalf("first copy disposition = %q, want queued", rec.Body.String()) } diff --git a/internal/cli/autodeploy_campaign_test.go b/internal/cli/autodeploy_campaign_test.go new file mode 100644 index 0000000..8d92ab8 --- /dev/null +++ b/internal/cli/autodeploy_campaign_test.go @@ -0,0 +1,97 @@ +package cli + +import ( + "errors" + "github.com/useteploy/teploy/internal/autodeploy" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +type campaignGateLedger struct { + entered chan struct{} + release chan struct{} + fail bool + once sync.Once +} + +func (l *campaignGateLedger) Append(_ autodeploy.AdmissionRecord) error { + l.once.Do(func() { close(l.entered) }) + <-l.release + if l.fail { + return errors.New("fsync failed") + } + return nil +} +func TestCampaignDuplicateWaitsForDurability(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{true: "fail", false: "success"}[fail], func(t *testing.T) { + l := &campaignGateLedger{entered: make(chan struct{}), release: make(chan struct{}), fail: fail} + ledger := &memLedger{} + run := newCountingRun() + q := newAdmissionQueue(ledger, run.run, nil) + h := newWebhookHandler(webhookHandlerConfig{secret: "dummy", branch: "main", app: "app", dedup: autodeploy.NewDeliveryDedup(), ledger: l, queue: q}) + done := make(chan int, 2) + request := func() { + body := `{"ref":"refs/heads/main","after":"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}` + r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + r.Header.Set("X-Hub-Signature-256", githubSign("dummy", []byte(body))) + w := httptest.NewRecorder() + h(w, r) + done <- w.Code + } + go request() + <-l.entered + go request() + select { + case code := <-done: + t.Fatalf("early ack %d", code) + case <-time.After(30 * time.Millisecond): + } + close(l.release) + for i := 0; i < 2; i++ { + select { + case code := <-done: + if fail && code != 503 { + t.Fatalf("failed admission acknowledged %d", code) + } + if !fail && code != 200 { + t.Fatalf("successful admission %d", code) + } + case <-time.After(time.Second): + t.Fatal("duplicate stuck") + } + } + run.waitIdle(t, q) + }) + } +} +func TestCampaignResidentContextContainment(t *testing.T) { + root := t.TempDir() + os.MkdirAll(filepath.Join(root, "api"), 0700) + os.WriteFile(filepath.Join(root, "api", "Dockerfile"), []byte("FROM scratch"), 0600) + got, err := buildContextDir(root, "api") + if err != nil || filepath.Base(got) != "api" { + t.Fatalf("%s %v", got, err) + } + outside := t.TempDir() + os.Symlink(outside, filepath.Join(root, "escape")) + for _, p := range []string{"../outside", outside, "escape"} { + if _, err := buildContextDir(root, p); err == nil { + t.Fatalf("escape admitted %s", p) + } + } +} +func TestCampaignBuildVersionRefusesBeforeConfigEffects(t *testing.T) { + for _, v := range []string{"../bad", strings.Repeat("a", 129), "a:b"} { + err := runBuild(&Flags{}, v, "") + if err == nil || !strings.Contains(err.Error(), "invalid --version") { + t.Fatalf("%q %v", v, err) + } + } +} diff --git a/internal/cli/autodeploy_serve.go b/internal/cli/autodeploy_serve.go index f045026..7c9143c 100644 --- a/internal/cli/autodeploy_serve.go +++ b/internal/cli/autodeploy_serve.go @@ -24,6 +24,7 @@ import ( "github.com/useteploy/teploy/internal/releasemeta" "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" + "github.com/useteploy/teploy/internal/trigger" ) // newAutoDeployServeCmd is the resident webhook listener `teploy autodeploy @@ -129,6 +130,10 @@ func runAutoDeployServe(app, branch string, port int, strictEnv bool) error { queue := newAdmissionQueue(ledger, func(changedFiles []string, filesKnown bool, commit string) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() + if !trigger.ValidCommit(commit) { + logf("refusing nonimmutable authenticated admission; explicit scheduler tip mode is separate") + return + } if err := triggerAutoDeploy(ctx, executor, app, branch, buildDir, commit, out, changedFiles, filesKnown, strictEnv); err != nil { logf("deploy failed: %v", err) } else { @@ -234,6 +239,7 @@ type webhookHandlerConfig struct { // C02), and enqueues them on the bounded queue. A persistence failure is // a 503 + Retry-After: never ack what isn't durable. func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { + var admissionMu sync.Mutex writeJSON := func(w http.ResponseWriter, code int, v any) { body, err := json.Marshal(v) if err != nil { @@ -300,6 +306,10 @@ func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { contentSum := sha256.Sum256(body) digest := hex.EncodeToString(contentSum[:]) contentID := autodeploy.ContentIDFromDigest(digest) + // Serialize dedup through durable admission: concurrent duplicates wait + // for the first admission outcome before they can be acknowledged. + admissionMu.Lock() + defer admissionMu.Unlock() if cfg.dedup.SeenAndRecord(contentID) { writeJSON(w, http.StatusOK, admissionResponse{Status: "duplicate"}) if cfg.logf != nil { @@ -328,6 +338,13 @@ func newWebhookHandler(cfg webhookHandlerConfig) http.HandlerFunc { // after/checkout_sha names the exact commit this event built; the // fetch pins the checkout to it instead of the moving tip. commit := autodeploy.PushCommit(body) + if commit == "" { + // Authenticated delivery does not imply an immutable revision. Never + // send malformed/missing/zero after values into explicit tip mode. + cfg.dedup.Unrecord(contentID) + writeJSON(w, http.StatusUnprocessableEntity, admissionResponse{Status: "error", Error: "push requires a full nonzero authenticated commit"}) + return + } // DURABLE ADMISSION (C02): the record is fsynced BEFORE the 200. // A crash immediately after the response still leaves the admitted @@ -636,6 +653,14 @@ func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, if err := expandEnvTemplates(appCfg.Env, strictEnv); err != nil { return err } + if len(appCfg.EnvLiteral) > 0 { + if appCfg.Env == nil { + appCfg.Env = map[string]string{} + } + for k, v := range appCfg.EnvLiteral { + appCfg.Env[k] = v + } + } if len(appCfg.EnvFiles) > 0 { fileVars, err := env.LoadLocalEnvFiles(ctx, buildDir, appCfg.EnvFiles) if err != nil { @@ -668,18 +693,31 @@ func triggerAutoDeploy(ctx context.Context, executor ssh.Executor, app, branch, return fmt.Errorf("type:static apps are not supported by `teploy autodeploy serve` yet — use a scheduled/manual deploy") } - version, err := gitShortHashIn(buildDir) + revision, err := executor.Run(ctx, "cd "+ssh.ShellQuote(buildDir)+" && git rev-parse HEAD") if err != nil { - return fmt.Errorf("resolving version: %w", err) + return fmt.Errorf("resolving fetched revision: %w", err) + } + revision = strings.TrimSpace(revision) + if len(revision) != 40 && len(revision) != 64 { + return fmt.Errorf("invalid fetched revision") + } + if _, decodeErr := hex.DecodeString(revision); decodeErr != nil { + return fmt.Errorf("invalid fetched revision") } - if revision, revisionErr := gitRevisionIn(buildDir); revisionErr == nil { - appCfg.SourceRevision = revision + if commit != "" && revision != commit { + return fmt.Errorf("fetched revision does not match authenticated commit") } + version := revision + appCfg.SourceRevision = revision var image string needsBuild := appCfg.Image == "" if needsBuild { - buildMode, detectErr := build.DetectAt(buildDir, appCfg.Dockerfile) + contextDir, contextErr := buildContextDir(buildDir, appCfg.Context) + if contextErr != nil { + return contextErr + } + buildMode, detectErr := build.DetectAt(contextDir, appCfg.Dockerfile) if detectErr != nil { return detectErr } @@ -782,3 +820,29 @@ func resolveTLSFromRoot(tls *config.TLSConfig, root string) *config.TLSConfig { out.Key = resolve(out.Key) return &out } + +func buildContextDir(checkout, configured string) (string, error) { + if configured == "" { + configured = "." + } + if filepath.IsAbs(configured) { + return "", fmt.Errorf("build context must be relative to checkout") + } + root, err := filepath.Abs(checkout) + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(filepath.Join(root, configured)) + if err != nil { + return "", err + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return "", err + } + rel, err := filepath.Rel(root, resolved) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("build context escapes checkout") + } + return resolved, nil +} diff --git a/internal/cli/autodeploy_serve_test.go b/internal/cli/autodeploy_serve_test.go index 6c60c0e..4234c42 100644 --- a/internal/cli/autodeploy_serve_test.go +++ b/internal/cli/autodeploy_serve_test.go @@ -156,7 +156,7 @@ func TestWebhookHandler_ValidSignatureTriggersDeployOnce(t *testing.T) { run := newCountingRun() handler, ledger, queue := newAdmissionStack("s3cret", "", "myapp", run.run) - rec := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) + rec := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa"}`) if rec.Code != http.StatusOK { t.Errorf("status = %d, want 200", rec.Code) @@ -178,7 +178,7 @@ func TestWebhookHandler_InvalidSignatureNeverTriggers(t *testing.T) { run := newCountingRun() handler, _, _ := newAdmissionStack("s3cret", "", "myapp", run.run) - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"ref":"refs/heads/main"}`)) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa"}`)) req.Header.Set("X-Hub-Signature-256", "sha256=0000000000000000000000000000000000000000000000000000000000000000") rec := httptest.NewRecorder() @@ -217,8 +217,8 @@ func TestWebhookHandler_ReplayedDeliveryIgnored(t *testing.T) { run := newCountingRun() handler, _, queue := newAdmissionStack("s3cret", "", "myapp", run.run) - rec1 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) - rec2 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) + rec1 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa"}`) + rec2 := postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa"}`) run.waitCall(t) run.waitIdle(t, queue) @@ -241,7 +241,7 @@ func TestWebhookHandler_GitLabToken(t *testing.T) { run := newCountingRun() handler, _, queue := newAdmissionStack("s3cret", "", "myapp", run.run) - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"ref":"refs/heads/main"}`)) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa"}`)) req.Header.Set("X-Gitlab-Token", "s3cret") rec := httptest.NewRecorder() @@ -314,7 +314,7 @@ func TestWebhookHandler_OnDedupChangedCalledAfterDurableAdmission(t *testing.T) onDedupChanged: func() { changedCount++ }, }) - postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"a"}`) + postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main","after":"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}`) if changedCount != 1 { t.Errorf("onDedupChanged called %d times after durable admission, want 1", changedCount) } @@ -341,7 +341,7 @@ func TestWebhookHandler_OnlyWatchedBranchPushes(t *testing.T) { body string want bool }{ - {"watched branch", `{"ref":"refs/heads/main","commits":[{"added":["a"]}]}`, true}, + {"watched branch", `{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa","commits":[{"added":["a"]}]}`, true}, {"other branch", `{"ref":"refs/heads/develop","commits":[{"added":["a"]}]}`, false}, {"tag push", `{"ref":"refs/tags/v1.0.0"}`, false}, {"ping", `{}`, false}, @@ -372,7 +372,7 @@ func TestWebhookHandler_OnlyWatchedBranchPushes(t *testing.T) { func TestWebhookHandler_ContentReplayRejected(t *testing.T) { run := newCountingRun() handler, _, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) - body := `{"ref":"refs/heads/main"}` + body := `{"ref":"refs/heads/main","after":"51cff3c1f0bc59f6187e7040cc12a4e9b1eca7aa"}` if rec := postSigned(t, handler, "s3cret", "delivery-1", body); rec.Code != http.StatusOK { t.Fatalf("first delivery status = %d, want 200", rec.Code) @@ -401,16 +401,16 @@ func TestWebhookHandler_ReusedDeliveryIDDifferentContentNotSuppressed(t *testing run := newCountingRun() handler, _, queue := newAdmissionStack("s3cret", "", "myapp", run.run) - postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"aaaa"}`) + postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"70c881d4a26984ddce795f6f71817c9cf4480e79"}`) run.waitCall(t) - postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"bbbb"}`) + postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"8aed1322e5450badb078e1fb60a817a1df25a2ca"}`) run.waitCall(t) run.waitIdle(t, queue) if run.count() != 2 { t.Errorf("distinct authenticated content under a reused delivery ID must both deploy, got %d", run.count()) } // The SAME content replays to a no-op regardless of the header. - postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"aaaa"}`) + postSigned(t, handler, "s3cret", "same-delivery-id", `{"ref":"refs/heads/main","after":"70c881d4a26984ddce795f6f71817c9cf4480e79"}`) run.waitIdle(t, queue) if run.count() != 2 { t.Errorf("replayed content must be a no-op, got %d", run.count()) @@ -466,25 +466,42 @@ func TestWebhookHandler_ThreadsCommitToTrigger(t *testing.T) { } } -// A delivery with NO usable commit still deploys — pinned to nothing (tip), -// stated as such to the trigger. -func TestWebhookHandler_NoCommitMeansTip(t *testing.T) { - var gotCommit string +// C02: an authenticated delivery without a usable immutable commit is +// REFUSED (422, delivery unrecorded so a corrected retry can admit) — it +// must never fall through to moving-tip deployment. Explicit scheduler or +// manual tip mode is a separate admission that cannot inherit this event's +// authentication. +func TestWebhookHandler_NoCommitRefusedNeverTip(t *testing.T) { + var gotCommit = "unset" var gotMu sync.Mutex + ran := false run := func(_ []string, _ bool, commit string) { + ran = true gotMu.Lock() gotCommit = commit gotMu.Unlock() } - handler, _, queue := newAdmissionStack("s3cret", "main", "myapp", run) + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run) - postSigned(t, handler, "s3cret", "delivery-1", `{"ref":"refs/heads/main"}`) + for _, payload := range []string{ + `{"ref":"refs/heads/main"}`, // no after/checkout_sha at all + `{"ref":"refs/heads/main","after":""}`, // empty + `{"ref":"refs/heads/main","after":"abc"}`, // malformed hash + `{"ref":"refs/heads/main","after":"` + strings.Repeat("0", 40) + `"}`, // all-zero deletion marker + } { + rec := postSigned(t, handler, "s3cret", "delivery-nocommit", payload) + if rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("payload %s: status = %d, want 422 refusing nonimmutable admission", payload, rec.Code) + } + } waitQueueIdle(t, queue) - gotMu.Lock() defer gotMu.Unlock() - if gotCommit != "" { - t.Errorf("payload without a commit pinned trigger to %q, want empty (tip)", gotCommit) + if ran { + t.Errorf("nonimmutable delivery reached the deploy trigger pinned to %q", gotCommit) + } + if got := len(ledger.snapshot()); got != 0 { + t.Errorf("refused delivery left %d durable records; the retry path must be able to admit a corrected payload", got) } } diff --git a/internal/cli/binarydist.go b/internal/cli/binarydist.go index f396143..3b3190d 100644 --- a/internal/cli/binarydist.go +++ b/internal/cli/binarydist.go @@ -5,7 +5,9 @@ import ( "context" "crypto/sha256" "fmt" + "path" "strings" + "time" "github.com/useteploy/teploy/internal/ssh" ) @@ -18,7 +20,7 @@ import ( // (update.go's fetchLatestRelease/downloadToBytes/checksumFor/ // extractBinary), just targeting the server's platform instead of the // operator's local one, and uploading instead of self-replacing. -func deployTeployBinaryToServer(ctx context.Context, exec ssh.Executor, remotePath string) (version string, err error) { +func deployTeployBinaryToServer(ctx context.Context, exec ssh.Executor, remotePath string, capabilities ...string) (version string, err error) { goos, goarch, err := serverPlatform(ctx, exec) if err != nil { return "", fmt.Errorf("detecting server platform: %w", err) @@ -64,29 +66,8 @@ func deployTeployBinaryToServer(ctx context.Context, exec ssh.Executor, remotePa return "", fmt.Errorf("extracting binary: %w", err) } - // Upload to a staging path and rename into place, rather than writing - // remotePath directly: if `teploy autodeploy serve` is already running - // from remotePath (re-running `autodeploy setup` to pick up a newer - // release), an SFTP write straight to that path fails with ETXTBSY - // ("text file busy") — Linux refuses to open-for-write a file that's - // currently mapped as a running executable. rename() has no such - // restriction: it only touches the directory entry, so it succeeds - // even while the old inode is still executing. The running process - // keeps its old code until the systemd restart later in Setup picks - // up the new binary at the same path. Found live: re-running - // `autodeploy setup` against an already-configured app failed outright. - stagingPath := remotePath + ".new" - if err := exec.Upload(ctx, bytes.NewReader(binData), stagingPath, "0755"); err != nil { - return "", fmt.Errorf("uploading teploy binary: %w", err) - } - if _, err := exec.Run(ctx, fmt.Sprintf("mv %s %s", ssh.ShellQuote(stagingPath), ssh.ShellQuote(remotePath))); err != nil { - return "", fmt.Errorf("installing teploy binary: %w", err) - } - - // Sanity check the uploaded binary actually runs before wiring a - // systemd unit up to depend on it. - if _, err := exec.Run(ctx, ssh.ShellQuote(remotePath)+" version"); err != nil { - return "", fmt.Errorf("uploaded binary failed to run: %w", err) + if err := installServerBinary(ctx, exec, remotePath, binData, latestVersion, capabilities...); err != nil { + return "", err } return latestVersion, nil @@ -124,3 +105,70 @@ func serverPlatform(ctx context.Context, exec ssh.Executor) (goos, goarch string return goos, goarch, nil } + +// installServerBinary validates a private candidate before one atomic, +// serialized publication. The incumbent is never replaced by a refused +// candidate; privileged destinations use the supported passwordless sudo path. +func installServerBinary(ctx context.Context, executor ssh.Executor, destination string, data []byte, version string, capabilities ...string) error { + if !path.IsAbs(destination) || path.Clean(destination) != destination { + return fmt.Errorf("binary destination must be a clean absolute path") + } + out, err := executor.Run(ctx, "umask 077; mktemp -d /tmp/teploy-install.XXXXXXXX") + if err != nil { + return fmt.Errorf("creating binary staging directory: %w", err) + } + dir := strings.TrimSpace(out) + if !strings.HasPrefix(dir, "/tmp/teploy-install.") || strings.ContainsAny(dir, "\n\r") || path.Dir(dir) != "/tmp" { + return fmt.Errorf("invalid binary staging directory") + } + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _, _ = executor.Run(cleanup, "rm -rf -- "+ssh.ShellQuote(dir)) + }() + candidate := path.Join(dir, "teploy") + if err := executor.Upload(ctx, bytes.NewReader(data), candidate, "0700"); err != nil { + return fmt.Errorf("uploading teploy candidate: %w", err) + } + checkCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + actual, err := executor.Run(checkCtx, ssh.ShellQuote(candidate)+" version") + if err != nil { + return fmt.Errorf("candidate binary failed to run: %w", err) + } + if strings.TrimSpace(actual) != "teploy "+version { + return fmt.Errorf("candidate version mismatch: expected teploy %s", version) + } + for _, capability := range capabilities { + var args []string + for _, arg := range strings.Fields(capability) { + args = append(args, ssh.ShellQuote(arg)) + } + if len(args) == 0 { + return fmt.Errorf("empty binary capability") + } + if _, err := executor.Run(checkCtx, ssh.ShellQuote(candidate)+" "+strings.Join(args, " ")+" --help >/dev/null 2>&1"); err != nil { + return fmt.Errorf("candidate does not support %s: %w", capability, err) + } + } + prefix := "" + if strings.HasPrefix(destination, "/usr/") || strings.HasPrefix(destination, "/opt/") { + prefix = sudoPrefixFor(ctx, executor) + if prefix != "" { + prefix = "sudo -n " + } + } + script := fmt.Sprintf(`set -eu +stage=$(mktemp %s) +trap 'rm -f -- "$stage"' EXIT HUP INT TERM +cp -- %s "$stage" +chmod 0755 "$stage" +mv -f -- "$stage" %s`, ssh.ShellQuote(path.Join(path.Dir(destination), ".teploy-candidate.XXXXXXXX")), ssh.ShellQuote(candidate), ssh.ShellQuote(destination)) + // A unique sibling gives atomic rename on the destination filesystem. + // flock prevents concurrent installers from interleaving publication. + command := prefix + "flock -w 30 " + ssh.ShellQuote(destination+".install.lock") + " sh -c " + ssh.ShellQuote(script) + if _, err := executor.Run(ctx, command); err != nil { + return fmt.Errorf("publishing verified binary: %w", err) + } + return nil +} diff --git a/internal/cli/binarydist_test.go b/internal/cli/binarydist_test.go index b6d1dff..af6a18f 100644 --- a/internal/cli/binarydist_test.go +++ b/internal/cli/binarydist_test.go @@ -2,6 +2,11 @@ package cli import ( "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" "testing" "github.com/useteploy/teploy/internal/ssh" @@ -53,3 +58,111 @@ func TestServerPlatform_UnsupportedArch(t *testing.T) { t.Error("expected an error for an unsupported architecture") } } + +func TestInstallServerBinaryValidatesBeforePublication(t *testing.T) { + for _, tt := range []struct { + name, binary string + capability []string + }{ + {"wrong-version", "#!/bin/sh\necho teploy 8.0.0\n", nil}, + {"cannot-run", "#!/bin/sh\nexit 2\n", nil}, + {"missing-capability", "#!/bin/sh\nif [ \"$1\" = version ]; then echo teploy 9.0.0; else exit 1; fi\n", []string{"autodeploy redeploy"}}, + } { + t.Run(tt.name, func(t *testing.T) { + dst := filepath.Join(t.TempDir(), "teploy") + if err := os.WriteFile(dst, []byte("incumbent"), 0700); err != nil { + t.Fatal(err) + } + err := installServerBinary(context.Background(), ssh.NewLocalExecutor(), dst, []byte(tt.binary), "9.0.0", tt.capability...) + if err == nil { + t.Fatal("rejected candidate accepted") + } + got, err := os.ReadFile(dst) + if err != nil || string(got) != "incumbent" { + t.Fatalf("incumbent changed: %q %v", got, err) + } + }) + } +} + +func TestInstallServerBinaryConcurrentPublication(t *testing.T) { + if _, err := exec.LookPath("flock"); err != nil { + t.Skip("flock unavailable (Linux server publication contract)") + } + dst := filepath.Join(t.TempDir(), "teploy") + failures := make(chan error, 4) + for i := 0; i < 4; i++ { + go func(i int) { + failures <- installServerBinary(context.Background(), ssh.NewLocalExecutor(), dst, []byte(fmt.Sprintf("#!/bin/sh\n# candidate %d\necho teploy 9.0.0\n", i)), "9.0.0") + }(i) + } + for i := 0; i < 4; i++ { + if err := <-failures; err != nil { + t.Fatal(err) + } + } + output, err := exec.Command(dst, "version").Output() + if err != nil || strings.TrimSpace(string(output)) != "teploy 9.0.0" { + t.Fatalf("published candidate: %q %v", output, err) + } +} + +func TestInstallServerBinaryUsesSudoForProtectedDestination(t *testing.T) { + executor := ssh.NewMockExecutor("test", + ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/tmp/teploy-install.test"}, + ssh.MockCommand{Match: "'/tmp/teploy-install.test/teploy' version", Output: "teploy 9.0.0"}, + ssh.MockCommand{Match: "id -u", Output: "1000"}, + ssh.MockCommand{Match: "sudo -n flock -w 30", Output: ""}, + ssh.MockCommand{Match: "rm -rf --", Output: ""}, + ) + if err := installServerBinary(context.Background(), executor, "/usr/local/bin/teploy", []byte("binary"), "9.0.0"); err != nil { + t.Fatal(err) + } + var published bool + for _, command := range executor.Calls { + if strings.HasPrefix(command, "sudo -n flock -w 30 ") { + published = true + } + } + if !published { + t.Fatal("protected binary publication bypassed sudo") + } + for uploaded := range executor.Files { + if !strings.HasPrefix(uploaded, "/tmp/teploy-install.test/") { + t.Fatalf("privileged direct upload %s", uploaded) + } + } +} + +func TestInstallServerBinaryNonRootIntegration(t *testing.T) { + if os.Getenv("TEPLOY_INSTALL_INTEGRATION") != "1" { + t.Skip("requires a disposable Linux host with passwordless sudo") + } + if os.Geteuid() == 0 { + t.Fatal("integration must exercise a non-root installer") + } + output, err := exec.Command("sudo", "-n", "mktemp", "-d", "/usr/local/bin/teploy-campaign.XXXXXXXX").Output() + if err != nil { + t.Fatal(err) + } + dir := strings.TrimSpace(string(output)) + if filepath.Dir(dir) != "/usr/local/bin" || !strings.HasPrefix(filepath.Base(dir), "teploy-campaign.") { + t.Fatal("invalid disposable destination") + } + defer exec.Command("sudo", "-n", "rm", "-rf", "--", dir).Run() + if err := exec.Command("sudo", "-n", "chmod", "0755", dir).Run(); err != nil { + t.Fatal(err) + } + destination := filepath.Join(dir, "teploy") + good := []byte("#!/bin/sh\necho teploy 9.0.0\n") + if err := installServerBinary(context.Background(), ssh.NewLocalExecutor(), destination, good, "9.0.0"); err != nil { + t.Fatal(err) + } + if err := installServerBinary(context.Background(), ssh.NewLocalExecutor(), destination, []byte("#!/bin/sh\nexit 3\n"), "9.0.0"); err == nil { + t.Fatal("broken candidate accepted") + } + actual, err := exec.Command(destination, "version").Output() + if err != nil || strings.TrimSpace(string(actual)) != "teploy 9.0.0" { + t.Fatalf("incumbent damaged: %q %v", actual, err) + } +} diff --git a/internal/cli/build.go b/internal/cli/build.go index cce1363..0a88b20 100644 --- a/internal/cli/build.go +++ b/internal/cli/build.go @@ -66,6 +66,11 @@ Example: } func runBuild(flags *Flags, version, destination string) error { + if version != "" { + if err := validateVersionArg(version); err != nil { + return err + } + } var appCfg *config.AppConfig var err error if destination != "" { @@ -176,7 +181,10 @@ func runBuild(flags *Flags, version, destination string) error { // attempt's build dir is the --link-dest basis so transfer stays // incremental. The directory is scratch — the next deploy of this app // prunes it once its hash leaves the protection window. - att := releasemeta.MustAttempt(appCfg.App, version) + att, err := releasemeta.NewAttempt(appCfg.App, version) + if err != nil { + return err + } remoteDir, err := syncAttemptBuildContext(ctx, executor, appCfg, att, buildMode, host, user, key, out, os.Stderr) if err != nil { return err diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index ae7beda..669fc67 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -14,6 +14,7 @@ import ( "errors" "os" "path/filepath" + "regexp" "strings" "testing" "time" @@ -232,10 +233,12 @@ func TestContractsAttemptNameGolden(t *testing.T) { writeFixture(t, "attempt-name/valid/examples.json", []string{ "abc1234.deadb17ecafef00d", "9f31c02.0123456789abcdef", + "ABC1234.deadb17ecafef00d", + "v1.2.3.deadb17ecafef00d", + "release_1.deadb17ecafef00d", }) writeFixture(t, "attempt-name/invalid/examples.json", []string{ "deadb17ecafef00d", // missing the hash half - "ABC1234.deadb17ecafef00d", // uppercase "abc1234.DeadB17eCafef00d", // uppercase hex half "abc1234.deadb17ecafef00", // 15 hex chars "../escape.attempt0000000", // path characters @@ -351,3 +354,33 @@ func TestContractsPreviewStateListRowGolden(t *testing.T) { writeFixture(t, name, row) } } + +func TestRuntimeIdentifierSchemaParity(t *testing.T) { + for _, hash := range []string{"v1", "v1.2.3", "1750000000", "abcdef12", "release_1", "ABC1234", strings.Repeat("a", 128)} { + attempt, err := releasemeta.NewAttempt("demo", hash) + if err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + schema, value string + nested bool + }{{"attempt-name", attempt.Name(), false}, {"release-record", hash, true}} { + data, err := os.ReadFile(filepath.Join(contractsDir, "schema", tt.schema+".schema.json")) + if err != nil { + t.Fatal(err) + } + var schema map[string]any + if err := json.Unmarshal(data, &schema); err != nil { + t.Fatal(err) + } + pattern, _ := schema["pattern"].(string) + if tt.nested { + pattern = schema["properties"].(map[string]any)["hash"].(map[string]any)["pattern"].(string) + } + matched, err := regexp.MatchString(pattern, tt.value) + if err != nil || !matched { + t.Fatalf("%s rejects runtime %s: %v", tt.schema, tt.value, err) + } + } + } +} diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 8b1b53c..c4150bf 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -31,10 +31,12 @@ import ( "github.com/useteploy/teploy/internal/secret" "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" + "github.com/useteploy/teploy/internal/trigger" ) func newDeployCmd(flags *Flags) *cobra.Command { var ( + triggerStdin bool image string version string skipDNSCheck bool @@ -72,6 +74,11 @@ a machine error envelope on stderr whose code distinguishes the classes (config-invalid, conflict, uncertain-outcome, internal).`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if triggerStdin { + return triggerCommand(cmd, "deploy", func(r trigger.Request) (trigger.Result, error) { + return runTriggerCLI(cmd, flags, "deploy", "", image, version, previewDeployOpts{}, r) + }) + } var serverName string if len(args) > 0 { serverName = args[0] @@ -87,6 +94,7 @@ a machine error envelope on stderr whose code distinguishes the classes }, } + cmd.Flags().BoolVar(&triggerStdin, "trigger-stdin", false, "read a bounded immutable trigger request from stdin") cmd.Flags().StringVar(&image, "image", "", "Docker image to deploy (skips build if set)") cmd.Flags().StringVar(&version, "version", "", "version identifier (default: git short hash)") cmd.Flags().BoolVar(&skipDNSCheck, "skip-dns-check", false, "skip DNS validation (for proxied domains like Cloudflare)") @@ -226,6 +234,22 @@ func resolveDeployEnv(ctx context.Context, appCfg *config.AppConfig, strict bool } } } + if appCfg.Env == nil { + appCfg.Env = map[string]string{} + } + for k, v := range appCfg.EnvLiteral { + appCfg.Env[k] = v + } + for name, acc := range appCfg.Accessories { + if acc.Env == nil { + acc.Env = map[string]string{} + } + for k, v := range acc.EnvLiteral { + acc.Env[k] = v + } + appCfg.Accessories[name] = acc + } + return nil } @@ -308,7 +332,7 @@ func selectServersByRoleTag(names []string, all map[string]config.Server, role s // deployAppConfig runs a single-server deploy from an already-loaded AppConfig. // Used by runDeploy (from teploy.yml), runTemplateInstall (from template), // and runApply (a verified C05 plan record — planID stamps the receipt). -func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, version string, skipDNSCheck, migrateVolumes bool, planID string) error { +func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, version string, skipDNSCheck, migrateVolumes bool, planID string, plans ...*PlanRecord) error { var err error // 2. Resolve server (single-server deploy). @@ -334,7 +358,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, // no health checks) so we branch here before the container build/run // machinery runs. if appCfg.IsStatic() { - return runStaticDeploy(appCfg, host, user, key) + return runStaticDeploy(appCfg, host, user, key, plans...) } // 3. Resolve image. @@ -406,7 +430,7 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, // serialize at the source instead of interleaving writes onto shared // per-app paths. The lease is fenced (F16) and renewed in the // background; it is released when deployAppConfig returns. - if err := state.EnsureAppDir(ctx, executor, appCfg.App); err != nil { + if err := docker.NewClient(executor).EnsureManagedDirectory(ctx, appCfg.App, "", ""); err != nil { return fmt.Errorf("creating app directory: %w", err) } lk, err := state.AcquireLockFenced(ctx, executor, appCfg.App) @@ -415,6 +439,15 @@ func deployAppConfig(flags *Flags, appCfg *config.AppConfig, serverName, image, } defer state.ReleaseLockFenced(executor, lk, appCfg.App) lk.StartRenewal(executor) + if len(plans) > 0 && plans[0] != nil { + if err := revalidateLockedPlan(ctx, executor, appCfg, image, version, plans[0]); err != nil { + return refuseApply(err) + } + } + + if err := admitDeploymentVolumes(ctx, &state.FencedExecutor{Executor: executor, Lock: lk}, appCfg); err != nil { + return err + } // The attempt keys every artifact this deploy generates (F08): // immutable per (release, attempt), so the F14 record's references @@ -522,15 +555,47 @@ func deployBuiltImage(ctx context.Context, executor ssh.Executor, appCfg *config // deployBuiltImageFenced is deployBuiltImage with the caller's lease and // attempt: lk is a lock the caller already owns (the terminal path's early // lease and the resident autodeploy path — audits F07/F08) and att keys the -// attempt-scoped artifacts (env file, TLS). lk == nil means Deployer.Deploy -// acquires the lock itself (att must still be non-nil for the env file). +// attempt-scoped artifacts (env file, TLS). A nil lease is acquired here +// before ownership admission and preparation effects. // sourceRoot is the directory the source was synced/built from ("." for // manual deploys, the fetched checkout for autodeploy) — it keys the // plan-time provenance (C04); empty means no build provenance. planID, // when non-empty, is the C05 plan record the caller verified before // executing; it is stamped into the provenance receipt so releasemeta // ties the release back to the reviewed plan. -func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg *config.AppConfig, image, version, serverDisplay string, migrateVolumes, needsBuild bool, sourceRoot string, lk *state.Lock, att *releasemeta.Attempt, planID string) error { +func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg *config.AppConfig, image, version, serverDisplay string, migrateVolumes, needsBuild bool, sourceRoot string, lk *state.Lock, att *releasemeta.Attempt, planID string) (retErr error) { + out := deployOutput(ctx) + if out == nil { + out = os.Stdout + } + if lk == nil { + if err := docker.NewClient(executor).EnsureManagedDirectory(ctx, appCfg.App, "", ""); err != nil { + return err + } + var err error + lk, err = state.AcquireLockFenced(ctx, executor, appCfg.App) + if err != nil { + return err + } + defer state.ReleaseLockFenced(executor, lk, appCfg.App) + lk.StartRenewal(executor) + } + rawExecutor := executor + executor = &state.FencedExecutor{Executor: executor, Lock: lk} + if err := admitDeploymentVolumes(ctx, executor, appCfg); err != nil { + return err + } + var migration *docker.VolumeMigration + defer func() { + if migration == nil { + return + } + if retErr == nil || state.PreservePublishedState(retErr) { + migration.Commit() + } else { + retErr = errors.Join(retErr, migration.Restore(context.WithoutCancel(ctx))) + } + }() if att == nil { attVal := releasemeta.MustAttempt(appCfg.App, version) att = &attVal @@ -547,17 +612,17 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * // attempt's write-once namespace, next to the build context it // describes. Best-effort resolution, but the receipt itself must // land: a missing provenance file is an unwitnessed plan (warned). - prov := resolveDeployProvenance(ctx, executor, os.Stdout, appCfg, sourceRoot, image, version, manifestSHA256, needsBuild) + prov := resolveDeployProvenance(ctx, executor, out, appCfg, sourceRoot, image, version, manifestSHA256, needsBuild) prov.PlanID = planID if err := releasemeta.WriteAttemptProvenance(ctx, executor, *att, prov); err != nil { - fmt.Printf("Warning: could not persist the deploy provenance receipt for %s@%s: %v\n", appCfg.App, version, err) + fmt.Fprintf(out, "Warning: could not persist the deploy provenance receipt for %s@%s: %v\n", appCfg.App, version, err) } // 9. Ensure accessories are running. var envFile string if len(appCfg.Accessories) > 0 { - fmt.Println("Ensuring accessories...") - accMgr := accessories.NewManager(executor, os.Stdout) + fmt.Fprintln(out, "Ensuring accessories...") + accMgr := accessories.NewManager(executor, out) allVars := make(map[string]string) for _, name := range sortedAccessoryNames(appCfg.Accessories) { vars, err := accMgr.EnsureRunning(ctx, appCfg.App, name, appCfg.Accessories[name]) @@ -602,9 +667,15 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * var volumes map[string]string if len(appCfg.Volumes) > 0 { volumes = plannedVolumeMounts(appCfg.App, appCfg.Volumes) - for _, hostPath := range managedVolumeHostPaths(appCfg.App, appCfg.Volumes) { - if _, err := executor.Run(ctx, "mkdir -p "+hostPath); err != nil { - return fmt.Errorf("creating volume directory %s: %w", hostPath, err) + for _, name := range volumeKeys(appCfg.Volumes) { + if config.IsHostBindVolume(name) { + continue + } + if _, owned := appCfg.VolumeOwnership[name]; owned { + continue + } + if err := docker.NewClient(executor).EnsureManagedDirectory(ctx, appCfg.App, "", name); err != nil { + return err } } } @@ -631,7 +702,8 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * if !migrateVolumes { return docker.FormatMismatchError(appCfg.App, mismatches) } - if err := docker.MigrateVolumes(ctx, executor, appCfg.App, mismatches, os.Stdout); err != nil { + migration, err = docker.BeginVolumeMigration(ctx, executor, appCfg.App, mismatches, out) + if err != nil { return fmt.Errorf("migrating volumes: %w", err) } } @@ -646,7 +718,7 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * return err } if tlsCert != "" { - fmt.Println(" TLS certificate uploaded") + fmt.Fprintln(out, " TLS certificate uploaded") } // Container env: teploy.yml's `env:` block plus decrypted secrets, @@ -658,15 +730,16 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * } // 11. Deploy. - deployer := deploy.NewDeployer(executor, os.Stdout) + deployer := deploy.NewDeployer(rawExecutor, out) deployCfg := deployConfigFromApp(appCfg, image, version, envFiles, volumes, tlsCert, tlsKey, tlsInternal, appliedManifest, manifestSHA256) deployCfg.Provenance = prov + attachTrigger(ctx, &deployCfg) // Vulnerability gate: scan the image on the server before any container // starts — fixable CRITICALs block the deploy. if appCfg.Scan { - fmt.Println("Scanning image for vulnerabilities (trivy)...") - if err := docker.NewClient(executor).ScanImage(ctx, image, os.Stdout); err != nil { + fmt.Fprintln(out, "Scanning image for vulnerabilities (trivy)...") + if err := docker.NewClient(executor).ScanImage(ctx, image, out); err != nil { return err } } @@ -709,7 +782,7 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * // 13. Prune old build images (best-effort). if needsBuild { - builder := build.NewBuilder(executor, os.Stdout) + builder := build.NewBuilder(executor, out) builder.PruneImages(ctx, appCfg.App) } @@ -889,8 +962,8 @@ func runMultiDeploy(flags *Flags, appCfg *config.AppConfig, image, version strin fmt.Print(multideploy.FormatResults(canaryResults)) if failed := rollbackFailedWave(ctx, appCfg, canary, canaryResults, parallel, migrateVolumes); failed > 0 { // A canary failure halts everything: the rest of the fleet was - // never touched and the canary is back on the old version. - return fmt.Errorf("rollout halted: %d of %d canary server(s) failed; rest of fleet untouched", failed, len(canary)) + // never touched; canary authority may require reconciliation. + return fmt.Errorf("rollout halted: %d of %d canary server(s) failed; rest of fleet untouched; reconcile failed canary authority", failed, len(canary)) } fmt.Printf("Rollout: canary healthy — deploying remaining %d server(s) (parallel=%d, max_failures=%d)...\n", len(rest), parallel, maxFailures) @@ -926,6 +999,9 @@ func runMultiDeploy(flags *Flags, appCfg *config.AppConfig, image, version strin // main wave busts the failure budget. successTargets = append(canarySucceeded, successTargets...) + if err := fleetPublicationError(results); err != nil { + return err + } if failCount == 0 { if len(successTargets) > 0 { // Front-door activation is a required deployment phase (audit @@ -1031,6 +1107,10 @@ func rollbackFailedWave(ctx context.Context, appCfg *config.AppConfig, wave []mu failed++ } } + if err := fleetPublicationError(results); err != nil { + fmt.Fprintln(os.Stderr, err) + return failed + } if failed == 0 { return 0 } @@ -1162,7 +1242,7 @@ func gitRevisionIn(dir string) (string, error) { // runStaticDeploy handles the type:static deploy path: build (locally) → // rsync → symlink → Caddyfile mirror. Container build/run/health-check // machinery is intentionally bypassed. -func runStaticDeploy(cfg *config.AppConfig, host, user, key string) error { +func runStaticDeploy(cfg *config.AppConfig, host, user, key string, plans ...*PlanRecord) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -1198,6 +1278,13 @@ func runStaticDeploy(cfg *config.AppConfig, host, user, key string) error { SourceRevision: cfg.SourceRevision, } + if len(plans) > 0 && plans[0] != nil { + rec := plans[0] + staticCfg.ExpectedState = &state.GenerationExpectation{Exists: rec.TargetState.Deployed, Generation: rec.TargetState.Generation} + staticCfg.UnderLockPreflight = func(ctx context.Context) error { + return revalidateLockedPlan(ctx, executor, cfg, "", rec.TargetVersion, rec) + } + } d := deploy.NewStaticDeployer(executor, os.Stdout) d.SSHKeyPath = key if err := d.Deploy(ctx, staticCfg); err != nil { @@ -1384,3 +1471,16 @@ func ensureImage(ctx context.Context, dk *docker.Client, image string, out io.Wr fmt.Fprintln(out, " Image pulled") return nil } + +func fleetPublicationError(results []multideploy.Result) error { + var debt error + for _, result := range results { + if result.PublicationNeedsReconciliation || state.PreservePublishedState(result.Error) { + debt = errors.Join(debt, fmt.Errorf("%s: %w", result.Server, result.Error)) + } + } + if debt != nil { + return fmt.Errorf("fleet publication requires reconciliation; automatic load-balancer changes and convergence rollback withheld: %w", debt) + } + return nil +} diff --git a/internal/cli/deployconfig.go b/internal/cli/deployconfig.go index 6e81ab4..9d9b8ee 100644 --- a/internal/cli/deployconfig.go +++ b/internal/cli/deployconfig.go @@ -20,6 +20,7 @@ func deployConfigFromApp(appCfg *config.AppConfig, image, version string, envFil Version: version, EnvFiles: envFiles, Volumes: volumes, + VolumeOwnership: appCfg.VolumeOwnership, Processes: appCfg.Processes, NoHealthcheck: disabledHealthchecks(appCfg.Healthcheck), Health: healthConfigFrom(appCfg.Health), diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index b27aded..85a4f96 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -634,13 +634,10 @@ const doctorSecretFilesFind = "find /deployments \\( -name .git -o -name node_mo "! -name 'teploy.example.*' ! -name .env.example ! -name .env.sample ! -name .env.template " + "-print" -const doctorSecretFilesProbe = doctorSecretFilesFind + " 2>/dev/null | head -n 201" +const doctorSecretFilesProbe = "scan=$(" + doctorSecretFilesFind + `) || exit $?; printf '%s\n' "$scan" | head -n 201` -// doctorOpenBuildDirsProbe lists build-context directories other host -// users can traverse (o+x): the legacy shared build dir and the attempt -// root. Read-only. -const doctorOpenBuildDirsProbe = "find /deployments -mindepth 2 -maxdepth 3 -type d " + - "\\( -path '/deployments/*/build' -o -path '/deployments/*/meta/att' \\) -perm -001 -print 2>/dev/null | head -n 201" +// doctorOpenBuildDirsProbe preserves find's status while bounding output. +const doctorOpenBuildDirsProbe = `scan=$(find /deployments -mindepth 2 -maxdepth 3 -type d \( -path '*/build' -o -path '*/meta/att' \) -perm -001 -print) || exit $?; printf '%s\n' "$scan" | head -n 201` // doctorSecretExposureCheck finds what teploy CLIs before L14 left behind: // they uploaded the whole source tree, gitignored files included, so diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index 957ee7e..a89a594 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -31,7 +31,7 @@ func doctorHappyMock() *ssh.MockExecutor { ssh.MockCommand{Match: "docker manifest inspect", Output: `{"schemaVersion":2}`}, ssh.MockCommand{Match: "docker exec caddy", Output: `{}`}, ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Output: "teploy v0.1.37"}, - ssh.MockCommand{Match: "find /deployments ", Output: ""}, + ssh.MockCommand{Match: "scan=$(find /deployments ", Output: ""}, ) } @@ -180,7 +180,7 @@ func TestDoctorHumanTable(t *testing.T) { ssh.MockCommand{Match: "docker manifest inspect", Output: `{}`}, ssh.MockCommand{Match: "docker exec caddy", Output: `{}`}, ssh.MockCommand{Match: "'/deployments/.bin/teploy' version", Output: "teploy v0.1.37"}, - ssh.MockCommand{Match: "find /deployments ", Output: ""}, + ssh.MockCommand{Match: "scan=$(find /deployments ", Output: ""}, ) report, _ := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) var out bytes.Buffer @@ -677,7 +677,7 @@ func TestDoctorEndToEndAllOK(t *testing.T) { // outside doctor's read-only allowlist — the no-deployment-effects proof. func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { t.Helper() - readOnly := []string{ + readOnly := []string{"scan=$(find", "docker version", "docker manifest inspect", "docker exec caddy", @@ -707,7 +707,7 @@ func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { func TestDoctorSecretExposureCheck(t *testing.T) { ctx := context.Background() t.Run("clean host", func(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "find /deployments ", Output: ""}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "scan=$(find /deployments ", Output: ""}) if check := doctorSecretExposureCheck(ctx, mock); check.Result != "ok" { t.Fatalf("check = %+v, want ok", check) } @@ -715,8 +715,8 @@ func TestDoctorSecretExposureCheck(t *testing.T) { }) t.Run("leaked overlay fails with the paths", func(t *testing.T) { mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "find /deployments \\( -name .git", Output: "/deployments/dash/meta/att/v1.0123456789abcdef/build/teploy.home.yml\n/deployments/dash/build/.env\n"}, - ssh.MockCommand{Match: "find /deployments -mindepth 2", Output: "/deployments/dash/build\n"}, + ssh.MockCommand{Match: "scan=$(find /deployments \\( -name .git", Output: "/deployments/dash/meta/att/v1.0123456789abcdef/build/teploy.home.yml\n/deployments/dash/build/.env\n"}, + ssh.MockCommand{Match: "scan=$(find /deployments -mindepth 2", Output: "/deployments/dash/build\n"}, ) check := doctorSecretExposureCheck(ctx, mock) if check.Result != "fail" { @@ -741,8 +741,8 @@ func TestDoctorSecretExposureCheck(t *testing.T) { }) t.Run("open build dir alone warns", func(t *testing.T) { mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "find /deployments \\( -name .git", Output: ""}, - ssh.MockCommand{Match: "find /deployments -mindepth 2", Output: "/deployments/ship/meta/att\n"}, + ssh.MockCommand{Match: "scan=$(find /deployments \\( -name .git", Output: ""}, + ssh.MockCommand{Match: "scan=$(find /deployments -mindepth 2", Output: "/deployments/ship/meta/att\n"}, ) check := doctorSecretExposureCheck(ctx, mock) if check.Result != "warn" || !strings.Contains(check.Detail, "/deployments/ship/meta/att") { @@ -750,7 +750,7 @@ func TestDoctorSecretExposureCheck(t *testing.T) { } }) t.Run("scan failure warns", func(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "find /deployments ", Err: errors.New("ssh: connection lost")}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "scan=$(find /deployments ", Err: errors.New("ssh: connection lost")}) if check := doctorSecretExposureCheck(ctx, mock); check.Result != "warn" { t.Fatalf("check = %+v, want warn", check) } @@ -761,8 +761,8 @@ func TestDoctorSecretExposureCheck(t *testing.T) { fmt.Fprintf(&many, "/deployments/a/build/x%d/.env\n", i) } mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "find /deployments \\( -name .git", Output: many.String()}, - ssh.MockCommand{Match: "find /deployments -mindepth 2", Output: ""}, + ssh.MockCommand{Match: "scan=$(find /deployments \\( -name .git", Output: many.String()}, + ssh.MockCommand{Match: "scan=$(find /deployments -mindepth 2", Output: ""}, ) check := doctorSecretExposureCheck(ctx, mock) if !strings.Contains(check.Detail, "200+ secret-bearing") || !strings.Contains(check.Detail, "(+196 more)") { diff --git a/internal/cli/dr.go b/internal/cli/dr.go index 7185bb9..62a4bdf 100644 --- a/internal/cli/dr.go +++ b/internal/cli/dr.go @@ -3,7 +3,9 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" + "io" "os" "os/signal" @@ -109,7 +111,11 @@ func newDRCreateCmd(flags *Flags, version string) *cobra.Command { return err } defer executor.Close() - client := backup.NewClient(executor, os.Stdout) + progress := io.Writer(os.Stdout) + if flags.JSON { + progress = os.Stderr + } + client := backup.NewClient(executor, progress) m, err := client.CreateBundle(ctx, backup.BundleOptions{ App: appCfg.App, Config: *appCfg, @@ -274,7 +280,11 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { return err } defer executor.Close() - client := backup.NewClient(executor, os.Stdout) + progress := io.Writer(os.Stdout) + if flags.JSON { + progress = os.Stderr + } + client := backup.NewClient(executor, progress) receipt, err := client.RestoreBundleIsolated(ctx, backup.BundleRestoreOptions{ App: appCfg.App, ID: args[0], @@ -284,7 +294,11 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { return err } if flags.JSON { - return json.NewEncoder(os.Stdout).Encode(receipt) + var outcome error + if !receipt.OK { + outcome = fmt.Errorf("restore validation failed — see the receipt; nothing live was touched") + } + return errors.Join(outcome, json.NewEncoder(os.Stdout).Encode(receipt)) } printReceipt(receipt) if !receipt.OK { @@ -331,7 +345,11 @@ func newDRCutoverCmd(flags *Flags) *cobra.Command { return err } defer executor.Close() - client := backup.NewClient(executor, os.Stdout) + progress := io.Writer(os.Stdout) + if flags.JSON { + progress = os.Stderr + } + client := backup.NewClient(executor, progress) receipt, err := client.CutoverBundle(ctx, backup.BundleRestoreOptions{ App: appCfg.App, ID: args[0], @@ -341,7 +359,11 @@ func newDRCutoverCmd(flags *Flags) *cobra.Command { return err } if flags.JSON { - return json.NewEncoder(os.Stdout).Encode(receipt) + var outcome error + if !receipt.OK { + outcome = fmt.Errorf("cutover did not complete — inspect the receipt and recovery directories") + } + return errors.Join(outcome, json.NewEncoder(os.Stdout).Encode(receipt)) } fmt.Printf("Cutover receipt for bundle %s: %d path(s) promoted, originals kept in %v\n", receipt.BundleID, len(receipt.Promoted), receipt.RecoveryDirs) diff --git a/internal/cli/drift.go b/internal/cli/drift.go index a0ea233..e24d7d6 100644 --- a/internal/cli/drift.go +++ b/internal/cli/drift.go @@ -213,7 +213,7 @@ func reportDrift(ctx context.Context, flags *Flags, appCfg *config.AppConfig, ex // the empty loop used to report "in sync" for a fully removed // deployment (audit F76). State says this app IS deployed, so an // empty live inventory is maximal drift. - if len(items) == 0 && len(containers) == 0 { + if len(items) == 0 && !hasDeployedWorkload(containers) { items = append(items, driftItem{ Kind: "missing", Name: appCfg.App, @@ -261,3 +261,12 @@ func reportDrift(ctx context.Context, flags *Flags, appCfg *config.AppConfig, ex fmt.Println("(This is report-only — no changes were made.)") return true, nil } + +func hasDeployedWorkload(containers []docker.Container) bool { + for _, c := range containers { + if c.Labels["teploy.role"] != "accessory" && !strings.HasPrefix(c.Labels["teploy.process"], "preview-") && c.Labels["teploy.version"] != "" { + return true + } + } + return false +} diff --git a/internal/cli/heal.go b/internal/cli/heal.go index 6fbb457..3378569 100644 --- a/internal/cli/heal.go +++ b/internal/cli/heal.go @@ -89,7 +89,8 @@ type HealConfig struct { Processes []string `json:"processes"` // opted-in process names (web is the only one with a probe surface) MaxAttempts int `json:"max_attempts"` // consecutive restarts before giving up + alerting BackoffSecs int `json:"backoff_seconds"` // minimum seconds between restarts of the same container - HealthPath string `json:"health_path"` // probe path (default /health) + HealthMode string `json:"health_mode,omitempty"` + HealthPath string `json:"health_path"` // probe path (default /health) } func (c HealConfig) withDefaults() HealConfig { @@ -99,7 +100,7 @@ func (c HealConfig) withDefaults() HealConfig { if c.BackoffSecs == 0 { c.BackoffSecs = 60 } - if c.HealthPath == "" { + if c.HealthPath == "" && c.HealthMode != deploy.HealthModeTCP { c.HealthPath = "/health" } if len(c.Processes) == 0 { @@ -251,6 +252,12 @@ func runHealEnable(flags *Flags, cfg HealConfig, intervalSec int, noBinary bool) return err } defer executor.Close() + if cfg.HealthPath == "" { + cfg.HealthPath = appCfg.Health.Path + } + if cfg.HealthMode == "" { + cfg.HealthMode = appCfg.Health.Mode + } if intervalSec < 5 { intervalSec = 30 } @@ -356,7 +363,7 @@ func runHealPass(ctx context.Context, exec ssh.Executor, app string, out io.Writ if err != nil || port == 0 { continue // can't probe without a published port } - if probeHealthy(ctx, exec, port, cfg.HealthPath) { + if deploy.NewDeployer(exec, io.Discard).HealthCheckAtWithConfig(ctx, port, c.Name, deploy.HealthConfig{Path: cfg.HealthPath, Mode: cfg.HealthMode, Timeout: 5 * time.Second, Interval: time.Second}) == nil { delete(hs, c.Name) // recovered — reset attempt state continue } @@ -411,23 +418,7 @@ func runHealPass(ctx context.Context, exec ssh.Executor, app string, out io.Writ // probeHealthy runs a single host-side health check, mirroring // internal/deploy/health.go: 200 = healthy; 404/3xx falls back to a TCP check. func probeHealthy(ctx context.Context, exec ssh.Executor, port int, path string) bool { - out, err := exec.Run(ctx, fmt.Sprintf("curl -s -o /dev/null -w '%%{http_code}' http://localhost:%d%s", port, path)) - if err != nil { - return false - } - code := strings.TrimSpace(out) - if code == "200" { - return true - } - if code == "404" || strings.HasPrefix(code, "3") { - cmd, ok := deploy.TCPProbeCommand("localhost", port) - if !ok { - return false - } - _, terr := exec.Run(ctx, cmd) - return terr == nil - } - return false + return deploy.NewDeployer(exec, io.Discard).HealthCheckAtWithConfig(ctx, port, "", deploy.HealthConfig{Path: path, Timeout: 5 * time.Second, Interval: time.Second}) == nil } func readHealConf(ctx context.Context, exec ssh.Executor, app string) (HealConfig, bool, error) { diff --git a/internal/cli/kv.go b/internal/cli/kv.go index 9558cb9..2183359 100644 --- a/internal/cli/kv.go +++ b/internal/cli/kv.go @@ -277,8 +277,12 @@ func runKvQuery(flags *Flags, t *kvTarget, sql string) (string, error) { dk := docker.NewClient(executor) container := accessories.ContainerName(appCfg.App, t.accessory) - command := fmt.Sprintf("nucleus shell -c %s --json", shellSingleQuote(sql)) - output, err := dk.Exec(ctx, container, command) + help, err := dk.Exec(ctx, container, "nucleus shell --help") + if err != nil || !strings.Contains(help, "--command-stdin") { + return "", fmt.Errorf("Nucleus accessory requires shell --command-stdin support for private SQL transport; upgrade the Nucleus image") + } + command := "nucleus shell --command-stdin --json" + output, err := dk.ExecInput(ctx, container, command, strings.NewReader(sql)) if err != nil { msg := strings.TrimSpace(output) if msg != "" { diff --git a/internal/cli/lifecycle_admission_regression_test.go b/internal/cli/lifecycle_admission_regression_test.go new file mode 100644 index 0000000..fd727e3 --- /dev/null +++ b/internal/cli/lifecycle_admission_regression_test.go @@ -0,0 +1,26 @@ +package cli + +import ( + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "testing" +) + +func TestLogsRejectsSelectorBeforeResolution(t *testing.T) { + if err := runLogs(&Flags{}, "demo", "web;touch /tmp/marker", 50, false); err == nil { + t.Fatal("unsafe selector accepted") + } +} +func TestFleetEndpointSeparatesManagementPort(t *testing.T) { + for _, test := range []struct{ endpoint, want string }{{"192.0.2.10:2222", "192.0.2.10:8080"}, {"[2001:db8::1]:2222", "[2001:db8::1]:8080"}, {"2001:db8::1", "[2001:db8::1]:8080"}} { + if got := fleetServiceAddress(test.endpoint, &config.AppConfig{Ingress: "host", Port: 8080}); got != test.want { + t.Fatalf("%s => %s", test.endpoint, got) + } + } +} +func TestAccessoriesAndPreviewsAreNotDeployedWorkload(t *testing.T) { + inventory := []docker.Container{{Labels: map[string]string{"teploy.role": "accessory", "teploy.version": "v1"}}, {Labels: map[string]string{"teploy.process": "preview-main", "teploy.version": "v1"}}} + if hasDeployedWorkload(inventory) { + t.Fatal("accessory-only inventory hides missing application") + } +} diff --git a/internal/cli/logs.go b/internal/cli/logs.go index f4741c6..1d8c114 100644 --- a/internal/cli/logs.go +++ b/internal/cli/logs.go @@ -8,7 +8,9 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" "github.com/useteploy/teploy/internal/state" ) @@ -49,6 +51,12 @@ func tailToLines(_ *pflag.FlagSet, name string) pflag.NormalizedName { } func runLogs(flags *Flags, appName, process string, lines int, follow bool) error { + if err := config.ValidateIdentifier("process", process); err != nil { + return err + } + if lines < 0 { + return fmt.Errorf("lines must be nonnegative") + } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -74,5 +82,5 @@ func logsCommand(containerName string, lines int, follow bool) string { if follow { followFlag = " -f" } - return fmt.Sprintf("docker logs%s --tail %d %s", followFlag, lines, containerName) + return fmt.Sprintf("docker logs%s --tail %d %s", followFlag, lines, ssh.ShellQuote(containerName)) } diff --git a/internal/cli/machine_contract_test.go b/internal/cli/machine_contract_test.go index 8d53785..fa08912 100644 --- a/internal/cli/machine_contract_test.go +++ b/internal/cli/machine_contract_test.go @@ -150,11 +150,11 @@ func TestServerListJSONWithoutConfigFile(t *testing.T) { } func TestLogsCommandFollowModes(t *testing.T) { - if got := logsCommand("app-web-v1", 50, true); got != "docker logs -f --tail 50 app-web-v1" { + if got := logsCommand("app-web-v1", 50, true); got != "docker logs -f --tail 50 'app-web-v1'" { t.Fatalf("default logs command = %q", got) } got := logsCommand("app-web-v1", 100, false) - if got != "docker logs --tail 100 app-web-v1" { + if got != "docker logs --tail 100 'app-web-v1'" { t.Fatalf("bounded logs command = %q", got) } if strings.Contains(got, " -f") { diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go index bf43f54..b73b23d 100644 --- a/internal/cli/machineinterface.go +++ b/internal/cli/machineinterface.go @@ -100,6 +100,18 @@ const ( // updates); `preview list --json` rows carry url with the served // scheme (DELEGATED_DECISIONS §10). CapPreviewExposure = "preview-exposure" + // `deploy|preview deploy|preview destroy --trigger-stdin --json`: + // bounded immutable trigger admission converges on the TARGET's + // resident state — durable per-operation receipts under the app + // lease, replay reconciliation before any preparation, truthful + // committed/not_committed/unknown results (final JSON on nonzero). + // Agreed reserved interface: cli-trigger-convergence-proposal.json. + CapTriggerConvergence = "trigger-convergence-v1" + // Preview destruction compares the admitted ownership ID/generation + // (and optional updated_at token) against authority re-read under the + // same lease; a renewed preview is refused before any route, container + // or authority effect — including between prune's listing and destroy. + CapPreviewCompareDestroy = "preview-compare-destroy-v1" ) // MachineCapabilities returns every capability token this build @@ -124,6 +136,8 @@ func MachineCapabilities() []string { CapDoctorDiagnostics, CapPlanApply, CapPreviewExposure, + CapTriggerConvergence, + CapPreviewCompareDestroy, } sort.Strings(tokens) return tokens diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go index 29f851b..da6e07d 100644 --- a/internal/cli/machineinterface_test.go +++ b/internal/cli/machineinterface_test.go @@ -92,6 +92,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { "plan-apply", "preview-blue-green", "preview-canonical-id", + "preview-compare-destroy-v1", "preview-exposure", "provenance-records", "readiness-receipts", @@ -100,6 +101,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { "server-status-machine", "server-update", "template-var-stdin", + "trigger-convergence-v1", } got := MachineCapabilities() if len(got) != len(want) { diff --git a/internal/cli/network_grant.go b/internal/cli/network_grant.go index 554288e..a736e87 100644 --- a/internal/cli/network_grant.go +++ b/internal/cli/network_grant.go @@ -55,9 +55,9 @@ func newNetworkGrantCmd(flags *Flags) *cobra.Command { ) cmd := &cobra.Command{ Use: "grant", - Short: "Mint a time-boxed mesh access key (auto-revokes)", - Long: "Create a pre-auth key that expires on its own — \"give the contractor two\n" + - "hours of access\". The node it enrolls is ephemeral (drops off the mesh when it\n" + + Short: "Mint a mesh enrollment key with an expiry", + Long: "Create a pre-auth key with a bounded enrollment window. Expiry does not\n" + + "revoke enrolled devices; remove those devices to end access. The node is ephemeral (drops off the mesh when it\n" + "disconnects) and carries the tags you set; what those tags can reach is decided\n" + "by your tailnet/headscale ACL policy, which teploy never edits.\n\n" + "Credentials: TAILSCALE_API_KEY (+ optional TAILSCALE_TAILNET), or\n" + @@ -92,12 +92,12 @@ func newNetworkGrantCmd(flags *Flags) *cobra.Command { default: fmt.Println(" tailscale up --auth-key ") } - fmt.Println("The key expires automatically; revoke early with: teploy network revoke", grant.ID) + fmt.Println("Enrollment expires automatically; enrolled devices require separate removal. Revoke the key early with: teploy network revoke", grant.ID) return nil }, } cmd.Flags().StringVar(&provider, "provider", "", "mesh provider (tailscale, headscale); default from teploy.yml") - cmd.Flags().DurationVar(&ttl, "ttl", time.Hour, "grant lifetime (e.g. 30m, 2h, 24h)") + cmd.Flags().DurationVar(&ttl, "ttl", time.Hour, "enrollment window (e.g. 30m, 2h, 24h); enrolled access does not expire") cmd.Flags().StringArrayVar(&tags, "tag", nil, "ACL tag(s) for the enrolled node (e.g. tag:contractor); repeatable") cmd.Flags().BoolVar(&reusable, "reusable", false, "allow multiple devices to enroll with this key") return cmd diff --git a/internal/cli/plan.go b/internal/cli/plan.go index 0d4241f..3a96b17 100644 --- a/internal/cli/plan.go +++ b/internal/cli/plan.go @@ -3,11 +3,14 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "os" "os/signal" + "path/filepath" "sort" + "strings" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/config" @@ -145,7 +148,7 @@ func buildPlanRecord(ctx context.Context, exec ssh.Executor, out io.Writer, appC } needsBuild := image == "" - _, manifestSHA, err := config.NormalizeAndDigest(appCfg, image) + manifestSHA, err := config.ExecutionBindingDigest(appCfg, image) if err != nil { return nil, false, fmt.Errorf("normalizing planned manifest: %w", err) } @@ -340,6 +343,9 @@ func runPlan(flags *Flags, version, image, destination, outFile string) error { return err } + if err := bindPlanOutput(rec, appCfg, outFile); err != nil { + return err + } if err := renderPlan(flags, rec, sameVersion); err != nil { return err } @@ -533,14 +539,16 @@ func desiredContainers(appCfg *config.AppConfig, version string) []desiredContai // provenance receipt, so apply executes them without a plan-id stamp — // recorded as a C05 tail. func planStatic(ctx context.Context, flags *Flags, appCfg *config.AppConfig, executor ssh.Executor, destination, outFile string) error { - _ = ctx - current, _ := state.Read(ctx, executor, appCfg.App) + current, err := state.Read(ctx, executor, appCfg.App) + if err != nil { + return err + } version, err := gitShortHash() if err != nil { version = "" } - _, manifestSHA, err := config.NormalizeAndDigest(appCfg, "") + manifestSHA, err := config.ExecutionBindingDigest(appCfg, "") if err != nil { return fmt.Errorf("normalizing planned manifest: %w", err) } @@ -563,7 +571,15 @@ func planStatic(ctx context.Context, flags *Flags, appCfg *config.AppConfig, exe if current != nil { rec.TargetState = PlanTargetState{Deployed: true, Generation: current.Generation, CurrentHash: current.CurrentHash, ManifestSHA256: current.ManifestSHA256} } - rec.PlanID = computePlanID(rec) + if err := bindPlanOutput(rec, appCfg, outFile); err != nil { + return err + } + + if outFile != "" { + if err := savePlanFile(outFile, rec); err != nil { + return err + } + } if flags.JSON { // Old static keys (app/server/type/note) unchanged; record @@ -586,11 +602,71 @@ func planStatic(ctx context.Context, flags *Flags, appCfg *config.AppConfig, exe fmt.Printf("? %s\n", n) } if outFile != "" { - if err := savePlanFile(outFile, rec); err != nil { - return err - } fmt.Printf("\nPlan %s written to %s — apply with: teploy apply %s\n", rec.PlanID, outFile, outFile) } fmt.Println("\nThis is a preview. No changes were made.") return nil } + +func staticPlanFingerprint(exclusions []string, configs ...*config.AppConfig) (string, error) { + var cfg *config.AppConfig + if len(configs) > 0 { + cfg = configs[0] + } + src, err := resolveReviewedSource(cfg) + if err != nil { + return "", err + } + excluded := make(map[string]bool, len(exclusions)) + for _, name := range exclusions { + excluded[name] = true + } + entries := src.Entries[:0] + for _, entry := range src.Entries { + if !excluded[entry] { + entries = append(entries, entry) + } + } + src.Entries = entries + contextPath := "" + if cfg != nil && !cfg.IsStatic() { + contextPath = cfg.Context + } + return src.ReviewedFingerprint(contextPath) +} + +func bindPlanOutput(rec *PlanRecord, cfg *config.AppConfig, outFile string) error { + if outFile != "" { + abs, err := filepath.Abs(outFile) + if err != nil { + return err + } + root, err := filepath.Abs(".") + if err != nil { + return err + } + rel, err := filepath.Rel(root, abs) + if err != nil { + return err + } + if rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + rec.InputExclusions = []string{filepath.ToSlash(rel)} + for parent := filepath.Dir(rel); parent != "."; parent = filepath.Dir(parent) { + if _, err := os.Stat(parent); errors.Is(err, os.ErrNotExist) { + rec.InputExclusions = append(rec.InputExclusions, filepath.ToSlash(parent)) + } else if err != nil { + return err + } + } + } + } + if rec.Image.NeedsBuild { + fp, err := staticPlanFingerprint(rec.InputExclusions, cfg) + if err != nil { + return fmt.Errorf("binding reviewed source inputs: %w", err) + } + rec.Image.ContextFingerprint = fp + } + rec.PlanID = computePlanID(rec) + return nil +} diff --git a/internal/cli/plan_campaign_test.go b/internal/cli/plan_campaign_test.go new file mode 100644 index 0000000..9fcca00 --- /dev/null +++ b/internal/cli/plan_campaign_test.go @@ -0,0 +1,158 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func TestLockedPlanRejectsInterveningGeneration(t *testing.T) { + cfg := &config.AppConfig{App: "demo", Image: "image:v1"} + digest, err := config.ExecutionBindingDigest(cfg, cfg.Image) + if err != nil { + t.Fatal(err) + } + rec := &PlanRecord{App: cfg.App, Server: "host", ConfigDigest: digest, TargetVersion: "v1", VersionExplicit: true, TargetState: PlanTargetState{Deployed: true, Generation: 4}} + mock := ssh.NewMockExecutor("host") + mock.Files["/deployments/demo/state.json"] = []byte(`{"schema_version":2,"generation":5,"current_hash":"v2"}`) + err = revalidateLockedPlan(context.Background(), mock, cfg, cfg.Image, "v1", rec) + if !errors.Is(err, errPlanDrift) { + t.Fatalf("stale reviewed generation accepted: %v", err) + } + for _, call := range mock.Calls { + if call == "docker run" { + t.Fatal("unexpected effect") + } + } +} + +func TestStaticJSONPlanWritesAndBindsInputs(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + if err := os.WriteFile("index.html", []byte("safe"), 0600); err != nil { + t.Fatal(err) + } + cfg := &config.AppConfig{App: "demo", Type: config.TypeStatic, Source: ".", Domain: "example.test"} + mock := ssh.NewMockExecutor("host") + // JSON is a presentation mode: the reviewed record must still be saved. + original := os.Stdout + file, err := os.Create(filepath.Join(t.TempDir(), "stdout")) + if err != nil { + t.Fatal(err) + } + os.Stdout = file + defer func() { os.Stdout = original; file.Close() }() + if err := planStatic(context.Background(), &Flags{JSON: true}, cfg, mock, "", "plans/plan.json"); err != nil { + t.Fatal(err) + } + rec, err := loadPlanFile("plans/plan.json") + if err != nil { + t.Fatal(err) + } + fp, err := staticPlanFingerprint(rec.InputExclusions, cfg) + if err != nil { + t.Fatal(err) + } + if fp == "" || fp != rec.Image.ContextFingerprint { + t.Fatal("unchanged static inputs falsely drifted") + } + if err := os.WriteFile("index.html", []byte("changed"), 0600); err != nil { + t.Fatal(err) + } + after, _ := staticPlanFingerprint(rec.InputExclusions, cfg) + if after == fp { + t.Fatal("static content change was not bound") + } + file.Sync() + file.Seek(0, io.SeekStart) + var result map[string]any + if err := json.NewDecoder(file).Decode(&result); err != nil { + t.Fatal(err) + } + if result["plan_id"] != rec.PlanID { + t.Fatal("saved and displayed identity differ") + } +} + +func TestReviewedInputSnapshotIsIndependentAndMatchesAdmission(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + gitRun(t, dir, "init", "-q") + os.WriteFile(".gitignore", []byte("private.txt\n"), 0600) + os.WriteFile("private.txt", []byte("dummy confidential marker"), 0600) + os.WriteFile("Dockerfile", []byte("FROM scratch\nCOPY app /app\n"), 0600) + os.WriteFile("app", []byte("safe"), 0755) + os.Mkdir("assets", 0755) + os.WriteFile("assets/icon", []byte("asset"), 0644) + cfg := &config.AppConfig{App: "demo"} + before, err := staticPlanFingerprint(nil) + if err != nil { + t.Fatal(err) + } + snapshot, err := freezePlanInputs(nil, cfg) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(filepath.Dir(snapshot)) + if _, err := os.Stat(filepath.Join(snapshot, "private.txt")); !os.IsNotExist(err) { + t.Fatal("ignored private file copied") + } + os.WriteFile("app", []byte("changed after verification"), 0755) + if err := os.Chdir(snapshot); err != nil { + t.Fatal(err) + } + after, err := staticPlanFingerprint(nil) + if err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("snapshot does not preserve admitted input identity") + } + info, _ := os.Stat("assets") + if info.Mode().Perm() != 0755 { + t.Fatalf("image directory permissions changed: %o", info.Mode().Perm()) + } + got, _ := os.ReadFile("app") + if string(got) != "safe" { + t.Fatal("source mutation changed reviewed bytes") + } +} + +func TestStaticReviewedSourceIncludesIgnoredDist(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + gitRun(t, dir, "init", "-q") + os.WriteFile(".gitignore", []byte("dist/\n"), 0600) + os.Mkdir("dist", 0755) + os.WriteFile("dist/index.html", []byte("safe"), 0644) + cfg := &config.AppConfig{App: "demo", Type: config.TypeStatic, Source: "dist"} + before, err := staticPlanFingerprint(nil, cfg) + if err != nil { + t.Fatal(err) + } + snapshot, err := freezePlanInputs(nil, cfg) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(filepath.Dir(snapshot)) + os.WriteFile("dist/index.html", []byte("changed"), 0644) + changed, _ := staticPlanFingerprint(nil, cfg) + if changed == before { + t.Fatal("ignored serving source not bound") + } + os.Chdir(snapshot) + frozen, err := staticPlanFingerprint(nil, cfg) + if err != nil { + t.Fatal(err) + } + if frozen != before { + t.Fatal("ignored serving source not frozen") + } +} diff --git a/internal/cli/planrecord.go b/internal/cli/planrecord.go index 08276e1..cd65eba 100644 --- a/internal/cli/planrecord.go +++ b/internal/cli/planrecord.go @@ -28,15 +28,21 @@ import ( "encoding/json" "errors" "fmt" + "io" + "io/fs" "os" "path/filepath" + "sort" + "strings" "time" + "github.com/useteploy/teploy/internal/build" + "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/state" ) // PlanRecordSchemaVersion is the plan record's schema version. -const PlanRecordSchemaVersion = 1 +const PlanRecordSchemaVersion = 2 // Image resolution classes: what the plan could and could not bind. const ( @@ -80,12 +86,13 @@ type PlanRecord struct { VersionExplicit bool `json:"version_explicit,omitempty"` // ConfigDigest is the effective-config digest - // (config.NormalizeAndDigest) over the plan's app config with the + // (config.ExecutionBindingDigest) over the plan's app config with the // image reference the plan resolved — "" for a build deploy, whose // image does not exist at plan time. The DEPLOYED receipt carries // its own manifest digest computed with the built image; the plan id // (stamped into that receipt) is the tie between the two. - ConfigDigest string `json:"config_digest"` + ConfigDigest string `json:"config_digest"` + InputExclusions []string `json:"input_exclusions,omitempty"` Image PlanImageIdentity `json:"image"` TargetState PlanTargetState `json:"target_state"` @@ -160,6 +167,9 @@ func computePlanID(rec *PlanRecord) string { fmt.Fprintf(h, "destination=%s\n", rec.Destination) fmt.Fprintf(h, "target_version=%s\n", rec.TargetVersion) fmt.Fprintf(h, "config_digest=%s\n", rec.ConfigDigest) + for _, path := range rec.InputExclusions { + fmt.Fprintf(h, "input_exclusion=%q\n", path) + } fmt.Fprintf(h, "context_fingerprint=%s\n", rec.Image.ContextFingerprint) fmt.Fprintf(h, "dockerfile_sha256=%s\n", rec.Image.DockerfileSHA256) fmt.Fprintf(h, "generation=%d\n", rec.TargetState.Generation) @@ -292,3 +302,252 @@ func loadPlanFile(path string) (*PlanRecord, error) { } return &rec, nil } + +// freezePlanInputs copies the admitted source into a private tree. Apply then +// verifies and builds that same tree, so edits to the user's checkout cannot +// change the bytes between review validation and transfer/build. +func freezePlanInputs(exclusions []string, cfg *config.AppConfig) (string, error) { + src, err := resolveReviewedSource(cfg) + if err != nil { + return "", err + } + root, err := filepath.Abs(".") + if err != nil { + return "", err + } + snapshot, err := os.MkdirTemp("", "teploy-reviewed-inputs-*") + if err != nil { + return "", err + } + privateWorkspace := snapshot + snapshot = filepath.Join(privateWorkspace, "tree") + rootInfo, err := os.Stat(root) + if err != nil { + os.RemoveAll(privateWorkspace) + return "", err + } + if err := os.Mkdir(snapshot, 0700); err != nil { + os.RemoveAll(privateWorkspace) + return "", err + } + ok := false + defer func() { + if !ok { + os.RemoveAll(privateWorkspace) + } + }() + excluded := map[string]bool{} + for _, path := range exclusions { + excluded[path] = true + } + copyFile := func(rel string, private bool) error { + in, err := os.Open(filepath.Join(root, rel)) + if err != nil { + return err + } + defer in.Close() + info, err := in.Stat() + if err != nil { + return err + } + dest := filepath.Join(snapshot, rel) + if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { + return err + } + mode := info.Mode().Perm() + if private { + mode = 0600 + } + out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + modeErr := out.Chmod(mode) + closeErr := out.Close() + return errors.Join(copyErr, modeErr, closeErr) + } + admitted := map[string]bool{} + for _, rel := range src.Entries { + if !excluded[rel] { + admitted[rel] = true + } + } + for _, rel := range src.Entries { + if excluded[rel] { + continue + } + info, err := os.Lstat(filepath.Join(root, rel)) + if err != nil { + return "", err + } + dest := filepath.Join(snapshot, rel) + if info.Mode().IsRegular() { + if err := copyFile(rel, false); err != nil { + return "", err + } + continue + } + if info.IsDir() { + if err := os.MkdirAll(dest, 0700); err != nil { + return "", err + } + continue + } + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(filepath.Join(root, rel)) + if err != nil { + return "", err + } + if filepath.IsAbs(target) { + return "", fmt.Errorf("reviewed apply requires relative source symlinks: %s", rel) + } + resolved, err := filepath.EvalSymlinks(filepath.Join(root, rel)) + if err != nil { + return "", err + } + relative, err := filepath.Rel(root, resolved) + if err != nil { + return "", err + } + if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("source symlink escapes reviewed input: %s", rel) + } + if targetInfo, err := os.Stat(resolved); err != nil { + return "", err + } else if !targetInfo.IsDir() && !admitted[filepath.ToSlash(relative)] { + return "", fmt.Errorf("source symlink targets an unreviewed file: %s", rel) + } + if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { + return "", err + } + if err := os.Symlink(target, dest); err != nil { + return "", err + } + continue + } + return "", fmt.Errorf("unsupported reviewed source entry %s", rel) + } + // Admission controls and TLS input files stay private and are excluded from + // source transfer by the same protected-path rules as the original tree. + controls := []string{".teployignore", ".dockerignore", ".gitignore"} + if cfg.TLS != nil { + controls = append(controls, cfg.TLS.Cert, cfg.TLS.Key) + } + for _, name := range controls { + if name == "" || filepath.IsAbs(name) { + continue + } + clean := filepath.Clean(name) + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + continue + } + if _, err := os.Stat(filepath.Join(snapshot, clean)); err == nil { + continue + } + if _, err := os.Stat(filepath.Join(root, clean)); errors.Is(err, os.ErrNotExist) { + continue + } else if err != nil { + return "", err + } + if err := copyFile(clean, true); err != nil { + return "", err + } + } + // Preserve source directory modes inside the private root: these modes can + // become image metadata through Docker COPY, and must not become 0700 merely + // because the snapshot's admission workspace is private. + directories := map[string]bool{} + for rel := range admitted { + parent := filepath.Dir(rel) + if info, err := os.Lstat(filepath.Join(root, rel)); err == nil && info.IsDir() { + directories[rel] = true + } + for parent != "." { + directories[parent] = true + parent = filepath.Dir(parent) + } + } + for rel := range directories { + info, err := os.Stat(filepath.Join(root, rel)) + if err != nil { + return "", err + } + if err := os.Chmod(filepath.Join(snapshot, rel), info.Mode().Perm()); err != nil { + return "", err + } + } + + if err := os.Chmod(snapshot, rootInfo.Mode().Perm()); err != nil { + return "", err + } + ok = true + return snapshot, nil +} + +// A prebuilt static source is uploaded even when Git ignores it. Include that +// actual serving tree alongside project build inputs so dist/ is both bound +// and frozen; protected configuration and secret files still never transfer. +func resolveReviewedSource(cfg *config.AppConfig) (*build.Source, error) { + src, err := build.ResolveSource(".") + if err != nil { + return nil, err + } + if cfg == nil || !cfg.IsStatic() { + return src, nil + } + root, err := filepath.Abs(".") + if err != nil { + return nil, err + } + source, err := filepath.Abs(cfg.Source) + if err != nil { + return nil, err + } + rel, err := filepath.Rel(root, source) + if err != nil { + return nil, err + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("reviewed static source must be inside the project directory; place the serving tree under the project before planning") + } + if _, err := os.Stat(source); errors.Is(err, os.ErrNotExist) && len(cfg.Build) > 0 { + return src, nil + } else if err != nil { + return nil, err + } + entries := map[string]bool{} + for _, entry := range src.Entries { + entries[entry] = true + } + err = filepath.WalkDir(source, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + relative = filepath.ToSlash(relative) + if relative == "." { + return nil + } + if src.Rules.IsProtected(relative, entry.IsDir()) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + entries[relative] = true + return nil + }) + if err != nil { + return nil, err + } + src.Entries = src.Entries[:0] + for entry := range entries { + src.Entries = append(src.Entries, entry) + } + sort.Strings(src.Entries) + return src, nil +} diff --git a/internal/cli/preview.go b/internal/cli/preview.go index afecce0..20ac04c 100644 --- a/internal/cli/preview.go +++ b/internal/cli/preview.go @@ -14,6 +14,7 @@ import ( "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/notify" "github.com/useteploy/teploy/internal/preview" + "github.com/useteploy/teploy/internal/trigger" ) func newPreviewCmd(flags *Flags) *cobra.Command { @@ -42,9 +43,38 @@ type previewDeployOpts struct { } func newPreviewDeployCmd(flags *Flags) *cobra.Command { - return newPreviewDeployCmdWith(func(branch string, opts previewDeployOpts) error { - return runPreviewDeploy(flags, branch, opts) - }) + cmd := newPreviewDeployCmdWith(func(branch string, opts previewDeployOpts) error { return runPreviewDeploy(flags, branch, opts) }) + var triggerStdin bool + cmd.Flags().BoolVar(&triggerStdin, "trigger-stdin", false, "read a bounded immutable trigger request from stdin") + ordinary := cmd.RunE + cmd.Args = func(cmd *cobra.Command, args []string) error { + if triggerStdin { + return cobra.MaximumNArgs(1)(cmd, args) + } + return cobra.ExactArgs(1)(cmd, args) + } + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if !triggerStdin { + return ordinary(cmd, args) + } + return triggerCommand(cmd, "preview_create", func(r trigger.Request) (trigger.Result, error) { + branch := strings.TrimPrefix(r.Ref, "refs/heads/") + if len(args) > 0 && args[0] != branch { + return trigger.Initial(r), fmt.Errorf("preview branch mismatch") + } + opts := previewDeployOpts{} + opts.ttl, _ = cmd.Flags().GetString("ttl") + opts.image, _ = cmd.Flags().GetString("image") + opts.baseDomain, _ = cmd.Flags().GetString("base-domain") + httpOnly, _ := cmd.Flags().GetBool("http-only") + allowIPs, _ := cmd.Flags().GetStringSlice("allow-ip") + if err := finishPreviewDeployOpts(cmd, &opts, httpOnly, allowIPs); err != nil { + return trigger.Initial(r), err + } + return runTriggerCLI(cmd, flags, "preview_create", branch, opts.image, "", opts, r) + }) + } + return cmd } // newPreviewDeployCmdWith builds the command around run (the test seam: @@ -295,14 +325,28 @@ func previewListRows(previews []preview.State) []previewListRow { } func newPreviewDestroyCmd(flags *Flags) *cobra.Command { - return &cobra.Command{ - Use: "destroy ", - Short: "Tear down a preview environment", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - return runPreviewDestroy(flags, args[0]) - }, + var triggerStdin bool + cmd := &cobra.Command{Use: "destroy [branch]", Short: "Tear down a preview environment"} + cmd.Flags().BoolVar(&triggerStdin, "trigger-stdin", false, "read a bounded immutable trigger request from stdin") + cmd.Args = func(cmd *cobra.Command, args []string) error { + if triggerStdin { + return cobra.MaximumNArgs(1)(cmd, args) + } + return cobra.ExactArgs(1)(cmd, args) + } + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if triggerStdin { + return triggerCommand(cmd, "preview_destroy", func(r trigger.Request) (trigger.Result, error) { + branch := strings.TrimPrefix(r.Ref, "refs/heads/") + if len(args) > 0 && args[0] != branch { + return trigger.Initial(r), fmt.Errorf("preview branch mismatch") + } + return runTriggerCLI(cmd, flags, "preview_destroy", branch, "", "", previewDeployOpts{}, r) + }) + } + return runPreviewDestroy(flags, args[0]) } + return cmd } func runPreviewDestroy(flags *Flags, branch string) error { diff --git a/internal/cli/root.go b/internal/cli/root.go index c8785ff..48b9070 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -1,7 +1,9 @@ package cli import ( + "encoding/json" "fmt" + "github.com/useteploy/teploy/internal/trigger" "os" "github.com/spf13/cobra" @@ -37,7 +39,15 @@ func NewRootCmd(version string) *cobra.Command { return nil } if err := os.Chdir(flags.ProjectDir); err != nil { - return fmt.Errorf("changing to project directory %q: %w", flags.ProjectDir, err) + failure := fmt.Errorf("changing to project directory %q: %w", flags.ProjectDir, err) + if requested, _ := cmd.Flags().GetBool("trigger-stdin"); requested { + result := trigger.Initial(trigger.Request{}) + result.ErrorCode = "admission_failed" + if encodeErr := json.NewEncoder(cmd.OutOrStdout()).Encode(result); encodeErr != nil { + return encodeErr + } + } + return failure } return nil }, @@ -51,6 +61,7 @@ func NewRootCmd(version string) *cobra.Command { root.PersistentFlags().BoolVar(&flags.StrictEnv, "strict-env", false, "strict env/overlay mode: fail on unset ${VAR} in env:, and let an empty map/list in a destination overlay explicitly clear the base value") root.AddCommand(newDeployCmd(flags)) + root.AddCommand(newTriggerCmd(flags)) root.AddCommand(newBuildCmd(flags)) root.AddCommand(newExecCmd(flags)) root.AddCommand(newAppCmd(flags)) diff --git a/internal/cli/round3_regression_test.go b/internal/cli/round3_regression_test.go new file mode 100644 index 0000000..4de21fc --- /dev/null +++ b/internal/cli/round3_regression_test.go @@ -0,0 +1,153 @@ +package cli + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/multideploy" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +func TestRound3FleetMigrationPreservesPublication(t *testing.T) { + for _, outcome := range []error{&state.PublicationError{Committed: true, Err: errors.New("sidecar repair failed")}, &state.PublicationError{Unknown: true, Err: errors.New("authority read lost")}, errors.New("before publication")} { + t.Run(outcome.Error(), func(t *testing.T) { + mock := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "mkdir /deployments/demo/.lock"}, + ssh.MockCommand{Match: "teploy_volume_actor", Output: ""}, + ssh.MockCommand{Match: "err=$(mktemp)", Output: "exists"}, + ssh.MockCommand{Match: "docker image inspect", Output: "sha256:" + strings.Repeat("a", 64)}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker start", Output: ""}, + ssh.MockCommand{Match: "mkdir -p", Output: ""}, + ssh.MockCommand{Match: "cp -a", Output: ""}, + ssh.MockCommand{Match: "docker ps -aq", Output: "old-web"}, + ssh.MockCommand{Match: "docker inspect old-web", Output: "bind|/old/data|/data"}, + ssh.MockCommand{Match: "docker ps -q", Output: "old-web old-worker old-db"}) + cfg := &config.AppConfig{App: "demo", Image: "image:v2", Ingress: "external", Volumes: map[string]string{"data": "/data"}} + server := newSingleServerDeployer(mock, io.Discard, "", true) + reached := false + server.deployEngine = func(context.Context, deploy.Config, *state.Lock) error { reached = true; return outcome } + err := server.deployApp(context.Background(), cfg, nil, "", "v2") + if !reached || !errors.Is(err, outcome) { + t.Fatalf("did not reach engine: %v calls=%v", err, mock.Calls) + } + restored := map[string]bool{} + for _, command := range mock.Calls { + if name, quoted := strings.CutPrefix(command, "docker start 'old-"); quoted { + if end := strings.Index(name, "'"); end >= 0 { + restored["old-"+name[:end]] = true + } + } + } + if state.PreservePublishedState(outcome) && len(restored) != 0 { + t.Fatalf("published old writers resurrected: %v", mock.Calls) + } + if !state.PreservePublishedState(outcome) && len(restored) != 3 { + t.Fatalf("ordinary failure lost recovery (restarted %v): %v", restored, mock.Calls) + } + }) + } +} +func TestRound3FleetWithholdsAutomaticConvergence(t *testing.T) { + cause := &state.PublicationError{Unknown: true, Err: errors.New("lost authority")} + results := []multideploy.Result{{Server: "healthy", Success: true}, {Server: "unknown", Error: cause}} + if !state.PreservePublishedState(fleetPublicationError(results)) { + t.Fatal("fleet discarded publication outcome") + } +} +func TestRound3ReviewedModesFreezeAndLockedValidation(t *testing.T) { + t.Chdir(t.TempDir()) + if err := os.Chmod(".", 0755); err != nil { + t.Fatal(err) + } + if err := os.Mkdir("src", 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile("src/run", []byte("same bytes"), 0755); err != nil { + t.Fatal(err) + } + cfg := &config.AppConfig{App: "demo", Context: ".", Type: config.TypeStatic, Source: "."} + original, err := staticPlanFingerprint(nil, cfg) + if err != nil { + t.Fatal(err) + } + digest, _ := config.ExecutionBindingDigest(cfg, "") + record := &PlanRecord{App: "demo", Server: "host", TargetVersion: "v1", ConfigDigest: digest, Image: PlanImageIdentity{NeedsBuild: true, ContextFingerprint: original}} + // Both file and directory chmod must move the authorization identity. + for _, path := range []string{"src/run", "src", "."} { + if err := os.Chmod(path, 0700); err != nil { + t.Fatal(err) + } + frozen, err := freezePlanInputs(nil, cfg) + if err != nil { + t.Fatal(err) + } + work, _ := os.Getwd() + if err := os.Chdir(frozen); err != nil { + t.Fatal(err) + } + err = revalidateLockedPlan(context.Background(), ssh.NewMockExecutor("host"), cfg, "", "v1", record) + os.Chdir(work) + os.RemoveAll(filepath.Dir(frozen)) + if !errors.Is(err, errPlanDrift) { + t.Fatalf("chmod %s accepted: %v", path, err) + } + if err := os.Chmod(path, 0755); err != nil { + t.Fatal(err) + } + } + frozen, err := freezePlanInputs(nil, cfg) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(filepath.Dir(frozen)) + work, _ := os.Getwd() + os.Chdir(frozen) + defer os.Chdir(work) + got, err := staticPlanFingerprint(nil, cfg) + if err != nil || got != original { + t.Fatalf("unchanged frozen modes differ: %s %s %v", got, original, err) + } + if _, err := os.Stat(filepath.Join(frozen, "src/run")); err != nil { + t.Fatal(err) + } +} +func TestRound3OwnershipFailurePrecedesSharedAndFleetEffects(t *testing.T) { + uid, gid := uint32(10001), uint32(10001) + cfg := &config.AppConfig{App: "demo", Image: "image:v1", Volumes: map[string]string{"data": "/data"}, VolumeOwnership: map[string]config.VolumeOwnership{"data": {UID: &uid, GID: &gid, Mode: "0700"}}, Accessories: map[string]config.AccessoryConfig{"db": {Image: "postgres:16"}}} + for _, fleet := range []bool{false, true} { + mock := &round3VolumeExecutor{ssh.NewMockExecutor("host", ssh.MockCommand{Match: "mkdir /deployments/demo/.lock"})} + var err error + if fleet { + err = newSingleServerDeployer(mock, io.Discard, "", true).deployApp(context.Background(), cfg, nil, "", "v1") + } else { + err = deployBuiltImageFenced(context.Background(), mock, cfg, cfg.Image, "v1", "host", true, false, "", nil, nil, "") + } + if err == nil { + t.Fatal("ownership refusal lost") + } + for _, command := range mock.Calls { + for _, effect := range []string{"docker run", "docker rm", "docker stop", "cp -a", "mkdir -p /deployments/demo/volumes"} { + if strings.Contains(command, effect) { + t.Fatalf("effect before admission: %s", command) + } + } + } + } +} + +type round3VolumeExecutor struct{ *ssh.MockExecutor } + +func (e *round3VolumeExecutor) Run(ctx context.Context, command string) (string, error) { + if strings.Contains(command, "' validate ") { + return "", errors.New("populated mismatch") + } + return e.MockExecutor.Run(ctx, command) +} diff --git a/internal/cli/runtime_campaign_test.go b/internal/cli/runtime_campaign_test.go new file mode 100644 index 0000000..1d8771c --- /dev/null +++ b/internal/cli/runtime_campaign_test.go @@ -0,0 +1,49 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" + "io" + "os" + "testing" +) + +func TestCampaignNetbirdSetupUsesTargetProvider(t *testing.T) { + m := ssh.NewMockExecutor("dummy", ssh.MockCommand{Match: "whoami", Output: "root"}, ssh.MockCommand{Match: "which netbird", Output: "/usr/bin/netbird"}, ssh.MockCommand{Match: "netbird status", Output: "Management: Connected\nSignal: Connected\nNetBird IP: 100.80.1.2/16"}) + got, err := setupNetwork(context.Background(), m, &bytes.Buffer{}, "netbird", "dummy-key") + if err != nil || got != "100.80.1.2" { + t.Fatalf("%s %v", got, err) + } +} +func TestCampaignInvalidJSONValidationFails(t *testing.T) { + original := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = w + defer func() { os.Stdout = original }() + err = outputResult(&Flags{JSON: true}, &validationResult{Valid: false, Errors: []string{"dummy invalid"}}) + w.Close() + b, _ := io.ReadAll(r) + r.Close() + if err == nil { + t.Fatal("invalid config exited successfully") + } + var result validationResult + if json.Unmarshal(b, &result) != nil || result.Valid { + t.Fatalf("invalid JSON result %s", b) + } +} + +func TestCampaignLiteralVaultValueDoesNotResolve(t *testing.T) { + m := ssh.NewMockExecutor("host") + cfg := &config.AppConfig{App: "demo", Env: map[string]string{"PASSWORD": "vault:literal#password"}, EnvLiteral: map[string]string{"PASSWORD": "vault:literal#password"}} + got, err := mergeSecretVaultRefs(context.Background(), m, cfg, nil) + if err != nil || len(got) != 0 || len(m.Calls) != 0 { + t.Fatalf("literal reference resolved: %v %v calls=%v", got, err, m.Calls) + } +} diff --git a/internal/cli/scale.go b/internal/cli/scale.go index f41a5ff..a2a8986 100644 --- a/internal/cli/scale.go +++ b/internal/cli/scale.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "io" + "net" "os" "os/signal" "sort" "strconv" + "strings" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/caddy" @@ -46,6 +48,12 @@ func runScale(flags *Flags, count, parallel int) error { return err } + if appCfg.Type == "static" { + return fmt.Errorf("fleet scaling static applications is unsupported; deploy each static target explicitly") + } + if err := resolveDeployEnv(context.Background(), appCfg, flags.StrictEnv); err != nil { + return err + } // 2. Get app-role servers. serversPath, err := config.DefaultServersPath() if err != nil { @@ -134,10 +142,13 @@ func runScale(flags *Flags, count, parallel int) error { } } + if err := fleetPublicationError(results); err != nil { + return err + } // 6. Update LB if any servers succeeded. if len(successTargets) > 0 { if err := updateLoadBalancer(ctx, flags, appCfg, serversPath, successTargets); err != nil { - fmt.Fprintf(os.Stderr, "Warning: LB update failed: %v\n", err) + return fmt.Errorf("backends deployed but load balancer activation failed: %w", err) } } @@ -229,7 +240,7 @@ func updateLoadBalancer(ctx context.Context, flags *Flags, appCfg *config.AppCon upstreams := make([]caddy.Upstream, len(targets)) for i, t := range targets { // Default HTTP port for app containers. - upstreams[i] = caddy.Upstream{Dial: fmt.Sprintf("%s:80", t.Host)} + upstreams[i] = caddy.Upstream{Dial: fleetServiceAddress(t.Host, appCfg)} } // Update each LB server. @@ -266,7 +277,7 @@ func updateLoadBalancer(ctx context.Context, flags *Flags, appCfg *config.AppCon tls := caddy.TLS{Cert: cert, Key: key, Internal: internal} client := caddy.NewClient(executor) - err = client.SetLoadBalancer(ctx, appCfg.App, appCfg.Domain, upstreams, tls, appCfg.CaddyExtra, appCfg.Cache, caddyFirewall(appCfg.Firewall), caddyAccess(appCfg.Access)) + err = client.SetLoadBalancerHealth(ctx, appCfg.App, appCfg.Domain, upstreams, appCfg.Health.Path, tls, appCfg.CaddyExtra, appCfg.Cache, caddyFirewall(appCfg.Firewall), caddyAccess(appCfg.Access)) executor.Close() if err != nil { return fmt.Errorf("updating LB %s: %w", name, err) @@ -277,3 +288,16 @@ func updateLoadBalancer(ctx context.Context, flags *Flags, appCfg *config.AppCon return nil } + +func fleetServiceAddress(endpoint string, app *config.AppConfig) string { + host := endpoint + if parsed, _, err := net.SplitHostPort(endpoint); err == nil { + host = parsed + } + host = strings.Trim(host, "[]") + port := 80 + if app.Ingress == "host" && app.Port > 0 { + port = app.Port + } + return net.JoinHostPort(host, strconv.Itoa(port)) +} diff --git a/internal/cli/secret.go b/internal/cli/secret.go index ada766d..cdaf284 100644 --- a/internal/cli/secret.go +++ b/internal/cli/secret.go @@ -227,7 +227,7 @@ container. A key that is not set is reported as such and is not an error, so re-running the removal of an already-revoked credential is safe. Containers keep the old value -in their environment until they are restarted.`, +in their environment until the app is redeployed with teploy deploy.`, Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runSecretRm(flags, *provider, args) @@ -270,7 +270,7 @@ func runSecretRm(flags *Flags, providerFlag string, keys []string) error { fmt.Printf(" No secret %s set — nothing to remove\n", key) } if removed > 0 { - fmt.Println(" (Restart containers to drop the value from their environment)") + fmt.Println(" (Redeploy the app to drop the value from their environment)") } return nil } @@ -306,6 +306,6 @@ func runSecretRotate(flags *Flags, key string) error { } fmt.Printf(" Rotated %s\n", key) fmt.Printf(" New value: %s\n", newVal) - fmt.Println(" (Restart containers to apply)") + fmt.Println(" (Redeploy the app to apply)") return nil } diff --git a/internal/cli/secret_openbao.go b/internal/cli/secret_openbao.go index c3b2b2d..f639b31 100644 --- a/internal/cli/secret_openbao.go +++ b/internal/cli/secret_openbao.go @@ -315,10 +315,16 @@ func mergeSecretVaultRefs(ctx context.Context, exec ssh.Executor, appCfg *config if dst == nil { dst = make(map[string]string) } - if len(openbao.CollectRefs(appCfg.Env)) == 0 { + directives := make(map[string]string, len(appCfg.Env)) + for k, v := range appCfg.Env { + if _, literal := appCfg.EnvLiteral[k]; !literal { + directives[k] = v + } + } + if len(openbao.CollectRefs(directives)) == 0 { return dst, nil } - resolved, err := openbao.NewClient(exec, os.Stderr).ResolveEnvRefs(ctx, appCfg.App, appCfg.Secret.Accessory, appCfg.Env) + resolved, err := openbao.NewClient(exec, os.Stderr).ResolveEnvRefs(ctx, appCfg.App, appCfg.Secret.Accessory, directives) if err != nil { return nil, fmt.Errorf("resolving vault references: %w", err) } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 4f999b8..b7a60ca 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "net" "os" osexec "os/exec" "os/signal" @@ -202,7 +203,6 @@ func runSetup(flags *Flags, host string, name string, noHarden bool, networkProv if vpnIP != "" { serverHost = vpnIP fmt.Printf("\nVPN connected — reconnecting via %s\n", vpnIP) - executor.Close() reconnectCfg := ssh.ConnectConfig{ Host: vpnIP, User: user, @@ -212,10 +212,13 @@ func runSetup(flags *Flags, host string, name string, noHarden bool, networkProv vpnDial := func(ctx context.Context) (ssh.Executor, error) { return ssh.Connect(ctx, reconnectCfg) } - executor, err = ssh.NewReconnectingExecutor(ctx, vpnDial, setupReconnectBudget) + replacement, reconnectErr := ssh.NewReconnectingExecutor(ctx, vpnDial, setupReconnectBudget) + err = reconnectErr if err != nil { return fmt.Errorf("reconnecting via VPN IP %s: %w", vpnIP, err) } + executor.Close() + executor = replacement defer executor.Close() } @@ -308,10 +311,6 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN return "", fmt.Errorf("installing %s: %w", providerName, err) } - // Get the server's hostname before joining — we'll use it to find the node locally. - hostname, _ := exec.Run(ctx, "hostname") - hostname = strings.TrimSpace(hostname) - // Reset Tailscale state if present — cloned VMs inherit the previous machine's identity // which causes IP conflicts. Stop tailscaled, wipe state, restart. if providerName == "tailscale" || providerName == "headscale" { @@ -334,36 +333,32 @@ func setupNetwork(ctx context.Context, exec ssh.Executor, w io.Writer, providerN // Fire VPN join in the background and don't wait for it. // Tailscale/Headscale modifies iptables which can kill the SSH connection, - // so we detach the command and poll from the local machine instead. + // so detach the join and use the reconnecting authenticated executor to observe it. fmt.Fprintf(w, "Joining %s mesh...\n", providerName) if err := joinVPNMesh(ctx, exec, sudo, providerName, cfg); err != nil { return "", err } - // Poll locally for the node to appear on our tailnet. - fmt.Fprintf(w, " Waiting for node to appear on tailnet...\n") - tsBinary := findTailscaleBinary() + // Query the authenticated target through its selected provider, rather + // than trusting a same-named peer on an unrelated local control plane. + cfg.Sudo = sudo + provider, err = network.NewProvider(cfg) + if err != nil { + return "", err + } + waitCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() var vpnIP string - for i := 0; i < 30; i++ { // 30 attempts, 2 seconds each = 60 second timeout - time.Sleep(2 * time.Second) - out, err := runLocal(tsBinary, "status") - if err != nil { - continue - } - for _, line := range strings.Split(out, "\n") { - fields := strings.Fields(line) - if len(fields) >= 2 && strings.EqualFold(fields[1], hostname) { - vpnIP = fields[0] - break - } - } - if vpnIP != "" { + for { + vpnIP, err = provider.GetIP(waitCtx, exec) + if err == nil && net.ParseIP(vpnIP) != nil { break } - } - - if vpnIP == "" { - return "", fmt.Errorf("timed out waiting for %s to join tailnet (expected hostname: %s)", providerName, hostname) + select { + case <-waitCtx.Done(): + return "", fmt.Errorf("waiting for %s address: %w", providerName, waitCtx.Err()) + case <-time.After(2 * time.Second): + } } fmt.Fprintf(w, " VPN IP: %s\n", vpnIP) diff --git a/internal/cli/singledeploy.go b/internal/cli/singledeploy.go index e464799..1119e72 100644 --- a/internal/cli/singledeploy.go +++ b/internal/cli/singledeploy.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "io" "time" @@ -14,6 +15,7 @@ import ( "github.com/useteploy/teploy/internal/releasemeta" "github.com/useteploy/teploy/internal/secret" "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" ) // singleServerDeployer wraps the deploy logic for a single server. @@ -22,6 +24,7 @@ type singleServerDeployer struct { exec ssh.Executor out io.Writer keyPath string + deployEngine func(context.Context, deploy.Config, *state.Lock) error migrateVolumes bool // auto-migrate foreign volume sources instead of aborting } @@ -39,11 +42,42 @@ func newSingleServerDeployer(exec ssh.Executor, out io.Writer, keyPath string, m // build-from-source falls back to git (audit F10 — this path used to ignore // the flags entirely and always take gitShortHash, which failed outside a // checkout and mislabeled prebuilt deploys). -func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.AppConfig, tags map[string]string, imageOverride, versionOverride string) error { +func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.AppConfig, tags map[string]string, imageOverride, versionOverride string) (retErr error) { + if appCfg.Type == "static" { + return fmt.Errorf("fleet deployment of static applications is unsupported; deploy each static target explicitly") + } + if err := docker.NewClient(s.exec).EnsureManagedDirectory(ctx, appCfg.App, "", ""); err != nil { + return err + } + rawExecutor := s.exec + lk, err := state.AcquireLockFenced(ctx, rawExecutor, appCfg.App) + if err != nil { + return err + } + defer state.ReleaseLockFenced(rawExecutor, lk, appCfg.App) + lk.StartRenewal(rawExecutor) + invocation := *s + s = &invocation + s.exec = &state.FencedExecutor{Executor: rawExecutor, Lock: lk} + if err := admitDeploymentVolumes(ctx, s.exec, appCfg); err != nil { + return err + } + var migration *docker.VolumeMigration + defer func() { + if migration != nil { + if retErr == nil || state.PreservePublishedState(retErr) { + migration.Commit() + } else { + retErr = errors.Join(retErr, migration.Restore(ctx)) + } + } + }() image := imageOverride + if image == "" { + image = appCfg.Image + } // Resolve version (mirrors deployAppConfig's ordering and rationale). - var err error if versionOverride != "" { if err := validateVersionArg(versionOverride); err != nil { return err @@ -176,8 +210,10 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App } hostPath := fmt.Sprintf("/deployments/%s/volumes/%s", appCfg.App, name) volumes[hostPath] = containerPath - if _, err := s.exec.Run(ctx, "mkdir -p "+hostPath); err != nil { - return fmt.Errorf("creating volume directory %s: %w", hostPath, err) + if _, owned := appCfg.VolumeOwnership[name]; !owned { + if err := docker.NewClient(s.exec).EnsureManagedDirectory(ctx, appCfg.App, "", name); err != nil { + return err + } } } } @@ -205,7 +241,8 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App if !s.migrateVolumes { return docker.FormatMismatchError(appCfg.App, mismatches) } - if err := docker.MigrateVolumes(ctx, s.exec, appCfg.App, mismatches, s.out); err != nil { + migration, err = docker.BeginVolumeMigration(ctx, s.exec, appCfg.App, mismatches, s.out) + if err != nil { return fmt.Errorf("migrating volumes: %w", err) } } @@ -258,7 +295,7 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App if err := releasemeta.WriteAttemptProvenance(ctx, s.exec, att, prov); err != nil { fmt.Fprintf(s.out, "Warning: could not persist the deploy provenance receipt for %s@%s: %v\n", appCfg.App, version, err) } - deployer := deploy.NewDeployer(s.exec, s.out) + deployer := deploy.NewDeployer(rawExecutor, s.out) deployCfg := deployConfigFromApp(appCfg, image, version, envFiles, volumes, tlsCert, tlsKey, tlsInternal, appliedManifest, manifestSHA256) deployCfg.Provenance = prov @@ -272,5 +309,8 @@ func (s *singleServerDeployer) deployApp(ctx context.Context, appCfg *config.App } } - return deployer.Deploy(ctx, deployCfg) + if s.deployEngine != nil { + return s.deployEngine(ctx, deployCfg, lk) + } + return deployer.DeployFenced(ctx, deployCfg, lk) } diff --git a/internal/cli/template.go b/internal/cli/template.go index 55b1387..383d880 100644 --- a/internal/cli/template.go +++ b/internal/cli/template.go @@ -1,11 +1,14 @@ package cli import ( + "bytes" "context" "encoding/json" "fmt" + "gopkg.in/yaml.v3" "os" "os/signal" + "path/filepath" "strings" "github.com/spf13/cobra" @@ -147,16 +150,11 @@ func runTemplateDeploy(flags *Flags, name, domain, server string, port int, extr return err } - // The written teploy.yml is what `teploy deploy` re-reads, so a --port - // override must land in the FILE, not just the in-memory config (there is - // no AppConfig serializer; a targeted top-level line patch is the same - // shape GenerateSecrets uses for its substitutions). - if port != 0 { - content = replaceTopLevelPort(content, port) + content, err = persistTemplateOverrides(content, domain, server, port) + if err != nil { + return err } - - // Write to teploy.yml in current directory. - if err := os.WriteFile("teploy.yml", []byte(content), 0644); err != nil { + if err := writePrivateTemplate("teploy.yml", []byte(content)); err != nil { return fmt.Errorf("writing teploy.yml: %w", err) } @@ -291,6 +289,10 @@ func runTemplateInstall(flags *Flags, name, domain, server string, port int, ext } fmt.Printf(" Server: %s\n", server) + if err := resolveDeployEnv(ctx, appCfg, flags.StrictEnv); err != nil { + return err + } + // Templates are first-deploys by definition, so volume mismatch can't apply yet. if err := deployAppConfig(flags, appCfg, server, appCfg.Image, "", false, false, ""); err != nil { return err @@ -347,3 +349,65 @@ func joinStrings(ss []string) string { } return result } + +func persistTemplateOverrides(content, domain, server string, port int) (string, error) { + var doc yaml.Node + if err := yaml.Unmarshal([]byte(content), &doc); err != nil { + return "", err + } + if len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode { + return "", fmt.Errorf("template must be a YAML mapping") + } + root := doc.Content[0] + set := func(key, value, tag string) { + for i := 0; i < len(root.Content); i += 2 { + if root.Content[i].Value == key { + root.Content[i+1] = &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value} + return + } + } + root.Content = append(root.Content, &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, &yaml.Node{Kind: yaml.ScalarNode, Tag: tag, Value: value}) + } + if domain != "" { + set("domain", domain, "!!str") + } + if server != "" { + set("server", server, "!!str") + } + if port != 0 { + set("port", fmt.Sprint(port), "!!int") + } + var out bytes.Buffer + enc := yaml.NewEncoder(&out) + enc.SetIndent(2) + if err := enc.Encode(&doc); err != nil { + return "", err + } + if err := enc.Close(); err != nil { + return "", err + } + return out.String(), nil +} + +// Link publication is atomic and refuses any preexisting file or symlink. +func writePrivateTemplate(path string, content []byte) error { + f, err := os.CreateTemp(filepath.Dir(path), ".teploy-template-*") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err = f.Write(content); err == nil { + err = f.Sync() + } + closeErr := f.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if err := os.Link(f.Name(), path); err != nil { + return fmt.Errorf("refusing to replace existing output or unable to publish %s: %w", path, err) + } + return nil +} diff --git a/internal/cli/template_test.go b/internal/cli/template_test.go index b5d4b15..20c1423 100644 --- a/internal/cli/template_test.go +++ b/internal/cli/template_test.go @@ -1,6 +1,8 @@ package cli import ( + "os" + "path/filepath" "strings" "testing" @@ -116,3 +118,42 @@ func TestTemplateVars(t *testing.T) { t.Errorf("nil stdin vars: %v", vars) } } + +func TestPrivateTemplatePublicationAndOverrides(t *testing.T) { + content, err := persistTemplateOverrides("app: demo\ningress: host\nport: 80\n", "example.test", "production", 8080) + if err != nil { + t.Fatal(err) + } + cfg, err := config.ParseAppBytes([]byte(content)) + if err != nil { + t.Fatal(err) + } + if cfg.Server != "production" || cfg.Domain != "example.test" || cfg.Port != 8080 { + t.Fatalf("lost overrides: %+v", cfg) + } + path := filepath.Join(t.TempDir(), "teploy.yml") + if err := writePrivateTemplate(path, []byte(content)); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("mode %o", info.Mode().Perm()) + } + if err := writePrivateTemplate(path, []byte("replacement")); err == nil { + t.Fatal("existing output replaced") + } + got, _ := os.ReadFile(path) + if string(got) != content { + t.Fatal("existing file changed") + } + link := filepath.Join(filepath.Dir(path), "link.yml") + if err := os.Symlink(path, link); err != nil { + t.Fatal(err) + } + if err := writePrivateTemplate(link, []byte("replacement")); err == nil { + t.Fatal("symlink replaced") + } +} diff --git a/internal/cli/trigger.go b/internal/cli/trigger.go new file mode 100644 index 0000000..747b252 --- /dev/null +++ b/internal/cli/trigger.go @@ -0,0 +1,385 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "github.com/useteploy/teploy/internal/caddy" + "io" + "reflect" + "strings" + "time" + + "encoding/json" + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/build" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/preview" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" + "github.com/useteploy/teploy/internal/trigger" +) + +type triggerContextKey struct{} +type triggerOutputKey struct{} +type triggerTransportKey struct{} + +func deployOutput(ctx context.Context) io.Writer { + if out, ok := ctx.Value(triggerOutputKey{}).(io.Writer); ok { + return out + } + return nil +} + +// TriggerBindingDigest uses the SAME private PlanRecord2 algorithm with the +// admitted target substituted for mutable connection aliases. It never uses +// the public, redacted ManifestSHA256. Existing PlanRecord2 stays unchanged. +func TriggerBindingDigest(cfg *config.AppConfig, image, target string) (string, error) { + bound := *cfg + bound.Server = target + bound.Servers = nil + bound.User = "" + return config.ExecutionBindingDigest(&bound, image) +} + +// triggerCommand is the actual Cobra stdin/final-result boundary. Diagnostic +// writers are passed explicitly; production effect wrappers do not write to +// stdout. The final result is emitted even when decode/admission/effects fail. +func triggerCommand(cmd *cobra.Command, action string, run func(trigger.Request) (trigger.Result, error)) error { + r, err := trigger.Decode(cmd.InOrStdin()) + result := trigger.Initial(r) + if err == nil && r.Action != action { + err = fmt.Errorf("request action disagrees with command") + } + if err != nil { + result.ErrorCode = "invalid_request" + } else { + result, err = run(r) + } + if err != nil && result.ErrorCode == "" { + result.ErrorCode = "admission_failed" + } + if encodeErr := json.NewEncoder(cmd.OutOrStdout()).Encode(result); encodeErr != nil { + return errors.Join(err, encodeErr) + } + return err +} + +func observeDeploy(ctx context.Context, exec ssh.Executor, app string, r trigger.Request) (*trigger.Result, error) { + actual, err := state.Read(ctx, exec, app) + if err != nil { + return nil, err + } + if actual == nil || actual.TriggerOperationKey != r.OperationKey { + return nil, nil + } + if actual.ExecutionBindingDigest != r.ExecutionBindingDigest || actual.SourceRevision != r.Commit || actual.ImageDigest == "" { + return nil, fmt.Errorf("deployed authority binding/image mismatch") + } + + observed := time.Now().UTC() + gen := actual.Generation + result := trigger.Result{SchemaVersion: 1, OperationKey: r.OperationKey, Publication: "committed", Reconciliation: "required", Generation: &gen, ReleaseHash: actual.CurrentHash, ImageDigest: actual.ImageDigest, AuthorityObservedAt: &observed} + release, err := releasemeta.Read(ctx, exec, app, actual.CurrentHash) + if err != nil { + return &result, err + } + if release == nil || release.TriggerOperationKey != r.OperationKey || release.ExecutionBindingDigest != r.ExecutionBindingDigest || release.Hash != actual.CurrentHash || release.Generation != actual.Generation || release.ImageDigest != actual.ImageDigest { + return &result, fmt.Errorf("immutable release completion unproven") + } + inventory, err := docker.NewClient(exec).ListContainers(ctx, app) + if err != nil { + return &result, err + } + web := false + for _, container := range inventory { + if container.Labels["teploy.version"] != actual.CurrentHash || container.Labels["teploy.process"] != "web" { + continue + } + if container.State != "running" || container.ID == "" { + return &result, fmt.Errorf("serving workload unproven") + } + digest, err := docker.NewClient(exec).ContainerImageDigest(ctx, container.ID) + if err != nil || digest != actual.ImageDigest { + return &result, errors.Join(fmt.Errorf("actual workload image mismatch"), err) + } + web = true + } + if !web { + return &result, fmt.Errorf("serving web workload absent") + } + result.Reconciliation = "complete" + + // A committed state with failed generation-sidecar repair is not complete. + sidecar, err := state.ReadCommittedGeneration(ctx, exec, app) + if err != nil { + result.Reconciliation = "required" + return &result, err + } + if sidecar != gen { + result.Reconciliation = "required" + return &result, fmt.Errorf("generation repair required") + } + return &result, nil +} + +// runTriggerDeployment keeps admission, preparations and publication under +// one target lease. The injected command tests exercise this wrapper through +// triggerCommand rather than bypassing the production adapter. +func runTriggerDeployment(ctx context.Context, exec ssh.Executor, lock *state.Lock, cfg *config.AppConfig, image, version string, r trigger.Request, out io.Writer) (trigger.Result, error) { + result := trigger.Initial(r) + if err := state.RequireLease(lock); err != nil { + return result, err + } + fenced := &state.FencedExecutor{Executor: exec, Lock: lock} + target, err := trigger.EnrolledTarget(ctx, fenced, cfg.App) + if err != nil { + return result, err + } + binding, err := TriggerBindingDigest(cfg, image, target) + if err != nil { + return result, err + } + if r.TargetID != target || r.ExecutionBindingDigest != binding || r.Commit != cfg.SourceRevision { + result.ErrorCode = "identity_mismatch" + return result, fmt.Errorf("target/source/private execution binding mismatch") + } + adapter := trigger.Adapter{Store: trigger.RemoteStore{Exec: fenced, App: cfg.App}, Evidence: func(ctx context.Context, r trigger.Request) (json.RawMessage, error) { + e, err := readDeploymentEvidence(ctx, fenced, cfg.App) + if e.Authority != nil && e.Authority.Generation != r.ExpectedGeneration { + return nil, fmt.Errorf("application generation changed") + } + if e.Authority == nil && r.ExpectedGeneration != 0 { + return nil, fmt.Errorf("application authority absent") + } + if err != nil { + return nil, err + } + return json.Marshal(e) + }, Observe: func(ctx context.Context, receipt trigger.Receipt) (*trigger.Result, error) { + return observeDeploymentReceipt(ctx, fenced, cfg.App, receipt) + }} + needsBuild := image == "" + return adapter.Run(ctx, r, func(ctx context.Context) error { + if err := admitDeploymentVolumes(ctx, fenced, cfg); err != nil { + return err + } + ctx = context.WithValue(ctx, triggerContextKey{}, r) + ctx = context.WithValue(ctx, triggerOutputKey{}, out) + if image == "" { + mode, err := build.DetectAt(cfg.Context, cfg.Dockerfile) + if err != nil { + return err + } + _, _, key, err := config.ResolveServer(cfg.Server, "", "", "") + host, user := exec.Host(), exec.User() + if override, ok := ctx.Value(triggerTransportKey{}).(string); ok && override != "" { + key = override + } + if err != nil { + return err + } + att := releasemeta.MustAttempt(cfg.App, version) + if cfg.BuildLocal { + image, err = build.LocalBuild(ctx, build.LocalBuildConfig{App: cfg.App, Version: version, Mode: mode, Dir: ".", Context: cfg.Context, Dockerfile: cfg.Dockerfile, Host: host, User: user, KeyPath: key, Platform: cfg.Platform, Exec: fenced}, out) + } else { + dir, syncErr := syncAttemptBuildContext(ctx, fenced, cfg, att, mode, host, user, key, out, out) + if syncErr != nil { + return syncErr + } + image, err = build.NewBuilder(fenced, out).Build(ctx, build.BuildConfig{App: cfg.App, Version: version, Mode: mode, BuildDir: dir, Context: cfg.Context, Dockerfile: cfg.Dockerfile, Platform: cfg.Platform}) + } + if err != nil { + return err + } + } + if err := ensureImage(ctx, docker.NewClient(fenced), image, out); err != nil { + return err + } + att := releasemeta.MustAttempt(cfg.App, version) + return deployBuiltImageFenced(ctx, exec, cfg, image, version, exec.Host(), false, needsBuild, ".", lock, &att, "") + }) +} + +func runTriggerCLI(cmd *cobra.Command, flags *Flags, action, branch, image, version string, opts previewDeployOpts, r trigger.Request) (trigger.Result, error) { + result := trigger.Initial(r) + ctx := cmd.Context() + cfg, err := config.LoadApp(".") + if err != nil { + return result, err + } + if err = resolveDeployEnv(ctx, cfg, flags.StrictEnv); err != nil { + return result, err + } + if action != "preview_destroy" { + if err := verifyCheckoutRequest(ctx, r); err != nil { + return result, err + } + revision, revisionErr := gitRevisionIn(".") + if revisionErr != nil || revision != r.Commit { + return result, fmt.Errorf("checkout does not match full authenticated commit") + } + cfg.SourceRevision = revision + } + if image == "" { + image = cfg.Image + } + if version == "" { + version = r.Commit + "-" + r.OperationKey[:16] + } + exec, err := connectForApp(ctx, flags, cfg) + if err != nil { + return result, err + } + defer exec.Close() + if err = docker.NewClient(exec).EnsureManagedDirectory(ctx, cfg.App, "", ""); err != nil { + return result, err + } + lock, err := state.AcquireLockFenced(ctx, exec, cfg.App) + if err != nil { + return result, err + } + defer state.ReleaseLockFenced(exec, lock, cfg.App) + lock.StartRenewal(exec) + if action == "deploy" { + if cfg.IsStatic() { + result.ErrorCode = "unsupported_static_trigger" + return result, fmt.Errorf("static trigger adapter is not yet implemented") + } + ctx = context.WithValue(ctx, triggerTransportKey{}, flags.Key) + return runTriggerDeployment(ctx, exec, lock, cfg, image, version, r, cmd.ErrOrStderr()) + } + fenced := &state.FencedExecutor{Executor: exec, Lock: lock} + target, err := trigger.EnrolledTarget(ctx, fenced, cfg.App) + if err != nil { + return result, err + } + binding, err := TriggerBindingDigest(cfg, image, target) + if err != nil { + return result, err + } + if r.TargetID != target || (action != "preview_destroy" && r.ExecutionBindingDigest != binding) { + return result, fmt.Errorf("target/private binding mismatch") + } + if strings.TrimPrefix(r.Ref, "refs/heads/") != branch { + return result, fmt.Errorf("preview branch does not match exact request ref") + } + mgr := preview.NewManager(exec, cmd.ErrOrStderr()).WithLease(lock) + adapter := trigger.Adapter{Store: trigger.RemoteStore{Exec: fenced, App: cfg.App}, Evidence: func(ctx context.Context, request trigger.Request) (json.RawMessage, error) { + if request.Action != "preview_destroy" { + return nil, nil + } + s, err := mgr.Observe(ctx, cfg.App, branch) + if err != nil { + return nil, err + } + if s == nil || s.OwnershipID != request.PreviewIdentity || s.Generation != request.ExpectedGeneration || s.ExecutionBindingDigest != request.ExecutionBindingDigest || s.SourceRevision != request.Commit || s.Repo != request.RepositoryID { + return nil, fmt.Errorf("retained preview authority mismatch") + } + return json.Marshal(s) + }, Observe: func(ctx context.Context, receipt trigger.Receipt) (*trigger.Result, error) { + request := receipt.Request + s, err := mgr.Observe(ctx, cfg.App, strings.TrimPrefix(request.Ref, "refs/heads/")) + if err != nil { + return nil, err + } + if request.Action == "preview_destroy" { + var admitted preview.State + if err := json.Unmarshal(receipt.Evidence, &admitted); err != nil { + return nil, err + } + absent, err := mgr.ProveDestroyed(ctx, cfg.App, admitted) + if err != nil || !absent { + return nil, err + } + now := time.Now().UTC() + gen := admitted.Generation + return &trigger.Result{SchemaVersion: 1, OperationKey: request.OperationKey, Publication: "committed", Reconciliation: "complete", Generation: &gen, AuthorityObservedAt: &now}, nil + } + if s == nil { + return nil, nil + } + if s.OperationKey != request.OperationKey { + return nil, nil + } + if s.ExecutionBindingDigest != request.ExecutionBindingDigest || s.SourceRevision != request.Commit || s.OwnershipID != request.PreviewIdentity || s.Generation <= request.ExpectedGeneration || s.ImageDigest == "" { + return nil, fmt.Errorf("preview completion identity unproven") + } + if err := mgr.ProveServing(ctx, cfg.App, *s); err != nil { + return nil, err + } + now := time.Now().UTC() + gen := s.Generation + return &trigger.Result{SchemaVersion: 1, OperationKey: request.OperationKey, Publication: "committed", Reconciliation: "complete", Generation: &gen, ImageDigest: s.ImageDigest, AuthorityObservedAt: &now}, nil + }} + return adapter.Run(ctx, r, func(ctx context.Context) error { + if action == "preview_destroy" { + return mgr.CompareDestroyHeld(ctx, cfg.App, branch, r.PreviewIdentity, r.ExpectedGeneration, r.ExpectedPreviewUpdatedAt) + } + ttl, err := time.ParseDuration(opts.ttl) + if err != nil { + return err + } + if err := ensureImage(ctx, docker.NewClient(fenced), image, cmd.ErrOrStderr()); err != nil { + return err + } + return mgr.DeployHeld(ctx, preview.DeployConfig{App: cfg.App, Domain: cfg.Domain, Branch: branch, Image: image, Version: version, TTL: ttl, Repo: r.RepositoryID, OwnershipID: r.PreviewIdentity, ExpectedGeneration: &r.ExpectedGeneration, OperationKey: r.OperationKey, ExecutionBindingDigest: binding, SourceRevision: r.Commit, BaseDomain: opts.baseDomain, HTTPOnly: opts.httpOnly, AllowIPs: opts.allowIPs}) + }) +} + +// Assert the deployment metadata always reaches the authoritative engine. +func attachTrigger(ctx context.Context, cfg *deploy.Config) { + if r, ok := ctx.Value(triggerContextKey{}).(trigger.Request); ok { + cfg.TriggerOperationKey = r.OperationKey + cfg.ExecutionBindingDigest = r.ExecutionBindingDigest + cfg.SourceRevision = r.Commit + } +} + +// deploymentEvidence captures authority plus exact immutable workload inventory +// BEFORE any preparation. It supports honest ordinary failed-health results. +type deploymentEvidence struct { + Authority *state.AppState `json:"authority"` + Containers []docker.Container `json:"containers"` + Route string `json:"route"` + RoutePresent bool `json:"route_present"` +} + +func readDeploymentEvidence(ctx context.Context, exec ssh.Executor, app string) (deploymentEvidence, error) { + var e deploymentEvidence + var err error + e.Authority, err = state.Read(ctx, exec, app) + if err != nil { + return e, err + } + e.Containers, err = docker.NewClient(exec).ListContainers(ctx, app) + if err != nil { + return e, err + } + e.Route, e.RoutePresent, err = caddy.NewClient(exec).ReadManagedBlock(ctx, app) + return e, err +} +func observeDeploymentReceipt(ctx context.Context, exec ssh.Executor, app string, receipt trigger.Receipt) (*trigger.Result, error) { + result, err := observeDeploy(ctx, exec, app, receipt.Request) + if result != nil || err != nil { + return result, err + } + var before deploymentEvidence + if err = json.Unmarshal(receipt.Evidence, &before); err != nil { + return nil, err + } + actual, err := readDeploymentEvidence(ctx, exec, app) + if err != nil { + return nil, err + } + // Comparison includes immutable IDs and running states. A candidate, route + // switch, changed generation or unknown inventory cannot read as precommit. + if !reflect.DeepEqual(before, actual) { + return nil, nil + } + now := time.Now().UTC() + return &trigger.Result{SchemaVersion: 1, OperationKey: receipt.Request.OperationKey, Publication: "not_committed", Reconciliation: "complete", AuthorityObservedAt: &now}, nil +} diff --git a/internal/cli/trigger_cobra_test.go b/internal/cli/trigger_cobra_test.go new file mode 100644 index 0000000..d581d3d --- /dev/null +++ b/internal/cli/trigger_cobra_test.go @@ -0,0 +1,71 @@ +package cli + +import ( + "bytes" + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/trigger" +) + +func TestTriggerActualCobraNonzeroFinalJSON(t *testing.T) { + for _, args := range [][]string{{"deploy", "--trigger-stdin", "--json"}, {"preview", "deploy", "--trigger-stdin", "--json"}, {"preview", "destroy", "--trigger-stdin", "--json"}} { + for _, input := range []string{`{"schema_version":1,"operation_key":"dummy-secret","unknown":"private-value"}`, strings.Repeat("x", trigger.MaxRequestBytes+1), `{`, `{}`} { + cmd := NewRootCmd("fixture") + var out, stderr bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(&stderr) + cmd.SetIn(strings.NewReader(input)) + cmd.SetArgs(args) + if err := cmd.Execute(); err == nil { + t.Fatal("malformed trigger succeeded") + } + var result trigger.Result + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatalf("nonzero stdout not exactly JSON: %s %v", out.String(), err) + } + if result.SchemaVersion != 1 || result.Publication != "not_started" || result.Generation != nil { + t.Fatal(result) + } + if strings.Contains(out.String(), "private-value") || strings.Contains(out.String(), "dummy-secret") { + t.Fatal("private input echoed") + } + } + } +} +func TestRound5AuthenticatedUnusableCommitNeverAdmitsOrInvokes(t *testing.T) { + run := newCountingRun() + handler, ledger, queue := newAdmissionStack("s3cret", "main", "myapp", run.run) + for _, body := range []string{`{"ref":"refs/heads/main"}`, `{"ref":"refs/heads/main","after":"malformed"}`, `{"ref":"refs/heads/main","after":"` + strings.Repeat("0", 40) + `"}`, `{"ref":"refs/heads/main","deleted":true,"after":"` + strings.Repeat("a", 40) + `"}`} { + response := postSigned(t, handler, "s3cret", "", body) + if response.Code != http.StatusUnprocessableEntity && response.Code != http.StatusAccepted { + t.Fatal(response.Code, response.Body.String()) + } + } + run.waitIdle(t, queue) + if run.count() != 0 { + t.Fatal("malformed event reached production invocation") + } + if len(ledger.snapshot()) != 0 { + t.Fatal("nondeployable event durably admitted") + } +} +func TestRound5TriggerBindingSurvivesAliasRenameButNotTargetReuse(t *testing.T) { + cfg := &config.AppConfig{App: "demo", Server: "original"} + first, err := TriggerBindingDigest(cfg, "image:A", strings.Repeat("a", 32)+"/demo") + if err != nil { + t.Fatal(err) + } + cfg.Server = "renamed" + second, _ := TriggerBindingDigest(cfg, "image:A", strings.Repeat("a", 32)+"/demo") + if first != second { + t.Fatal("alias entered execution identity") + } + third, _ := TriggerBindingDigest(cfg, "image:A", strings.Repeat("b", 32)+"/demo") + if first == third { + t.Fatal("new target reused old private binding") + } +} diff --git a/internal/cli/trigger_identity.go b/internal/cli/trigger_identity.go new file mode 100644 index 0000000..241336e --- /dev/null +++ b/internal/cli/trigger_identity.go @@ -0,0 +1,223 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/state" + "github.com/useteploy/teploy/internal/trigger" +) + +// canonicalRepository records the forge origin and canonical repository path, +// with no authentication material. API origins are supplied by the source +// authority; a checkout can prove their repository mapping, not API credentials. +func canonicalRepository(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if strings.HasPrefix(raw, "git@") { + host, path, ok := strings.Cut(strings.TrimPrefix(raw, "git@"), ":") + if !ok { + return "", fmt.Errorf("invalid repository origin") + } + raw = "https://" + host + "/" + path + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" || u.RawQuery != "" || u.Fragment != "" { + return "", fmt.Errorf("repository requires a canonical forge origin and path") + } + if u.Scheme == "ssh" { + u.Scheme = "https" + } + if u.Scheme != "https" && u.Scheme != "http" { + return "", fmt.Errorf("unsupported repository origin") + } + u.User = nil + u.Scheme = strings.ToLower(u.Scheme) + origin, err := trigger.CanonicalAPIBase(u.Scheme + "://" + u.Host) + if err != nil { + return "", err + } + normalized, _ := url.Parse(origin) + u.Host = normalized.Host + u.Path = strings.TrimSuffix(strings.TrimSuffix(u.Path, "/"), ".git") + u.RawPath = "" + if u.Path == "" || strings.Contains(u.Path, "/../") || strings.Contains(u.Path, "/./") { + return "", fmt.Errorf("invalid repository identity") + } + return u.String(), nil +} +func checkoutRepository(dir string) (string, error) { + out, err := exec.Command("git", "-C", dir, "config", "--get", "remote.origin.url").Output() + if err != nil { + return "", err + } + return canonicalRepository(string(out)) +} +func checkoutRef(dir string) (string, error) { + out, err := exec.Command("git", "-C", dir, "symbolic-ref", "-q", "HEAD").Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} +func verifyCheckoutRequest(ctx context.Context, r trigger.Request) error { + repo, err := checkoutRepository(".") + if err != nil { + return err + } + if err := verifyRepositoryMapping(ctx, r.RepositoryID, repo, http.DefaultTransport); err != nil { + return err + } + // Detached checkouts carry no moving branch authority. Their exact ref is + // supplied by the authenticated producer and their full commit is re-proven. + ref, err := checkoutRef(".") + if err == nil && ref != r.Ref { + return fmt.Errorf("checkout ref differs from admitted exact ref") + } + return nil +} + +// Native repository proof is read from an explicitly configured authenticated +// API. A caller cannot redirect API credentials by changing repository_id. +// The same resolver must run before fetch in automatic checkout producers. +func verifyRepositoryMapping(ctx context.Context, identity, clone string, transport http.RoundTripper) error { + i, err := trigger.ParseRepositoryIdentity(identity) + if err != nil { + return err + } + if i.Forge == "generic" { + locator, err := canonicalRepository(strings.TrimPrefix(i.ID, "url:")) + if err != nil || locator != strings.TrimPrefix(i.ID, "url:") || locator != clone { + return fmt.Errorf("checkout differs from generic repository locator") + } + return nil + } + configuredBase, err := trigger.CanonicalAPIBase(os.Getenv("TEPLOY_TRIGGER_API_BASE")) + if err != nil || configuredBase != i.APIBase || os.Getenv("TEPLOY_TRIGGER_FORGE") != i.Forge { + return fmt.Errorf("native repository requires configured API authority") + } + if strings.HasPrefix(i.APIBase, "http:") && os.Getenv("TEPLOY_TRIGGER_ALLOW_HTTP") != "1" { + return fmt.Errorf("private HTTP API authority not admitted") + } + token := os.Getenv("TEPLOY_TRIGGER_API_TOKEN") + if token == "" { + return fmt.Errorf("native repository requires authenticated metadata") + } + var endpoint string + switch i.Forge { + case "github": + endpoint = "/repositories/" + strings.TrimPrefix(i.ID, "id:") + case "gitlab": + endpoint = "/projects/" + strings.TrimPrefix(i.ID, "id:") + default: + return fmt.Errorf("native repository metadata resolver not implemented for configured forge") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, i.APIBase+endpoint, nil) + if err != nil { + return fmt.Errorf("invalid repository metadata request") + } + if i.Forge == "gitlab" { + req.Header.Set("PRIVATE-TOKEN", token) + } else { + req.Header.Set("Authorization", "Bearer "+token) + } + req.Header.Set("Accept", "application/json") + client := &http.Client{Transport: transport, Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + response, err := client.Do(req) + if err != nil { + return fmt.Errorf("authenticated repository metadata unavailable") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("authenticated repository metadata refused") + } + body, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil || len(body) > 1<<20 { + return fmt.Errorf("repository metadata exceeds bound or unreadable") + } + var metadata struct { + ID json.Number `json:"id"` + CloneURL string `json:"clone_url"` + HTTPURL string `json:"http_url_to_repo"` + } + if err := json.Unmarshal(body, &metadata); err != nil || "id:"+metadata.ID.String() != i.ID { + return fmt.Errorf("authenticated native repository identity mismatch") + } + locator := metadata.CloneURL + if i.Forge == "gitlab" { + locator = metadata.HTTPURL + } + actual, err := canonicalRepository(locator) + if err != nil || actual != clone { + return fmt.Errorf("authenticated API repository does not map to checkout origin") + } + return nil +} +func newTriggerCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{Use: "trigger", Short: "Inspect immutable trigger admission identity"} + var image string + identity := &cobra.Command{Use: "identity", Args: cobra.NoArgs, Short: "Enroll target and read the opaque execution binding", RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.LoadApp(".") + if err != nil { + return err + } + if err = resolveDeployEnv(cmd.Context(), cfg, flags.StrictEnv); err != nil { + return err + } + if image == "" { + image = cfg.Image + } + exec, err := connectForApp(cmd.Context(), flags, cfg) + if err != nil { + return err + } + defer exec.Close() + if err = docker.NewClient(exec).EnsureManagedDirectory(cmd.Context(), cfg.App, "", ""); err != nil { + return err + } + lock, err := state.AcquireLockFenced(cmd.Context(), exec, cfg.App) + if err != nil { + return err + } + defer state.ReleaseLockFenced(exec, lock, cfg.App) + lock.StartRenewal(exec) + fenced := &state.FencedExecutor{Executor: exec, Lock: lock} + target, err := trigger.EnrolledTarget(cmd.Context(), fenced, cfg.App) + if err != nil { + return err + } + binding, err := TriggerBindingDigest(cfg, image, target) + if err != nil { + return err + } + actual, err := state.Read(cmd.Context(), fenced, cfg.App) + if err != nil { + return err + } + // A successful absence read is explicit; no generation is synthesized. + var generation *uint64 + if actual != nil { + gen := actual.Generation + generation = &gen + } + return json.NewEncoder(cmd.OutOrStdout()).Encode(struct { + SchemaVersion int `json:"schema_version"` + TargetID string `json:"target_id"` + Binding string `json:"execution_binding_digest"` + Generation *uint64 `json:"generation,omitempty"` + }{1, target, binding, generation}) + }} + identity.Flags().StringVar(&image, "image", "", "image reference used by the execution binding (empty for source build)") + cmd.AddCommand(identity) + return cmd +} diff --git a/internal/cli/validate.go b/internal/cli/validate.go index 62cadbc..db39e43 100644 --- a/internal/cli/validate.go +++ b/internal/cli/validate.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -320,7 +321,11 @@ func outputResult(flags *Flags, result *validationResult) error { if flags.JSON { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") - return enc.Encode(result) + var outcome error + if !result.Valid { + outcome = fmt.Errorf("validation failed with %d error(s)", len(result.Errors)) + } + return errors.Join(outcome, enc.Encode(result)) } if result.Valid { diff --git a/internal/cli/volume_admission.go b/internal/cli/volume_admission.go new file mode 100644 index 0000000..5fe9a39 --- /dev/null +++ b/internal/cli/volume_admission.go @@ -0,0 +1,51 @@ +package cli + +import ( + "context" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" + "sort" +) + +// First validate every explicit root without mutation, then provision. This +// precedes accessory startup, agent startup and migration stop/copy. +func admitDeploymentVolumes(ctx context.Context, exec ssh.Executor, cfg *config.AppConfig) error { + dk := docker.NewClient(exec) + for _, name := range volumeKeys(cfg.VolumeOwnership) { + if err := dk.ValidateManagedVolume(ctx, cfg.App, "", name, cfg.VolumeOwnership[name]); err != nil { + return err + } + } + for _, accessory := range sortedAccessoryNames(cfg.Accessories) { + acc := cfg.Accessories[accessory] + for _, name := range volumeKeys(acc.VolumeOwnership) { + if err := dk.ValidateManagedVolume(ctx, cfg.App, accessory, name, acc.VolumeOwnership[name]); err != nil { + return err + } + } + } + for _, name := range volumeKeys(cfg.VolumeOwnership) { + if err := dk.ProvisionManagedVolume(ctx, cfg.App, "", name, cfg.VolumeOwnership[name]); err != nil { + return err + } + } + for _, accessory := range sortedAccessoryNames(cfg.Accessories) { + acc := cfg.Accessories[accessory] + for _, name := range volumeKeys(acc.VolumeOwnership) { + if err := dk.ProvisionManagedVolume(ctx, cfg.App, accessory, name, acc.VolumeOwnership[name]); err != nil { + return err + } + } + } + return nil +} + +func volumeKeys[V any](values map[string]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/internal/config/app.go b/internal/config/app.go index aa02517..ae47d27 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -122,10 +122,12 @@ func (r *RolloutConfig) CanaryCount(total int) (int, error) { // AccessoryConfig represents a stateful service container (database, cache, etc.). type AccessoryConfig struct { - Image string `yaml:"image" toml:"image"` - Port int `yaml:"port,omitempty" toml:"port"` - Env map[string]string `yaml:"env,omitempty" toml:"env"` - Volumes map[string]string `yaml:"volumes,omitempty" toml:"volumes"` + Image string `yaml:"image" toml:"image"` + Port int `yaml:"port,omitempty" toml:"port"` + Env map[string]string `yaml:"env,omitempty" toml:"env"` + EnvLiteral map[string]string `yaml:"env_literal,omitempty" toml:"env_literal"` + Volumes map[string]string `yaml:"volumes,omitempty" toml:"volumes"` + VolumeOwnership map[string]VolumeOwnership `yaml:"volume_ownership,omitempty" toml:"volume_ownership"` // Command overrides the image's default command (docker run trailing // args) — required by images whose entrypoint needs an explicit verb, // e.g. MinIO (`server /data --console-address :9001`) or ntfy (`serve`). @@ -419,14 +421,16 @@ type AppConfig struct { // everything — historical behavior. Set to 2 or 3 to enable auto-prune // with a rollback window. Container-deploy only; static deploy uses // KeepReleases instead. - KeepVersions int `yaml:"keep_versions,omitempty" toml:"keep_versions"` - Hooks HooksConfig `yaml:"hooks,omitempty" toml:"hooks"` - Volumes map[string]string `yaml:"volumes,omitempty" toml:"volumes"` - Processes map[string]string `yaml:"processes,omitempty" toml:"processes"` + KeepVersions int `yaml:"keep_versions,omitempty" toml:"keep_versions"` + Hooks HooksConfig `yaml:"hooks,omitempty" toml:"hooks"` + Volumes map[string]string `yaml:"volumes,omitempty" toml:"volumes"` + VolumeOwnership map[string]VolumeOwnership `yaml:"volume_ownership,omitempty" toml:"volume_ownership"` + Processes map[string]string `yaml:"processes,omitempty" toml:"processes"` // Env are plain environment variables passed to the app container. Values // may reference ${VAR} from the local environment (expanded at deploy // time). Secrets set via `teploy secret` override these key-for-key. - Env map[string]string `yaml:"env,omitempty" toml:"env"` + Env map[string]string `yaml:"env,omitempty" toml:"env"` + EnvLiteral map[string]string `yaml:"env_literal,omitempty" toml:"env_literal"` // Health configures the teploy-level deploy health check (the HTTP poll // that gates traffic switch after a deploy, separate from the container // HEALTHCHECK). Path defaults to "/health" when unset. @@ -794,7 +798,11 @@ func ValidatePublishEntries(entries []string) error { if spec.HostPort == 0 { continue // ephemeral ports cannot collide with each other by name } - key := fmt.Sprintf("%d/%s", spec.HostPort, spec.Proto) + proto := spec.Proto + if proto == "" { + proto = "tcp" + } + key := fmt.Sprintf("%d/%s", spec.HostPort, proto) set, ok := seen[key] if !ok { set = &bindSet{binds: map[string]bool{}} @@ -1028,6 +1036,12 @@ func (c *AppConfig) validate() error { if c.Health.Mode == HealthModeTCP && c.Health.Path != "" { return fmt.Errorf("'health.path' has no effect with 'health.mode: tcp' (TCP readiness dials the port; nothing is fetched) — remove the path or use mode http/auto") } + if c.IsStatic() && len(c.VolumeOwnership) > 0 { + return fmt.Errorf("volume_ownership requires a container app") + } + if err := ValidateVolumeOwnership(c.Volumes, c.VolumeOwnership); err != nil { + return err + } for name, dest := range c.Volumes { if !validName.MatchString(name) && !IsHostBindVolume(name) { return fmt.Errorf("volume name %q must be lowercase alphanumeric with hyphens, or an absolute host path for a bind mount", name) @@ -1037,6 +1051,9 @@ func (c *AppConfig) validate() error { } } for name, acc := range c.Accessories { + if err := ValidateVolumeOwnership(acc.Volumes, acc.VolumeOwnership); err != nil { + return fmt.Errorf("accessory %s: %w", name, err) + } if !validName.MatchString(name) { return fmt.Errorf("accessory name %q must be lowercase alphanumeric with hyphens", name) } @@ -1239,6 +1256,9 @@ func LoadApp(dir string) (*AppConfig, error) { return nil, invalidConfig(err) } if composeCfg != nil { + if err := composeCfg.validate(); err != nil { + return nil, fmt.Errorf("invalid Compose config: %w", err) + } return composeCfg, nil } @@ -1326,6 +1346,9 @@ func clearExplicitEmpties(base, overlay *AppConfig, present map[string]any) { if _, ok := present["publish"]; ok && len(overlay.Publish) == 0 { base.Publish = nil } + if _, ok := present["volume_ownership"]; ok && len(overlay.VolumeOwnership) == 0 { + base.VolumeOwnership = nil + } if _, ok := present["volumes"]; ok && len(overlay.Volumes) == 0 { base.Volumes = nil } @@ -1450,6 +1473,14 @@ func mergeConfigs(base, overlay *AppConfig) { base.Hooks.PostDeploy = overlay.Hooks.PostDeploy } // Merge maps: overlay keys override base keys. + if len(overlay.VolumeOwnership) > 0 { + if base.VolumeOwnership == nil { + base.VolumeOwnership = map[string]VolumeOwnership{} + } + for k, v := range overlay.VolumeOwnership { + base.VolumeOwnership[k] = v + } + } if len(overlay.Volumes) > 0 { if base.Volumes == nil { base.Volumes = map[string]string{} @@ -1566,6 +1597,14 @@ func mergeConfigs(base, overlay *AppConfig) { base.Headers[k] = v } } + if len(overlay.EnvLiteral) > 0 { + if base.EnvLiteral == nil { + base.EnvLiteral = map[string]string{} + } + for k, v := range overlay.EnvLiteral { + base.EnvLiteral[k] = v + } + } if overlay.KeepReleases > 0 { base.KeepReleases = overlay.KeepReleases } @@ -1599,6 +1638,11 @@ func mergeAccessory(base, overlay AccessoryConfig) AccessoryConfig { } if overlay.Command != "" { out.Command = overlay.Command + out.CommandArgs = nil + } + if len(overlay.CommandArgs) > 0 { + out.CommandArgs = append([]string(nil), overlay.CommandArgs...) + out.Command = "" } if len(overlay.Publish) > 0 { out.Publish = overlay.Publish @@ -1619,6 +1663,14 @@ func mergeAccessory(base, overlay AccessoryConfig) AccessoryConfig { out.Env[k] = v } } + if len(overlay.VolumeOwnership) > 0 { + if out.VolumeOwnership == nil { + out.VolumeOwnership = map[string]VolumeOwnership{} + } + for k, v := range overlay.VolumeOwnership { + out.VolumeOwnership[k] = v + } + } if len(overlay.Volumes) > 0 { if out.Volumes == nil { out.Volumes = map[string]string{} @@ -1627,6 +1679,14 @@ func mergeAccessory(base, overlay AccessoryConfig) AccessoryConfig { out.Volumes[k] = v } } + if len(overlay.EnvLiteral) > 0 { + if out.EnvLiteral == nil { + out.EnvLiteral = map[string]string{} + } + for k, v := range overlay.EnvLiteral { + out.EnvLiteral[k] = v + } + } return out } diff --git a/internal/config/appliedview.go b/internal/config/appliedview.go index b9df1a6..fc2f564 100644 --- a/internal/config/appliedview.go +++ b/internal/config/appliedview.go @@ -59,11 +59,15 @@ func ParseAppliedManifest(data []byte) (*AppliedManifestView, error) { if err := json.Unmarshal(data, &root); err != nil { return nil, fmt.Errorf("parsing applied manifest: %w", err) } + if root == nil { + return nil, fmt.Errorf("applied manifest root must be an object") + } view := &AppliedManifestView{} - decodeString(root, "app", &view.App) - decodeString(root, "deployment_type", &view.DeploymentType) - decodeString(root, "domain", &view.Domain) - decodeString(root, "ingress_mode", &view.IngressMode) + for key, dst := range map[string]*string{"app": &view.App, "deployment_type": &view.DeploymentType, "domain": &view.Domain, "ingress_mode": &view.IngressMode} { + if err := decodeString(root, key, dst); err != nil { + return nil, err + } + } if raw, ok := root["container"]; ok && len(raw) > 0 && string(raw) != "null" { var c struct { @@ -122,8 +126,14 @@ func ParseAppliedManifest(data []byte) (*AppliedManifestView, error) { return view, nil } -func decodeString(root map[string]json.RawMessage, key string, dst *string) { +func decodeString(root map[string]json.RawMessage, key string, dst *string) error { if raw, ok := root[key]; ok { - _ = json.Unmarshal(raw, dst) + if string(raw) == "null" { + return fmt.Errorf("applied manifest %s must be a string", key) + } + if err := json.Unmarshal(raw, dst); err != nil { + return fmt.Errorf("parsing applied manifest %s: %w", key, err) + } } + return nil } diff --git a/internal/config/campaign_test.go b/internal/config/campaign_test.go new file mode 100644 index 0000000..705265a --- /dev/null +++ b/internal/config/campaign_test.go @@ -0,0 +1,33 @@ +package config + +import "testing" + +func TestPublishDefaultTCPConflict(t *testing.T) { + if err := ValidatePublishEntries([]string{"127.0.0.1:8080:80", "127.0.0.1:8080:81/tcp"}); err == nil { + t.Fatal("equivalent TCP bindings accepted") + } + if err := ValidatePublishEntries([]string{"127.0.0.1:8080:80", "127.0.0.1:8080:81/udp"}); err != nil { + t.Fatal(err) + } +} +func TestAccessoryOverlayCommandTransitions(t *testing.T) { + a := mergeAccessory(AccessoryConfig{Command: "old"}, AccessoryConfig{CommandArgs: []string{"new", "arg"}}) + if a.Command != "" || len(a.CommandArgs) != 2 { + t.Fatalf("argv overlay: %+v", a) + } + a = mergeAccessory(a, AccessoryConfig{Command: "replacement"}) + if a.Command != "replacement" || len(a.CommandArgs) != 0 { + t.Fatalf("string overlay: %+v", a) + } + a = mergeAccessory(AccessoryConfig{}, AccessoryConfig{CommandArgs: []string{"new"}}) + if len(a.CommandArgs) != 1 { + t.Fatal("new accessory lost argv") + } +} +func TestAppliedManifestRejectsInvalidBaseline(t *testing.T) { + for _, input := range []string{`null`, `{"app":42}`, `{"app":null}`, `{"domain":[]}`, `{"ingress_mode":{}}`} { + if _, err := ParseAppliedManifest([]byte(input)); err == nil { + t.Errorf("accepted %s", input) + } + } +} diff --git a/internal/config/compose.go b/internal/config/compose.go index 2d1699b..7ed945a 100644 --- a/internal/config/compose.go +++ b/internal/config/compose.go @@ -138,6 +138,11 @@ func LoadCompose(dir string) (*AppConfig, error) { } func mapCompose(dir string, compose composeFile) (*AppConfig, error) { + absoluteDir, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + dir = absoluteDir cfg := &AppConfig{ App: filepath.Base(dir), Processes: make(map[string]string), @@ -213,12 +218,18 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { } webBuildContext := parseBuildContext(webService.Build) + webDockerfile, err := composeDockerfile(webService.Build) + if err != nil { + return nil, err + } + cfg.Context = webBuildContext + cfg.Dockerfile = webDockerfile // Set domain placeholder — user must set this. cfg.Domain = cfg.App + ".example.com" // Web process gets empty command (use image CMD). - cfg.Processes["web"] = "" + cfg.Processes["web"] = parseCommand(webService.Command) // If web service has an image (not build), use it. if webService.Image != "" && webBuildContext == "" { @@ -281,6 +292,10 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { svc := services[name] svcBuildContext := parseBuildContext(svc.Build) + svcDockerfile, buildErr := composeDockerfile(svc.Build) + if buildErr != nil { + return nil, fmt.Errorf("service %s: %w", name, buildErr) + } // Check if it's a known accessory image. if isAccessoryImage(svc.Image) { @@ -291,8 +306,9 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { // rather than translated or rejected (classification table). acc := AccessoryConfig{ - Image: svc.Image, - Port: accessoryPort(svc.Image), + Image: svc.Image, + Command: parseCommand(svc.Command), + Port: accessoryPort(svc.Image), } // Map environment variables. @@ -312,7 +328,7 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { } // Same build context as web → worker process. - if svcBuildContext != "" && svcBuildContext == webBuildContext { + if svcBuildContext != "" && svcBuildContext == webBuildContext && svcDockerfile == webDockerfile { violations = append(violations, composeFieldViolations(name, svc)...) if svc.Healthcheck != nil { if composeHealthcheckDisabled(svc.Healthcheck) { @@ -340,7 +356,7 @@ func mapCompose(dir string, compose composeFile) (*AppConfig, error) { // Has a standalone image that isn't a known DB → treat as accessory. if svc.Image != "" { violations = append(violations, composeFieldViolations(name, svc)...) - acc := AccessoryConfig{Image: svc.Image} + acc := AccessoryConfig{Image: svc.Image, Command: parseCommand(svc.Command)} env := parseEnvironment(svc.Environment) if len(env) > 0 { acc.Env = env @@ -876,3 +892,20 @@ func isAccessoryImage(image string) bool { func accessoryPort(image string) int { return knownAccessoryImages[imageBaseName(image)] } + +func composeDockerfile(build interface{}) (string, error) { + if v, ok := build.(map[string]interface{}); ok { + for k := range v { + if k != "context" && k != "dockerfile" { + return "", fmt.Errorf("unsupported compose build field %q; write teploy.yml explicitly", k) + } + } + if d, exists := v["dockerfile"]; exists { + if text, ok := d.(string); ok { + return text, nil + } + return "", fmt.Errorf("compose dockerfile must be a string") + } + } + return "", nil +} diff --git a/internal/config/compose_lifecycle_regression_test.go b/internal/config/compose_lifecycle_regression_test.go new file mode 100644 index 0000000..15f9c26 --- /dev/null +++ b/internal/config/compose_lifecycle_regression_test.go @@ -0,0 +1,46 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestComposePreservesBuildAndCommands(t *testing.T) { + dir := t.TempDir() + source := `services: + web: + build: {context: ./api, dockerfile: Dockerfile.prod} + command: [node, "server file.js"] + ports: ["3000:3000"] + db: + image: redis:7 + command: [redis-server, --appendonly, "yes"] +` + if err := os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(source), 0600); err != nil { + t.Fatal(err) + } + cfg, err := LoadCompose(dir) + if err != nil { + t.Fatal(err) + } + if cfg.Context != "./api" || cfg.Dockerfile != "Dockerfile.prod" || cfg.Processes["web"] != "node 'server file.js'" || cfg.Accessories["db"].Command != "redis-server --appendonly yes" { + t.Fatalf("discarded build/command: %+v", cfg) + } +} +func TestComposeRejectsDifferentDockerfileWorkers(t *testing.T) { + dir := t.TempDir() + source := `services: + web: + build: {context: ., dockerfile: Dockerfile.web} + ports: ["3000:3000"] + worker: + build: {context: ., dockerfile: Dockerfile.worker} + command: node worker.js +` + os.WriteFile(filepath.Join(dir, "compose.yml"), []byte(source), 0600) + if _, err := LoadCompose(dir); err == nil || !strings.Contains(err.Error(), "independent build") { + t.Fatalf("different image flattened: %v", err) + } +} diff --git a/internal/config/compose_test.go b/internal/config/compose_test.go index 539d749..41c3baf 100644 --- a/internal/config/compose_test.go +++ b/internal/config/compose_test.go @@ -159,6 +159,7 @@ services: worker: build: context: . + dockerfile: Dockerfile.prod command: node worker.js ` os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644) diff --git a/internal/config/manifest.go b/internal/config/manifest.go index 618487f..60250ab 100644 --- a/internal/config/manifest.go +++ b/internal/config/manifest.go @@ -54,7 +54,7 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "dockerfile": cleanOptionalPath(cfg.Dockerfile), "keep_versions": cfg.KeepVersions, "processes": sortedMapKeys(cfg.Processes), - "env_keys": sortedMapKeys(cfg.Env), + "env_keys": bindingCombinedEnvKeys(cfg.Env, cfg.EnvLiteral), "env_files": append([]string(nil), cfg.EnvFiles...), "volumes": cfg.Volumes, "memory": cfg.Memory, @@ -85,6 +85,9 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s } } + if len(cfg.VolumeOwnership) > 0 { + manifest["volume_ownership"] = cfg.VolumeOwnership + } manifest["accessories"] = normalizedAccessories(cfg.Accessories) manifest["tls"] = normalizedTLS(cfg.TLS) manifest["firewall"] = normalizedFirewall(cfg.Firewall) @@ -168,11 +171,14 @@ func normalizedAccessories(accessories map[string]AccessoryConfig) map[string]an out[name] = map[string]any{ "image": accessory.Image, "port": accessory.Port, - "env_keys": sortedMapKeys(accessory.Env), + "env_keys": bindingCombinedEnvKeys(accessory.Env, accessory.EnvLiteral), "volumes": accessory.Volumes, "publish": accessory.Publish, "has_command": accessory.Command != "", } + if len(accessory.VolumeOwnership) > 0 { + out[name].(map[string]any)["volume_ownership"] = accessory.VolumeOwnership + } } return out } @@ -214,3 +220,49 @@ func normalizedAccess(access AccessConfig) map[string]any { } return out } + +// ExecutionBindingDigest binds a reviewed plan to execution-bearing config. +// The digest is private: its input is never emitted in the display manifest. +// Environment values remain deliberately outside plan authorization; their +// keys and references are bound, as is every other configuration field. +func ExecutionBindingDigest(cfg *AppConfig, image string) (string, error) { + copyCfg := *cfg + copyCfg.Image = image + copyCfg.SourceRevision = "" + copyCfg.Env = bindingEnvKeys(cfg.Env) + copyCfg.EnvLiteral = bindingEnvKeys(cfg.EnvLiteral) + copyCfg.Accessories = make(map[string]AccessoryConfig, len(cfg.Accessories)) + for name, acc := range cfg.Accessories { + acc.Env = bindingEnvKeys(acc.Env) + acc.EnvLiteral = bindingEnvKeys(acc.EnvLiteral) + copyCfg.Accessories[name] = acc + } + data, err := json.Marshal(copyCfg) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func bindingEnvKeys(env map[string]string) map[string]string { + if len(env) == 0 { + return nil + } + keys := make(map[string]string, len(env)) + for key := range env { + keys[key] = "" + } + return keys +} + +func bindingCombinedEnvKeys(env, literal map[string]string) []string { + keys := bindingEnvKeys(env) + if keys == nil { + keys = map[string]string{} + } + for key := range literal { + keys[key] = "" + } + return sortedMapKeys(keys) +} diff --git a/internal/config/manifest_binding_test.go b/internal/config/manifest_binding_test.go new file mode 100644 index 0000000..35086a0 --- /dev/null +++ b/internal/config/manifest_binding_test.go @@ -0,0 +1,45 @@ +package config + +import ( + "encoding/json" + "testing" +) + +func TestExecutionBindingIncludesCommandsAndPolicy(t *testing.T) { + base := AppConfig{App: "demo", Processes: map[string]string{"web": "server --safe"}, Hooks: HooksConfig{PreDeploy: "check safe"}, CaddyExtra: "header X-Mode safe", Env: map[string]string{"TOKEN": "dummy-one"}} + first, err := ExecutionBindingDigest(&base, "image:v1") + if err != nil { + t.Fatal(err) + } + changes := []func(*AppConfig){ + func(c *AppConfig) { c.Processes["web"] = "server --unsafe" }, + func(c *AppConfig) { c.Hooks.PreDeploy = "check unsafe" }, + func(c *AppConfig) { c.CaddyExtra = "header X-Mode unsafe" }, + func(c *AppConfig) { c.Build = []string{"echo unsafe"} }, + func(c *AppConfig) { c.Headers = map[string]string{"X-Policy": "changed"} }, + func(c *AppConfig) { + c.Accessories = map[string]AccessoryConfig{"db": {CommandArgs: []string{"redis-server", "--appendonly", "yes"}}} + }, + } + for i, change := range changes { + data, _ := json.Marshal(base) + var cfg AppConfig + json.Unmarshal(data, &cfg) + change(&cfg) + digest, err := ExecutionBindingDigest(&cfg, "image:v1") + if err != nil { + t.Fatal(err) + } + if digest == first { + t.Fatalf("change %d not bound", i) + } + } + base.Env["TOKEN"] = "dummy-two" + same, _ := ExecutionBindingDigest(&base, "image:v1") + if same != first { + t.Fatal("environment value policy unexpectedly changed") + } + if base.Env["TOKEN"] != "dummy-two" { + t.Fatal("digest mutated config") + } +} diff --git a/internal/config/servers.go b/internal/config/servers.go index 8477a22..6853c60 100644 --- a/internal/config/servers.go +++ b/internal/config/servers.go @@ -74,6 +74,16 @@ func LoadServers(path string) (*ServersConfig, error) { // ResolveServer looks up a server by name or treats the input as a raw host. // Priority: flags → env vars → servers.yml. func ResolveServer(name string, flagHost, flagUser, flagKey string) (host, user, keyPath string, err error) { + defer func() { + if err == nil { + if flagUser != "" { + user = flagUser + } + if flagKey != "" { + keyPath = flagKey + } + } + }() // 1. Flags override everything if flagHost != "" { // A --host value that names a servers.yml entry resolves to that diff --git a/internal/config/volume_ownership.go b/internal/config/volume_ownership.go new file mode 100644 index 0000000..b34f16d --- /dev/null +++ b/internal/config/volume_ownership.go @@ -0,0 +1,46 @@ +package config + +import ( + "fmt" + "regexp" + "strconv" +) + +// VolumeOwnership initializes the root of a Teploy-managed named volume. +// Pointers require explicit IDs, including a deliberately selected zero. +type VolumeOwnership struct { + UID *uint32 `yaml:"uid" toml:"uid" json:"uid"` + GID *uint32 `yaml:"gid" toml:"gid" json:"gid"` + Mode string `yaml:"mode" toml:"mode" json:"mode"` +} + +var volumeMode = regexp.MustCompile(`^0[0-7]{3}$`) + +func (v VolumeOwnership) Validate() error { + if v.UID == nil || v.GID == nil || *v.UID == ^uint32(0) || *v.GID == ^uint32(0) { + return fmt.Errorf("volume ownership requires numeric uid/gid in 0..4294967294") + } + if !volumeMode.MatchString(v.Mode) { + return fmt.Errorf("volume ownership mode must be a quoted four-digit octal mode, e.g. \"0700\"") + } + mode, _ := strconv.ParseUint(v.Mode, 8, 16) + if mode&0002 != 0 { + return fmt.Errorf("volume ownership mode must not be world-writable") + } + return nil +} + +func ValidateVolumeOwnership(volumes map[string]string, owners map[string]VolumeOwnership) error { + for name, permission := range owners { + if err := ValidateIdentifier("volume", name); err != nil { + return err + } + if _, ok := volumes[name]; !ok { + return fmt.Errorf("volume_ownership.%s must name an entry in volumes", name) + } + if err := permission.Validate(); err != nil { + return fmt.Errorf("volume_ownership.%s: %w", name, err) + } + } + return nil +} diff --git a/internal/config/volume_ownership_test.go b/internal/config/volume_ownership_test.go new file mode 100644 index 0000000..b66911b --- /dev/null +++ b/internal/config/volume_ownership_test.go @@ -0,0 +1,32 @@ +package config + +import "testing" + +func TestVolumeOwnershipAdmission(t *testing.T) { + base := "app: observe\ningress: host\nimage: observe:test\nport: 5080\nvolumes:\n observe-state: /var/lib/observe\n" + for _, tail := range []string{ + "volume_ownership:\n observe-state: {uid: 10001, gid: 10001, mode: \"0700\"}\n", + "accessories:\n db:\n image: postgres:17\n volumes: {data: /data}\n volume_ownership:\n data: {uid: 999, gid: 999, mode: \"0750\"}\n", + } { + if _, err := ParseAppBytes([]byte(base + tail)); err != nil { + t.Fatal(err) + } + } + for _, tail := range []string{ + "volume_ownership:\n observe-state: {uid: -1, gid: 10001, mode: \"0700\"}\n", + "volume_ownership:\n observe-state: {uid: 4294967295, gid: 10001, mode: \"0700\"}\n", + "volume_ownership:\n observe-state: {gid: 10001, mode: \"0700\"}\n", + "volume_ownership:\n observe-state: {uid: 10001, gid: 10001, mode: \"777\"}\n", + "volume_ownership:\n observe-state: {uid: 10001, gid: 10001, mode: \"4777\"}\n", + "volume_ownership:\n observe-state: {uid: 10001, gid: 10001, mode: \"0777\"}\n", + "volume_ownership:\n missing: {uid: 10001, gid: 10001, mode: \"0700\"}\n", + "volume_ownership:\n /tmp/outside: {uid: 10001, gid: 10001, mode: \"0700\"}\n", + } { + if _, err := ParseAppBytes([]byte(base + tail)); err == nil { + t.Fatalf("accepted invalid ownership: %s", tail) + } + } + if _, err := ParseAppBytes([]byte(base)); err != nil { + t.Fatalf("legacy volumes broken: %v", err) + } +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index ac66ad8..90e6862 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -22,15 +22,18 @@ import ( // Config holds all parameters for a deploy. type Config struct { - App string - Domain string - Image string - Version string // short git hash or tag - EnvFiles []string // paths to env files on the server, applied in order (later files' keys win) — see docker.RunConfig.EnvFiles - Env map[string]string - Volumes map[string]string - Cmd string // command override for single-process deploys - Processes map[string]string // process_name -> command (overrides Cmd) + TriggerOperationKey string + ExecutionBindingDigest string + App string + Domain string + Image string + Version string // short git hash or tag + EnvFiles []string // paths to env files on the server, applied in order (later files' keys win) — see docker.RunConfig.EnvFiles + Env map[string]string + Volumes map[string]string + VolumeOwnership map[string]config.VolumeOwnership + Cmd string // command override for single-process deploys + Processes map[string]string // process_name -> command (overrides Cmd) // NoHealthcheck disables the container HEALTHCHECK for specific processes. // Keyed by process name; true means pass --no-healthcheck to docker run. // Used when a process shouldn't be probed by the image's built-in @@ -216,6 +219,17 @@ func (c Config) validate() error { // Provenance identity (C04): a record describing another deploy than // the Config it rides on is a lie that would corrupt the plan/receipt // equality surfaces — refused before any effect. + for name, permission := range c.VolumeOwnership { + if err := config.ValidateName(name); err != nil { + return err + } + if err := permission.Validate(); err != nil { + return err + } + if _, ok := c.Volumes[fmt.Sprintf("/deployments/%s/volumes/%s", c.App, name)]; !ok { + return fmt.Errorf("volume ownership requires matching managed volume %s", name) + } + } if c.Provenance != nil && (c.Provenance.App != c.App || c.Provenance.Release != c.Version) { return fmt.Errorf("provenance identity mismatch: provenance describes %s@%s, deploy is %s@%s", c.Provenance.App, c.Provenance.Release, c.App, c.Version) } @@ -356,6 +370,17 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) return fmt.Errorf("refusing to deploy with unreadable state for %s: %w", cfg.App, err) } + volumeNames := make([]string, 0, len(cfg.VolumeOwnership)) + for name := range cfg.VolumeOwnership { + volumeNames = append(volumeNames, name) + } + sort.Strings(volumeNames) + for _, name := range volumeNames { + if err := d.docker.ProvisionManagedVolume(ctx, cfg.App, "", name, cfg.VolumeOwnership[name], lk.GuardPrefix()); err != nil { + return err + } + } + // 1z. Resolve the generation identities this deploy prepares against // (C01-8/9). Every successful operation bumps the state generation by // one; the candidates this deploy starts are labeled with the NEXT @@ -995,6 +1020,8 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) newState.ManifestSHA256 = cfg.ManifestSHA256 newState.AppliedManifest = append(json.RawMessage(nil), cfg.AppliedManifest...) newState.SourceRevision = cfg.SourceRevision + newState.TriggerOperationKey = cfg.TriggerOperationKey + newState.ExecutionBindingDigest = cfg.ExecutionBindingDigest newState.ImageRef = cfg.Image newState.ImageDigest = ImageDigestFromRef(cfg.Image) if newState.ImageDigest == "" { @@ -1013,6 +1040,9 @@ func (d *Deployer) DeployFenced(ctx context.Context, cfg Config, lk *state.Lock) // commands land inside the successor's window (WriteFencedGeneration // refuses with ErrGenerationFenced naming both generations). if err := state.WriteFencedGeneration(ctx, d.exec, cfg.App, newState, lk, fromGeneration); err != nil { + if state.PreservePublishedState(err) { + return fmt.Errorf("state publication requires reconciliation; serving workload preserved: %w", err) + } return d.abortStateCommit(ctx, cfg, current, started, startedIDs, displacedHostWeb, assetAttempt, restoreRouteCAS(resolvedRouteHash, switchedRouteHash), start, err) } @@ -1645,26 +1675,28 @@ func (d *Deployer) recordRelease(ctx context.Context, cfg Config, att releasemet healthCfg := cfg.Health.withDefaults() rec := &releasemeta.Record{ - App: cfg.App, - Hash: cfg.Version, - Generation: applied.Generation, - DeploymentType: "container", - IngressMode: cfg.Ingress, - Domain: cfg.Domain, - ImageRef: cfg.Image, - ImageDigest: applied.ImageDigest, - ManifestSHA256: cfg.ManifestSHA256, - Replicas: len(ports), - Processes: maps.Clone(cfg.Processes), - Cmd: cfg.Cmd, - Env: maps.Clone(cfg.Env), - EnvFiles: append([]string(nil), cfg.EnvFiles...), - Volumes: maps.Clone(cfg.Volumes), - Publish: append([]string(nil), cfg.Publish...), - Memory: cfg.Memory, - CPU: cfg.CPU, - StopTimeout: cfg.StopTimeout, - Bind: cfg.Bind, + TriggerOperationKey: cfg.TriggerOperationKey, ExecutionBindingDigest: cfg.ExecutionBindingDigest, + App: cfg.App, + Hash: cfg.Version, + Generation: applied.Generation, + DeploymentType: "container", + IngressMode: cfg.Ingress, + Domain: cfg.Domain, + ImageRef: cfg.Image, + ImageDigest: applied.ImageDigest, + ManifestSHA256: cfg.ManifestSHA256, + Replicas: len(ports), + Processes: maps.Clone(cfg.Processes), + Cmd: cfg.Cmd, + Env: maps.Clone(cfg.Env), + EnvFiles: append([]string(nil), cfg.EnvFiles...), + Volumes: maps.Clone(cfg.Volumes), + VolumeOwnership: maps.Clone(cfg.VolumeOwnership), + Publish: append([]string(nil), cfg.Publish...), + Memory: cfg.Memory, + CPU: cfg.CPU, + StopTimeout: cfg.StopTimeout, + Bind: cfg.Bind, Health: &releasemeta.Health{ Mode: healthCfg.Mode, Path: healthCfg.Path, diff --git a/internal/deploy/lifecycle.go b/internal/deploy/lifecycle.go index f322486..fd08e1e 100644 --- a/internal/deploy/lifecycle.go +++ b/internal/deploy/lifecycle.go @@ -3,7 +3,12 @@ package deploy import ( "context" "fmt" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/releasemeta" "io" + "sort" + "strconv" + "strings" "time" "github.com/useteploy/teploy/internal/docker" @@ -29,6 +34,13 @@ func NewLifecycle(exec ssh.Executor, out io.Writer) *Lifecycle { // Stop stops all running containers for the app. func (l *Lifecycle) Stop(ctx context.Context, app string, timeout int) error { + lk, err := state.AcquireLockFenced(ctx, l.exec, app) + if err != nil { + return err + } + lk.StartRenewal(l.exec) + defer state.ReleaseLockFenced(l.exec, lk, app) + containers, err := l.appContainers(ctx, app) if err != nil { return err @@ -39,7 +51,7 @@ func (l *Lifecycle) Stop(ctx context.Context, app string, timeout int) error { for _, c := range containers { fmt.Fprintf(l.out, "Stopping %s...\n", c.Name) - if err := l.docker.Stop(ctx, c.Name, timeout); err != nil { + if err := l.stop(ctx, lk, c, timeout); err != nil { return err } } @@ -51,6 +63,13 @@ func (l *Lifecycle) Stop(ctx context.Context, app string, timeout int) error { // Start starts all stopped containers for the app and runs a health check. func (l *Lifecycle) Start(ctx context.Context, app string) error { + lk, err := state.AcquireLockFenced(ctx, l.exec, app) + if err != nil { + return err + } + lk.StartRenewal(l.exec) + defer state.ReleaseLockFenced(l.exec, lk, app) + current, err := state.Read(ctx, l.exec, app) if err != nil || current == nil { return fmt.Errorf("no deploy state found for %s — deploy first", app) @@ -66,7 +85,7 @@ func (l *Lifecycle) Start(ctx context.Context, app string) error { for _, c := range containers { fmt.Fprintf(l.out, "Starting %s...\n", c.Name) - if err := l.docker.Start(ctx, c.Name); err != nil { + if err := l.start(ctx, lk, c); err != nil { return err } } @@ -76,8 +95,26 @@ func (l *Lifecycle) Start(ctx context.Context, app string) error { fmt.Fprintln(l.out, "Running health check...") deployer := &Deployer{exec: l.exec, out: l.out} // Probe the bound address, not localhost — see healthProbeHost. - bindHost := l.docker.HostBindIP(ctx, containers[0].Name) - if err := deployer.healthCheck(ctx, current.CurrentPort, defaultHealthConfig(), bindHost); err != nil { + webName := "" + for _, c := range containers { + if c.Labels["teploy.process"] == "web" { + webName = c.Name + break + } + } + if webName == "" { + return fmt.Errorf("current release has no web container") + } + bindHost := l.docker.HostBindIP(ctx, webName) + health := defaultHealthConfig() + rec, recErr := releasemeta.Read(ctx, l.exec, app, current.CurrentHash) + if recErr != nil { + return recErr + } + if rec != nil { + applyRecordToRollback(&RollbackConfig{}, rec, &health) + } + if err := deployer.healthCheck(ctx, current.CurrentPort, health, bindHost); err != nil { return fmt.Errorf("health check failed after start: %w", err) } fmt.Fprintln(l.out, " Health check passed") @@ -90,6 +127,13 @@ func (l *Lifecycle) Start(ctx context.Context, app string) error { // Restart stops then starts all containers for the app. func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error { + lk, err := state.AcquireLockFenced(ctx, l.exec, app) + if err != nil { + return err + } + lk.StartRenewal(l.exec) + defer state.ReleaseLockFenced(l.exec, lk, app) + current, err := state.Read(ctx, l.exec, app) if err != nil || current == nil { return fmt.Errorf("no deploy state found for %s — deploy first", app) @@ -106,7 +150,7 @@ func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error // Stop all. for _, c := range containers { fmt.Fprintf(l.out, "Stopping %s...\n", c.Name) - if err := l.docker.Stop(ctx, c.Name, timeout); err != nil { + if err := l.stop(ctx, lk, c, timeout); err != nil { return err } } @@ -114,7 +158,7 @@ func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error // Start all. for _, c := range containers { fmt.Fprintf(l.out, "Starting %s...\n", c.Name) - if err := l.docker.Start(ctx, c.Name); err != nil { + if err := l.start(ctx, lk, c); err != nil { return err } } @@ -124,8 +168,26 @@ func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error fmt.Fprintln(l.out, "Running health check...") deployer := &Deployer{exec: l.exec, out: l.out} // Probe the bound address, not localhost — see healthProbeHost. - bindHost := l.docker.HostBindIP(ctx, containers[0].Name) - if err := deployer.healthCheck(ctx, current.CurrentPort, defaultHealthConfig(), bindHost); err != nil { + webName := "" + for _, c := range containers { + if c.Labels["teploy.process"] == "web" { + webName = c.Name + break + } + } + if webName == "" { + return fmt.Errorf("current release has no web container") + } + bindHost := l.docker.HostBindIP(ctx, webName) + health := defaultHealthConfig() + rec, recErr := releasemeta.Read(ctx, l.exec, app, current.CurrentHash) + if recErr != nil { + return recErr + } + if rec != nil { + applyRecordToRollback(&RollbackConfig{}, rec, &health) + } + if err := deployer.healthCheck(ctx, current.CurrentPort, health, bindHost); err != nil { return fmt.Errorf("health check failed after restart: %w", err) } fmt.Fprintln(l.out, " Health check passed") @@ -136,40 +198,105 @@ func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error return nil } -// appContainers returns all running containers for the app. +// Only current recorded processes and currently configured accessories belong +// to lifecycle. Start activates configured accessories, including previously +// stopped ones; removed/unknown accessories and historical workloads stay alone. func (l *Lifecycle) appContainers(ctx context.Context, app string) ([]docker.Container, error) { - containers, err := l.docker.ListContainers(ctx, app) + current, err := state.Read(ctx, l.exec, app) + if err != nil { + return nil, err + } + if current == nil { + return nil, fmt.Errorf("no deploy state for %s", app) + } + containers, err := l.appContainersByVersion(ctx, app, current.CurrentHash) if err != nil { return nil, err } - var running []docker.Container - for _, c := range containers { - if c.State == "running" { - running = append(running, c) + for _, container := range containers { + if container.State == "running" { + running = append(running, container) } } return running, nil } - -// appContainersByVersion returns all containers (any state) matching the given version. func (l *Lifecycle) appContainersByVersion(ctx context.Context, app, version string) ([]docker.Container, error) { + current, err := state.Read(ctx, l.exec, app) + if err != nil { + return nil, err + } + if current == nil || current.CurrentHash != version { + return nil, fmt.Errorf("current release changed") + } + record, err := releasemeta.Read(ctx, l.exec, app, version) + if err != nil { + return nil, err + } + if record == nil || record.App != app || record.Hash != version { + return nil, fmt.Errorf("current release record required for lifecycle") + } + accessories := map[string]bool{} + if len(current.AppliedManifest) > 0 { + manifest, err := config.ParseAppliedManifest(current.AppliedManifest) + if err != nil { + return nil, err + } + for name := range manifest.Accessories { + accessories[name] = true + } + } containers, err := l.docker.ListContainers(ctx, app) if err != nil { return nil, err } - - // Match by the teploy.version label, not a name suffix — replica containers - // ("---1") don't end in "-", so a suffix match - // silently skips them (restart/start would miss every web replica). var matched []docker.Container - for _, c := range containers { - if c.Labels["teploy.version"] == version { - matched = append(matched, c) + for _, container := range containers { + if currentLifecycleContainer(container, app, version, current.Generation, record.Generation, record.Processes, accessories) { + matched = append(matched, container) } } + sort.SliceStable(matched, func(i, j int) bool { + return matched[i].Labels["teploy.role"] == "accessory" && matched[j].Labels["teploy.role"] != "accessory" + }) return matched, nil } +func currentLifecycleContainer(container docker.Container, app, version string, generation, recordedGeneration uint64, processes map[string]string, accessories map[string]bool) bool { + labels := container.Labels + if container.ID == "" || labels["teploy.app"] != app { + return false + } + if labels["teploy.preview"] != "" { + return false + } + role := labels["teploy.role"] + if role == "accessory" { + name := labels["teploy.accessory"] + return name != "" && accessories[name] && container.Name == app+"-"+name && labels["teploy.process"] == "" + } + if role != "" && role != "app" { + return false + } + process := labels["teploy.process"] + if labels["teploy.version"] != version { + return false + } + if process != "web" { + if _, recorded := processes[process]; !recorded || process == "" { + return false + } + } + if label := labels["teploy.generation"]; label != "" && label != strconv.FormatUint(generation, 10) && label != strconv.FormatUint(recordedGeneration, 10) { + return false + } + name := app + "-" + process + "-" + version + if container.Name == name { + return true + } + suffix, ok := strings.CutPrefix(container.Name, name+"-") + replica, err := strconv.Atoi(suffix) + return ok && err == nil && replica > 0 && strconv.Itoa(replica) == suffix +} func (l *Lifecycle) logAction(ctx context.Context, app, action string) { state.AppendLog(ctx, l.exec, state.LogEntry{ @@ -179,3 +306,20 @@ func (l *Lifecycle) logAction(ctx context.Context, app, action string) { Success: true, }) } + +func (l *Lifecycle) stop(ctx context.Context, lk *state.Lock, c docker.Container, timeout int) error { + ref := c.ID + if ref == "" { + return fmt.Errorf("immutable container ID required") + } + _, err := lk.Guarded(ctx, l.exec, fmt.Sprintf("docker stop --time %d %s", timeout, ssh.ShellQuote(ref))) + return err +} +func (l *Lifecycle) start(ctx context.Context, lk *state.Lock, c docker.Container) error { + ref := c.ID + if ref == "" { + return fmt.Errorf("immutable container ID required") + } + _, err := lk.Guarded(ctx, l.exec, "docker start "+ssh.ShellQuote(ref)) + return err +} diff --git a/internal/deploy/lifecycle_campaign_test.go b/internal/deploy/lifecycle_campaign_test.go new file mode 100644 index 0000000..60df2dc --- /dev/null +++ b/internal/deploy/lifecycle_campaign_test.go @@ -0,0 +1,50 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "github.com/useteploy/teploy/internal/deploy/recovery" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" + "strings" + "testing" +) + +func TestCampaignLifecycleStartsAccessoryBeforeCurrentRelease(t *testing.T) { + inventory := `{"ID":"web","Names":"demo-web-v1","State":"exited","Labels":"teploy.app=demo,teploy.process=web,teploy.version=v1"} +{"ID":"old","Names":"demo-web-v0","State":"exited","Labels":"teploy.app=demo,teploy.process=web,teploy.version=v0"} +{"ID":"db","Names":"demo-db","State":"exited","Labels":"teploy.app=demo,teploy.role=accessory,teploy.accessory=db"}` + m := ssh.NewMockExecutor("dummy", ssh.MockCommand{Match: "mkdir /deployments/demo/.lock"}, ssh.MockCommand{Match: "docker start"}, ssh.MockCommand{Match: "docker ps --all", Output: inventory}) + seedRound3Lifecycle(m, "demo") + s := state.NewAppliedState(nil, "container", "external", "") + s.CurrentHash = "v1" + b, _ := json.Marshal(s) + _ = b + if err := NewLifecycle(m, &bytes.Buffer{}).Start(context.Background(), "demo"); err != nil { + t.Fatal(err) + } + var starts []string + for _, c := range m.Calls { + if strings.HasPrefix(c, "docker start ") { + starts = append(starts, c) + } + } + if len(starts) != 2 || starts[0] != "docker start 'db'" || starts[1] != "docker start 'web'" { + t.Fatalf("start ownership/order %v", starts) + } +} +func TestCampaignObserveManagedWorkerAndAccessory(t *testing.T) { + inv := `{"ID":"web","Names":"demo-web-v1","State":"running","Labels":"teploy.app=demo,teploy.process=web,teploy.version=v1"} +{"ID":"worker","Names":"demo-worker-v1","State":"running","Labels":"teploy.app=demo,teploy.process=worker,teploy.version=v1"} +{"ID":"db","Names":"demo-db","State":"running","Labels":"teploy.app=demo,teploy.role=accessory,teploy.accessory=db"}` + m := ssh.NewMockExecutor("dummy", ssh.MockCommand{Match: "docker ps --all", Output: inv}) + rec := &releasemeta.Record{SchemaVersion: 1, App: "demo", Hash: "v1", Processes: map[string]string{"web": "", "worker": "work"}} + b, _ := json.Marshal(rec) + m.Files["/deployments/demo/meta/v1.json"] = b + o := Observe(context.Background(), m, "demo", "v2", "v1") + if o.ForeignCandidates == recovery.Present { + t.Fatalf("managed processes marked foreign: %+v", o) + } +} diff --git a/internal/deploy/lifecycle_test.go b/internal/deploy/lifecycle_test.go index 5941280..7a9f620 100644 --- a/internal/deploy/lifecycle_test.go +++ b/internal/deploy/lifecycle_test.go @@ -12,6 +12,7 @@ import ( func TestStop(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: `{"ID":"abc123","Names":"myapp-web-v1","Image":"myapp:latest","State":"running","Status":"Up 2h","Labels":"teploy.app=myapp,teploy.version=v1,teploy.process=web"}` + "\n" + `{"ID":"def456","Names":"myapp-worker-v1","Image":"myapp:latest","State":"running","Status":"Up 2h","Labels":"teploy.app=myapp,teploy.version=v1,teploy.process=worker"}`, @@ -22,6 +23,7 @@ func TestStop(t *testing.T) { ) var buf bytes.Buffer + seedRound3Lifecycle(mock, "myapp") lc := NewLifecycle(mock, &buf) if err := lc.Stop(context.Background(), "myapp", 10); err != nil { t.Fatalf("unexpected error: %v", err) @@ -45,9 +47,11 @@ func TestStop(t *testing.T) { func TestStop_NoContainers(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: ""}, ) + seedRound3Lifecycle(mock, "myapp") lc := NewLifecycle(mock, &bytes.Buffer{}) err := lc.Stop(context.Background(), "myapp", 10) if err == nil { @@ -58,6 +62,7 @@ func TestStop_NoContainers(t *testing.T) { func TestStart(t *testing.T) { stateContent := "current_port=49152\ncurrent_hash=v1\n" mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\n" + stateContent}, ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", @@ -70,6 +75,7 @@ func TestStart(t *testing.T) { ) var buf bytes.Buffer + seedRound3Lifecycle(mock, "myapp") lc := NewLifecycle(mock, &buf) if err := lc.Start(context.Background(), "myapp"); err != nil { t.Fatalf("unexpected error: %v", err) @@ -82,6 +88,7 @@ func TestStart(t *testing.T) { func TestStart_NoState(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "absent"}, ) @@ -99,6 +106,7 @@ func TestStart_NoState(t *testing.T) { func TestRestart(t *testing.T) { stateContent := "current_port=49152\ncurrent_hash=v1\n" mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\n" + stateContent}, ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", @@ -112,6 +120,7 @@ func TestRestart(t *testing.T) { ) var buf bytes.Buffer + seedRound3Lifecycle(mock, "myapp") lc := NewLifecycle(mock, &buf) if err := lc.Restart(context.Background(), "myapp", 10); err != nil { t.Fatalf("unexpected error: %v", err) @@ -131,6 +140,7 @@ func TestRestart(t *testing.T) { func TestStop_LogsAction(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: `{"ID":"abc123","Names":"myapp-web-v1","Image":"myapp:latest","State":"running","Status":"Up 2h","Labels":"teploy.app=myapp,teploy.version=v1,teploy.process=web"}`, }, @@ -139,6 +149,7 @@ func TestStop_LogsAction(t *testing.T) { ssh.MockCommand{Match: "UPLOAD:", Output: ""}, ) + seedRound3Lifecycle(mock, "myapp") lc := NewLifecycle(mock, &bytes.Buffer{}) lc.Stop(context.Background(), "myapp", 10) diff --git a/internal/deploy/reconcile.go b/internal/deploy/reconcile.go index d69564b..beabd55 100644 --- a/internal/deploy/reconcile.go +++ b/internal/deploy/reconcile.go @@ -65,10 +65,31 @@ func Observe(ctx context.Context, exec ssh.Executor, app, attempted, predecessor predName = fmt.Sprintf("%s-web-%s", app, predecessor) } for _, c := range containers { - isCandidate := strings.HasPrefix(c.Name, candPrefix) - isPredecessor := predName != "" && - (strings.HasPrefix(c.Name, predName) || strings.HasPrefix(c.Name, predName+"_replaced")) + version := c.Labels["teploy.version"] + process := c.Labels["teploy.process"] + isCandidate := version == attempted && process == "web" + if version == "" { + isCandidate = c.Name == candPrefix || strings.HasPrefix(c.Name, candPrefix+"-") + } + isPredecessor := predecessor != "" && version == predecessor && process == "web" + if version == "" { + isPredecessor = predName != "" && (c.Name == predName || c.Name == predName+"_replaced" || strings.HasPrefix(c.Name, predName+"-")) + } switch { + case c.Labels["teploy.role"] == "accessory" && c.Labels["teploy.accessory"] != "": + // Independently managed accessories have no app-release version. + continue + case process != "" && process != "web" && (version == attempted || version == predecessor): + rec, recErr := releasemeta.Read(ctx, exec, app, version) + if recErr == nil && rec != nil { + if _, ok := rec.Processes[process]; ok { + continue + } + } + if c.State == "running" { + o.ForeignCandidates = recovery.Present + } + continue case isCandidate: if c.State == "running" { o.Candidates = recovery.Present diff --git a/internal/deploy/rollback.go b/internal/deploy/rollback.go index 5ed6c41..5e422a0 100644 --- a/internal/deploy/rollback.go +++ b/internal/deploy/rollback.go @@ -264,11 +264,13 @@ func Rollback(ctx context.Context, exec ssh.Executor, out io.Writer, cfg Rollbac // newer generation is left strictly alone; recovery of OUR effects must // not destroy the new owner's workload. Holdership is deliberately NOT // checked here (A07: recovery is never fenced). - restoreDisplaced := func() { + restoreDisplaced := func() error { + var failures []error recoveryCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() for _, name := range displacedHostWeb { if rerr := dk.RestartFenced(recoveryCtx, name, nil, fromGeneration, ""); rerr != nil { + failures = append(failures, fmt.Errorf("restoring %s: %w", name, rerr)) if state.GenerationFenced(rerr) { fmt.Fprintf(out, " WARNING: could not restore %s after the failed rollback — a newer generation owns the name\n", name) continue @@ -278,6 +280,7 @@ func Rollback(ctx context.Context, exec ssh.Executor, out io.Writer, cfg Rollbac fmt.Fprintf(out, " Restored %s\n", name) } } + return errors.Join(failures...) } if fixedPorts { // Fence + generation check composed into the stop (F16 + C01-8/9): @@ -557,19 +560,36 @@ func Rollback(ctx context.Context, exec ssh.Executor, out io.Writer, cfg Rollbac // cannot commit over a successor's newer generation (ErrGenerationFenced // naming both), so a stale rollback can never become authority. if err := state.WriteFencedGeneration(ctx, exec, cfg.App, newState, lk, fromGeneration); err != nil { + if state.PreservePublishedState(err) { + return fmt.Errorf("state publication requires reconciliation; serving workload preserved: %w", err) + } + // Fixed host ports: the target holds them. Stop it, restore the // displaced workload, then remove the uncommitted target — previously // this branch skipped the restore and still claimed "the original // workload was left running" (audit F12). if fixedPorts { + recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() + if cfg.usesCaddy() { + if restoreErr := restoreRollbackRoute(recoveryCtx, exec, out, cd, dk, cfg, current, containers, restoreRouteCAS(resolvedRouteHash, switchedRouteHash), fromGeneration); restoreErr != nil { + return fmt.Errorf("committing authoritative state: %w; restoring route failed: %v; target preserved to avoid routing to a removed container", err, restoreErr) + } + } for _, name := range started { - dk.Stop(ctx, name, 5) + if stopErr := dk.Stop(recoveryCtx, name, 5); stopErr != nil { + return fmt.Errorf("state commit failed: %w; stopping target before fixed-port recovery: %v", err, stopErr) + } + } + if restoreErr := restoreDisplaced(); restoreErr != nil { + return fmt.Errorf("state commit failed: %w; restoring fixed-port workload failed: %v", err, restoreErr) } - restoreDisplaced() for _, name := range started { - dk.Remove(ctx, name) + if removeErr := dk.Remove(recoveryCtx, name); removeErr != nil { + return fmt.Errorf("state commit failed: %w; workload/route restored but target cleanup failed: %v", err, removeErr) + } } - return fmt.Errorf("committing authoritative applied state after rollback: %w; the fixed-port workload was restored and the uncommitted target was removed", err) + return fmt.Errorf("committing authoritative applied state after rollback: %w; the fixed-port workload and route were restored and the uncommitted target was removed", err) } if cfg.usesCaddy() { // The restore is CAS-fenced (A12/T05): it may only overwrite diff --git a/internal/deploy/rollback_test.go b/internal/deploy/rollback_test.go index 528dd3e..16ce2e2 100644 --- a/internal/deploy/rollback_test.go +++ b/internal/deploy/rollback_test.go @@ -700,3 +700,54 @@ func TestRollback_RoutePhaseFailureUnwinds(t *testing.T) { t.Error("the uncommitted target container was left running after the route failure") } } + +func TestRollback_FixedPublishStateFailureRestoresCaddyRoute(t *testing.T) { + stateContent := "current_port=49153\ncurrent_hash=v2\nprevious_port=49152\nprevious_hash=v1\ndomain=myapp.com\n" + mock := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "mkdir -p /deployments/myapp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, + ssh.MockCommand{Match: "cat /deployments/myapp/.lock/info", Err: fmt.Errorf("none")}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/state' ]", Output: "present\n" + stateContent}, + ssh.MockCommand{Match: "mkdir -p /deployments/myapp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/myapp/.lock", Output: ""}, + ssh.MockCommand{Match: "cat /deployments/myapp/.lock/info", Err: fmt.Errorf("none")}, + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", + Output: `{"ID":"aaa","Names":"myapp-web-v1","Image":"myapp:latest","State":"exited","Status":"Exited","Labels":"teploy.app=myapp,teploy.version=v1,teploy.process=web"}` + "\n" + + `{"ID":"bbb","Names":"myapp-web-v2","Image":"myapp:latest","State":"running","Status":"Up 1h","Labels":"teploy.app=myapp,teploy.version=v2,teploy.process=web"}`, + }, + ssh.MockCommand{Match: "docker inspect 'myapp-web-v1'", Output: `[{"Config":{"Image":"myapp:latest","Labels":{"teploy.app":"myapp"}},"HostConfig":{"NetworkMode":"teploy","PortBindings":{"3000/tcp":[{"HostIp":"127.0.0.1","HostPort":"49152"}]},"RestartPolicy":{"Name":"no"}},"NetworkSettings":{"Networks":{"teploy":{"Aliases":["myapp"]}}}}]`}, + ssh.MockCommand{Match: "docker rm", Output: ""}, + ssh.MockCommand{Match: "docker inspect 'myapp-web-v2'", Output: `[{"Config":{"Image":"myapp:latest","Labels":{"teploy.app":"myapp"}},"HostConfig":{"NetworkMode":"teploy","PortBindings":{"3000/tcp":[{"HostIp":"127.0.0.1","HostPort":"49153"}]},"RestartPolicy":{"Name":"no"}},"NetworkSettings":{"Networks":{"teploy":{"Aliases":["myapp"]}}}}]`}, + ssh.MockCommand{Match: "docker run", Output: ""}, + // HostBindIP fixture: the bind IP the health probe actually dials (TCL-16 fail-closed host validation makes the old artifact — a bare + // port number reused from the HostPort fixture — an invalid probe host). + ssh.MockCommand{Match: "docker inspect -f '{{range $p, $b := .NetworkSettings.Ports}}{{range $b}}{{.HostIp}}", Output: "127.0.0.1 "}, + ssh.MockCommand{Match: "docker inspect -f '{{range $p, $b := .NetworkSettings.Ports}}", Output: "49152"}, + ssh.MockCommand{Match: "docker inspect -f '{{range $p, $_ := .NetworkSettings.Ports}}", Output: "3000/tcp"}, + ssh.MockCommand{Match: "curl", Output: "200"}, + ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Output: "{\n\tadmin 0.0.0.0:2019\n}\n"}, + ssh.MockCommand{Match: "mv /tmp/teploy_caddyfile.tmp", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "UPLOAD:/deployments/myapp/state.json.tmp-", Err: fmt.Errorf("disk full")}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ) + mock.Files["/deployments/myapp/meta/v1.json"] = []byte(`{"schema_version":1,"app":"myapp","hash":"v1","deployment_type":"container","ingress_mode":"caddy","domain":"myapp.com","ports":[{"host_port":49152,"container_port":3000}],"publish":["127.0.0.1:6380:6379"]}`) + var out bytes.Buffer + + err := Rollback(context.Background(), mock, &out, rollbackCfg()) + if err == nil || !strings.Contains(err.Error(), "fixed-port workload and route were restored") { + t.Fatalf("expected fail-closed rollback state error, got %v", err) + } + if strings.Contains(out.String(), "Rolled back") { + t.Fatalf("failed rollback reported success: %s", out.String()) + } + + restoredCaddyfile := string(mock.Files["/deployments/caddy/Caddyfile"]) + if !strings.Contains(restoredCaddyfile, "myapp-web-v2:3000") { + t.Fatalf("original route was not restored: %s", restoredCaddyfile) + } +} diff --git a/internal/deploy/round3_regression_test.go b/internal/deploy/round3_regression_test.go new file mode 100644 index 0000000..59085ee --- /dev/null +++ b/internal/deploy/round3_regression_test.go @@ -0,0 +1,72 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/releasemeta" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +func seedRound3Lifecycle(mock *ssh.MockExecutor, app string) { + cfg := &config.AppConfig{App: app, Image: "image:v1", Accessories: map[string]config.AccessoryConfig{"db": {Image: "postgres:16"}}} + manifest, digest, _ := config.NormalizeAndDigest(cfg, cfg.Image) + applied := state.NewAppliedState(nil, "container", "external", "") + applied.CurrentHash = "v1" + applied.AppliedManifest = manifest + applied.ManifestSHA256 = digest + data, _ := json.Marshal(applied) + mock.Files["/deployments/"+app+"/state.json"] = data + record := &releasemeta.Record{SchemaVersion: 1, App: app, Hash: "v1", Processes: map[string]string{"web": "serve", "worker": "work"}} + data, _ = json.Marshal(record) + mock.Files["/deployments/"+app+"/meta/v1.json"] = data +} +func TestRound3LifecycleSharedWorldExcludesPreviewOldAndUnknown(t *testing.T) { + inventory := `{"ID":"web","Names":"demo-web-v1","State":"running","Labels":{"teploy.app":"demo","teploy.process":"web","teploy.version":"v1"}} +{"ID":"worker","Names":"demo-worker-v1","State":"running","Labels":{"teploy.app":"demo","teploy.process":"worker","teploy.version":"v1"}} +{"ID":"db","Names":"demo-db","State":"running","Labels":{"teploy.app":"demo","teploy.role":"accessory","teploy.accessory":"db"}} +{"ID":"old","Names":"demo-web-v0","State":"running","Labels":{"teploy.app":"demo","teploy.process":"web","teploy.version":"v0"}} +{"ID":"preview","Names":"demo-web-v1-preview","State":"running","Labels":{"teploy.app":"demo","teploy.process":"web","teploy.version":"v1","teploy.preview":"feature"}} +{"ID":"unknown","Names":"demo-mystery-v1","State":"running","Labels":{"teploy.app":"demo","teploy.process":"mystery","teploy.version":"v1"}} +{"ID":"removed-db","Names":"demo-removed","State":"exited","Labels":{"teploy.app":"demo","teploy.role":"accessory","teploy.accessory":"removed"}}` + for _, operation := range []string{"stop", "start", "restart"} { + mock := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "mkdir /deployments/demo/.lock"}, ssh.MockCommand{Match: "docker stop --time 5", Output: ""}, ssh.MockCommand{Match: "docker start", Output: ""}, ssh.MockCommand{Match: "docker ps --all", Output: inventory}) + seedRound3Lifecycle(mock, "demo") + lifecycle := NewLifecycle(mock, &bytes.Buffer{}) + var err error + switch operation { + case "stop": + err = lifecycle.Stop(context.Background(), "demo", 5) + case "start": + err = lifecycle.Start(context.Background(), "demo") + case "restart": + err = lifecycle.Restart(context.Background(), "demo", 5) + } + if err != nil { + t.Fatalf("%s: %v", operation, err) + } + effects := 0 + for _, command := range mock.Calls { + if strings.HasPrefix(command, "docker start ") || strings.HasPrefix(command, "docker stop ") { + effects++ + for _, excluded := range []string{"'old'", "'preview'", "'unknown'", "'removed-db'"} { + if strings.Contains(command, excluded) { + t.Fatalf("%s touched %s", operation, command) + } + } + } + } + want := 3 + if operation == "restart" { + want = 6 + } + if effects != want { + t.Fatalf("%s effects %d want %d", operation, effects, want) + } + } +} diff --git a/internal/deploy/static.go b/internal/deploy/static.go index 9302a38..fda2b24 100644 --- a/internal/deploy/static.go +++ b/internal/deploy/static.go @@ -43,17 +43,19 @@ import ( // config.AppConfig that matter for static and translates them into the form // the deployer needs (resolved hosts, container-side root path, etc.). type StaticConfig struct { - App string - Domain string // raw "a.com, www.a.com" — caddy.SetStaticRoute splits it - Source string // local path to the directory to upload (the built dist/) - Build []string - BuildRemote bool - SPA bool - SPAFallback string - Cache map[string]string - Headers map[string]string - KeepReleases int // 0 = use default - CaddyExtra string + ExpectedState *state.GenerationExpectation + UnderLockPreflight func(context.Context) error + App string + Domain string // raw "a.com, www.a.com" — caddy.SetStaticRoute splits it + Source string // local path to the directory to upload (the built dist/) + Build []string + BuildRemote bool + SPA bool + SPAFallback string + Cache map[string]string + Headers map[string]string + KeepReleases int // 0 = use default + CaddyExtra string // Server-side mount path — the directory Caddy is configured to serve // from, inside the Caddy container. See DefaultStaticMount. @@ -136,6 +138,35 @@ func (d *StaticDeployer) Deploy(ctx context.Context, cfg StaticConfig) error { start := time.Now() fmt.Fprintf(d.out, "Deploying %s (static)\n", cfg.App) + // 4. Lock the app to avoid concurrent deploys racing on the symlink swap + // (F16: fenced, so a broken holder's late writes are refused). + if err := state.EnsureAppDir(ctx, d.exec, cfg.App); err != nil { + return fmt.Errorf("ensure app dir: %w", err) + } + lk, err := state.AcquireLockFenced(ctx, d.exec, cfg.App) + if err != nil { + return fmt.Errorf("acquire lock: %w", err) + } + defer state.ReleaseLockFenced(d.exec, lk, cfg.App) + lk.StartRenewal(d.exec) + + // 5. Read prior state for rollback bookkeeping. A read failure aborts — + // treating unreadable state as "no state" would drop the rollback + // history and mislabel a replacement as a first deploy (audit F15). + prior, err := state.Read(ctx, d.exec, cfg.App) + if err != nil { + return fmt.Errorf("refusing to deploy with unreadable state for %s: %w", cfg.App, err) + } + + if err := cfg.ExpectedState.Check(cfg.App, prior); err != nil { + return err + } + if cfg.UnderLockPreflight != nil { + if err := cfg.UnderLockPreflight(ctx); err != nil { + return err + } + } + // 1. Build (locally for now; remote build is a deferred feature). if len(cfg.Build) > 0 { if cfg.BuildRemote { @@ -163,26 +194,6 @@ func (d *StaticDeployer) Deploy(ctx context.Context, cfg StaticConfig) error { shortHash := hash[:12] fmt.Fprintf(d.out, " release %s\n", shortHash) - // 4. Lock the app to avoid concurrent deploys racing on the symlink swap - // (F16: fenced, so a broken holder's late writes are refused). - if err := state.EnsureAppDir(ctx, d.exec, cfg.App); err != nil { - return fmt.Errorf("ensure app dir: %w", err) - } - lk, err := state.AcquireLockFenced(ctx, d.exec, cfg.App) - if err != nil { - return fmt.Errorf("acquire lock: %w", err) - } - defer state.ReleaseLockFenced(d.exec, lk, cfg.App) - lk.StartRenewal(d.exec) - - // 5. Read prior state for rollback bookkeeping. A read failure aborts — - // treating unreadable state as "no state" would drop the rollback - // history and mislabel a replacement as a first deploy (audit F15). - prior, err := state.Read(ctx, d.exec, cfg.App) - if err != nil { - return fmt.Errorf("refusing to deploy with unreadable state for %s: %w", cfg.App, err) - } - // 6. Ensure the releases dir exists, then rsync to a temp release dir and // atomically rename. If the release for this hash already exists we // skip the upload (idempotent re-deploy). @@ -270,6 +281,9 @@ func (d *StaticDeployer) Deploy(ctx context.Context, cfg StaticConfig) error { // The commit runs under the fence (F16): the rename that makes the new // release authoritative is a guarded effect. if err := state.WriteFenced(ctx, d.exec, cfg.App, newState, lk); err != nil { + if state.PreservePublishedState(err) { + return err + } return d.abortStaticStateCommit(ctx, cfg.App, cfg.StateDir, currentLink, prior, err) } @@ -730,6 +744,9 @@ func (d *StaticDeployer) Rollback(ctx context.Context, cfg StaticRollbackConfig) newState.ApplyRelease(prior.PreviousRelease) } if err := state.WriteFenced(ctx, d.exec, cfg.App, newState, lk); err != nil { + if state.PreservePublishedState(err) { + return err + } if restoreErr := d.restoreStaticLink(ctx, cfg.App, cfg.StateDir, prior); restoreErr != nil { return fmt.Errorf("committing authoritative applied state after static rollback route switch: %w; restoring release %s failed: %v; the target release was left active", err, prior.CurrentHash, restoreErr) } @@ -832,6 +849,9 @@ func (d *StaticDeployer) RollbackStateOnly(ctx context.Context, app, toHash stri newState.ApplyRelease(prior.PreviousRelease) } if err := state.WriteFenced(ctx, d.exec, app, newState, lk); err != nil { + if state.PreservePublishedState(err) { + return err + } if restoreErr := d.restoreStaticLink(ctx, app, DefaultStateDir, prior); restoreErr != nil { return fmt.Errorf("committing authoritative applied state after static rollback route switch: %w; restoring release %s failed: %v; the target release was left active", err, prior.CurrentHash, restoreErr) } @@ -886,3 +906,7 @@ func (d *StaticDeployer) ListReleases(ctx context.Context, app, stateDir string) } return releases, nil } + +// StaticSourceFingerprint binds the actual static files, including generated +// output excluded from the project build context, to a reviewed apply. +func StaticSourceFingerprint(dir string) (string, error) { return hashDir(dir) } diff --git a/internal/deploy/volume_ownership_test.go b/internal/deploy/volume_ownership_test.go new file mode 100644 index 0000000..23d9205 --- /dev/null +++ b/internal/deploy/volume_ownership_test.go @@ -0,0 +1,35 @@ +package deploy + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func TestVolumeOwnershipFailurePrecedesContainerStart(t *testing.T) { + uid, gid := uint32(10001), uint32(10001) + m := ssh.NewMockExecutor("host", + ssh.MockCommand{Match: "docker image inspect", Output: "sha256:" + strings.Repeat("a", 64)}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/observe/state.json' ]", Output: "absent"}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/observe/state' ]", Output: "absent"}, + ssh.MockCommand{Match: "teploy_volume_actor=", Err: errors.New("provision refused")}, + ) + cfg := Config{App: "observe", Image: "observe:test", Version: "v1", Ingress: "host", ContainerPort: 5080, + Volumes: map[string]string{"/deployments/observe/volumes/observe-state": "/var/lib/observe"}, + VolumeOwnership: map[string]config.VolumeOwnership{"observe-state": {UID: &uid, GID: &gid, Mode: "0700"}}, + } + err := NewDeployer(m, &bytes.Buffer{}).DeployLocked(context.Background(), cfg) + if err == nil || !strings.Contains(err.Error(), "admitting managed directory") { + t.Fatalf("ownership refusal: %v calls=%v", err, m.Calls) + } + for _, command := range m.Calls { + if strings.Contains(command, "docker run") || strings.Contains(command, "docker stop") { + t.Fatalf("container affected before ownership established: %s", command) + } + } +} diff --git a/internal/docker/migration_regression_test.go b/internal/docker/migration_regression_test.go new file mode 100644 index 0000000..2721168 --- /dev/null +++ b/internal/docker/migration_regression_test.go @@ -0,0 +1,46 @@ +package docker + +import ( + "context" + "errors" + "github.com/useteploy/teploy/internal/ssh" + "io" + "strings" + "testing" +) + +func TestMigrationFailureResumesEveryWriter(t *testing.T) { + mock := ssh.NewMockExecutor("test", ssh.MockCommand{Match: "docker ps -q", Output: "web1\nworker1\n"}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "cp -a", Err: errors.New("copy failed")}, ssh.MockCommand{Match: "docker start", Output: ""}) + _, err := BeginVolumeMigration(context.Background(), mock, "demo", []VolumeMismatch{{ExistingSource: "/old", ExpectedSource: "/new"}}, io.Discard) + if err == nil { + t.Fatal("copy failure must fail") + } + all := strings.Join(mock.Calls, "\n") + for _, id := range []string{"web1", "worker1"} { + if !strings.Contains(all, "docker start '"+id+"'") { + t.Fatalf("writer %s not recovered: %s", id, all) + } + } +} + +func TestMigrationSuccessfulCopyStaysQuiescedUntilCommit(t *testing.T) { + mock := ssh.NewMockExecutor("test", ssh.MockCommand{Match: "docker ps -q", Output: "web1"}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "cp -a", Output: ""}, ssh.MockCommand{Match: "docker start", Output: ""}) + migration, err := BeginVolumeMigration(context.Background(), mock, "demo", []VolumeMismatch{{ExistingSource: "/old", ExpectedSource: "/new"}}, io.Discard) + if err != nil { + t.Fatal(err) + } + for _, call := range mock.Calls { + if strings.HasPrefix(call, "docker start") { + t.Fatal("reopened writes before commit") + } + } + migration.Commit() + if err := migration.Restore(context.Background()); err != nil { + t.Fatal(err) + } + for _, call := range mock.Calls { + if strings.HasPrefix(call, "docker start") { + t.Fatal("committed migration restarted displaced source") + } + } +} diff --git a/internal/docker/recreate.go b/internal/docker/recreate.go index 870c8e9..7221c1b 100644 --- a/internal/docker/recreate.go +++ b/internal/docker/recreate.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "net" + "os" "sort" "strconv" "strings" @@ -48,41 +49,41 @@ type RecreateMount struct { // represented through the docker CLI (which takes a single --entrypoint // string); recreate fails closed instead of silently mangling it. type RecreateSpec struct { - Name string `json:"name"` - ImageID string `json:"image_id,omitempty"` // immutable sha256:… (top-level .Image) - ImageRef string `json:"image_ref,omitempty"` // Config.Image original tag reference - Entrypoint []string `json:"entrypoint,omitempty"` - Cmd []string `json:"cmd,omitempty"` - Env []string `json:"env,omitempty"` + Name string `json:"name"` + ImageID string `json:"image_id,omitempty"` // immutable sha256:… (top-level .Image) + ImageRef string `json:"image_ref,omitempty"` // Config.Image original tag reference + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env []string `json:"env,omitempty"` // EnvFile, when set, names a private on-target env file rendered from // Env; Recreate publishes it via --env-file instead of -e arguments so // resolved values never appear in the host process list / command // diagnostics (audit T19). Empty at inspect time. - EnvFile string `json:"-"` - WorkingDir string `json:"working_dir,omitempty"` - User string `json:"user,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - NoHealthcheck bool `json:"no_healthcheck,omitempty"` - Networks []string `json:"networks,omitempty"` // non-default networks, primary first - Aliases []string `json:"aliases,omitempty"` // primary network's aliases, sorted - PortBindings []RecreateBinding `json:"port_bindings,omitempty"` - Binds []string `json:"binds,omitempty"` - Mounts []RecreateMount `json:"mounts,omitempty"` - MemoryBytes int64 `json:"memory_bytes,omitempty"` - NanoCPUs int64 `json:"nano_cpus,omitempty"` - RestartPolicy string `json:"restart_policy,omitempty"` - StopTimeout int `json:"stop_timeout,omitempty"` - StopSignal string `json:"stop_signal,omitempty"` - LogDriver string `json:"log_driver,omitempty"` - LogOpts []string `json:"log_opts,omitempty"` // sorted "k=v" - ExtraHosts []string `json:"extra_hosts,omitempty"` - Sysctls map[string]string `json:"sysctls,omitempty"` - Tmpfs map[string]string `json:"tmpfs,omitempty"` - CapAdd []string `json:"cap_add,omitempty"` - CapDrop []string `json:"cap_drop,omitempty"` - SecurityOpt []string `json:"security_opt,omitempty"` - Privileged bool `json:"privileged,omitempty"` - ReadonlyRootfs bool `json:"readonly_rootfs,omitempty"` + EnvFile string `json:"-"` + WorkingDir string `json:"working_dir,omitempty"` + User string `json:"user,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + NoHealthcheck bool `json:"no_healthcheck,omitempty"` + Networks []string `json:"networks,omitempty"` // non-default networks, primary first + Aliases []string `json:"aliases,omitempty"` // primary network's aliases, sorted + PortBindings []RecreateBinding `json:"port_bindings,omitempty"` + Binds []string `json:"binds,omitempty"` + Mounts []RecreateMount `json:"mounts,omitempty"` + MemoryBytes int64 `json:"memory_bytes,omitempty"` + NanoCPUs int64 `json:"nano_cpus,omitempty"` + RestartPolicy string `json:"restart_policy,omitempty"` + StopTimeout int `json:"stop_timeout,omitempty"` + StopSignal string `json:"stop_signal,omitempty"` + LogDriver string `json:"log_driver,omitempty"` + LogOpts []string `json:"log_opts,omitempty"` // sorted "k=v" + ExtraHosts []string `json:"extra_hosts,omitempty"` + Sysctls map[string]string `json:"sysctls,omitempty"` + Tmpfs map[string]string `json:"tmpfs,omitempty"` + CapAdd []string `json:"cap_add,omitempty"` + CapDrop []string `json:"cap_drop,omitempty"` + SecurityOpt []string `json:"security_opt,omitempty"` + Privileged bool `json:"privileged,omitempty"` + ReadonlyRootfs bool `json:"readonly_rootfs,omitempty"` } // containerInspect mirrors the subset of `docker inspect` JSON used to build @@ -110,13 +111,13 @@ type containerInspect struct { } } HostConfig struct { - NetworkMode string - PortBindings map[string][]struct { + NetworkMode string + PortBindings map[string][]struct { HostIp string HostPort string } - Binds []string - Mounts []struct { + Binds []string + Mounts []struct { Type string // "bind" | "volume" | "tmpfs" Source string Target string @@ -125,19 +126,19 @@ type containerInspect struct { RestartPolicy struct { Name string } - Memory int64 // bytes - NanoCpus int64 // nano-CPUs (1 CPU = 1e9) - LogConfig struct { + Memory int64 // bytes + NanoCpus int64 // nano-CPUs (1 CPU = 1e9) + LogConfig struct { Type string Config map[string]string } - ExtraHosts []string - Sysctls map[string]string - Tmpfs map[string]string - CapAdd []string - CapDrop []string - SecurityOpt []string - Privileged bool + ExtraHosts []string + Sysctls map[string]string + Tmpfs map[string]string + CapAdd []string + CapDrop []string + SecurityOpt []string + Privileged bool ReadonlyRootfs bool } // EffectiveMounts is the container's EFFECTIVE mount inventory @@ -182,29 +183,29 @@ func (c *Client) InspectRecreate(ctx context.Context, name string) (*RecreateSpe // so tests can drive it without an executor. func specFromInspect(name string, in containerInspect) (*RecreateSpec, error) { spec := &RecreateSpec{ - Name: name, - ImageID: in.Image, - ImageRef: in.Config.Image, - Entrypoint: append([]string(nil), in.Config.Entrypoint...), - Cmd: append([]string(nil), in.Config.Cmd...), - Env: append([]string(nil), in.Config.Env...), - WorkingDir: in.Config.WorkingDir, - User: in.Config.User, - Labels: in.Config.Labels, - StopSignal: in.Config.StopSignal, - StopTimeout: in.Config.StopTimeout, - Binds: append([]string(nil), in.HostConfig.Binds...), - MemoryBytes: in.HostConfig.Memory, - NanoCPUs: in.HostConfig.NanoCpus, - RestartPolicy: in.HostConfig.RestartPolicy.Name, - LogDriver: in.HostConfig.LogConfig.Type, - ExtraHosts: append([]string(nil), in.HostConfig.ExtraHosts...), - Sysctls: in.HostConfig.Sysctls, - Tmpfs: in.HostConfig.Tmpfs, - CapAdd: append([]string(nil), in.HostConfig.CapAdd...), - CapDrop: append([]string(nil), in.HostConfig.CapDrop...), - SecurityOpt: append([]string(nil), in.HostConfig.SecurityOpt...), - Privileged: in.HostConfig.Privileged, + Name: name, + ImageID: in.Image, + ImageRef: in.Config.Image, + Entrypoint: append([]string(nil), in.Config.Entrypoint...), + Cmd: append([]string(nil), in.Config.Cmd...), + Env: append([]string(nil), in.Config.Env...), + WorkingDir: in.Config.WorkingDir, + User: in.Config.User, + Labels: in.Config.Labels, + StopSignal: in.Config.StopSignal, + StopTimeout: in.Config.StopTimeout, + Binds: append([]string(nil), in.HostConfig.Binds...), + MemoryBytes: in.HostConfig.Memory, + NanoCPUs: in.HostConfig.NanoCpus, + RestartPolicy: in.HostConfig.RestartPolicy.Name, + LogDriver: in.HostConfig.LogConfig.Type, + ExtraHosts: append([]string(nil), in.HostConfig.ExtraHosts...), + Sysctls: in.HostConfig.Sysctls, + Tmpfs: in.HostConfig.Tmpfs, + CapAdd: append([]string(nil), in.HostConfig.CapAdd...), + CapDrop: append([]string(nil), in.HostConfig.CapDrop...), + SecurityOpt: append([]string(nil), in.HostConfig.SecurityOpt...), + Privileged: in.HostConfig.Privileged, ReadonlyRootfs: in.HostConfig.ReadonlyRootfs, } if in.Config.Healthcheck != nil && len(in.Config.Healthcheck.Test) == 1 && in.Config.Healthcheck.Test[0] == "NONE" { @@ -619,10 +620,12 @@ func (c *Client) recreate(ctx context.Context, spec *RecreateSpec, avoidPorts ma if err != nil { return err } - cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second) defer func() { - c.exec.Run(cleanupCtx, "rm -f -- "+ssh.ShellQuote(envPath)) - cleanupCancel() + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cleanupCancel() + if _, cleanupErr := c.exec.Run(cleanupCtx, "rm -f -- "+ssh.ShellQuote(envPath)); cleanupErr != nil { + fmt.Fprintf(os.Stderr, "teploy: failed to remove private recreation environment file: %v\n", cleanupErr) + } }() spec.EnvFile = envPath } diff --git a/internal/docker/recreate_campaign_test.go b/internal/docker/recreate_campaign_test.go new file mode 100644 index 0000000..d84c6a9 --- /dev/null +++ b/internal/docker/recreate_campaign_test.go @@ -0,0 +1,35 @@ +package docker + +import ( + "context" + "github.com/useteploy/teploy/internal/ssh" + "strings" + "testing" + "time" +) + +type campaignSlowRecreate struct { + *ssh.MockExecutor + cleanupLive bool +} + +func (e *campaignSlowRecreate) Run(ctx context.Context, command string) (string, error) { + if strings.HasPrefix(command, "docker run ") { + time.Sleep(5100 * time.Millisecond) + } + if strings.HasPrefix(command, "rm -f -- '/tmp/.teploy-recreate-env-") { + _, bounded := ctx.Deadline() + e.cleanupLive = ctx.Err() == nil && bounded + } + return e.MockExecutor.Run(ctx, command) +} +func TestCampaignRecreateCleanupLifetime(t *testing.T) { + exec := &campaignSlowRecreate{MockExecutor: ssh.NewMockExecutor("dummy", ssh.MockCommand{Match: "docker rm -f"}, ssh.MockCommand{Match: "docker run"}, ssh.MockCommand{Match: "rm -f --"})} + spec := &RecreateSpec{Name: "demo", ImageRef: "demo:latest", Env: []string{"DUMMY=value"}} + if err := NewClient(exec).Recreate(context.Background(), spec, nil); err != nil { + t.Fatal(err) + } + if !exec.cleanupLive { + t.Fatal("secret cleanup context already expired") + } +} diff --git a/internal/docker/round5_migration_test.go b/internal/docker/round5_migration_test.go new file mode 100644 index 0000000..6871363 --- /dev/null +++ b/internal/docker/round5_migration_test.go @@ -0,0 +1,58 @@ +package docker + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestRound5ActualManualRecoveryScriptStopsAllAndRefusesCopyAfterFailure(t *testing.T) { + for _, fail := range []bool{true, false} { + t.Run(map[bool]string{true: "failed-stop", false: "all-stopped"}[fail], func(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "source's data") + dest := filepath.Join(root, "new's data") + os.Mkdir(source, 0700) + os.WriteFile(filepath.Join(source, "record"), []byte("original"), 0600) + ids := strings.Repeat("a", 64) + "\n" + strings.Repeat("b", 64) + log := filepath.Join(root, "stop.log") + script := `#!/bin/sh +case "$1" in + ps) if test -f "$FIXTURE_LOG" && test "$(wc -l < "$FIXTURE_LOG" | tr -d ' ')" = 2; then exit 0; fi; printf '%s\n' "$FIXTURE_IDS";; + stop) printf '%s\n' "$3" >> "$FIXTURE_LOG"; if test "$FAIL_STOP" = yes; then exit 1; fi;; + *) exit 99;; +esac +` + os.WriteFile(filepath.Join(root, "docker"), []byte(script), 0700) + command := exec.Command("sh", "-c", manualMigrationScript("demo", []VolumeMismatch{{ExistingSource: source, ExpectedSource: dest}})) + failMode := "no" + if fail { + failMode = "yes" + } + command.Env = append(os.Environ(), "PATH="+root+":"+os.Getenv("PATH"), "FIXTURE_LOG="+log, "FIXTURE_IDS="+ids, "FAIL_STOP="+failMode) + output, err := command.CombinedOutput() + if fail { + if err == nil { + t.Fatal("failed stop succeeded") + } + if _, statErr := os.Stat(dest); !os.IsNotExist(statErr) { + t.Fatal("copy ran after failed stop") + } + } else { + if err != nil { + t.Fatal(err, string(output)) + } + data, err := os.ReadFile(filepath.Join(dest, "record")) + if err != nil || string(data) != "original" { + t.Fatal("quoted checked copy failed", err) + } + stops, _ := os.ReadFile(log) + if strings.Count(string(stops), "\n") != 2 { + t.Fatal("not all writers stopped") + } + } + }) + } +} diff --git a/internal/docker/volume_ownership.go b/internal/docker/volume_ownership.go new file mode 100644 index 0000000..9cf1bf4 --- /dev/null +++ b/internal/docker/volume_ownership.go @@ -0,0 +1,132 @@ +package docker + +import ( + "context" + "fmt" + "path" + "strconv" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +// Ancestors must be owned by root or the SSH deployment administrator and +// not group/world writable. The SSH administrator has passwordless sudo and +// is trusted as root; its own processes and root can mutate ancestors. Root and sudo +// administrators are trusted; an app lease cannot fence unrelated root actors. +// All lookups are descriptor-relative O_NOFOLLOW, and fchown/fchmod use the +// exact directory fd admitted. Requires existing Python 3 on the Linux host. +const managedDirectoryPython = `import os, stat, sys +parts = sys.argv[1].strip('/').split('/') +operation = sys.argv[2] +uid, gid, mode = int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5], 8) +actor = int(sys.argv[6]) if len(sys.argv) > 6 else os.getuid() +flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW +fd = os.open('/', flags) +try: + for index, part in enumerate(parts): + if part in ('', '.', '..'): raise RuntimeError('invalid managed path') + parent = os.fstat(fd) + if parent.st_uid not in (0, actor) or parent.st_mode & 0o022: + raise RuntimeError('unsafe mutable volume ancestor; requires trusted administrator ownership and no group/world write') + final = index == len(parts)-1 + try: + child = os.open(part, flags, dir_fd=fd) + except FileNotFoundError: + if operation == 'validate': sys.exit(0) + os.mkdir(part, 0o700 if final and uid >= 0 else 0o755, dir_fd=fd) + if not (final and uid >= 0): os.chown(part, actor, -1, dir_fd=fd, follow_symlinks=False) + child = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = child + if uid >= 0: + current = os.fstat(fd) + matches = (current.st_uid, current.st_gid, stat.S_IMODE(current.st_mode)) == (uid, gid, mode) + if not matches: + if os.listdir(fd): raise RuntimeError('populated volume ownership mismatch; explicit operator migration required') + if operation != 'validate': + os.fchown(fd, uid, gid) + os.fchmod(fd, mode) +finally: + os.close(fd) +` + +func (c *Client) managedDirectory(ctx context.Context, dir, operation string, permission *config.VolumeOwnership, prefix string) error { + uid, gid, mode := "-1", "-1", "0755" + if permission != nil { + if err := permission.Validate(); err != nil { + return err + } + uid, gid, mode = strconv.FormatUint(uint64(*permission.UID), 10), strconv.FormatUint(uint64(*permission.GID), 10), permission.Mode + } + command := "python3 -c " + ssh.ShellQuote(managedDirectoryPython) + " " + ssh.ShellQuote(dir) + " " + operation + " " + uid + " " + gid + " " + mode + _, err := c.exec.Run(ctx, prefix+"teploy_volume_actor=$(id -u); if [ \"$(id -u)\" = 0 ]; then "+command+" \"$teploy_volume_actor\"; else sudo -n "+command+" \"$teploy_volume_actor\"; fi") + if err != nil { + return fmt.Errorf("admitting managed directory %s (requires Python 3 and root/passwordless sudo): %w", dir, err) + } + return nil +} +func managedVolumePath(app, accessory, name string) (string, error) { + for _, value := range []string{app, name} { + if err := config.ValidateName(value); err != nil { + return "", err + } + } + root := path.Join("/deployments", app, "volumes") + if accessory != "" { + if err := config.ValidateName(accessory); err != nil { + return "", err + } + root = path.Join("/deployments", app, "accessories", accessory) + } + return path.Join(root, name), nil +} +func (c *Client) ValidateManagedVolume(ctx context.Context, app, accessory, name string, permission config.VolumeOwnership, prefix ...string) error { + dir, err := managedVolumePath(app, accessory, name) + if err != nil { + return err + } + return c.managedDirectory(ctx, dir, "validate", &permission, firstPrefix(prefix)) +} +func (c *Client) ProvisionManagedVolume(ctx context.Context, app, accessory, name string, permission config.VolumeOwnership, prefix ...string) error { + dir, err := managedVolumePath(app, accessory, name) + if err != nil { + return err + } + return c.managedDirectory(ctx, dir, "provision", &permission, firstPrefix(prefix)) +} + +// EnsureManagedDirectory is also used for legacy volumes without ownership +// metadata and accessory control directories. It never traverses a symlink. +func (c *Client) EnsureManagedDirectory(ctx context.Context, app, accessory, name string, prefix ...string) error { + if err := config.ValidateName(app); err != nil { + return err + } + dir := path.Join("/deployments", app) + if accessory != "" { + if err := config.ValidateName(accessory); err != nil { + return err + } + dir = path.Join(dir, "accessories", accessory) + } else if name != "" { + dir = path.Join(dir, "volumes") + } + if name != "" { + if err := config.ValidateName(name); err != nil { + return err + } + dir = path.Join(dir, name) + } + return c.managedDirectory(ctx, dir, "provision", nil, firstPrefix(prefix)) +} +func firstPrefix(prefix []string) string { + if len(prefix) > 0 { + return prefix[0] + } + return "" +} + +// Retained for command fixtures; production uses the same descriptor helper. +func managedVolumeOwnershipScript(root, name string, permission config.VolumeOwnership) string { + return "python3 -c " + ssh.ShellQuote(managedDirectoryPython) + " " + ssh.ShellQuote(path.Join(root, name)) + " provision " + fmt.Sprintf("%d %d %s", *permission.UID, *permission.GID, permission.Mode) +} diff --git a/internal/docker/volume_ownership_test.go b/internal/docker/volume_ownership_test.go new file mode 100644 index 0000000..ab69ab7 --- /dev/null +++ b/internal/docker/volume_ownership_test.go @@ -0,0 +1,107 @@ +package docker + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func volumePermissions(uid, gid uint32, mode string) config.VolumeOwnership { + return config.VolumeOwnership{UID: &uid, GID: &gid, Mode: mode} +} + +func TestVolumeOwnershipCommandIsBounded(t *testing.T) { + m := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "if [", Output: ""}, ssh.MockCommand{Match: "teploy_volume_actor", Output: ""}) + if err := NewClient(m).ProvisionManagedVolume(context.Background(), "observe", "", "observe-state", volumePermissions(10001, 10001, "0700")); err != nil { + t.Fatal(err) + } + command := m.Calls[0] + for _, bad := range []string{"chown -R", "chmod -R", "docker run", "/etc/"} { + if strings.Contains(command, bad) { + t.Fatalf("unsafe ownership command: %s", bad) + } + } + if !strings.Contains(command, "sudo -n") || !strings.Contains(command, "populated volume ownership mismatch") { + t.Fatal("missing privilege/refusal contract") + } + m.Calls = nil + if err := NewClient(m).ProvisionManagedVolume(context.Background(), "observe", "", "/tmp/escape", volumePermissions(10001, 10001, "0700")); err == nil || len(m.Calls) > 0 { + t.Fatal("host path ownership admitted") + } +} + +func TestVolumeOwnershipRealDirectoryLifecycle(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("descriptor fixture requires Linux directory descriptors") + } + root := t.TempDir() + p := volumePermissions(uint32(os.Getuid()), uint32(os.Getgid()), "0700") + // Relocate only the trusted starting descriptor: t.TempDir lives under a + // world-writable ancestor which production correctly refuses. This fixture + // exercises the descriptor core, not production ancestor admission. + fixture := func(name string, permission config.VolumeOwnership) string { + python := strings.Replace(managedDirectoryPython, "fd = os.open('/', flags)", "fd = os.open("+fmt.Sprintf("%q", root)+", flags)", 1) + return "python3 -c " + ssh.ShellQuote(python) + " " + ssh.ShellQuote(name) + " provision " + fmt.Sprintf("%d %d %s", *permission.UID, *permission.GID, permission.Mode) + } + script := fixture("data", p) + run := func(script string) error { + cmd := exec.Command("sh", "-c", script) + out, err := cmd.CombinedOutput() + if err != nil { + t.Logf("script refusal: %s", out) + } + return err + } + if err := run(script); err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "data") + info, err := os.Stat(dir) + if err != nil || info.Mode().Perm() != 0700 { + t.Fatalf("fresh mode: %v %v", info, err) + } + data := filepath.Join(dir, "retained") + if err := os.WriteFile(data, []byte("existing user data"), 0640); err != nil { + t.Fatal(err) + } + if err := run(script); err != nil { + t.Fatalf("idempotent populated matching directory: %v", err) + } + mismatch := fixture("data", volumePermissions(uint32(os.Getuid()), uint32(os.Getgid()), "0750")) + if run(mismatch) == nil { + t.Fatal("populated mismatch changed") + } + info, _ = os.Stat(data) + if info.Mode().Perm() != 0640 { + t.Fatal("descendant permissions changed") + } + contents, _ := os.ReadFile(data) + if string(contents) != "existing user data" { + t.Fatal("descendant bytes changed") + } + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + if run(fixture("link", p)) == nil { + t.Fatal("symlink volume accepted") + } + parentLink := filepath.Join(root, "parent") + if err := os.Symlink(outside, parentLink); err != nil { + t.Fatal(err) + } + if run(fixture("parent/data", p)) == nil { + t.Fatal("symlink parent accepted") + } + if _, err := os.Stat(filepath.Join(outside, "data")); !os.IsNotExist(err) { + t.Fatal("arbitrary host path touched") + } +} diff --git a/internal/docker/volumes.go b/internal/docker/volumes.go index a5bfa9d..0dd262f 100644 --- a/internal/docker/volumes.go +++ b/internal/docker/volumes.go @@ -2,9 +2,11 @@ package docker import ( "context" + "errors" "fmt" "io" "strings" + "time" "github.com/useteploy/teploy/internal/ssh" ) @@ -117,10 +119,11 @@ func FormatMismatchError(app string, mismatches []VolumeMismatch) error { fmt.Fprintln(&b, " teploy-expected path before deploy. This stops the running container") fmt.Fprintln(&b, " first and preserves file ownership via cp -a.") fmt.Fprintln(&b, " (b) Migrate manually on the server, then re-run teploy deploy:") - for _, m := range mismatches { - fmt.Fprintf(&b, " docker stop $(docker ps -q --filter label=teploy.app=%s --filter label=teploy.process=web)\n", shellEscape(app)) - fmt.Fprintf(&b, " mkdir -p %s && cp -a %s/. %s/\n", m.ExpectedSource, m.ExistingSource, m.ExpectedSource) - } + fmt.Fprintln(&b, " Hold a manual app lock (teploy lock) and an exclusive administrator window first.") + fmt.Fprintln(&b, " Run this ONE checked script. A failed inventory/stop refuses every copy.") + fmt.Fprintf(&b, " sh -c %s\n", ssh.ShellQuote(manualMigrationScript(app, mismatches))) + fmt.Fprintln(&b, " Keep the lock until redeploy/recovery; use the printed immutable IDs to restart old writers if needed, then teploy unlock.") + return fmt.Errorf("%s", b.String()) } @@ -132,38 +135,46 @@ func FormatMismatchError(app string, mismatches []VolumeMismatch) error { // // On any error the function returns immediately — the caller should treat // a partial migration as a failure state and not proceed with the deploy. -func MigrateVolumes(ctx context.Context, exec ssh.Executor, app string, mismatches []VolumeMismatch, out io.Writer) error { +func BeginVolumeMigration(ctx context.Context, exec ssh.Executor, app string, mismatches []VolumeMismatch, out io.Writer) (migration *VolumeMigration, retErr error) { if len(mismatches) == 0 { - return nil + return nil, nil } cidCmd := fmt.Sprintf( - "docker ps -aq --filter label=teploy.app=%s --filter label=teploy.process=web | head -n 1", + "docker ps -q --filter label=teploy.app=%s", shellEscape(app), ) cidOut, err := exec.Run(ctx, cidCmd) if err != nil { - return fmt.Errorf("looking up existing container: %w", err) + return nil, fmt.Errorf("looking up existing container: %w", err) } - cid := strings.TrimSpace(cidOut) + ids := strings.Fields(cidOut) + cid := strings.Join(ids, " ") if cid == "" { // No running container to migrate from — strange, but means there's // nothing to copy. Treat as no-op rather than error. fmt.Fprintln(out, " (no running container found — nothing to migrate)") - return nil + return nil, nil } fmt.Fprintf(out, "Migrating %d volume(s) for %s...\n", len(mismatches), app) fmt.Fprintf(out, " Stopping container %s before copy...\n", cid[:min(12, len(cid))]) + migration = &VolumeMigration{exec: exec, ids: ids} + recoveryHandle := migration + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, recoveryHandle.Restore(ctx)) + } + }() if _, err := exec.Run(ctx, fmt.Sprintf("docker stop %s", cid)); err != nil { - return fmt.Errorf("stopping container before migration: %w", err) + return nil, fmt.Errorf("stopping container before migration: %w", err) } for _, m := range mismatches { fmt.Fprintf(out, " %s -> %s\n", m.ExistingSource, m.ExpectedSource) mkdir := fmt.Sprintf("mkdir -p %s", shellEscape(m.ExpectedSource)) if _, err := exec.Run(ctx, mkdir); err != nil { - return fmt.Errorf("creating destination %s: %w", m.ExpectedSource, err) + return nil, fmt.Errorf("creating destination %s: %w", m.ExpectedSource, err) } // cp -a preserves ownership/perms/times; using "/." copies contents // rather than the directory itself so the destination already-exists @@ -173,12 +184,12 @@ func MigrateVolumes(ctx context.Context, exec ssh.Executor, app string, mismatch shellEscape(m.ExistingSource), shellEscape(m.ExpectedSource), ) if _, err := exec.Run(ctx, copyCmd); err != nil { - return fmt.Errorf("copying %s -> %s: %w", m.ExistingSource, m.ExpectedSource, err) + return nil, fmt.Errorf("copying %s -> %s: %w", m.ExistingSource, m.ExpectedSource, err) } } fmt.Fprintln(out, " Migration complete. Proceeding with deploy.") - return nil + return migration, nil } // shellEscape wraps a value in single quotes for safe shell embedding, @@ -191,3 +202,59 @@ func shellEscape(s string) string { } return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" } + +type VolumeMigration struct { + exec ssh.Executor + ids []string + committed bool +} + +func (m *VolumeMigration) Commit() { + if m != nil { + m.committed = true + } +} +func (m *VolumeMigration) Restore(ctx context.Context) error { + if m == nil || m.committed { + return nil + } + recovery, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + var result error + for _, id := range m.ids { + if _, err := m.exec.Run(recovery, "docker start "+ssh.ShellQuote(id)); err != nil { + result = errors.Join(result, err) + } + } + return result +} +func MigrateVolumes(ctx context.Context, exec ssh.Executor, app string, mismatches []VolumeMismatch, out io.Writer) error { + m, err := BeginVolumeMigration(ctx, exec, app, mismatches, out) + if err != nil { + return err + } + return m.Restore(ctx) +} + +// manualMigrationScript inventories ALL app-labelled running writers, never +// the first web process. Every path is quoted and every stop is checked. +func manualMigrationScript(app string, mismatches []VolumeMismatch) string { + var b strings.Builder + fmt.Fprintf(&b, "set -eu\nids=$(docker ps --no-trunc -q --filter %s) || exit 1\nstopped=''\n", ssh.ShellQuote("label=teploy.app="+app)) + b.WriteString(`for id in $ids; do + case $id in *[!0-9a-f]*|'') echo 'invalid writer ID' >&2; exit 1;; esac + [ ${#id} -eq 64 ] || exit 1 + docker stop -- "$id" || { echo "stop failed; no copy performed; recovery IDs: $stopped $id" >&2; exit 1; } + stopped="$stopped $id" +done +`) + fmt.Fprintf(&b, `remaining=$(docker ps --no-trunc -q --filter %s) || exit 1 +[ -z "$remaining" ] || { echo 'writers remain; no copy performed' >&2; exit 1; } +`, ssh.ShellQuote("label=teploy.app="+app)) + for _, m := range mismatches { + fmt.Fprintf(&b, `mkdir -p -- %s && cp -a -- %s %s || { echo "copy failed; recovery IDs: $stopped" >&2; exit 1; } +`, ssh.ShellQuote(m.ExpectedSource), ssh.ShellQuote(m.ExistingSource+"/."), ssh.ShellQuote(m.ExpectedSource+"/")) + } + b.WriteString("echo \"copy complete; keep writers stopped until deploy; recovery IDs: $stopped\"\n") + return b.String() +} diff --git a/internal/docker/volumes_test.go b/internal/docker/volumes_test.go index d8bd907..58a12e3 100644 --- a/internal/docker/volumes_test.go +++ b/internal/docker/volumes_test.go @@ -184,8 +184,9 @@ func TestFormatMismatchError_IncludesEverythingUserNeeds(t *testing.T) { func TestMigrateVolumes_HappyPath(t *testing.T) { mock := ssh.NewMockExecutor("1.2.3.4", - ssh.MockCommand{Match: "docker ps -aq", Output: "deadbeef"}, + ssh.MockCommand{Match: "docker ps -q", Output: "deadbeef"}, ssh.MockCommand{Match: "docker stop deadbeef", Output: ""}, + ssh.MockCommand{Match: "docker start", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "cp -a", Output: ""}, ) @@ -228,7 +229,7 @@ func TestMigrateVolumes_NoExistingContainer(t *testing.T) { // User passed --migrate-volumes but the existing container was already gone. // Should no-op rather than error. mock := ssh.NewMockExecutor("1.2.3.4", - ssh.MockCommand{Match: "docker ps -aq", Output: ""}, + ssh.MockCommand{Match: "docker ps -q", Output: ""}, ) var buf bytes.Buffer diff --git a/internal/env/env.go b/internal/env/env.go index db362ad..d238842 100644 --- a/internal/env/env.go +++ b/internal/env/env.go @@ -145,7 +145,7 @@ func (m *Manager) writeEnv(ctx context.Context, app string, vars map[string]stri func parseEnv(content string) map[string]string { vars := make(map[string]string) for _, line := range strings.Split(content, "\n") { - line = strings.TrimSpace(line) + line = strings.TrimLeft(line, " \t") if line == "" || strings.HasPrefix(line, "#") { continue } @@ -155,7 +155,6 @@ func parseEnv(content string) map[string]string { } key := line[:idx] val := line[idx+1:] - val = unquote(val) vars[key] = val } return vars diff --git a/internal/env/env_test.go b/internal/env/env_test.go index 50dcf08..ecdf88c 100644 --- a/internal/env/env_test.go +++ b/internal/env/env_test.go @@ -220,8 +220,8 @@ EMPTY= vars := parseEnv(content) tests := map[string]string{ - "DB_URL": "postgres://user:pass@host/db", - "SECRET": "my secret value", + "DB_URL": `"postgres://user:pass@host/db"`, + "SECRET": "'my secret value'", "PLAIN": "simple", "EMPTY": "", } @@ -283,3 +283,15 @@ func TestSerializeEnv_SortedKeys(t *testing.T) { t.Errorf("expected third key Z, got: %s", lines[2]) } } + +func TestLiteralEnvRoundtrip(t *testing.T) { + input := map[string]string{"QUOTE": `"quoted"`, "SPACE": "value ", "B": "old"} + vars := parseEnv(serializeEnv(input)) + vars["B"] = "new" + again := parseEnv(serializeEnv(vars)) + for _, k := range []string{"QUOTE", "SPACE"} { + if again[k] != input[k] { + t.Fatalf("%s changed: %q", k, again[k]) + } + } +} diff --git a/internal/harden/harden.go b/internal/harden/harden.go index d130dbb..2ab140c 100644 --- a/internal/harden/harden.go +++ b/internal/harden/harden.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "io" + "strconv" "strings" + "time" "github.com/useteploy/teploy/internal/ssh" ) @@ -29,6 +31,21 @@ func ConfigureUFW(ctx context.Context, exec ssh.Executor, w io.Writer, sudo stri } } + conn, err := exec.Run(ctx, "printf '%s' \"$SSH_CONNECTION\"") + if err != nil { + return fmt.Errorf("reading management connection: %w", err) + } + fields := strings.Fields(conn) + if len(fields) != 4 { + return fmt.Errorf("cannot establish management SSH port; refusing to enable firewall") + } + port, err := strconv.Atoi(fields[3]) + if err != nil || port < 1 || port > 65535 { + return fmt.Errorf("invalid management SSH port") + } + if _, err := exec.Run(ctx, sudo+fmt.Sprintf("ufw allow %d/tcp", port)); err != nil { + return fmt.Errorf("preserving SSH management port: %w", err) + } if _, err := exec.Run(ctx, sudo+"ufw default deny incoming && "+sudo+"ufw default allow outgoing"); err != nil { return fmt.Errorf("setting ufw defaults: %w", err) } @@ -113,20 +130,28 @@ func HardenSSH(ctx context.Context, exec ssh.Executor, w io.Writer, sudo string) return nil } - if _, err := exec.Run(ctx, sudo+`sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config`); err != nil { - return fmt.Errorf("setting PermitRootLogin: %w", err) + // Put global policy before Include/Match: sshd uses the first value found. + policy := "PermitRootLogin prohibit-password\nPasswordAuthentication no\nKbdInteractiveAuthentication no\nPubkeyAuthentication yes\n" + workspace, err := exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-sshd.XXXXXXXXXXXX") + if err != nil { + return fmt.Errorf("creating private SSH policy workspace: %w", err) } - - if _, err := exec.Run(ctx, sudo+`sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config`); err != nil { - return fmt.Errorf("setting PasswordAuthentication: %w", err) + workspace = strings.TrimSpace(workspace) + if !strings.HasPrefix(workspace, "/tmp/teploy-sshd.") || strings.ContainsAny(workspace, "\n\r ") { + return fmt.Errorf("invalid private SSH workspace") } - - if _, err := exec.Run(ctx, sudo+`sed -i 's/^#\?PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config`); err != nil { - return fmt.Errorf("setting PubkeyAuthentication: %w", err) + policyPath := workspace + "/policy" + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + exec.Run(cleanup, "rm -f -- "+ssh.ShellQuote(policyPath)+"; rmdir -- "+ssh.ShellQuote(workspace)) + }() + if err := exec.Upload(ctx, strings.NewReader(policy), policyPath, "0600"); err != nil { + return err } - - if _, err := exec.Run(ctx, sudo+"systemctl restart sshd"); err != nil { - return fmt.Errorf("restarting sshd: %w", err) + command := sudo + "sh -c " + ssh.ShellQuote(sshPolicyScript(policyPath)) + if _, err := exec.Run(ctx, command); err != nil { + return fmt.Errorf("validating and installing effective SSH policy: %w", err) } fmt.Fprintln(w, " PermitRootLogin prohibit-password") @@ -267,3 +292,28 @@ func splitFirstLine(s string) string { } return s } + +// Quote only once, at the invocation boundary. The trap is ordinary shell syntax. +func sshPolicyScript(policyPath string) string { + return "policy=" + ssh.ShellQuote(policyPath) + ` +set -eu +backup=$(mktemp /etc/ssh/sshd_config.teploy-old.XXXXXXXX) +candidate=$(mktemp /etc/ssh/sshd_config.teploy-new.XXXXXXXX) +trap 'rm -f "$candidate"' EXIT +cp -p /etc/ssh/sshd_config "$backup" +cat "$policy" /etc/ssh/sshd_config > "$candidate" +chmod 600 "$candidate" +sshd -t -f "$candidate" +effective=$(sshd -T -f "$candidate") +for setting in "permitrootlogin without-password" "passwordauthentication no" "kbdinteractiveauthentication no" "pubkeyauthentication yes"; do + printf "%s\n" "$effective" | grep -Fx "$setting" >/dev/null || exit 1 +done +mv "$candidate" /etc/ssh/sshd_config +if ! systemctl reload sshd; then + cp -p "$backup" /etc/ssh/sshd_config + systemctl reload sshd + exit 1 +fi +rm -f "$backup" +` +} diff --git a/internal/harden/harden_test.go b/internal/harden/harden_test.go index 5fa87b3..3771d23 100644 --- a/internal/harden/harden_test.go +++ b/internal/harden/harden_test.go @@ -12,6 +12,12 @@ import ( func TestConfigureUFW_AlreadyInstalled(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "which ufw", Output: "/usr/sbin/ufw"}, ssh.MockCommand{Match: "ufw default deny incoming && ufw default allow outgoing", Output: ""}, ssh.MockCommand{Match: "ufw allow 22/tcp && ufw allow 80/tcp && ufw allow 443/tcp", Output: ""}, @@ -37,6 +43,12 @@ func TestConfigureUFW_AlreadyInstalled(t *testing.T) { func TestConfigureUFW_FreshInstall(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "which ufw", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "DEBIAN_FRONTEND=noninteractive apt-get update", Output: ""}, ssh.MockCommand{Match: "ufw default deny incoming", Output: ""}, @@ -57,6 +69,12 @@ func TestConfigureUFW_FreshInstall(t *testing.T) { func TestConfigureUFW_WithSudo(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "which ufw", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "sudo DEBIAN_FRONTEND=noninteractive apt-get update", Output: ""}, ssh.MockCommand{Match: "sudo ufw default deny incoming", Output: ""}, @@ -83,6 +101,12 @@ func TestConfigureUFW_WithSudo(t *testing.T) { func TestInstallFail2ban_AlreadyInstalled(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "which fail2ban-server", Output: "/usr/bin/fail2ban-server"}, ssh.MockCommand{Match: "systemctl enable --now fail2ban", Output: ""}, ssh.MockCommand{Match: "echo $SSH_CONNECTION", Output: "203.0.113.7 54321 10.0.0.5 22"}, @@ -123,6 +147,12 @@ func TestInstallFail2ban_AlreadyInstalled(t *testing.T) { func TestInstallFail2ban_FreshInstall(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "which fail2ban-server", Err: fmt.Errorf("not found")}, ssh.MockCommand{Match: "DEBIAN_FRONTEND=noninteractive apt-get update", Output: ""}, ssh.MockCommand{Match: "systemctl enable --now fail2ban", Output: ""}, @@ -144,6 +174,12 @@ func TestInstallFail2ban_FreshInstall(t *testing.T) { func TestHardenSSH_NoAuthorizedKeys(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "cat ~/.ssh/authorized_keys", Err: fmt.Errorf("no such file")}, ssh.MockCommand{Match: "cat /root/.ssh/authorized_keys", Err: fmt.Errorf("no such file")}, ) @@ -166,6 +202,12 @@ func TestHardenSSH_NoAuthorizedKeys(t *testing.T) { func TestHardenSSH_EmptyAuthorizedKeys(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, ssh.MockCommand{Match: "cat ~/.ssh/authorized_keys", Output: " \n\n"}, ssh.MockCommand{Match: "cat /root/.ssh/authorized_keys", Output: " \n\n"}, ) @@ -182,6 +224,15 @@ func TestHardenSSH_EmptyAuthorizedKeys(t *testing.T) { func TestHardenSSH_KeysExist(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + // R2-01: the policy install is ONE whole-script command that + // validates, installs and rolls back (see round3_regression_test + // for the script body's real execution). + ssh.MockCommand{Match: "sh -c 'policy=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'policy=", Output: ""}, ssh.MockCommand{Match: "cat ~/.ssh/authorized_keys", Output: "ssh-ed25519 AAAAC3... user@host"}, ssh.MockCommand{Match: "sed -i", Output: ""}, ssh.MockCommand{Match: "sed -i", Output: ""}, @@ -205,19 +256,27 @@ func TestHardenSSH_KeysExist(t *testing.T) { t.Error("should report PubkeyAuthentication change") } - var sedCount int + var validated bool for _, call := range mock.Calls { - if strings.Contains(call, "sed -i") { - sedCount++ + if strings.Contains(call, "sshd -t -f") && strings.Contains(call, "sshd -T -f") { + validated = true } } - if sedCount != 3 { - t.Errorf("expected 3 sed calls, got %d", sedCount) + if !validated { + t.Fatal("effective policy must be validated before reload") } + } func TestHarden_RunsAllSteps_AsRoot(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + // R2-01: whole-script policy install/validate/rollback. + ssh.MockCommand{Match: "sh -c 'policy=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'policy=", Output: ""}, // detectSudo ssh.MockCommand{Match: "whoami", Output: "root"}, // UFW @@ -267,6 +326,12 @@ func TestHarden_RunsAllSteps_AsRoot(t *testing.T) { func TestHarden_RunsAllSteps_WithSudo(t *testing.T) { mock := ssh.NewMockExecutor("server1", + ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-sshd.", Output: "/tmp/teploy-sshd.fixture"}, + ssh.MockCommand{Match: "printf '%s'", Output: "203.0.113.7 54321 10.0.0.5 22"}, + ssh.MockCommand{Match: "ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sudo ufw allow 22/tcp", Output: ""}, + ssh.MockCommand{Match: "sh -c 'set -eu; backup=", Output: ""}, + ssh.MockCommand{Match: "sudo sh -c 'set -eu; backup=", Output: ""}, // detectSudo ssh.MockCommand{Match: "whoami", Output: "tyler"}, // UFW diff --git a/internal/harden/round3_regression_test.go b/internal/harden/round3_regression_test.go new file mode 100644 index 0000000..185bca6 --- /dev/null +++ b/internal/harden/round3_regression_test.go @@ -0,0 +1,62 @@ +package harden + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" +) + +func TestRound3SSHWholeScriptExecutesAndRollsBack(t *testing.T) { + for _, mode := range []string{"success", "validate-fails", "reload-fails"} { + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "bin") + os.Mkdir(bin, 0700) + config := filepath.Join(root, "sshd_config") + original := []byte("# old config\n") + os.WriteFile(config, original, 0600) + policy := filepath.Join(root, "policy with ' quote") + os.WriteFile(policy, []byte("PasswordAuthentication no\n"), 0600) + os.WriteFile(filepath.Join(bin, "sshd"), []byte(`#!/bin/sh +printf 'sshd %s\n' "$*" >> "$R3_LOG" +if [ "$R3_MODE" = validate-fails ]; then exit 1; fi +if [ "$1" = -T ]; then printf 'permitrootlogin without-password\npasswordauthentication no\nkbdinteractiveauthentication no\npubkeyauthentication yes\n'; fi +`), 0700) + os.WriteFile(filepath.Join(bin, "systemctl"), []byte(`#!/bin/sh +printf 'reload\n' >> "$R3_LOG" +if [ "$R3_MODE" = reload-fails ] && [ ! -e "$R3_ONCE" ]; then touch "$R3_ONCE"; exit 1; fi +`), 0700) + script := strings.ReplaceAll(sshPolicyScript(policy), "/etc/ssh/sshd_config", config) + command := exec.Command("sh", "-c", "sh -c "+ssh.ShellQuote(script)) + command.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "R3_MODE="+mode, "R3_LOG="+filepath.Join(root, "events"), "R3_ONCE="+filepath.Join(root, "once")) + output, err := command.CombinedOutput() + data, _ := os.ReadFile(config) + events, _ := os.ReadFile(filepath.Join(root, "events")) + if mode == "success" { + if err != nil || !strings.Contains(string(data), "PasswordAuthentication no") || !strings.Contains(string(events), "reload") { + t.Fatalf("whole script did not install/reload: %v %s %s", err, output, events) + } + } else { + if err == nil || string(data) != string(original) { + t.Fatalf("failure changed incumbent: %v %s", err, data) + } + // Count exact lines: the fixture paths embed the subtest + // name, so a substring count of "reload" would count the + // temp directory name too. + reloads := 0 + for _, line := range strings.Split(string(events), "\n") { + if line == "reload" { + reloads++ + } + } + if mode == "reload-fails" && reloads != 2 { + t.Fatalf("rollback did not reload old config (reloads=%d)", reloads) + } + } + }) + } +} diff --git a/internal/multideploy/multideploy.go b/internal/multideploy/multideploy.go index ddefe2b..9aad6b8 100644 --- a/internal/multideploy/multideploy.go +++ b/internal/multideploy/multideploy.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/useteploy/teploy/internal/state" "io" "sync" ) @@ -20,9 +21,10 @@ type ServerTarget struct { // Result tracks the outcome of a single-server deploy. type Result struct { - Server string - Success bool - Error error + Server string + Success bool + Error error + PublicationNeedsReconciliation bool } // PrefixWriter wraps a writer to prefix each line with a server name. @@ -197,9 +199,10 @@ func parallelDeploy(ctx context.Context, servers []ServerTarget, parallel int, f } results[idx] = Result{ - Server: srv.Name, - Success: err == nil, - Error: err, + Server: srv.Name, + Success: err == nil, + Error: err, + PublicationNeedsReconciliation: state.PreservePublishedState(err), } if err != nil { diff --git a/internal/openbao/audit.go b/internal/openbao/audit.go index 266a41d..d892c04 100644 --- a/internal/openbao/audit.go +++ b/internal/openbao/audit.go @@ -2,13 +2,17 @@ package openbao import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "errors" "fmt" "strconv" "strings" "github.com/useteploy/teploy/internal/accessories" teplaudit "github.com/useteploy/teploy/internal/audit" + "github.com/useteploy/teploy/internal/state" ) // OpenBaoAuditEntry is the subset of an OpenBao audit log line we forward. @@ -77,7 +81,13 @@ func ToObserveEvent(app string, e OpenBaoAuditEntry) (teplaudit.Event, bool) { // instance's tamper-evident trail. It tracks how many lines have already been // shipped (in a server-side marker file) so repeated runs are idempotent and // only new access events are forwarded. Returns the number shipped. -func (c *Client) ShipAudit(ctx context.Context, app, accessory, observeEndpoint, observeToken, observeSite string) (int, error) { +func (c *Client) ShipAudit(ctx context.Context, app, accessory, observeEndpoint, observeToken, observeSite string) (shipped int, retErr error) { + lock, err := state.AcquireLockFenced(ctx, c.exec, app) + if err != nil { + return 0, err + } + defer state.ReleaseLockFenced(c.exec, lock, app) + lock.StartRenewal(c.exec) if accessory == "" { accessory = defaultAccessory } @@ -87,27 +97,50 @@ func (c *Client) ShipAudit(ctx context.Context, app, accessory, observeEndpoint, container := accessories.ContainerName(app, accessory) markerFile := fmt.Sprintf("/deployments/%s/accessories/%s/.audit-shipped", app, accessory) - // How many lines shipped previously (0 if none). - prev := 0 - if out, err := c.exec.Run(ctx, "cat "+markerFile+" 2>/dev/null || echo 0"); err == nil { - if n, e := strconv.Atoi(strings.TrimSpace(out)); e == nil { - prev = n + type cursor struct { + Lines int `json:"lines"` + Prefix string `json:"prefix"` + } + var previous cursor + marker, err := c.exec.Run(ctx, "if test -f "+markerFile+"; then cat "+markerFile+"; elif test -e "+markerFile+"; then exit 1; else printf '{}'; fi") + if err != nil { + return 0, fmt.Errorf("reading audit cursor: %w", err) + } + if err := json.Unmarshal([]byte(marker), &previous); err != nil { + if _, legacyErr := strconv.Atoi(strings.TrimSpace(marker)); legacyErr != nil { + return 0, fmt.Errorf("invalid audit cursor: %w", err) } + // A legacy line count cannot establish log identity; replay once rather than lose rotated events. + fmt.Fprintln(c.out, "Migrating legacy audit cursor: existing events may be replayed with stable source_event_id") } + prev := previous.Lines // Read the whole audit log (JSON-per-line) from the container. - out, err := c.docker.Exec(ctx, container, "cat /openbao/data/audit.log 2>/dev/null || true") + out, err := c.docker.Exec(ctx, container, "cat /openbao/data/audit.log") if err != nil { return 0, fmt.Errorf("reading audit log: %w", err) } lines := strings.Split(strings.TrimRight(out, "\n"), "\n") total := len(lines) - if total <= prev { - return 0, nil // nothing new + prefix := func(n int) string { + sum := sha256.Sum256([]byte(strings.Join(lines[:n], "\n"))) + return hex.EncodeToString(sum[:]) + } + if prev < 0 || prev > total || (prev > 0 && previous.Prefix != prefix(prev)) { + prev = 0 + } + checkpoint := func(n int) error { + data, _ := json.Marshal(cursor{Lines: n, Prefix: prefix(n)}) + if err := lock.Check(ctx, c.exec); err != nil { + return err + } + return c.exec.Upload(ctx, strings.NewReader(string(data)+"\n"), markerFile, "0600") + } + if total == prev { + return 0, nil } - shipped := 0 - for _, line := range lines[prev:] { + for offset, line := range lines[prev:] { line = strings.TrimSpace(line) if line == "" || !strings.HasPrefix(line, "{") { continue @@ -120,19 +153,18 @@ func (c *Client) ShipAudit(ctx context.Context, app, accessory, observeEndpoint, if !ok { continue } + sum := sha256.Sum256([]byte(app + "\x00" + accessory + "\x00" + line)) + ev.Metadata["source_event_id"] = hex.EncodeToString(sum[:]) if err := teplaudit.Emit(ctx, observeEndpoint, observeToken, observeSite, ev); err != nil { // Persist progress up to the last successful ship so we don't // re-send, then surface the error. - c.persistShipMarker(ctx, markerFile, prev+shipped) - return shipped, fmt.Errorf("emitting audit event: %w", err) + return shipped, errors.Join(fmt.Errorf("emitting audit event: %w", err), checkpoint(prev+offset)) } shipped++ } - c.persistShipMarker(ctx, markerFile, total) + if err := checkpoint(total); err != nil { + return shipped, fmt.Errorf("persisting audit cursor: %w", err) + } return shipped, nil } - -func (c *Client) persistShipMarker(ctx context.Context, markerFile string, count int) { - _ = c.exec.Upload(ctx, strings.NewReader(strconv.Itoa(count)+"\n"), markerFile, "0600") -} diff --git a/internal/openbao/audit_cursor_regression_test.go b/internal/openbao/audit_cursor_regression_test.go new file mode 100644 index 0000000..6ec08ab --- /dev/null +++ b/internal/openbao/audit_cursor_regression_test.go @@ -0,0 +1,64 @@ +package openbao + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/useteploy/teploy/internal/ssh" + "net/http" + "net/http/httptest" + "testing" +) + +func auditCursorMock(marker, log string) *ssh.MockExecutor { + return ssh.NewMockExecutor("test", ssh.MockCommand{Match: "mkdir /deployments/demo/.lock", Output: ""}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ssh.MockCommand{Match: "if test -f /deployments/demo/accessories/openbao/.audit-shipped", Output: marker}, ssh.MockCommand{Match: "docker exec", Output: log}) +} +func auditResponse(path string) string { + return fmt.Sprintf(`{"type":"response","auth":{"display_name":"demo","policy_results":{"allowed":true}},"request":{"operation":"read","mount_type":"kv","path":%q}}`, path) +} + +func TestAuditCursorCountsConsumedLinesOnEmissionFailure(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls == 2 { + w.WriteHeader(500) + } else { + w.WriteHeader(201) + } + })) + defer server.Close() + log := `{"type":"request"}` + "\n" + auditResponse("A") + "\n" + `{"type":"request"}` + "\n" + auditResponse("B") + "\n" + mock := auditCursorMock("{}", log) + n, err := NewClient(mock, &bytes.Buffer{}).ShipAudit(context.Background(), "demo", "openbao", server.URL, "", "default") + if err == nil || n != 1 { + t.Fatalf("shipment failure: %d %v", n, err) + } + var cursor struct { + Lines int `json:"lines"` + Prefix string `json:"prefix"` + } + if err := json.Unmarshal(mock.Files["/deployments/demo/accessories/openbao/.audit-shipped"], &cursor); err != nil { + t.Fatal(err) + } + if cursor.Lines != 3 || cursor.Prefix == "" { + t.Fatalf("event count confused with consumed lines: %+v", cursor) + } +} +func TestAuditCursorDetectsSameLengthRotation(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls++; w.WriteHeader(201) })) + defer server.Close() + first := auditCursorMock("{}", auditResponse("A")+"\n") + if n, err := NewClient(first, &bytes.Buffer{}).ShipAudit(context.Background(), "demo", "openbao", server.URL, "", "default"); err != nil || n != 1 { + t.Fatalf("first: %d %v", n, err) + } + next := auditCursorMock(string(first.Files["/deployments/demo/accessories/openbao/.audit-shipped"]), auditResponse("B")+"\n") + if n, err := NewClient(next, &bytes.Buffer{}).ShipAudit(context.Background(), "demo", "openbao", server.URL, "", "default"); err != nil || n != 1 { + t.Fatalf("rotation stalled: %d %v", n, err) + } + if calls != 2 { + t.Fatalf("rotation delivered %d", calls) + } +} diff --git a/internal/openbao/database.go b/internal/openbao/database.go index dacb5e6..0595cc6 100644 --- a/internal/openbao/database.go +++ b/internal/openbao/database.go @@ -63,11 +63,11 @@ func (c *Client) EnableDatabaseSecrets(ctx context.Context, opts DBSetupOptions) // (`write -`), never the docker exec argv (C08). connURL := fmt.Sprintf("postgresql://{{username}}:{{password}}@%s:5432/%s?sslmode=disable", dbHost, opts.DBName) cfg, err := json.Marshal(map[string]string{ - "plugin_name": "postgresql-database-plugin", - "allowed_roles": dbRoleName(opts.App), - "connection_url": connURL, - "username": opts.AdminUser, - "password": opts.AdminPass, + "plugin_name": "postgresql-database-plugin", + "allowed_roles": dbRoleName(opts.App), + "connection_url": connURL, + "username": opts.AdminUser, + "password": opts.AdminPass, }) if err != nil { return fmt.Errorf("encoding db connection config: %w", err) @@ -245,9 +245,39 @@ func (c *Client) writeAppPolicy(ctx context.Context, container, root, app string // hasDBRole reports whether a dynamic DB role is already configured for the app // (so a policy re-write can preserve the DB grant). Best-effort: any error = // treat as absent. -func (c *Client) hasDBRole(ctx context.Context, container, root, app string) bool { - out, err := c.bao(ctx, container, root, "read database/roles/"+dbRoleName(app)) - return err == nil && !strings.Contains(out, "No value found") && !strings.Contains(out, "Error") +func (c *Client) discoverDBRoles(ctx context.Context, container, root, app string) (bool, error) { + found := false + for _, path := range []string{"database/roles/" + dbRoleName(app), "database/static-roles"} { + verb := "read " + if path == "database/static-roles" { + verb = "list -format=json " + } + out, err := c.bao(ctx, container, root, verb+path) + if strings.Contains(out, "No value found") || strings.Contains(out, "No value") { + continue + } + if err != nil { + return false, fmt.Errorf("discovering database policy before rewrite: %w", err) + } + if path == "database/static-roles" { + var result struct { + Data struct { + Keys []string `json:"keys"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(extractJSON(out)), &result); err != nil { + return false, fmt.Errorf("decoding static role inventory: %w", err) + } + for _, key := range result.Data.Keys { + if strings.HasPrefix(key, app+"-") { + found = true + } + } + } else { + found = true + } + } + return found, nil } // AppReadPolicy renders the app's least-privilege HCL policy (pure/testable): diff --git a/internal/openbao/secrets.go b/internal/openbao/secrets.go index 07915ef..8a61814 100644 --- a/internal/openbao/secrets.go +++ b/internal/openbao/secrets.go @@ -139,7 +139,11 @@ func (c *Client) EnsureAppRole(ctx context.Context, app, accessory string) (*App // 1. Least-privilege policy: read only this app's secrets. Shared with // EnableDatabaseSecrets via AppReadPolicy. Detect an already-configured DB // role so re-running setup after `vault db setup` preserves the DB grant. - if err := c.writeAppPolicy(ctx, container, root, app, c.hasDBRole(ctx, container, root, app)); err != nil { + withDB, err := c.discoverDBRoles(ctx, container, root, app) + if err != nil { + return nil, err + } + if err := c.writeAppPolicy(ctx, container, root, app, withDB); err != nil { return nil, err } _ = policyName @@ -167,8 +171,12 @@ func (c *Client) EnsureAppRole(ctx context.Context, app, accessory string) (*App } // 5. Persist for the deploy/agent paths. - _ = c.secrets.Set(ctx, app, secretRoleID, roleID) - _ = c.secrets.Set(ctx, app, secretSecretID, secretID) + if err := c.secrets.Set(ctx, app, secretRoleID, roleID); err != nil { + return nil, fmt.Errorf("persisting AppRole role ID: %w", err) + } + if err := c.secrets.Set(ctx, app, secretSecretID, secretID); err != nil { + return nil, fmt.Errorf("persisting AppRole secret ID (retry setup to repair partial credentials): %w", err) + } return &AppRoleCreds{RoleID: roleID, SecretID: secretID}, nil } diff --git a/internal/preview/admission_regression_test.go b/internal/preview/admission_regression_test.go new file mode 100644 index 0000000..138e88c --- /dev/null +++ b/internal/preview/admission_regression_test.go @@ -0,0 +1,125 @@ +package preview + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "github.com/useteploy/teploy/internal/ssh" + "sort" + "strings" + "sync" + "testing" +) + +func TestPreviewAuthorityFailurePreservesAccess(t *testing.T) { + mock := previewMockExecutor("test", ssh.MockCommand{Match: "cat " + previewStatePath("demo", "feature"), Err: errors.New("permission denied")}) + mgr := NewManager(mock, &bytes.Buffer{}) + if err := mgr.Deploy(context.Background(), DeployConfig{App: "demo", Branch: "feature", Repo: "repo", Image: "nginx", BaseDomain: "example.com"}); err == nil { + t.Fatal("unknown authority must fail closed") + } + for _, call := range mock.Calls { + if !strings.HasPrefix(call, "if test -f ") { + t.Fatalf("effect after failed authority read: %s", call) + } + } +} + +func TestPreviewKnownShortHashCollisionRefusesOldIdentity(t *testing.T) { + left, right := "audit-branch-68411", "audit-branch-91902" + if PreviewID("demo", left) == PreviewID("demo", right) { + t.Fatal("known 32-bit collision survived") + } + mock := previewMockExecutor("test") + record, _ := json.Marshal(State{Branch: left, Repo: "repo", ID: "demo-p-65337a47"}) + mock.Files[previewDir("demo")+"/demo-p-65337a47.json"] = record + mgr := NewManager(mock, &bytes.Buffer{}) + err := mgr.Destroy(context.Background(), "demo", right) + var ambiguous *AmbiguousPreviewError + if !errors.As(err, &ambiguous) { + t.Fatalf("old canonical collision must refuse: %v", err) + } + if len(mock.Files) != 1 { + t.Fatal("collision record changed") + } +} + +func TestPreviewTeardownFailureRetainsAuthority(t *testing.T) { + mock := previewMockExecutor("test", ssh.MockCommand{Match: "cat /deployments/caddy/Caddyfile", Err: errors.New("transport down")}) + path := previewStatePath("demo", "feature") + record, _ := json.Marshal(State{Branch: "feature", ID: PreviewID("demo", "feature"), Container: "demo-candidate", Route: "demo-route"}) + mock.Files[path] = record + if err := NewManager(mock, &bytes.Buffer{}).Destroy(context.Background(), "demo", "feature"); err == nil { + t.Fatal("failed route teardown reported success") + } + if string(mock.Files[path]) != string(record) { + t.Fatal("failed teardown erased authority") + } +} + +func previewMockExecutor(host string, commands ...ssh.MockCommand) *ssh.MockExecutor { + // The destroy path proves teardown (R2-07/r5): route absence activation, + // full immutable-ID inventory, and idempotent stop/remove of whatever + // the inventory still shows. An empty inventory models a server whose + // preview container is already gone. + commands = append(commands, ssh.MockCommand{Match: "docker inspect -f '{{.Image}}'", Output: "sha256:" + strings.Repeat("d", 64)}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, ssh.MockCommand{Match: "mkdir /deployments/", Output: ""}, ssh.MockCommand{Match: "mv -f --", Output: ""}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ssh.MockCommand{Match: "docker ps --all --no-trunc --format", Output: ""}, ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, ssh.MockCommand{Match: "docker stop", Output: ""}, ssh.MockCommand{Match: "docker rm", Output: ""}) + return ssh.NewMockExecutor(host, commands...) +} + +// previewInventoryExecutor models the destroy path's real dependency: the +// full immutable-ID inventory changes as effects land. A stopped container +// reports exited; a removed one leaves the inventory. Tests that assert a +// preview's container was (not) stopped wrap previewMockExecutor with it. +type previewInventoryExecutor struct { + *ssh.MockExecutor + mu sync.Mutex + containers map[string]string // container name -> immutable 64-hex ID + stopped map[string]bool +} + +func newPreviewInventoryExecutor(mock *ssh.MockExecutor, names ...string) *previewInventoryExecutor { + e := &previewInventoryExecutor{MockExecutor: mock, containers: map[string]string{}, stopped: map[string]bool{}} + for i, name := range names { + e.containers[name] = strings.Repeat("a", 62) + fmt.Sprintf("%02d", i+1) + } + return e +} +func (e *previewInventoryExecutor) Run(ctx context.Context, cmd string) (string, error) { + e.mu.Lock() + defer e.mu.Unlock() + // Effects arrive composed behind the app-lease fence guard; the effect + // is the tail after the guard fragment. + if i := strings.LastIndex(cmd, "exit 75; }; "); i >= 0 && strings.HasPrefix(cmd, "grep -q ") { + cmd = cmd[i+len("exit 75; }; "):] + } + if strings.Contains(cmd, "docker ps --all --no-trunc --format") { + var rows []string + for name, id := range e.containers { + state := "running" + if e.stopped[id] { + state = "exited" + } + rows = append(rows, fmt.Sprintf(`{"ID":%q,"Names":%q,"State":%q,"Labels":{"teploy.app":"demo"}}`, id, name, state)) + } + sort.Strings(rows) + return strings.Join(rows, "\n"), nil + } + for prefix, apply := range map[string]func(string){ + "docker stop -t 5 ": func(id string) { e.stopped[id] = true }, + "docker rm ": func(id string) { delete(e.containers, nameOfID(e.containers, id)) }, + } { + if rest, ok := strings.CutPrefix(cmd, prefix); ok { + apply(strings.Trim(strings.TrimSpace(rest), "'")) + } + } + return e.MockExecutor.Run(ctx, cmd) +} +func nameOfID(containers map[string]string, id string) string { + for name, containerID := range containers { + if containerID == id { + return name + } + } + return "" +} diff --git a/internal/preview/exposure_test.go b/internal/preview/exposure_test.go index e2ea92d..9256652 100644 --- a/internal/preview/exposure_test.go +++ b/internal/preview/exposure_test.go @@ -58,7 +58,7 @@ func managedBlock(t *testing.T, mock *ssh.MockExecutor, key string) string { // line) with the allowlist as its firewall, under the explicit base; the // record carries the mode and the output names the http:// URL. func TestDeploy_TailnetModeWritesHTTPOnlyGatedRoute(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -95,7 +95,7 @@ func TestDeploy_TailnetModeWritesHTTPOnlyGatedRoute(t *testing.T) { // hostname, still HTTP-only, still gated — never silently back to HTTPS or // open. func TestDeploy_UpdateInheritsTailnetMode(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -123,7 +123,7 @@ func TestDeploy_UpdateInheritsTailnetMode(t *testing.T) { // off keeps the recorded base and allowlist; an empty non-nil allowlist // clears it. func TestDeploy_UpdateOverridesFieldByField(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -157,7 +157,7 @@ func TestDeploy_UpdateOverridesFieldByField(t *testing.T) { // byte-shaped like before), and updating a record written before the // fields existed keeps automatic HTTPS, no gate, and the app-domain host. func TestDeploy_DefaultAndLegacyRecordsUnchanged(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -239,7 +239,7 @@ func TestDeploy_InvalidExposureRefusesBeforeMutation(t *testing.T) { "no-base": func(c *DeployConfig) { c.Domain = "" }, } { t.Run(name, func(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) mgr := NewManager(mock, &bytes.Buffer{}) cfg := deployCfg(loginBranch, "v1") mutate(&cfg) @@ -247,7 +247,7 @@ func TestDeploy_InvalidExposureRefusesBeforeMutation(t *testing.T) { t.Fatal("invalid exposure must fail the deploy") } for _, c := range mock.Calls { - if !strings.HasPrefix(c, "cat ") { + if !strings.HasPrefix(c, "cat ") && !strings.HasPrefix(c, "if test -f ") { t.Errorf("mutated before validating: %q", c) } } diff --git a/internal/preview/lifecycle_test.go b/internal/preview/lifecycle_test.go index 1674861..70ff940 100644 --- a/internal/preview/lifecycle_test.go +++ b/internal/preview/lifecycle_test.go @@ -32,7 +32,7 @@ func indexOf(calls []string, needle string) int { // switch — that ordering was the downtime window the main deploy path // doesn't have. func TestDeploy_BlueGreenOrdering(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", append(previewDeployMocks(), ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"})...) var buf bytes.Buffer @@ -93,7 +93,7 @@ func TestDeploy_FailedCandidateLeavesPredecessor(t *testing.T) { } mocks = append(mocks, mc) } - mock := ssh.NewMockExecutor("1.2.3.4", mocks...) + mock := previewMockExecutor("1.2.3.4", mocks...) var buf bytes.Buffer mgr := NewManager(mock, &buf) mgr.healthTimeout = 150 * time.Millisecond @@ -146,7 +146,7 @@ func TestDeploy_FailedCandidateLeavesPredecessor(t *testing.T) { // candidate's name, so it is renamed aside before the candidate starts and // retired after the switch (the main engine's same-version pattern). func TestDeploy_SameVersionUpdate(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", append(previewDeployMocks(), ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"}, ssh.MockCommand{Match: "docker rename", Output: ""})...) @@ -200,6 +200,8 @@ func TestPrune_RemovesExactlyExpiredBothErasIdempotent(t *testing.T) { otherFresh := previewStatePath("otherapp", "fresh-feature") destroyMocks := []ssh.MockCommand{ + {Match: "docker stop", Output: ""}, + {Match: "docker rm", Output: ""}, // Per-app record listings (PruneAll enumerates apps first). ssh.MockCommand{Match: "ls -d /deployments/*/previews", Output: "/deployments/myapp/previews\n/deployments/otherapp/previews"}, ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", Output: myExpired + "\n" + myFresh}, @@ -211,7 +213,7 @@ func TestPrune_RemovesExactlyExpiredBothErasIdempotent(t *testing.T) { ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, ssh.MockCommand{Match: "rmdir /deployments/caddy/.lock", Output: ""}, } - mock := ssh.NewMockExecutor("1.2.3.4", destroyMocks...) + mock := previewMockExecutor("1.2.3.4", destroyMocks...) mock.Files[myExpired] = []byte(canonicalRecordJSON("myapp", loginIDHex, loginBranch, "myapp-preview-p-"+loginIDHex+"-v1", "preview-feature-login-"+loginIDHex+".myapp.com", expiredAt)) mock.Files[myFresh] = []byte(canonicalRecordJSON("myapp", dashIDHex, dashBranch, @@ -222,8 +224,21 @@ func TestPrune_RemovesExactlyExpiredBothErasIdempotent(t *testing.T) { // A non-preview deployment resource that must never be touched. mock.Files["/deployments/myapp/state.json"] = []byte(`{"app":"myapp"}`) + // The server still runs both expired previews' containers; the fresh + // one is listed but must never be acted on. + inventory := newPreviewInventoryExecutor(mock, + "myapp-preview-p-"+loginIDHex+"-v1", "otherapp-preview-legacy-feature-v1", + "myapp-preview-p-"+dashIDHex+"-v1", "otherapp-preview-p-abcd1234-v1") + var buf bytes.Buffer - mgr := NewManager(mock, &buf) + mgr := NewManager(inventory, &buf) + // Capture the inventoried immutable IDs before the effects delete them. + expiredIDs := map[string]string{ + "myapp-preview-p-" + loginIDHex + "-v1": inventory.containers["myapp-preview-p-"+loginIDHex+"-v1"], + "otherapp-preview-legacy-feature-v1": inventory.containers["otherapp-preview-legacy-feature-v1"], + } + freshID := inventory.containers["myapp-preview-p-"+dashIDHex+"-v1"] + pruned, err := mgr.PruneAll(context.Background()) if err != nil { t.Fatalf("PruneAll: %v", err) @@ -241,17 +256,17 @@ func TestPrune_RemovesExactlyExpiredBothErasIdempotent(t *testing.T) { t.Errorf("fresh record removed: %s", path) } } - // The expired previews' containers were stopped (canonical + legacy - // both by their stored container names). - for _, container := range []string{ - "docker stop -t 5 'myapp-preview-p-" + loginIDHex + "-v1'", - "docker stop -t 5 'otherapp-preview-legacy-feature-v1'", - } { - if indexOf(mock.Calls, container) == -1 { - t.Errorf("expired preview's container not stopped: %q, calls: %v", container, mock.Calls) + // The expired previews' containers were stopped by their IMMUTABLE IDs + // (R2-07: effects address the inventoried ID, never a reusable name). + for name, id := range expiredIDs { + if indexOf(mock.Calls, "docker stop -t 5 '"+id+"'") == -1 { + t.Errorf("expired preview's container not stopped by immutable ID: %q, calls: %v", name, mock.Calls) + } + if indexOf(mock.Calls, "docker rm '"+id+"'") == -1 { + t.Errorf("expired preview's container not removed by immutable ID: %q, calls: %v", name, mock.Calls) } } - if indexOf(mock.Calls, "docker stop -t 5 'myapp-preview-p-"+dashIDHex+"-v1'") != -1 { + if indexOf(mock.Calls, "docker stop -t 5 '"+freshID+"'") != -1 { t.Error("fresh preview's container was stopped") } if indexOf(mock.Calls, "/deployments/myapp/state.json") != -1 { @@ -281,7 +296,7 @@ func TestPrune_RemovesExactlyExpiredBothErasIdempotent(t *testing.T) { // The TTL default is applied at create AND refreshed at update when the // caller doesn't pass one: ExpiresAt is exactly CreatedAt + 72h. func TestDeploy_TTLDefaultApplied(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", append(previewDeployMocks(), ssh.MockCommand{Match: "curl -s -o /dev/null", Output: "200"})...) var buf bytes.Buffer diff --git a/internal/preview/preview.go b/internal/preview/preview.go index 1c7a2aa..45653a0 100644 --- a/internal/preview/preview.go +++ b/internal/preview/preview.go @@ -2,6 +2,7 @@ package preview import ( "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" @@ -18,6 +19,7 @@ import ( "github.com/useteploy/teploy/internal/deploy" "github.com/useteploy/teploy/internal/docker" "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" ) const deploymentsDir = "/deployments" @@ -27,7 +29,17 @@ type State struct { // ID is the canonical preview identifier (-p-<8hex>, see // PreviewID). Empty on records written before the canonical-ID // migration (legacy slug-keyed records). - ID string `json:"id,omitempty"` + ID string `json:"id,omitempty"` + OwnershipID string `json:"ownership_id,omitempty"` + Generation uint64 `json:"generation,omitempty"` + // UpdatedAt is the change token compared by preview compare-destroy. + // A pointer so legacy records without one stay absent on the wire + // (encoding/json does not omit zero time.Time values). + UpdatedAt *time.Time `json:"updated_at,omitempty"` + OperationKey string `json:"operation_key,omitempty"` + ExecutionBindingDigest string `json:"execution_binding_digest,omitempty"` + SourceRevision string `json:"source_revision,omitempty"` + ImageDigest string `json:"image_digest,omitempty"` // Branch is the FULL, unsanitized branch name. Records are keyed by // the canonical ID, not by this value's sanitized form. Branch string `json:"branch"` @@ -80,14 +92,19 @@ func (s State) URL() string { // DeployConfig holds parameters for creating a preview. type DeployConfig struct { - App string - Domain string // base domain (e.g., myapp.com) - Branch string - Image string - Version string - EnvFile string - Env map[string]string - Volumes map[string]string + OwnershipID string + ExpectedGeneration *uint64 + OperationKey string + ExecutionBindingDigest string + SourceRevision string + App string + Domain string // base domain (e.g., myapp.com) + Branch string + Image string + Version string + EnvFile string + Env map[string]string + Volumes map[string]string // TTL is how long the preview lives; 0 means the documented default of // 72h. Applied on every Deploy (create AND update — an update refreshes // the deadline), recorded as the absolute State.ExpiresAt. @@ -246,7 +263,7 @@ func SanitizeBranch(branch string) string { // existing previews, while the app name is stable. func previewIDHex(app, branch string) string { sum := sha256.Sum256([]byte(app + "\x00" + branch)) - return hex.EncodeToString(sum[:4]) + return hex.EncodeToString(sum[:16]) } // PreviewID returns the canonical preview identifier: -p-<8hex>, @@ -331,10 +348,16 @@ func (e *AmbiguousPreviewError) Error() string { // error naming the path (identity cannot be established — fail closed // rather than guessing or silently adopting). func (m *Manager) readRecord(ctx context.Context, path string) (*State, error) { - content, err := m.exec.Run(ctx, "cat "+path) - if err != nil || strings.TrimSpace(content) == "" { + content, err := m.exec.Run(ctx, "if test -f "+ssh.ShellQuote(path)+"; then cat "+path+"; elif test -e "+ssh.ShellQuote(path)+"; then exit 1; else printf '__TEPLOY_PREVIEW_ABSENT__'; fi") + if err != nil { + return nil, fmt.Errorf("reading preview authority %s: %w", path, err) + } + if strings.TrimSpace(content) == "__TEPLOY_PREVIEW_ABSENT__" { return nil, nil } + if strings.TrimSpace(content) == "" { + return nil, fmt.Errorf("empty preview authority %s", path) + } var s State if err := json.Unmarshal([]byte(strings.TrimSpace(content)), &s); err != nil { return nil, fmt.Errorf("reading preview record %s: %w", path, err) @@ -364,6 +387,21 @@ func (m *Manager) resolveRecord(ctx context.Context, app, branch, repo string) ( return nil, "", err } + if canon == nil { + sum := sha256.Sum256([]byte(app + "\x00" + branch)) + oldPath := previewDir(app) + "/" + app + "-p-" + hex.EncodeToString(sum[:4]) + ".json" + old, readErr := m.readRecord(ctx, oldPath) + if readErr != nil { + return nil, "", readErr + } + if old != nil { + canon = old + canonPath = oldPath + } + } + if canon != nil && (canon.Branch != branch || (canon.Repo != "" && repo != "" && canon.Repo != repo)) { + return nil, "", &AmbiguousPreviewError{App: app, Path: canonPath, StoredBranch: canon.Branch, RequestedBranch: branch, StoredRepo: canon.Repo, RequestedRepo: repo} + } legacyPath := legacyPreviewStatePath(app, branch) legacy, err := m.readRecord(ctx, legacyPath) if err != nil { @@ -396,9 +434,7 @@ func (m *Manager) resolveRecord(ctx context.Context, app, branch, repo string) ( // this branch is a stale duplicate of the canonical one (interrupted // migration) — the full-Branch match established it is this branch's // own, so remove it. Any other legacy file stays untouched above. - if legacy.Branch == branch { - m.exec.Run(ctx, "rm -f -- "+legacyPath) - } + return canon, canonPath, nil } @@ -408,7 +444,7 @@ func (m *Manager) writeRecord(ctx context.Context, s *State, path string) error if err != nil { return err } - if err := m.exec.Upload(ctx, strings.NewReader(string(data)), path, "0644"); err != nil { + if err := state.WriteAtomicPrivate(ctx, m.exec, path, strings.NewReader(string(data))); err != nil { return err } return nil @@ -424,7 +460,7 @@ func (m *Manager) writeRecord(ctx context.Context, s *State, path string) error // running, routed, and recorded; only the failed candidate is cleaned up. // The canonical ID, state path, route key, and domain are stable across // updates — only the upstream container moves. -func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { +func (m *Manager) deployLocked(ctx context.Context, cfg DeployConfig) error { if cfg.TTL == 0 { cfg.TTL = 72 * time.Hour } @@ -439,6 +475,33 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { if err != nil { return err } + if cfg.ExpectedGeneration != nil { + actual := uint64(0) + if existing != nil { + actual = existing.Generation + if existing.OwnershipID != cfg.OwnershipID { + return fmt.Errorf("preview ownership changed") + } + } + if actual != *cfg.ExpectedGeneration { + return fmt.Errorf("preview generation changed") + } + } + ownership := cfg.OwnershipID + if existing != nil && existing.OwnershipID != "" { + ownership = existing.OwnershipID + } + if ownership == "" { + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return err + } + ownership = hex.EncodeToString(id[:]) + } + generation, err := m.reserveGeneration(ctx, cfg.App, existing) + if err != nil { + return err + } // The exposure mode is resolved (and validated) before anything is // mutated, against the record as found: an update inherits whatever // the deploy does not override. @@ -575,7 +638,9 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { // Write state. now := time.Now().UTC() state := State{ - ID: PreviewID(cfg.App, cfg.Branch), + ID: PreviewID(cfg.App, cfg.Branch), + OwnershipID: ownership, Generation: generation, UpdatedAt: &now, + OperationKey: cfg.OperationKey, ExecutionBindingDigest: cfg.ExecutionBindingDigest, SourceRevision: cfg.SourceRevision, Branch: cfg.Branch, Repo: cfg.Repo, Route: routeApp, @@ -589,8 +654,22 @@ func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { HTTPOnly: exp.httpOnly, AllowIPs: exp.allowIPs, } + state.ImageDigest, err = m.docker.ContainerImageDigest(ctx, containerName) + if err != nil { + return publicationUnknown(err) + } if err := m.writeRecord(ctx, &state, previewStatePath(cfg.App, cfg.Branch)); err != nil { - return fmt.Errorf("writing preview state: %w", err) + actual, readErr := m.readRecord(context.WithoutCancel(ctx), previewStatePath(cfg.App, cfg.Branch)) + if readErr == nil && actual != nil && actual.Generation == generation && actual.OwnershipID == ownership && actual.OperationKey == cfg.OperationKey { + return publicationCommitted(err) + } + return publicationUnknown(errors.Join(err, readErr)) + } + + if existingPath != "" && existingPath != previewStatePath(cfg.App, cfg.Branch) { + if _, err := m.exec.Run(ctx, "rm -f -- "+ssh.ShellQuote(existingPath)); err != nil { + return fmt.Errorf("retiring old preview authority: %w", err) + } } // Retire the predecessor — strictly AFTER the route serves the @@ -695,7 +774,7 @@ func (m *Manager) List(ctx context.Context, app string) ([]State, error) { } // Destroy tears down a preview environment. -func (m *Manager) Destroy(ctx context.Context, app, branch string) error { +func (m *Manager) destroyLocked(ctx context.Context, app, branch string, expected *State) error { s, path, err := m.resolveRecord(ctx, app, branch, "") if err != nil { return err @@ -704,15 +783,48 @@ func (m *Manager) Destroy(ctx context.Context, app, branch string) error { return nil // no preview to destroy } - // Stop and remove container. - m.docker.Stop(ctx, s.Container, 5) - m.docker.Remove(ctx, s.Container) - - // Remove Caddy route (keyed by the record's own era). - m.caddy.RemoveRoute(ctx, previewRouteKey(app, s)) + if expected == nil || s.OwnershipID != expected.OwnershipID || s.Generation != expected.Generation || !sameUpdatedAt(s.UpdatedAt, expected.UpdatedAt) { + return fmt.Errorf("preview ownership/generation changed; refusing teardown") + } + // Preserve authority until every required teardown effect succeeds. + if err := m.caddy.RemoveRoute(ctx, previewRouteKey(app, s)); err != nil { + return fmt.Errorf("removing preview route: %w", err) + } + // Reconcile route absence into the active Caddy config even on a retry. + if _, present, err := m.caddy.ReadManagedBlock(ctx, previewRouteKey(app, s)); err != nil || present { + return fmt.Errorf("preview route absence unproven: %v", err) + } + if _, err := m.exec.Run(ctx, "docker exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile"); err != nil { + return fmt.Errorf("activating preview route absence: %w", err) + } + container, err := m.previewContainer(ctx, s.Container) + if err != nil { + return err + } + if container != nil { + if container.State == "running" { + if err := m.docker.Stop(ctx, container.ID, 5); err != nil { + observed, inspectErr := m.previewContainer(ctx, s.Container) + if inspectErr != nil || (observed != nil && observed.State == "running") { + return errors.Join(err, inspectErr) + } + } + } + if err := m.docker.Remove(ctx, container.ID); err != nil { + observed, inspectErr := m.previewContainer(ctx, s.Container) + if inspectErr != nil || observed != nil { + return errors.Join(err, inspectErr) + } + } + observed, inspectErr := m.previewContainer(ctx, s.Container) + if inspectErr != nil || observed != nil { + return fmt.Errorf("preview container absence unproven: %v", inspectErr) + } + } - // Remove state file. - m.exec.Run(ctx, "rm -f -- "+path) + if _, err := m.exec.Run(ctx, "rm -f -- "+ssh.ShellQuote(path)); err != nil { + return fmt.Errorf("removing preview authority: %w", err) + } fmt.Fprintf(m.out, "Destroyed preview for branch %q\n", branch) return nil @@ -729,18 +841,20 @@ func (m *Manager) Prune(ctx context.Context, app string) (int, error) { now := time.Now().UTC() pruned := 0 + var failures error for _, p := range previews { if !now.After(p.ExpiresAt) { continue } - if err := m.Destroy(ctx, app, p.Branch); err != nil { + if err := m.CompareDestroy(ctx, app, p.Branch, p.OwnershipID, p.Generation, p.UpdatedAt); err != nil { + failures = errors.Join(failures, err) fmt.Fprintf(m.out, "Warning: failed to prune preview %s: %v\n", p.Branch, err) continue } fmt.Fprintf(m.out, "Pruned expired preview %q (%s)\n", p.Branch, p.Domain) pruned++ } - return pruned, nil + return pruned, failures } // PruneAll removes expired previews across EVERY app on the target server, @@ -783,3 +897,234 @@ func (m *Manager) PruneAll(ctx context.Context) (int, error) { } return total, errors.Join(errs...) } + +// Serialize preview updates and teardown with the app's existing renewable fence. +func (m *Manager) Deploy(ctx context.Context, cfg DeployConfig) error { + existing, _, err := m.resolveRecord(ctx, cfg.App, cfg.Branch, cfg.Repo) + if err != nil { + return err + } + if _, err := resolveExposure(cfg, existing); err != nil { + return err + } + locked, release, err := m.locked(ctx, cfg.App) + if err != nil { + return err + } + defer release() + return locked.deployLocked(ctx, cfg) +} +func (m *Manager) Destroy(ctx context.Context, app, branch string) error { + existing, _, err := m.resolveRecord(ctx, app, branch, "") + if err != nil { + return err + } + if existing == nil { + return nil + } + locked, release, err := m.locked(ctx, app) + if err != nil { + return err + } + defer release() + return locked.destroyLocked(ctx, app, branch, existing) +} +func (m *Manager) locked(ctx context.Context, app string) (*Manager, func(), error) { + if _, err := m.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(deploymentsDir+"/"+app)); err != nil { + return nil, nil, err + } + lock, err := state.AcquireLockFenced(ctx, m.exec, app) + if err != nil { + return nil, nil, err + } + lock.StartRenewal(m.exec) + exec := &previewExecutor{Executor: m.exec, lock: lock} + manager := NewManager(exec, m.out) + manager.healthInterval = m.healthInterval + manager.healthTimeout = m.healthTimeout + return manager, func() { state.ReleaseLockFenced(m.exec, lock, app) }, nil +} + +type previewExecutor struct { + ssh.Executor + lock *state.Lock +} + +func (e *previewExecutor) Run(ctx context.Context, cmd string) (string, error) { + return e.lock.Guarded(ctx, e.Executor, cmd) +} +func (e *previewExecutor) Upload(ctx context.Context, r io.Reader, path, mode string) error { + temporary := path + ".preview-" + e.lock.Owner() + if err := e.Executor.Upload(ctx, r, temporary, mode); err != nil { + return err + } + _, err := e.lock.Guarded(ctx, e.Executor, "mv -f -- "+ssh.ShellQuote(temporary)+" "+ssh.ShellQuote(path)) + return err +} + +func (m *Manager) previewContainer(ctx context.Context, name string) (*docker.Container, error) { + output, err := m.exec.Run(ctx, `docker ps --all --no-trunc --format '{"ID":{{json .ID}},"Names":{{json .Names}},"State":{{json .State}},"Labels":{{json .Labels}}}'`) + if err != nil { + return nil, fmt.Errorf("preview inventory unknown: %w", err) + } + containers, err := docker.ParseContainers(output) + if err != nil { + return nil, err + } + for _, container := range containers { + if container.Name == name { + if container.ID == "" { + return nil, fmt.Errorf("preview immutable ID missing") + } + return &container, nil + } + } + return nil, nil +} + +func publicationUnknown(err error) error { return &state.PublicationError{Unknown: true, Err: err} } +func publicationCommitted(err error) error { return &state.PublicationError{Committed: true, Err: err} } + +// sameUpdatedAt compares two optional change tokens: absent equals absent, +// and any present token must match exactly. +func sameUpdatedAt(a, b *time.Time) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return a.Equal(*b) +} + +func (m *Manager) reserveGeneration(ctx context.Context, app string, existing *State) (uint64, error) { + if _, err := m.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(previewDir(app))); err != nil { + return 0, err + } + path := previewDir(app) + "/.generation" + data, present, err := state.ReadRemoteFile(ctx, m.exec, path) + if err != nil { + return 0, err + } + var generation uint64 + if present { + generation, err = strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, err + } + } + if existing != nil && existing.Generation > generation { + generation = existing.Generation + } + if generation == ^uint64(0) { + return 0, fmt.Errorf("preview generation exhausted") + } + generation++ + if err = state.WriteAtomicPrivate(ctx, m.exec, path, strings.NewReader(strconv.FormatUint(generation, 10))); err != nil { + return 0, err + } + return generation, nil +} + +// CompareDestroy retains the listed identity through acquisition, then reads +// actual authority under the same renewable lease used by preview renewal. +// The expected triple IS the comparison: a preview renewed between the +// listing and this call rewrites ownership/generation/updated_at and is +// refused before any route, container or authority effect. Legacy records +// compare their all-empty triple exactly — any renewal stamps canonical +// fields, so a stale listing can never authorize teardown of a record that +// changed. The trigger path additionally requires a nonzero observed +// generation at request decode. +func (m *Manager) CompareDestroy(ctx context.Context, app, branch, ownership string, generation uint64, updatedAt *time.Time) error { + locked, release, err := m.locked(ctx, app) + if err != nil { + return err + } + defer release() + return locked.CompareDestroyHeld(ctx, app, branch, ownership, generation, updatedAt) +} +func (m *Manager) CompareDestroyHeld(ctx context.Context, app, branch, ownership string, generation uint64, updatedAt *time.Time) error { + actual, _, err := m.resolveRecord(ctx, app, branch, "") + if err != nil { + return err + } + // An absent authority alone does not prove teardown: the durable adapter + // must retain the previous effect evidence and prove route/container absence. + if actual == nil { + return fmt.Errorf("preview authority absent; teardown proof required") + } + if actual.OwnershipID != ownership || actual.Generation != generation || !sameUpdatedAt(actual.UpdatedAt, updatedAt) { + return fmt.Errorf("preview ownership/generation changed; refusing teardown") + } + return m.destroyLocked(ctx, app, branch, actual) +} + +// Observe reads the real owning record, never a guessed generation. +func (m *Manager) Observe(ctx context.Context, app, branch string) (*State, error) { + s, _, err := m.resolveRecord(ctx, app, branch, "") + return s, err +} + +// WithLease lets the trigger adapter and preview effects share ONE lease. +func (m *Manager) WithLease(lock *state.Lock) *Manager { + return NewManager(&previewExecutor{Executor: m.exec, lock: lock}, m.out) +} +func (m *Manager) DeployHeld(ctx context.Context, cfg DeployConfig) error { + return m.deployLocked(ctx, cfg) +} +func (e *previewExecutor) RunInput(ctx context.Context, cmd string, input io.Reader) error { + return e.Executor.RunInput(ctx, e.lock.GuardPrefix()+cmd, input) +} +func (e *previewExecutor) RunStream(ctx context.Context, cmd string, out, stderr io.Writer) error { + return e.Executor.RunStream(ctx, e.lock.GuardPrefix()+cmd, out, stderr) +} + +// ProveDestroyed reconciles from the immutable pre-effect evidence retained +// in the operation journal. Missing authority without this evidence is unknown. +func (m *Manager) ProveDestroyed(ctx context.Context, app string, admitted State) (bool, error) { + actual, _, err := m.resolveRecord(ctx, app, admitted.Branch, admitted.Repo) + if err != nil { + return false, err + } + if actual != nil { + if actual.OwnershipID != admitted.OwnershipID || actual.Generation != admitted.Generation || !sameUpdatedAt(actual.UpdatedAt, admitted.UpdatedAt) { + return false, fmt.Errorf("preview renewed; retained teardown evidence cannot authorize destruction") + } + if _, ok := m.exec.(*previewExecutor); !ok { + return false, fmt.Errorf("teardown reconciliation requires app lease") + } + if err := m.destroyLocked(ctx, app, admitted.Branch, &admitted); err != nil { + return false, err + } + } + if admitted.OwnershipID == "" || admitted.Generation == 0 { + return false, fmt.Errorf("missing admitted teardown evidence") + } + if _, present, err := m.caddy.ReadManagedBlock(ctx, previewRouteKey(app, &admitted)); err != nil || present { + return false, fmt.Errorf("route absence unproven: %v", err) + } + if _, err := m.exec.Run(ctx, "docker exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile"); err != nil { + return false, err + } + container, err := m.previewContainer(ctx, admitted.Container) + if err != nil { + return false, err + } + return container == nil, nil +} + +func (m *Manager) ProveServing(ctx context.Context, app string, authority State) error { + container, err := m.previewContainer(ctx, authority.Container) + if err != nil { + return err + } + if container == nil || container.State != "running" || container.ID == "" { + return fmt.Errorf("preview serving container unproven") + } + digest, err := m.docker.ContainerImageDigest(ctx, container.ID) + if err != nil || digest != authority.ImageDigest { + return errors.Join(fmt.Errorf("preview immutable image mismatch"), err) + } + route, present, err := m.caddy.ReadManagedBlock(ctx, previewRouteKey(app, &authority)) + if err != nil || !present || !strings.Contains(route, authority.Container) { + return errors.Join(fmt.Errorf("preview serving route unproven"), err) + } + return nil +} diff --git a/internal/preview/preview_test.go b/internal/preview/preview_test.go index 42b9211..93accc7 100644 --- a/internal/preview/preview_test.go +++ b/internal/preview/preview_test.go @@ -45,13 +45,13 @@ func TestPreviewIDGolden(t *testing.T) { tests := []struct { app, branch, want string }{ - {"myapp", "feature/login", "myapp-p-08e81639"}, - {"myapp", "feature-login", "myapp-p-cb4bdf9a"}, - {"myapp", "main", "myapp-p-563059ce"}, - {"myapp", "old-feature", "myapp-p-491218d3"}, - {"myapp", "active-feature", "myapp-p-9fdbab8f"}, - {"market-eval", "feature/login", "market-eval-p-a575aaf7"}, - {"market-eval", "feature-login", "market-eval-p-84d280a4"}, + {"myapp", "feature/login", "myapp-p-08e8163960152ff19d4fc9c432d3270b"}, + {"myapp", "feature-login", "myapp-p-cb4bdf9a7dccde9a7efb87f30130dd51"}, + {"myapp", "main", "myapp-p-563059ce85fc40eb503e69fbd8d6e5ac"}, + {"myapp", "old-feature", "myapp-p-491218d3875552fa51c5745cd295c065"}, + {"myapp", "active-feature", "myapp-p-9fdbab8f86d005409fbb6ee5161f2bca"}, + {"market-eval", "feature/login", "market-eval-p-a575aaf7f4e9a0cdff3aa848bdca42d8"}, + {"market-eval", "feature-login", "market-eval-p-84d280a492101709ac6c9dfaddfb1bcb"}, } for _, tt := range tests { if got := PreviewID(tt.app, tt.branch); got != tt.want { @@ -93,9 +93,11 @@ func TestPreviewBranchIdentityIsDistinct(t *testing.T) { } func TestDeploy(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "mkdir -p /deployments/myapp/previews", Output: ""}, ssh.MockCommand{Match: "cat /deployments/myapp/previews/feature-login.json", Output: "", Err: nil}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker rm", Output: ""}, ssh.MockCommand{Match: "ss -tln", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, ssh.MockCommand{Match: "docker inspect -f '{{range $p", Output: "80/tcp"}, @@ -127,13 +129,13 @@ func TestDeploy(t *testing.T) { } output := buf.String() - if !bytes.Contains([]byte(output), []byte("preview-feature-login-08e81639.myapp.com")) { + if !bytes.Contains([]byte(output), []byte("preview-feature-login-08e8163960152ff19d4fc9c432d3270b.myapp.com")) { t.Errorf("expected preview domain in output, got: %s", output) } } func TestList_Empty(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "ls", Output: "", Err: nil}, ) @@ -163,7 +165,7 @@ func TestPrune_OnlyDestroysExpired(t *testing.T) { freshJSON := fmt.Sprintf(`{"branch":"active-feature","domain":"preview-active-feature.myapp.com","port":49201,"container":"myapp-preview-active-feature-v2","image":"myapp:v2","created_at":"2020-01-01T00:00:00Z","expires_at":%q}`, "2099-01-01T00:00:00Z") - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", Output: "/deployments/myapp/previews/old-feature.json\n/deployments/myapp/previews/active-feature.json"}, ssh.MockCommand{Match: "cat /deployments/myapp/previews/old-feature.json", Output: expiredJSON}, @@ -205,11 +207,11 @@ func TestPreviewDomain(t *testing.T) { tests := []struct { app, branch, domain, want string }{ - {"myapp", "feature/login", "myapp.com", "preview-feature-login-08e81639.myapp.com"}, - {"myapp", "main", "example.com", "preview-main-563059ce.example.com"}, + {"myapp", "feature/login", "myapp.com", "preview-feature-login-08e8163960152ff19d4fc9c432d3270b.myapp.com"}, + {"myapp", "main", "example.com", "preview-main-563059ce85fc40eb503e69fbd8d6e5ac.example.com"}, // A 70-char slug must truncate so the whole DNS label stays <= 63: // "preview-" (8) + 46 chars + "-" + 8 hex = 63. - {"myapp", strings.Repeat("a", 70) + "/x", "myapp.com", "preview-" + strings.Repeat("a", 46) + "-54119e3a.myapp.com"}, + {"myapp", strings.Repeat("a", 70) + "/x", "myapp.com", "preview-" + strings.Repeat("a", 22) + "-54119e3a866a54584c354461c6a4e1e7.myapp.com"}, } for _, tt := range tests { got := previewDomain(tt.app, tt.branch, tt.domain) @@ -232,6 +234,8 @@ func TestPreviewDomain(t *testing.T) { func previewDeployMocks() []ssh.MockCommand { return []ssh.MockCommand{ ssh.MockCommand{Match: "mkdir -p /deployments/myapp/previews", Output: ""}, + ssh.MockCommand{Match: "docker stop", Output: ""}, + ssh.MockCommand{Match: "docker rm", Output: ""}, ssh.MockCommand{Match: "ss -tln", Output: ""}, ssh.MockCommand{Match: "docker run", Output: "abc123"}, ssh.MockCommand{Match: "docker inspect -f '{{range $p", Output: "80/tcp"}, @@ -245,8 +249,8 @@ func previewDeployMocks() []ssh.MockCommand { } const ( - loginIDHex = "08e81639" // previewIDHex("myapp", "feature/login") - dashIDHex = "cb4bdf9a" // previewIDHex("myapp", "feature-login") + loginIDHex = "08e8163960152ff19d4fc9c432d3270b" // previewIDHex("myapp", "feature/login") + dashIDHex = "cb4bdf9a7dccde9a7efb87f30130dd51" // previewIDHex("myapp", "feature-login") loginBranch = "feature/login" dashBranch = "feature-login" ) @@ -272,7 +276,7 @@ func mustDeploy(t *testing.T, mgr *Manager, cfg DeployConfig) { func callsContaining(mock *ssh.MockExecutor, needle string) []string { var found []string for _, c := range mock.Calls { - if strings.Contains(c, needle) { + if strings.Contains(c, needle) && !strings.HasPrefix(c, "grep -q ") { found = append(found, c) } } @@ -283,7 +287,7 @@ func callsContaining(mock *ssh.MockExecutor, needle string) []string { // deploy side by side with distinct state records, containers, routes and // domains — neither deploy destroys or blocks the other. func TestDeployCoexistence(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -353,7 +357,7 @@ func TestDeployCoexistence(t *testing.T) { // Updating one preview of a colliding pair must not touch the other's // record or container. func TestDeployUpdateOneLeavesOther(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -379,7 +383,7 @@ func TestDeployUpdateOneLeavesOther(t *testing.T) { // Destroying one preview of a colliding pair leaves the other fully // intact. func TestDestroyOneLeavesOther(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -414,7 +418,7 @@ func TestDestroyOneLeavesOther(t *testing.T) { // Prune expires one canonical preview of a colliding pair and leaves the // other running. func TestPruneExpiresOneLeavesOther(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", append([]ssh.MockCommand{ ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", Output: previewStatePath("myapp", loginBranch) + "\n" + previewStatePath("myapp", dashBranch)}, @@ -476,7 +480,7 @@ func seedCaddyfileWithRoute(mock *ssh.MockExecutor, key, host string) { // down the artifacts the record actually names (the old slug-keyed // container and route), and deploys fresh canonical-keyed artifacts. func TestLegacyAdoptionDeploy(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -532,7 +536,7 @@ func TestLegacyAdoptionDeploy(t *testing.T) { // DIFFERENT branch. Deploy must refuse with an ambiguous-resource error // naming both branches and the record path, and must not mutate anything. func TestLegacyCollisionDeployIsAmbiguous(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -563,7 +567,7 @@ func TestLegacyCollisionDeployIsAmbiguous(t *testing.T) { if got := string(mock.Files[legacyPath]); got != string(legacy) { t.Errorf("ambiguous legacy record was mutated:\nbefore: %s\nafter: %s", legacy, got) } - if calls := mock.Calls; len(calls) > 2 { // the two record reads only + if calls := mock.Calls; len(calls) > 3 { // the two record reads only t.Errorf("ambiguous legacy record must abort before any mutation, calls: %v", calls) } } @@ -571,7 +575,7 @@ func TestLegacyCollisionDeployIsAmbiguous(t *testing.T) { // Destroy hits the same ambiguity wall: it must refuse, not guess, and // leave the record intact. func TestLegacyCollisionDestroyIsAmbiguous(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -598,7 +602,7 @@ func TestLegacyCollisionDestroyIsAmbiguous(t *testing.T) { // Repo provenance participates in legacy adoption when both sides record // one: same branch, different repo → ambiguous, untouched. func TestLegacyRepoMismatchIsAmbiguous(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) var buf bytes.Buffer mgr := NewManager(mock, &buf) @@ -624,9 +628,10 @@ func TestLegacyRepoMismatchIsAmbiguous(t *testing.T) { // this branch once this branch has its own canonical record: the legacy // file belongs to that branch and is left in place untouched. func TestLegacyOtherBranchNotBlocked(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", previewDeployMocks()...) + mock := previewMockExecutor("1.2.3.4", previewDeployMocks()...) + inventory := newPreviewInventoryExecutor(mock, "myapp-preview-feature-login-v9") var buf bytes.Buffer - mgr := NewManager(mock, &buf) + mgr := NewManager(inventory, &buf) // This branch's canonical record exists (a modern deploy happened). mustDeploy(t, mgr, deployCfg(loginBranch, "v1")) @@ -646,14 +651,15 @@ func TestLegacyOtherBranchNotBlocked(t *testing.T) { } // And destroying THAT branch still finds and removes its legacy record. + legacyID := inventory.containers["myapp-preview-feature-login-v9"] if err := mgr.Destroy(context.Background(), "myapp", dashBranch); err != nil { t.Fatalf("Destroy of the legacy branch: %v", err) } if _, ok := mock.Files[legacyPath]; ok { t.Errorf("legacy record for %s not removed by its own destroy", dashBranch) } - if len(callsContaining(mock, "docker stop -t 5 'myapp-preview-feature-login-v9'")) != 1 { - t.Errorf("legacy branch's container not stopped via its stored name: %v", callsContaining(mock, "docker stop")) + if len(callsContaining(mock, "docker stop -t 5 '"+legacyID+"'")) != 1 { + t.Errorf("legacy branch's container not stopped by its immutable ID: %v", callsContaining(mock, "docker stop")) } } @@ -661,7 +667,7 @@ func TestLegacyOtherBranchNotBlocked(t *testing.T) { func TestListIncludesLegacyAndCanonical(t *testing.T) { loginPath := previewStatePath("myapp", loginBranch) legacyPath := legacyPreviewStatePath("myapp", dashBranch) - mock := ssh.NewMockExecutor("1.2.3.4", + mock := previewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "ls /deployments/myapp/previews/*.json", Output: loginPath + "\n" + legacyPath}, ) @@ -697,11 +703,11 @@ func TestProbeTCP_RequiresLiveListener(t *testing.T) { if !ok { t.Fatal("TCPProbeCommand rejected localhost:8080") } - live := ssh.NewMockExecutor("1.2.3.4", ssh.MockCommand{Match: want}) + live := previewMockExecutor("1.2.3.4", ssh.MockCommand{Match: want}) if !NewManager(live, io.Discard).probeTCP(context.Background(), 8080) { t.Fatal("probeTCP must pass when the held-connection probe exits 0") } - connectOnly := ssh.NewMockExecutor("1.2.3.4", ssh.MockCommand{Match: "bash -c '` answers from the recorded file state when the mock has // one (the real server re-reads whatever earlier writes left); an // explicit registration still wins for paths the mock has no file for. @@ -586,16 +610,42 @@ func (m *MockExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr } func (m *MockExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + var payload []byte if stdin != nil { data, err := io.ReadAll(stdin) if err != nil { return err } + payload = data m.mu.Lock() m.Inputs = append(m.Inputs, string(data)) m.mu.Unlock() } _, err := m.Run(ctx, cmd) + if strings.Contains(cmd, "authority exceeds bound") && strings.Contains(cmd, "os.replace(tmp,name") { + if err != nil && !strings.HasPrefix(err.Error(), "mock: unexpected command:") { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + for _, registered := range m.commands { + if mockCommandMatches(cmd, registered.Match) && registered.Err != nil { + return registered.Err + } + } + // Evaluate the actual composed holdership guard before publishing. + if _, held, ok := evalFenceGuard(m.Files, cmd); ok && !held { + return fmt.Errorf("exit status 75: TEPLOY_FENCE_LOST") + } + parts := strings.Split(cmd, " ") + quoted := parts[len(parts)-1] + path := strings.Trim(quoted, "'") + if !strings.HasPrefix(path, "/deployments/") { + return fmt.Errorf("invalid atomic fixture path") + } + m.Files[path] = append([]byte(nil), payload...) + return nil + } return err } diff --git a/internal/ssh/result.go b/internal/ssh/result.go index fd6c3ea..673d792 100644 --- a/internal/ssh/result.go +++ b/internal/ssh/result.go @@ -209,6 +209,7 @@ func (b *limitedBuffer) Write(p []byte) (int, error) { if b.limit <= 0 { b.limit = DefaultResultLimit } + originalLen := len(p) room := b.limit - int64(b.buf.Len()) if room <= 0 { b.overflow = true @@ -218,7 +219,8 @@ func (b *limitedBuffer) Write(p []byte) (int, error) { b.overflow = true p = p[:room] } - return b.buf.Write(p) + _, err := b.buf.Write(p) + return originalLen, err } func (b *limitedBuffer) bytes() []byte { diff --git a/internal/ssh/result_test.go b/internal/ssh/result_test.go index 8b69939..4f4ec00 100644 --- a/internal/ssh/result_test.go +++ b/internal/ssh/result_test.go @@ -88,14 +88,17 @@ func TestRunDetailed_LocalExecutor_TimedOutVsCanceled(t *testing.T) { // beyond the limit is dropped and flagged, never buffered. func TestRunDetailed_LocalExecutor_Truncated(t *testing.T) { e := NewLocalExecutor() - res := runDetailedWithLimit(context.Background(), e, "yes a", nil, 64) + res := runDetailedWithLimit(context.Background(), e, "printf '%0100d' 0", nil, 64) + if res.ExitCode != 0 || res.Err != nil { + t.Fatalf("producer outcome changed: %+v", res) + } if !res.Truncated { t.Fatal("Truncated must be set when the capture limit is hit") } if int64(len(res.Stdout)) > 64 { t.Fatalf("captured %d bytes, limit was 64", len(res.Stdout)) } - if !strings.HasPrefix(string(res.Stdout), "a") { + if !strings.HasPrefix(string(res.Stdout), "0") { t.Fatalf("the FIRST bytes must be kept, got %q", res.Stdout) } } @@ -237,3 +240,16 @@ func (f *fakeFallbackExecutor) Upload(ctx context.Context, content io.Reader, re func (f *fakeFallbackExecutor) Close() error { return nil } func (f *fakeFallbackExecutor) Host() string { return "fake" } func (f *fakeFallbackExecutor) User() string { return "root" } + +func TestLimitedBufferDiscardsTailWithoutShortWrite(t *testing.T) { + b := &limitedBuffer{limit: 4} + for _, input := range []string{"abcdef", "gh"} { + n, err := b.Write([]byte(input)) + if n != len(input) || err != nil { + t.Fatalf("Write = %d, %v", n, err) + } + } + if string(b.bytes()) != "abcd" || !b.overflow { + t.Fatalf("capture = %q, overflow=%v", b.bytes(), b.overflow) + } +} diff --git a/internal/state/atomic_private.go b/internal/state/atomic_private.go new file mode 100644 index 0000000..4fdf8af --- /dev/null +++ b/internal/state/atomic_private.go @@ -0,0 +1,46 @@ +package state + +import ( + "context" + "fmt" + "io" + + "github.com/useteploy/teploy/internal/ssh" +) + +// WriteAtomicPrivate fsyncs a private sibling then atomically replaces the +// authority and fsyncs its directory. The caller supplies a fenced executor. +// A reply failure after rename is deliberately not interpreted as rollback. +func WriteAtomicPrivate(ctx context.Context, exec ssh.Executor, path string, input io.Reader) error { + return exec.RunInput(ctx, "python3 -c "+ssh.ShellQuote(atomicPrivateScript)+" "+ssh.ShellQuote(path), input) +} + +const atomicPrivateScript = `import os,sys,secrets,stat +path=sys.argv[1] +parts=path.split('/') +if not path.startswith('/deployments/') or any(p in ('.','..') for p in parts): raise RuntimeError('invalid authority path') +fd=os.open('/',os.O_RDONLY|os.O_DIRECTORY) +for part in parts[1:-1]: + nextfd=os.open(part,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=fd);os.close(fd);fd=nextfd + s=os.fstat(fd) + if s.st_uid not in (0,os.geteuid()) or s.st_mode & 0o022: raise RuntimeError('unsafe authority directory') +name=parts[-1];tmp='.atomic-'+secrets.token_hex(16) +data=sys.stdin.buffer.read(1048577) +if len(data)>1048576: raise RuntimeError('authority exceeds bound') +candidate=os.open(tmp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=fd) +try: + with os.fdopen(candidate,'wb') as f: f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(tmp,name,src_dir_fd=fd,dst_dir_fd=fd);os.fsync(fd) +finally: + try: os.unlink(tmp,dir_fd=fd) + except FileNotFoundError: pass +` + +// RequireLease rejects accidental reuse of a journal under an unfenced +// executor. It is useful at lifecycle entrypoints which already own a lease. +func RequireLease(lock *Lock) error { + if lock == nil || lock.Owner() == "" { + return fmt.Errorf("durable operation requires an app lease") + } + return nil +} diff --git a/internal/state/campaign_test.go b/internal/state/campaign_test.go new file mode 100644 index 0000000..e4a6985 --- /dev/null +++ b/internal/state/campaign_test.go @@ -0,0 +1,91 @@ +package state + +import ( + "context" + "errors" + "fmt" + "github.com/useteploy/teploy/internal/ssh" + "regexp" + "strings" + "testing" +) + +func TestCampaignPublicationRepairsSidecarWithoutCompensation(t *testing.T) { + s := NewAppliedState(nil, "container", "external", "") + s.CurrentHash = "v2" + s.Generation = 2 + data, err := prepareState(s) + if err != nil { + t.Fatal(err) + } + exec := ssh.NewMockExecutor("test", ssh.MockCommand{Match: "if [ ! -e", Output: "present\n" + string(data)}) + if err := resolvePublication(exec, "app", data, 2, nil, errors.New("second rename failed")); err != nil { + t.Fatal(err) + } + found := false + for _, c := range exec.Calls { + if c == "mv -fT -- '/deployments/app/.generation.tmp-commit' '/deployments/app/.generation'" { + found = true + } + } + if !found { + t.Fatalf("repair not published: %v", exec.Calls) + } + _ = context.Background() +} +func TestCampaignPublicationUnknownPreservesAuthority(t *testing.T) { + exec := ssh.NewMockExecutor("test", ssh.MockCommand{Match: "if [ ! -e", Err: errors.New("transport unavailable")}) + err := resolvePublication(exec, "app", []byte(`{}`), 2, nil, errors.New("lost reply")) + if !PreservePublishedState(err) { + t.Fatalf("unsafe compensation: %v", err) + } +} + +type campaignPartialPublication struct { + *ssh.MockExecutor + failRepair bool +} + +func (e *campaignPartialPublication) Run(ctx context.Context, command string) (string, error) { + if strings.Contains(command, " && mv -f -- ") && strings.Contains(command, "/state.json") { + re := regexp.MustCompile(`mv -f -- '([^']+)' '([^']+/state.json)'`) + match := re.FindStringSubmatch(command) + if len(match) != 3 { + return "", errors.New("unexpected commit shape") + } + e.Files[match[2]] = e.Files[match[1]] + delete(e.Files, match[1]) + return "", errors.New("generation rename failed after authority published") + } + if e.failRepair && strings.Contains(command, "mv -fT -- ") { + return "", errors.New("repair blocked") + } + return e.MockExecutor.Run(ctx, command) +} +func TestCampaignFencedWriterSecondRenameRecovery(t *testing.T) { + for _, failRepair := range []bool{false, true} { + t.Run(fmt.Sprint(failRepair), func(t *testing.T) { + exec := &campaignPartialPublication{MockExecutor: ssh.NewMockExecutor("dummy", ssh.MockCommand{Match: "mkdir /deployments/demo/.lock"}), failRepair: failRepair} + lk, err := AcquireLockFenced(context.Background(), exec, "demo") + if err != nil { + t.Fatal(err) + } + defer ReleaseLockFenced(exec, lk, "demo") + s := NewAppliedState(nil, "container", "external", "") + s.CurrentHash = "v2" + s.Generation = 2 + err = WriteFencedGeneration(context.Background(), exec, "demo", s, lk, 1) + if failRepair { + if !PreservePublishedState(err) { + t.Fatalf("compensation permitted after committed authority: %v", err) + } + } else if err != nil { + t.Fatal(err) + } + actual, readErr := Read(context.Background(), exec, "demo") + if readErr != nil || actual.CurrentHash != "v2" { + t.Fatalf("authority lost: %+v %v", actual, readErr) + } + }) + } +} diff --git a/internal/state/fenced_executor.go b/internal/state/fenced_executor.go new file mode 100644 index 0000000..4a52064 --- /dev/null +++ b/internal/state/fenced_executor.go @@ -0,0 +1,31 @@ +package state + +import ( + "context" + "github.com/useteploy/teploy/internal/ssh" + "io" +) + +// FencedExecutor keeps preparation effects under the caller's renewable lease. +// Upload writes only into already admitted private managed directories. +type FencedExecutor struct { + ssh.Executor + Lock *Lock +} + +func (e *FencedExecutor) UnderFence() bool { return true } +func (e *FencedExecutor) Run(ctx context.Context, command string) (string, error) { + return e.Lock.Guarded(ctx, e.Executor, command) +} +func (e *FencedExecutor) RunInput(ctx context.Context, command string, input io.Reader) error { + return e.Executor.RunInput(ctx, e.Lock.GuardPrefix()+command, input) +} +func (e *FencedExecutor) RunStream(ctx context.Context, command string, out, stderr io.Writer) error { + return e.Executor.RunStream(ctx, e.Lock.GuardPrefix()+command, out, stderr) +} +func (e *FencedExecutor) Upload(ctx context.Context, input io.Reader, path, mode string) error { + if err := e.Lock.Check(ctx, e.Executor); err != nil { + return err + } + return e.Executor.Upload(ctx, input, path, mode) +} diff --git a/internal/state/lock.go b/internal/state/lock.go index 1253808..7107ff2 100644 --- a/internal/state/lock.go +++ b/internal/state/lock.go @@ -459,8 +459,8 @@ func writeFenced(ctx context.Context, exec ssh.Executor, app string, s *AppState if GenerationFenced(err) { return fmt.Errorf("%w: refusing to commit generation %d for %s over a newer committed generation (expected predecessor %d)", ErrGenerationFenced, s.Generation, app, *expectedGeneration) } - // Leave the temp files for diagnosis; they are inert. - return err + // Re-observe authority before any caller can compensate. + return resolvePublication(exec, app, data, s.Generation, lk, err) } return nil } diff --git a/internal/state/publication.go b/internal/state/publication.go new file mode 100644 index 0000000..1ce3a69 --- /dev/null +++ b/internal/state/publication.go @@ -0,0 +1,72 @@ +package state + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/useteploy/teploy/internal/ssh" + "time" +) + +// PublicationError means an error was reported after authority may have changed. +// Destructive compensation is unsafe when Committed or Unknown is true. +type PublicationError struct { + Committed, Unknown bool + Err error +} + +func (e *PublicationError) Error() string { + return fmt.Sprintf("state publication needs reconciliation (committed=%t unknown=%t): %v", e.Committed, e.Unknown, e.Err) +} +func (e *PublicationError) Unwrap() error { return e.Err } +func PreservePublishedState(err error) bool { + var e *PublicationError + return errors.As(err, &e) && (e.Committed || e.Unknown) +} + +func resolvePublication(exec ssh.Executor, app string, data []byte, generation uint64, lk *Lock, cause error) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + actual, present, err := ReadRemoteFile(ctx, exec, fmt.Sprintf("%s/%s/state.json", deploymentsDir, app)) + if err != nil { + return &PublicationError{Unknown: true, Err: errors.Join(cause, err)} + } + if !present { + return cause + } + if !bytes.Equal(bytes.TrimSpace(actual), bytes.TrimSpace(data)) { + // A different, parseable predecessor proves this write did not publish. + prior, readErr := Read(ctx, exec, app) + if readErr == nil && prior != nil && prior.Generation < generation { + return cause + } + return &PublicationError{Unknown: true, Err: cause} + } + if lk != nil { + // Repair only while our lease still protects this exact authority. + if err := lk.Check(ctx, exec); err != nil { + return &PublicationError{Committed: true, Err: errors.Join(cause, err)} + } + } + if err := writeGenerationSidecarFenced(ctx, exec, app, generation, lk); err != nil { + return &PublicationError{Committed: true, Err: errors.Join(cause, err)} + } + return nil +} + +// GenerationExpectation binds application of a reviewed plan to its baseline. +type GenerationExpectation struct { + Exists bool + Generation uint64 +} + +func (e *GenerationExpectation) Check(app string, current *AppState) error { + if e == nil { + return nil + } + if e.Exists != (current != nil) || (current != nil && current.Generation != e.Generation) { + return fmt.Errorf("plan baseline changed for %s; review a fresh plan", app) + } + return nil +} diff --git a/internal/state/state.go b/internal/state/state.go index bd12ad7..b2bf21a 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -132,15 +132,17 @@ const staleHealLockTTL = 2 * time.Minute // ReleaseMetadata identifies an applied release. PreviousRelease retains the // one-level rollback target's identity without creating a second state store. type ReleaseMetadata struct { - Hash string `json:"hash,omitempty"` - ManifestSHA256 string `json:"manifest_sha256,omitempty"` - SourceRevision string `json:"source_revision,omitempty"` - ImageRef string `json:"image_ref,omitempty"` - ImageDigest string `json:"image_digest,omitempty"` - UpdatedAt time.Time `json:"updated_at,omitempty"` - OperationID string `json:"operation_id,omitempty"` - Generation uint64 `json:"generation,omitempty"` - AppliedManifest json.RawMessage `json:"applied_manifest,omitempty"` + TriggerOperationKey string `json:"trigger_operation_key,omitempty"` + ExecutionBindingDigest string `json:"execution_binding_digest,omitempty"` + Hash string `json:"hash,omitempty"` + ManifestSHA256 string `json:"manifest_sha256,omitempty"` + SourceRevision string `json:"source_revision,omitempty"` + ImageRef string `json:"image_ref,omitempty"` + ImageDigest string `json:"image_digest,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` + OperationID string `json:"operation_id,omitempty"` + Generation uint64 `json:"generation,omitempty"` + AppliedManifest json.RawMessage `json:"applied_manifest,omitempty"` } // MigrationMetadata records the first import from the legacy key=value file. @@ -152,17 +154,19 @@ type MigrationMetadata struct { // stored at /deployments//state.json; the old key=value state file is // read only when state.json does not yet exist. type AppState struct { - SchemaVersion int `json:"schema_version"` - DeploymentType string `json:"deployment_type"` - IngressMode string `json:"ingress_mode"` - Domain string `json:"domain,omitempty"` - UpdatedAt time.Time `json:"updated_at"` - ManifestSHA256 string `json:"manifest_sha256,omitempty"` - SourceRevision string `json:"source_revision,omitempty"` - ImageRef string `json:"image_ref,omitempty"` - ImageDigest string `json:"image_digest,omitempty"` - OperationID string `json:"operation_id"` - Generation uint64 `json:"generation"` + TriggerOperationKey string `json:"trigger_operation_key,omitempty"` + ExecutionBindingDigest string `json:"execution_binding_digest,omitempty"` + SchemaVersion int `json:"schema_version"` + DeploymentType string `json:"deployment_type"` + IngressMode string `json:"ingress_mode"` + Domain string `json:"domain,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + ManifestSHA256 string `json:"manifest_sha256,omitempty"` + SourceRevision string `json:"source_revision,omitempty"` + ImageRef string `json:"image_ref,omitempty"` + ImageDigest string `json:"image_digest,omitempty"` + OperationID string `json:"operation_id"` + Generation uint64 `json:"generation"` AppliedManifest json.RawMessage `json:"applied_manifest,omitempty"` PreviousRelease *ReleaseMetadata `json:"previous_release,omitempty"` @@ -383,15 +387,17 @@ func (s *AppState) ReleaseMetadata() *ReleaseMetadata { return nil } return &ReleaseMetadata{ - Hash: s.CurrentHash, - ManifestSHA256: s.ManifestSHA256, - SourceRevision: s.SourceRevision, - ImageRef: s.ImageRef, - ImageDigest: s.ImageDigest, - UpdatedAt: s.UpdatedAt, - OperationID: s.OperationID, - Generation: s.Generation, - AppliedManifest: cloneRawMessage(s.AppliedManifest), + TriggerOperationKey: s.TriggerOperationKey, + ExecutionBindingDigest: s.ExecutionBindingDigest, + Hash: s.CurrentHash, + ManifestSHA256: s.ManifestSHA256, + SourceRevision: s.SourceRevision, + ImageRef: s.ImageRef, + ImageDigest: s.ImageDigest, + UpdatedAt: s.UpdatedAt, + OperationID: s.OperationID, + Generation: s.Generation, + AppliedManifest: cloneRawMessage(s.AppliedManifest), } } @@ -400,6 +406,8 @@ func (s *AppState) ApplyRelease(release *ReleaseMetadata) { if release == nil { return } + s.TriggerOperationKey = release.TriggerOperationKey + s.ExecutionBindingDigest = release.ExecutionBindingDigest s.ManifestSHA256 = release.ManifestSHA256 s.SourceRevision = release.SourceRevision s.ImageRef = release.ImageRef @@ -472,20 +480,26 @@ func Write(ctx context.Context, exec ssh.Executor, app string, s *AppState) erro } path := fmt.Sprintf("%s/%s/state.json", deploymentsDir, app) if err := ssh.UploadAtomic(ctx, exec, bytes.NewReader(data), path, "0644"); err != nil { - return err + return resolvePublication(exec, app, data, s.Generation, nil, err) + } + if err := writeGenerationSidecar(ctx, exec, app, s.Generation); err != nil { + return resolvePublication(exec, app, data, s.Generation, nil, err) } - return writeGenerationSidecar(ctx, exec, app, s.Generation) + return nil } // writeGenerationSidecar publishes the committed generation atomically // (stage + rename) — the targetguard contract's write side. func writeGenerationSidecar(ctx context.Context, exec ssh.Executor, app string, generation uint64) error { + return writeGenerationSidecarFenced(ctx, exec, app, generation, nil) +} +func writeGenerationSidecarFenced(ctx context.Context, exec ssh.Executor, app string, generation uint64, lk *Lock) error { path := GenerationSidecarPath(app) tmp := path + ".tmp-commit" if err := exec.Upload(ctx, strings.NewReader(strconv.FormatUint(generation, 10)+"\n"), tmp, "0644"); err != nil { return fmt.Errorf("staging the committed-generation sidecar for %s: %w", app, err) } - if _, err := exec.Run(ctx, "mv -fT -- "+ssh.ShellQuote(tmp)+" "+ssh.ShellQuote(path)); err != nil { + if _, err := lk.Guarded(ctx, exec, "mv -fT -- "+ssh.ShellQuote(tmp)+" "+ssh.ShellQuote(path)); err != nil { return fmt.Errorf("publishing the committed-generation sidecar for %s: %w", app, err) } return nil diff --git a/internal/template/template.go b/internal/template/template.go index 079721e..48d5f7b 100644 --- a/internal/template/template.go +++ b/internal/template/template.go @@ -170,11 +170,9 @@ func (r *Registry) Fetch(ctx context.Context, name string, vars map[string]strin content = string(body) - // Apply variable substitution. - content = substituteVariables(content, vars) - - // Generate secrets. + // Generate only directives supplied by the template, before user literals. content, generated = GenerateSecrets(content) + content = substituteVariables(content, vars) if vars != nil { if missing := findMissingVariables(content); len(missing) > 0 { @@ -184,6 +182,12 @@ func (r *Registry) Fetch(ctx context.Context, name string, vars map[string]strin if err := yaml.Unmarshal([]byte(content), &doc); err != nil { return "", nil, fmt.Errorf("rendered template %q is not valid YAML: %w", name, err) } + makeTemplateEnvLiteral(&doc) + rendered, marshalErr := yaml.Marshal(&doc) + if marshalErr != nil { + return "", nil, fmt.Errorf("encoding template: %w", marshalErr) + } + content = string(rendered) } return content, generated, nil @@ -488,3 +492,21 @@ func RandomHex(n int) string { } return hex.EncodeToString(b) } + +// Rendered template environment is literal configuration, including user +// passwords named auto or containing dollar signs. Ordinary env keeps its +// interpolation/generation semantics for hand-authored configurations. +func makeTemplateEnvLiteral(node *yaml.Node) { + if node.Kind == yaml.MappingNode { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == "env" { + node.Content[i].Value = "env_literal" + } + makeTemplateEnvLiteral(node.Content[i+1]) + } + } else { + for _, child := range node.Content { + makeTemplateEnvLiteral(child) + } + } +} diff --git a/internal/template/template_test.go b/internal/template/template_test.go index 71ecbd4..70338cd 100644 --- a/internal/template/template_test.go +++ b/internal/template/template_test.go @@ -3,6 +3,7 @@ package template import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -231,7 +232,7 @@ accessories: t.Fatalf("rendered template must be valid YAML: %v\n%s", err, content) } - env := doc["env"].(map[string]interface{}) + env := doc["env_literal"].(map[string]interface{}) for _, key := range []string{"SECRET", "EMBEDDED", "QUOTED", "SINGLE", "URL", "BOOLEANISH"} { got, ok := env[key].(string) if !ok { @@ -263,7 +264,7 @@ accessories: } acc := doc["accessories"].(map[string]interface{})["db"].(map[string]interface{}) - accEnv := acc["env"].(map[string]interface{}) + accEnv := acc["env_literal"].(map[string]interface{}) if got := accEnv["POSTGRES_PASSWORD"]; got != password { t.Errorf("accessory POSTGRES_PASSWORD (with trailing comment) = %#v, want exact value", got) } @@ -289,7 +290,7 @@ func TestFetch_LookalikeValuesStayStrings(t *testing.T) { } var doc struct { - Env map[string]string `yaml:"env"` + Env map[string]string `yaml:"env_literal"` } if err := yaml.Unmarshal([]byte(content), &doc); err != nil { t.Fatalf("parse: %v\n%s", err, content) @@ -394,3 +395,35 @@ func TestRegistry_SizeLimits(t *testing.T) { t.Errorf("expected template size-limit error, got: %v", err) } } + +func TestFetchPreservesDirectiveLikeUserPasswords(t *testing.T) { + for _, password := range []string{"generate", "auto", "p$UNSET_AUDIT", "p${UNSET_AUDIT}"} { + t.Run(password, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "app: demo\ningress: external\nimage: example\nenv:\n PASSWORD: {{password}}\naccessories:\n db:\n image: postgres:17\n env:\n POSTGRES_PASSWORD: {{password}}\n") + })) + defer srv.Close() + reg := NewRegistry() + reg.SetBaseURL(srv.URL) + content, generated, err := reg.Fetch(context.Background(), "demo", map[string]string{"password": password}) + if err != nil { + t.Fatal(err) + } + if len(generated) != 0 { + t.Fatal("user literal generated") + } + var doc struct { + Env map[string]string `yaml:"env_literal"` + Accessories map[string]struct { + Env map[string]string `yaml:"env_literal"` + } `yaml:"accessories"` + } + if err := yaml.Unmarshal([]byte(content), &doc); err != nil { + t.Fatal(err) + } + if doc.Env["PASSWORD"] != password || doc.Accessories["db"].Env["POSTGRES_PASSWORD"] != password { + t.Fatalf("literal mismatch: %s", content) + } + }) + } +} diff --git a/internal/trigger/adapter.go b/internal/trigger/adapter.go new file mode 100644 index 0000000..ae30cee --- /dev/null +++ b/internal/trigger/adapter.go @@ -0,0 +1,180 @@ +package trigger + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "time" + + "github.com/useteploy/teploy/internal/state" +) + +// Receipt survives listener restart and local delivery retention. Pending +// receipts are never pruned. Identity is persisted, not merely its hash. +type Receipt struct { + Request Request `json:"request"` + Evidence json.RawMessage `json:"evidence,omitempty"` + Result Result `json:"result"` + Completed bool `json:"completed"` +} +type Store interface { + Load(context.Context) ([]Receipt, error) + Save(context.Context, Receipt) error +} + +// Observe must prove immutable release/image/private binding and actual +// authority. A nil observation means no completion is proven, not absence of +// effects. It must never convert transport/parse errors to absence. +type Observe func(context.Context, Receipt) (*Result, error) +type Adapter struct { + Store Store + Observe Observe + Evidence func(context.Context, Request) (json.RawMessage, error) +} + +var ErrDebt = errors.New("target operation requires reconciliation before preparation") + +// Run is called AFTER acquiring the renewable app lease and BEFORE any +// preparation. The caller retains the SAME lease until Run returns. +func (a Adapter) Run(ctx context.Context, r Request, effect func(context.Context) error) (Result, error) { + result := Initial(r) + if err := r.Validate(); err != nil { + result.ErrorCode = "invalid_request" + return result, err + } + receipts, err := a.Store.Load(ctx) + if err != nil { + result.Publication = "unknown" + result.ErrorCode = "journal_read_failed" + return result, err + } + for _, prior := range receipts { + if prior.Request.OperationKey == r.OperationKey && !sameIdentity(prior.Request, r) { + result.ErrorCode = "identity_conflict" + return result, fmt.Errorf("receipt identity mismatch") + } + if !prior.Completed && prior.Result.Publication == "not_committed" && prior.Result.Reconciliation == "complete" { + continue + } + if prior.Completed { + if prior.Request.OperationKey == r.OperationKey { + return prior.Result, nil + } + continue + } + observed, readErr := a.Observe(ctx, prior) + if readErr != nil || observed == nil || observed.Reconciliation != "complete" { + result.Publication = "unknown" + result.ErrorCode = "reconciliation_required" + if prior.Request.OperationKey == r.OperationKey && observed != nil { + result = *observed + result.ErrorCode = "reconciliation_required" + } + return result, errors.Join(ErrDebt, readErr) + } + repaired := Receipt{Request: prior.Request, Evidence: prior.Evidence, Result: *observed, Completed: observed.Publication == "committed"} + if err = a.Store.Save(ctx, repaired); err != nil { + result.Publication = "unknown" + result.ErrorCode = "journal_write_failed" + return result, err + } + if prior.Request.OperationKey == r.OperationKey && observed.Publication == "committed" { + return *observed, nil + } + } + pending := Receipt{Request: r, Result: result} + if a.Evidence != nil { + pending.Evidence, err = a.Evidence(ctx, r) + if err != nil { + result.ErrorCode = "authority_admission_failed" + return result, err + } + } + if err = a.Store.Save(ctx, pending); err != nil { + result.ErrorCode = "journal_write_failed" + return result, err + } + // The durable pending frame precedes the callback, including builds, pulls, + // migrations and candidate creation. A crash at any point leaves debt. + effectErr := effect(ctx) + recovery, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + observed, readErr := a.Observe(recovery, pending) + if observed != nil { + result = *observed + } else { + result.Publication = "unknown" + result.Reconciliation = "required" + } + var publication *state.PublicationError + if errors.As(effectErr, &publication) { + if publication.Committed { + result.Publication = "committed" + } else if publication.Unknown { + result.Publication = "unknown" + } + result.Reconciliation = "required" + result.ErrorCode = "publication_reconciliation_required" + } else if readErr != nil { + result.Publication = "unknown" + result.Reconciliation = "required" + result.ErrorCode = "authority_read_failed" + } else if effectErr != nil { + // An ordinary precommit failure may be reported as not_committed only + // when the observer proves the unchanged baseline. Observe owns that proof. + result.ErrorCode = "execution_failed" + } else if observed == nil { + result.ErrorCode = "completion_unproven" + } + completed := result.Publication == "committed" && result.Reconciliation == "complete" + // Even a clean precommit failure is kept as debt until the observer proves + // no effects remain; retries cannot infer safe re-preparation from an error. + if err = a.Store.Save(recovery, Receipt{Request: r, Evidence: pending.Evidence, Result: result, Completed: completed}); err != nil { + result.Reconciliation = "required" + result.ErrorCode = "journal_write_failed" + if result.Publication == "committed" || result.Publication == "unknown" { + return result, &state.PublicationError{Committed: result.Publication == "committed", Unknown: result.Publication == "unknown", Err: errors.Join(effectErr, readErr, err)} + } + return result, errors.Join(effectErr, readErr, err) + } + if effectErr != nil || readErr != nil { + return result, errors.Join(effectErr, readErr) + } + if !completed { + return result, ErrDebt + } + return result, nil +} +func sameIdentity(a, b Request) bool { + a.ExpectedPreviewUpdatedAt = nil + b.ExpectedPreviewUpdatedAt = nil + return reflect.DeepEqual(a, b) +} +func ValidateReceipt(r Receipt) error { + if err := r.Request.Validate(); err != nil { + return err + } + if r.Result.SchemaVersion != 1 || r.Result.OperationKey != r.Request.OperationKey { + return fmt.Errorf("invalid receipt result identity") + } + switch r.Result.Publication { + case "not_started", "not_committed", "committed", "unknown": + default: + return fmt.Errorf("invalid receipt publication") + } + if r.Result.Reconciliation != "required" && r.Result.Reconciliation != "complete" { + return fmt.Errorf("invalid receipt reconciliation") + } + if r.Completed && (r.Result.Publication != "committed" || r.Result.Reconciliation != "complete" || r.Result.AuthorityObservedAt == nil) { + return fmt.Errorf("unproven completed receipt") + } + return nil +} +func EncodeReceipt(r Receipt) ([]byte, error) { + if err := ValidateReceipt(r); err != nil { + return nil, err + } + return json.Marshal(r) +} diff --git a/internal/trigger/adapter_test.go b/internal/trigger/adapter_test.go new file mode 100644 index 0000000..8efbc83 --- /dev/null +++ b/internal/trigger/adapter_test.go @@ -0,0 +1,185 @@ +package trigger + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/useteploy/teploy/internal/state" +) + +// targetJournal is independent target storage: new Adapter instances and +// delivery ledgers have no shared local cache. Its mutex models the app lease +// held by production wrappers, not an imaginary delivery FIFO. +type targetJournal struct { + rows map[string]Receipt + failWrite bool +} + +func (s *targetJournal) Load(context.Context) ([]Receipt, error) { + var rows []Receipt + for _, r := range s.rows { + rows = append(rows, r) + } + return rows, nil +} +func (s *targetJournal) Save(_ context.Context, r Receipt) error { + if s.failWrite { + return errors.New("fsync failed") + } + if s.rows == nil { + s.rows = map[string]Receipt{} + } + s.rows[r.Request.OperationKey] = r + return nil +} +func requestFixture() Request { + r := Request{SchemaVersion: 1, TargetID: strings.Repeat("a", 32) + "/demo", SourceID: "source-stable", RepositoryID: `["generic","https://forge.example","url:https://forge.example/team/repo"]`, Ref: "refs/heads/main", Commit: strings.Repeat("b", 40), ExecutionBindingDigest: strings.Repeat("c", 64), Action: "deploy"} + r.OperationKey = r.Key() + return r +} +func completed(r Request) *Result { + now := time.Now().UTC() + gen := uint64(7) + return &Result{SchemaVersion: 1, OperationKey: r.OperationKey, Publication: "committed", Reconciliation: "complete", Generation: &gen, ReleaseHash: "actual-release", ImageDigest: "sha256:" + strings.Repeat("d", 64), AuthorityObservedAt: &now} +} + +func TestAdapterSixConcurrentTriggersRestartRetentionAndBinding(t *testing.T) { + ctx := context.Background() + store := &targetJournal{} + var lease sync.Mutex + var wg sync.WaitGroup + effects := 0 + actual := map[string]*Result{} + r := requestFixture() + for _, kind := range []string{"manual", "webhook", "schedule", "PR", "CI", "resident-listener"} { + wg.Add(1) + go func(kind string) { + defer wg.Done() + lease.Lock() + defer lease.Unlock() + adapter := Adapter{Store: store, Observe: func(_ context.Context, receipt Receipt) (*Result, error) { + return actual[receipt.Request.OperationKey], nil + }} + if _, err := adapter.Run(ctx, r, func(context.Context) error { + if _, ok := store.rows[r.OperationKey]; !ok { + t.Error("effect before durable admission") + } + effects++ + actual[r.OperationKey] = completed(r) + return nil + }); err != nil { + t.Errorf("%s: %v", kind, err) + } + }(kind) + } + wg.Wait() + if effects != 1 { + t.Fatalf("effects=%d", effects) + } + // No delivery history survives; only the target journal and real authority. + adapter := Adapter{Store: store, Observe: func(_ context.Context, receipt Receipt) (*Result, error) { + return actual[receipt.Request.OperationKey], nil + }} + if _, err := adapter.Run(ctx, r, func(context.Context) error { t.Fatal("restart replay prepared"); return nil }); err != nil { + t.Fatal(err) + } + changed := r + changed.ExecutionBindingDigest = strings.Repeat("e", 64) + changed.OperationKey = changed.Key() + if _, err := adapter.Run(ctx, changed, func(context.Context) error { effects++; actual[changed.OperationKey] = completed(changed); return nil }); err != nil { + t.Fatal(err) + } + if effects != 2 { + t.Fatal("same commit changed binding was swallowed") + } + reused := r + reused.TargetID = strings.Repeat("f", 32) + "/demo" + if err := reused.Validate(); err == nil { + t.Fatal("reused alias identity tamper accepted") + } +} +func TestAdapterCrashPhasesAndUnknownNeverPrepare(t *testing.T) { + for _, phase := range []string{"before-prepare", "after-candidate", "after-route", "after-state-rename"} { + t.Run(phase, func(t *testing.T) { + r := requestFixture() + s := &targetJournal{} + _ = s.Save(context.Background(), Receipt{Request: r, Result: Initial(r)}) + adapter := Adapter{Store: s, Observe: func(context.Context, Receipt) (*Result, error) { + if phase == "after-state-rename" { + return completed(r), nil + } + return nil, errors.New("actual authority unproven") + }} + result, err := adapter.Run(context.Background(), r, func(context.Context) error { t.Fatal("replay prepared before reconciliation"); return nil }) + if phase == "after-state-rename" { + if err != nil || result.Publication != "committed" { + t.Fatal(result, err) + } + } else if err == nil || result.Publication != "unknown" || result.Generation != nil { + t.Fatal(result, err) + } + if len(s.rows) != 1 { + t.Fatal("debt erased") + } + }) + } +} +func TestAdapterPublicationFailuresAndPrecommitHonesty(t *testing.T) { + for _, outcome := range []string{"committed-repair-failed", "unknown-read", "ordinary-health-failed"} { + t.Run(outcome, func(t *testing.T) { + r := requestFixture() + s := &targetJournal{} + effectError := errors.New("health gate refused") + observation := completed(r) + switch outcome { + case "committed-repair-failed": + effectError = &state.PublicationError{Committed: true, Err: errors.New("sidecar fsync")} + case "unknown-read": + effectError = &state.PublicationError{Unknown: true, Err: errors.New("read transport")} + observation = nil + case "ordinary-health-failed": + observation = &Result{SchemaVersion: 1, OperationKey: r.OperationKey, Publication: "not_committed", Reconciliation: "complete"} + } + adapter := Adapter{Store: s, Observe: func(context.Context, Receipt) (*Result, error) { return observation, nil }} + result, err := adapter.Run(context.Background(), r, func(context.Context) error { return effectError }) + if err == nil { + t.Fatal("nonzero lost") + } + if outcome == "ordinary-health-failed" { + if result.Publication != "not_committed" { + t.Fatal(result) + } + } else if result.Reconciliation != "required" || s.rows[r.OperationKey].Completed { + t.Fatal("repair debt deduplicated", result) + } + }) + } +} +func TestStrictBoundedPrivateDecode(t *testing.T) { + r := requestFixture() + good, _ := json.Marshal(r) + if _, err := Decode(strings.NewReader(string(good))); err != nil { + t.Fatal(err) + } + for _, bad := range []string{string(good) + " {}", strings.Replace(string(good), `"schema_version":1`, `"schema_version":1,"schema_version":1`, 1), strings.Replace(string(good), `"expected_generation":0`, `"expected_generation":null`, 1), strings.Replace(string(good), r.Commit, strings.Repeat("0", 40), 1), strings.Repeat("x", MaxRequestBytes+1)} { + if _, err := Decode(strings.NewReader(bad)); err == nil { + t.Fatal("ambiguous/malformed/deletion/oversize accepted") + } + } + tampered := r + tampered.Ref = "refs/heads/other" + if tampered.Validate() == nil { + t.Fatal("field tamper admitted") + } + invalid := r + invalid.OperationKey = "dummy-secret" + b, _ := json.Marshal(Initial(invalid)) + if strings.Contains(string(b), "dummy-secret") { + t.Fatal("unvalidated private input echoed") + } +} diff --git a/internal/trigger/contract.go b/internal/trigger/contract.go new file mode 100644 index 0000000..6082025 --- /dev/null +++ b/internal/trigger/contract.go @@ -0,0 +1,162 @@ +// Package trigger owns durable cross-delivery operation admission. A delivery +// ledger is deliberately not an authority for deployment completion. +package trigger + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "regexp" + "strconv" + "strings" + "time" +) + +const MaxRequestBytes = 32 << 10 +const Domain = "teploy-trigger-operation-v1\x00" + +type Request struct { + SchemaVersion int `json:"schema_version"` + OperationKey string `json:"operation_key"` + TargetID string `json:"target_id"` + SourceID string `json:"source_id"` + RepositoryID string `json:"repository_id"` + Ref string `json:"ref"` + Commit string `json:"commit"` + ExecutionBindingDigest string `json:"execution_binding_digest"` + Action string `json:"action"` + PreviewIdentity string `json:"preview_identity,omitempty"` + ExpectedGeneration uint64 `json:"expected_generation"` + ExpectedPreviewUpdatedAt *time.Time `json:"expected_preview_updated_at,omitempty"` +} +type Result struct { + SchemaVersion int `json:"schema_version"` + OperationKey string `json:"operation_key"` + Publication string `json:"publication"` + Reconciliation string `json:"reconciliation"` + Generation *uint64 `json:"generation,omitempty"` + ReleaseHash string `json:"release_hash,omitempty"` + ImageDigest string `json:"image_digest,omitempty"` + AuthorityObservedAt *time.Time `json:"authority_observed_at,omitempty"` + ErrorCode string `json:"error_code,omitempty"` +} + +var hex64 = regexp.MustCompile(`^[0-9a-f]{64}$`) +var commitID = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) +var targetID = regexp.MustCompile(`^[0-9a-f]{32}/[a-zA-Z0-9][a-zA-Z0-9_-]*$`) + +// Key uses Go encoding/json's compact string-array encoding (including its +// HTML escaping), UTF-8, no trailing newline. Generation is a decimal STRING +// within the array; this prevents language-specific JSON number formatting. +func (r Request) Key() string { + b, _ := json.Marshal([]string{r.TargetID, r.SourceID, r.RepositoryID, r.Ref, r.Commit, r.ExecutionBindingDigest, r.Action, r.PreviewIdentity, strconv.FormatUint(r.ExpectedGeneration, 10)}) + h := sha256.Sum256(append([]byte(Domain), b...)) + return hex.EncodeToString(h[:]) +} +func (r Request) Validate() error { + if r.SchemaVersion != 1 || !targetID.MatchString(r.TargetID) || !commitID.MatchString(r.Commit) || strings.Trim(r.Commit, "0") == "" || !hex64.MatchString(r.ExecutionBindingDigest) { + return fmt.Errorf("invalid immutable trigger identity") + } + for _, s := range []string{r.SourceID, r.RepositoryID, r.Ref, r.PreviewIdentity} { + if len(s) > 4096 || strings.ContainsAny(s, "\x00\r\n") { + return fmt.Errorf("invalid trigger identity field") + } + } + if r.SourceID == "" || r.RepositoryID == "" || r.Ref == "" { + return fmt.Errorf("missing source, repository or exact ref") + } + if _, err := ParseRepositoryIdentity(r.RepositoryID); err != nil { + return err + } + switch r.Action { + case "deploy": + if r.PreviewIdentity != "" { + return fmt.Errorf("deploy cannot name a preview") + } + case "preview_create", "preview_destroy": + if r.PreviewIdentity == "" { + return fmt.Errorf("preview ownership identity required") + } + if r.Action == "preview_destroy" && r.ExpectedGeneration == 0 { + return fmt.Errorf("destroy requires an observed generation") + } + default: + return fmt.Errorf("invalid trigger action") + } + if !hex64.MatchString(r.OperationKey) || r.OperationKey != r.Key() { + return fmt.Errorf("operation key does not bind canonical fields") + } + return nil +} +func Decode(in io.Reader) (Request, error) { + var r Request + b, err := io.ReadAll(io.LimitReader(in, MaxRequestBytes+1)) + if err != nil { + return r, err + } + if len(b) > MaxRequestBytes { + return r, fmt.Errorf("trigger request exceeds %d bytes", MaxRequestBytes) + } + // Reject duplicate JSON object keys as well as unknown fields: accepting + // ambiguous requests would give different consumers different identities. + d := json.NewDecoder(strings.NewReader(string(b))) + tok, err := d.Token() + if err != nil || tok != json.Delim('{') { + return r, fmt.Errorf("trigger request must be an object") + } + seen := map[string]bool{} + for d.More() { + t, e := d.Token() + if e != nil { + return r, e + } + k, ok := t.(string) + if !ok || seen[k] { + return r, fmt.Errorf("duplicate trigger field") + } + seen[k] = true + var v json.RawMessage + if e = d.Decode(&v); e != nil { + return r, e + } + } + if _, err = d.Token(); err != nil { + return r, err + } + if _, err = d.Token(); err != io.EOF { + return r, fmt.Errorf("trailing trigger JSON") + } + for _, key := range []string{"schema_version", "operation_key", "target_id", "source_id", "repository_id", "ref", "commit", "execution_binding_digest", "action", "expected_generation"} { + if !seen[key] { + return r, fmt.Errorf("missing required trigger field") + } + } + var raw map[string]json.RawMessage + if err = json.Unmarshal(b, &raw); err != nil { + return r, err + } + for _, v := range raw { + if string(v) == "null" { + return r, fmt.Errorf("null trigger fields are forbidden") + } + } + d = json.NewDecoder(strings.NewReader(string(b))) + d.DisallowUnknownFields() + if err = d.Decode(&r); err != nil { + return r, err + } + return r, r.Validate() +} +func Initial(r Request) Result { + key := r.OperationKey + if !hex64.MatchString(key) { + key = "" + } + return Result{SchemaVersion: 1, OperationKey: key, Publication: "not_started", Reconciliation: "required"} +} + +func ValidCommit(commit string) bool { + return commitID.MatchString(commit) && strings.Trim(commit, "0") != "" +} diff --git a/internal/trigger/remote.go b/internal/trigger/remote.go new file mode 100644 index 0000000..555137e --- /dev/null +++ b/internal/trigger/remote.go @@ -0,0 +1,149 @@ +package trigger + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +// RemoteStore uses only the admitted executor; production passes a +// state.FencedExecutor so all journal changes use the application's lease. +type RemoteStore struct { + Exec ssh.Executor + App string +} + +func (s RemoteStore) Load(ctx context.Context) ([]Receipt, error) { + if err := config.ValidateName(s.App); err != nil { + return nil, err + } + output, err := s.Exec.Run(ctx, "python3 -c "+ssh.ShellQuote(loadReceipts)+" "+ssh.ShellQuote("/deployments/"+s.App)) + if err != nil { + return nil, err + } + if len(output) > 8<<20 { + return nil, fmt.Errorf("journal inventory exceeds bound; debt retained") + } + var receipts []Receipt + d := json.NewDecoder(strings.NewReader(output)) + d.DisallowUnknownFields() + if err = d.Decode(&receipts); err != nil { + return nil, err + } + for _, r := range receipts { + if err = ValidateReceipt(r); err != nil { + return nil, err + } + } + return receipts, nil +} +func (s RemoteStore) Save(ctx context.Context, r Receipt) error { + if err := config.ValidateName(s.App); err != nil { + return err + } + data, err := EncodeReceipt(r) + if err != nil { + return err + } + return s.Exec.RunInput(ctx, "python3 -c "+ssh.ShellQuote(saveReceipt)+" "+ssh.ShellQuote("/deployments/"+s.App)+" "+ssh.ShellQuote(r.Request.OperationKey+".json"), bytes.NewReader(data)) +} + +// EnrolledTarget proves identity on the target. Atomic link rather than a +// caller-supplied ID or an alias makes parallel first enrollment safe. The +// enrollment file is permanent; replacing a server without its enrollment +// necessarily creates a different target identity. +func EnrolledTarget(ctx context.Context, exec ssh.Executor, app string) (string, error) { + if err := config.ValidateName(app); err != nil { + return "", err + } + out, err := exec.Run(ctx, "python3 -c "+ssh.ShellQuote(enrollTarget)) + if err != nil { + return "", err + } + id := strings.TrimSpace(out) + "/" + app + if !targetID.MatchString(id) { + return "", fmt.Errorf("invalid target enrollment") + } + return id, nil +} + +const directoryCore = `import os,sys,json,stat,tempfile,secrets +flags=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW +root=os.open('/deployments',flags) +s=os.fstat(root) +if s.st_uid not in (0,os.geteuid()) or s.st_mode & 0o022: raise RuntimeError('unsafe deployments authority') +` +const enrollTarget = directoryCore + ` +try: + fd=os.open('.teploy-enrollment-id',os.O_RDONLY|os.O_NOFOLLOW,dir_fd=root) +except FileNotFoundError: + name='.enrollment-'+secrets.token_hex(16) + fd=os.open(name,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=root) + try: + os.write(fd,secrets.token_hex(16).encode());os.fsync(fd) + finally: os.close(fd) + try: + os.link(name,'.teploy-enrollment-id',src_dir_fd=root,dst_dir_fd=root,follow_symlinks=False);os.fsync(root) + except FileExistsError: pass + finally: os.unlink(name,dir_fd=root) + fd=os.open('.teploy-enrollment-id',os.O_RDONLY|os.O_NOFOLLOW,dir_fd=root) +s=os.fstat(fd) +if not stat.S_ISREG(s.st_mode) or s.st_mode & 0o077 or s.st_uid not in (0,os.geteuid()): raise RuntimeError('unsafe enrollment') +print(os.read(fd,64).decode()) +` +const journalCore = directoryCore + ` +app=os.open(sys.argv[1].split('/')[-1],flags,dir_fd=root) +s=os.fstat(app) +if s.st_uid not in (0,os.geteuid()) or s.st_mode & 0o022: raise RuntimeError('unsafe app authority') +try: os.mkdir('.operations',0o700,dir_fd=app);os.fsync(app) +except FileExistsError: pass +journal=os.open('.operations',flags,dir_fd=app) +s=os.fstat(journal) +if s.st_uid != os.geteuid() or s.st_mode & 0o077: raise RuntimeError('unsafe private journal') +` +const loadReceipts = journalCore + ` +rows=[];size=0 +for name in sorted(os.listdir(journal)): + if not name.endswith('.json'): continue + if len(rows)>=10000: raise RuntimeError('journal requires archival without deleting debt') + fd=os.open(name,os.O_RDONLY|os.O_NOFOLLOW,dir_fd=journal) + try: + s=os.fstat(fd) + if not stat.S_ISREG(s.st_mode) or s.st_mode & 0o077 or s.st_uid!=os.geteuid(): raise RuntimeError('unsafe receipt') + if s.st_size>65536: raise RuntimeError('oversized receipt') + data=os.read(fd,65537);size+=len(data) + if size>8388608: raise RuntimeError('journal exceeds bound; retain debt') + row=json.loads(data) + if name!=row['request']['operation_key']+'.json': raise RuntimeError('receipt filename mismatch') + rows.append(row) + finally: os.close(fd) +print(json.dumps(rows,separators=(',',':'))) +` +const saveReceipt = journalCore + ` +data=sys.stdin.buffer.read(65537) +if len(data)>65536: raise RuntimeError('receipt too large') +row=json.loads(data);name=sys.argv[2] +if name!=row['request']['operation_key']+'.json' or len(name)!=69 or any(c not in '0123456789abcdef' for c in name[:-5]): raise RuntimeError('bad receipt name') +try: + fd=os.open(name,os.O_RDONLY|os.O_NOFOLLOW,dir_fd=journal) + try: old=json.loads(os.read(fd,65537)) + finally: os.close(fd) + # The optional extra timestamp is an admission check, not domain identity. + a=dict(old['request']);b=dict(row['request']);a.pop('expected_preview_updated_at',None);b.pop('expected_preview_updated_at',None) + if a!=b or old.get('evidence')!=row.get('evidence'): raise RuntimeError('immutable receipt identity conflict') + if old['completed'] and old!=row: raise RuntimeError('completed receipt is immutable') +except FileNotFoundError: pass +candidate='.pending-'+secrets.token_hex(16) +fd=os.open(candidate,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=journal) +try: + with os.fdopen(fd,'wb') as f: f.write(data);f.flush();os.fsync(f.fileno()) + os.replace(candidate,name,src_dir_fd=journal,dst_dir_fd=journal);os.fsync(journal) +finally: + try: os.unlink(candidate,dir_fd=journal) + except FileNotFoundError: pass +` diff --git a/internal/trigger/repository.go b/internal/trigger/repository.go new file mode 100644 index 0000000..d344101 --- /dev/null +++ b/internal/trigger/repository.go @@ -0,0 +1,89 @@ +package trigger + +import ( + "encoding/json" + "fmt" + "net" + "net/url" + "regexp" + "strings" +) + +// RepositoryIdentity is a repository authority, not a checkout locator. Native +// IDs must additionally be bound to a clone locator by authenticated metadata. +type RepositoryIdentity struct{ Forge, APIBase, ID string } + +var nativeRepositoryID = regexp.MustCompile(`^id:[1-9][0-9]*$`) + +func CanonicalAPIBase(raw string) (string, error) { + if strings.ContainsAny(raw, "\x00\r\n\t ") { + return "", fmt.Errorf("invalid API base") + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" { + return "", fmt.Errorf("invalid API base") + } + u.Scheme = strings.ToLower(u.Scheme) + if u.Scheme != "https" && u.Scheme != "http" { + return "", fmt.Errorf("invalid API scheme") + } + host, port := strings.ToLower(u.Hostname()), u.Port() + if host == "" { + return "", fmt.Errorf("missing API host") + } + if (u.Scheme == "https" && port == "443") || (u.Scheme == "http" && port == "80") { + port = "" + } + u.Host = host + if strings.Contains(host, ":") { + u.Host = "[" + host + "]" + } + if port != "" { + u.Host = net.JoinHostPort(host, port) + } + for _, part := range strings.Split(u.Path, "/") { + if part == "." || part == ".." { + return "", fmt.Errorf("ambiguous API path") + } + } + u.Path = strings.TrimRight(u.Path, "/") + u.RawPath = "" + return u.String(), nil +} + +func ParseRepositoryIdentity(raw string) (RepositoryIdentity, error) { + var fields []string + if err := json.Unmarshal([]byte(raw), &fields); err != nil || len(fields) != 3 { + return RepositoryIdentity{}, fmt.Errorf("repository_id must be a canonical three-string array") + } + b, _ := json.Marshal(fields) + if string(b) != raw { + return RepositoryIdentity{}, fmt.Errorf("noncanonical repository_id") + } + i := RepositoryIdentity{fields[0], fields[1], fields[2]} + switch i.Forge { + case "github", "gitlab", "gitea", "forgejo", "generic": + default: + return i, fmt.Errorf("unknown repository forge") + } + base, err := CanonicalAPIBase(i.APIBase) + if err != nil || base != i.APIBase { + return i, fmt.Errorf("noncanonical repository API base") + } + if i.Forge == "generic" { + if !strings.HasPrefix(i.ID, "url:") { + return i, fmt.Errorf("generic repository requires a locator") + } + u, err := url.Parse(strings.TrimPrefix(i.ID, "url:")) + if err != nil || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path == "" { + return i, fmt.Errorf("invalid generic locator") + } + origin, err := CanonicalAPIBase(u.Scheme + "://" + u.Host) + if err != nil || origin != i.APIBase { + return i, fmt.Errorf("generic locator origin mismatch") + } + } else if !nativeRepositoryID.MatchString(i.ID) { + return i, fmt.Errorf("native repository ID must be an authenticated decimal ID") + } + return i, nil +} diff --git a/release/RELEASE_RECEIPT.md b/release/RELEASE_RECEIPT.md index 9735303..56aa24c 100644 --- a/release/RELEASE_RECEIPT.md +++ b/release/RELEASE_RECEIPT.md @@ -56,12 +56,12 @@ compares via source identity, not strict HEAD equality). - ldflags : -s -w -X main.version=0.0.0-localverify-11d0709 - matrix : linux/darwin/windows × amd64/arm64 (goreleaser parity) - checksums : release/expectations/checksums-0.0.0-localverify-11d0709-go1-26-6.txt - - bf185800ea9b731849cd0a4c61940586bb4fdfc5875636966abe01aad4a34d8e teploy_darwin_amd64 - - 553044b9f5d81fc99fb7ebd2369f0435a41daff3abbfe3f27e0fc192024653f8 teploy_darwin_arm64 - - bacf79c2f18585676e64ebbc06a409304fdb91fb72b9c386ae0b51b4adbcc259 teploy_linux_amd64 - - 33446a81515d1d7c4026b5de04aa56e7216389f6833ac56d13c35a25a41fa208 teploy_linux_arm64 - - 7d0255f55e8f6e74e9c85acff6d57b634b323dd5ea84caeb2ce86da966362fc9 teploy_windows_amd64 - - 13d24237797c3484a40388fa21af4834c1affa3ba16beb2a139960764c23de56 teploy_windows_arm64 + - 9b41470e71554bb6a2899a5832ae20b67af06ec27aa6b3d31a99fa07a3706329 teploy_darwin_amd64 + - fa51620f2abbbdfa8d9db71c18470c327f1ce7be8065cac7fb5045c01c5eef22 teploy_darwin_arm64 + - 2e37ab081802d45638dd5a4d0e50f685c9ef27dd438eafcb078653539b2dcd37 teploy_linux_amd64 + - 1e22ca85320fdef40039a430248ac5b46366ba41904d33e449e34742f0896c73 teploy_linux_arm64 + - d0b2447f61520c97f4be25e9ef57cc9bc7ab8db253a3b5672a20413708e3cc5a teploy_windows_amd64 + - 7108ded33e71efc125601d7bbf93bb1beb57d51e360dbe2b763bdc8314f89bd5 teploy_windows_arm64 - verify : `./scripts/release-verify.sh verify 0.0.0-localverify-11d0709` → REPRODUCIBLE (all 6 binaries bit-identical to the recording; clean tree, Darwin/arm64 host) @@ -78,3 +78,5 @@ embed mtimes) and is recorded at release time. Bit-identical reproduction is toolchain-scoped: the expectation file pins the go version, and `release-verify` refuses to compare (exit 1, INPUTS DIFFER) rather than report a meaningless mismatch across toolchains. + +2026-10-07 reconciliation: the human checksum transcription above was corrected to the expectation file after rebuilding all six binaries at exact source commit `11d0709fbe19b642739809c614d1ff64a0812e41` with Go 1.26.6 and verifying byte equality. Future recordings explicitly clear GOFLAGS, disable VCS build-info variation, and bind the actual compiler-source digest (including dirty source). Historical expectations retain their original recipe. diff --git a/scripts/release-verify.sh b/scripts/release-verify.sh index 1c7b25a..6e459ea 100755 --- a/scripts/release-verify.sh +++ b/scripts/release-verify.sh @@ -45,6 +45,17 @@ GO_SLUG="$(echo "$GO_VERSION" | tr '.' '-')" COMMIT="$(git rev-parse HEAD)" GIT_DIRTY="$(git status --porcelain | wc -l | tr -d ' ')" GIT_DESCRIBE="$(git describe --tags 2>/dev/null || echo untagged)" +SOURCE_SHA256="$(python3 - <<'PYCODE' +import hashlib,pathlib +h=hashlib.sha256() +paths=[pathlib.Path("go.mod"),pathlib.Path("go.sum")] +for root in ["cmd","internal"]: + paths.extend(p for p in pathlib.Path(root).rglob("*") if p.is_file()) +for p in sorted(paths): + h.update(str(p).encode()+b"\0"+p.read_bytes()+b"\0") +print(h.hexdigest()) +PYCODE +)" EXPECT_FILE="$EXPECT_DIR/checksums-${VERSION}-${GO_SLUG}.txt" record_inputs() { @@ -52,10 +63,12 @@ record_inputs() { # teploy release-verify receipt # version label : $VERSION # commit : $COMMIT +# source sha256 : $SOURCE_SHA256 # git describe : $GIT_DESCRIBE # dirty files : $GIT_DIRTY # go toolchain : $GO_VERSION (go.mod: $(awk '/^go /{print $2}' go.mod)) # build env : CGO_ENABLED=0 GOFLAGS= +# build flags : -trimpath -buildvcs=false # ldflags : -s -w -X main.version=$VERSION # recorded : $(date -u +%Y-%m-%dT%H:%M:%SZ) on $(uname -s)/$(uname -m) EOF @@ -67,8 +80,8 @@ build_matrix() { for os_arch in $OS_ARCH_MATRIX; do GOOS="${os_arch%%/*}"; GOARCH="${os_arch##*/}" echo "building teploy_${GOOS}_${GOARCH}" - CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" \ - go build -trimpath -ldflags "-s -w -X main.version=$VERSION" \ + CGO_ENABLED=0 GOFLAGS= GOOS="$GOOS" GOARCH="$GOARCH" \ + go build -trimpath -buildvcs=false -ldflags "-s -w -X main.version=$VERSION" \ -o "$DIST_DIR/teploy_${GOOS}_${GOARCH}" ./cmd/teploy done } @@ -102,7 +115,12 @@ verify) echo "Bit-identical reproduction is toolchain-scoped — align toolchains or re-record." exit 1 fi - if [ "$RECORDED_COMMIT" != "$COMMIT" ]; then + RECORDED_SOURCE="$(sed -n 's/^# source sha256 : //p' "$EXPECT_FILE" | head -1)" + if [ -n "$RECORDED_SOURCE" ] && [ "$RECORDED_SOURCE" != "$SOURCE_SHA256" ]; then + echo "INPUTS DIFFER: compiler source snapshot differs from recorded digest" + exit 1 + fi + if [ -z "$RECORDED_SOURCE" ] && [ "$RECORDED_COMMIT" != "$COMMIT" ]; then # The expectation file itself lands in a commit AFTER the source it # records, so strict HEAD equality would make every recorded # expectation unverifiable. The honest test is source identity: if @@ -150,8 +168,8 @@ smoke) esac rm -rf "$DIST_DIR"; mkdir -p "$DIST_DIR" echo "building linux/$DOCKER_ARCH smoke binary ($VERSION)" - CGO_ENABLED=0 GOOS=linux GOARCH="$DOCKER_ARCH" \ - go build -trimpath -ldflags "-s -w -X main.version=$VERSION" \ + CGO_ENABLED=0 GOFLAGS= GOOS=linux GOARCH="$DOCKER_ARCH" \ + go build -trimpath -buildvcs=false -ldflags "-s -w -X main.version=$VERSION" \ -o "$DIST_DIR/teploy_linux_$DOCKER_ARCH" ./cmd/teploy IMAGE="teploy:release-smoke-$VERSION" docker build -q -f "$REPO_ROOT/Dockerfile" --build-arg BINARY="dist-verify/teploy_linux_$DOCKER_ARCH" -t "$IMAGE" "$REPO_ROOT" >/dev/null diff --git a/scripts/render-distribution.py b/scripts/render-distribution.py new file mode 100644 index 0000000..59e68c4 --- /dev/null +++ b/scripts/render-distribution.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Render Homebrew/Scoop manifests from the release's archive checksums.""" +import json +import pathlib +import re +import sys + + +def render(tag, checksums, tap, bucket): + if not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-[A-Za-z0-9.-]+)?", tag): + raise ValueError("expected a release version tag") + hashes = {} + for line in pathlib.Path(checksums).read_text().splitlines(): + digest, name = line.split() + if not re.fullmatch(r"[a-fA-F0-9]{64}", digest) or name in hashes: + raise ValueError("invalid or duplicate checksum") + hashes[name] = digest.lower() + base = f"https://github.com/useteploy/teploy-cli/releases/download/{tag}" + lines = ['# typed: false', '# frozen_string_literal: true', '', + '# Generated by teploy-cli scripts/render-distribution.py.', + 'class Teploy < Formula', + ' desc "Zero-downtime Docker deploys to any server via SSH"', + ' homepage "https://github.com/useteploy/teploy-cli"', + ' license "MIT"'] + for platform, os_name in [('macos', 'darwin'), ('linux', 'linux')]: + lines.extend(['', f' on_{platform} do']) + for cpu, arch in [('intel', 'amd64'), ('arm', 'arm64')]: + asset = f'teploy_{os_name}_{arch}.tar.gz' + lines.extend([f' if Hardware::CPU.{cpu}? && Hardware::CPU.is_64_bit?', + f' url "{base}/{asset}"', + f' sha256 "{hashes[asset]}"', '', + ' define_method(:install) do', + ' bin.install "teploy"', ' end', ' end']) + lines.append(' end') + lines.extend(['', ' test do', ' assert_match version.to_s, shell_output("#{bin}/teploy version")', ' end', 'end', '']) + manifest = {'version': tag[1:], 'architecture': {}, + 'homepage': 'https://github.com/useteploy/teploy-cli', + 'license': 'MIT', 'description': 'Zero-downtime Docker deploys to any server via SSH'} + for key, arch in [('64bit', 'amd64'), ('arm64', 'arm64')]: + asset = f'teploy_windows_{arch}.zip' + manifest['architecture'][key] = {'url': f'{base}/{asset}', 'bin': ['teploy.exe'], 'hash': hashes[asset]} + # Finish validation of the whole matrix before writing either destination. + pathlib.Path(tap, 'teploy.rb').write_text('\n'.join(lines)) + pathlib.Path(bucket, 'teploy.json').write_text(json.dumps(manifest, indent=4)+'\n') + + +if __name__ == '__main__': + if len(sys.argv) != 5: + raise SystemExit('usage: render-distribution.py ') + render(*sys.argv[1:])