diff --git a/.github/RELEASE_NOTES_TEMPLATE.md b/.github/RELEASE_NOTES_TEMPLATE.md index 736a9ba..8aaf24d 100644 --- a/.github/RELEASE_NOTES_TEMPLATE.md +++ b/.github/RELEASE_NOTES_TEMPLATE.md @@ -1,5 +1,79 @@ ## aws-sync {{VERSION}} +### Breaking changes + +Read this section before upgrading. These changes fail closed, so an automated +deployment that does not act on them will stop working rather than degrade. + +**1. NQE-based account removal has been removed.** + +`--prune-missing` remains a recognized option so existing automation receives +an actionable error, but it always refuses before credentials, NQE, planning, +or PATCH work. NQE is observed snapshot inventory, not an authoritative account +manifest, so an absent row cannot prove an account should be deleted. Replace +NQE prune workflows with `sync-accounts` and a complete human-reviewed manifest. +Manifest removals still require `--allow-removals` and both nonzero removal +ceilings. + +**2. `serve-webhook --apply` now requires authentication and an explicit network.** + +The server previously accepted unauthenticated requests when no webhook +credentials were configured, and let an event select any network or setup. It +now refuses to start in apply mode without all three of: + +```bash +awssync serve-webhook --apply --yes \ + --webhook-basic-username \ + --webhook-basic-password \ + --network-id +``` + +Configure Forward to send matching credentials (`awssync configure-webhook`). +Event scope is now intersected with configured scope: an event naming a +different network, or a setup outside `--setup-id`, is rejected with `403` +instead of being honored. + +The server also persists dedupe and snapshot-ordering state to +`$UserConfigDir/awssync/webhook-state.json`. Ensure the service user can write +that directory, or set `--webhook-state-file`. + +**3. Destructive applies in unattended contexts now require an explicit flag.** + +Forward's API exposes no compare-and-swap token, so a concurrent edit in the UI +cannot be detected before a full-list PATCH overwrites it. Removals and disables +requested without a human present now require `--allow-unattended-destructive`: + +```bash +awssync sync-accounts --apply --yes --allow-unattended-destructive ... +awssync apply-plan --allow-unattended-destructive ... # when removing/disabling +awssync serve-webhook --apply --yes --allow-unattended-destructive ... +``` + +`--yes` counts as unattended even in a terminal. The flag does not bypass +`--allow-removals`, evidence rules, or either removal ceiling — it is an +additional acknowledgement, not a replacement. `safe-sync` is unaffected, +being additive-only. Non-destructive applies are unaffected. + +### Safety changes + +- NQE reconciliation is unconditionally additive. Pagination completeness checks remain to diagnose truncated observed data, but completeness no longer authorizes absence-based deletion. +- A malformed account ID now fails the plan instead of being silently skipped, since skipping rows is how a partial inventory becomes a deletion. Use `--allow-malformed-rows` to skip and report them; doing so marks the inventory incomplete and therefore blocks removals. +- Setting an account to `enabled: false` is now classified as destructive. It consumes the same authorization and removal ceilings as deletion, closing a path where `apply-plan` could disable every account in a setup without tripping any removal guard. +- All account-list writes go through one guarded apply path, enforced by a test that fails if any other caller appears. +- External ID rotation now writes a pre-change rollback artifact, re-reads before PATCH, and binds confirmation to the computed payload. +- A partial multi-setup apply reports per-setup disposition (applied, pending, conflicted, failed) and a result-journal path instead of a bare error. +- Planning is deterministic: preview and apply produce identical digests for identical inputs. +- Cross-setup account moves are refused. Sequential per-setup PATCHes cannot guarantee an account ends up in exactly one setup if the run fails midway. + +### Known limitation + +Forward's cloud-account API provides no ETag, version field, or other +compare-and-swap token. A concurrent edit made in the Forward UI between this +tool's final read and its PATCH will be overwritten, and this is deterministic +rather than a narrow race. The pre-PATCH re-read narrows the window but does not +close it. Prefer `safe-sync` for routine work, and avoid unattended destructive +runs on setups that people also edit by hand. + ### Highlights - New `awssync safe-sync` command provides a one-command routine workflow: 24-hour snapshot freshness, preflight, compact preview, additive-only enforcement, one confirmation, rollback, and apply. @@ -8,12 +82,12 @@ - The README is now novice-first, with the routine workflow, count definitions, expected output, common stop conditions, and a short decision diagram before expert features. - A one-page routine operator handoff is available at `docs/routine-safe-sync.md`. - NQE reconciliation is additive by default: configured accounts missing from the current NQE result remain in the setup, while discovered disabled accounts are re-enabled. -- NQE-based deletion now requires `--prune-missing`, `--allow-removals`, and both nonzero `--max-removals` and `--max-removal-percent` bounds. -- Every apply writes a complete pre-change `.rollback.json` payload and verifies that the selected setup state has not changed before the first PATCH. +- NQE-based deletion is retired; `--prune-missing` returns an actionable refusal and reviewed manifest removal remains available through `sync-accounts`. +- Every apply writes a pre-change `.rollback.json` PATCH payload containing the account list and PATCHable setup fields, not a full setup backup, and verifies that the selected setup state has not changed before the first PATCH. - CLI runs pin the latest processed snapshot so planning and apply use one immutable NQE inventory. - Invalid NQE account-ID placeholders are ignored and reported instead of becoming AWS accounts. - Human-readable output is now the default; use `--json` or `--format json` for automation. -- Regression coverage includes 0, 1, 10, half, and all-enabled account states; additive and explicit-prune paths; multi-setup isolation; concurrent setup changes; rollback; and snapshot pinning. +- Regression coverage includes 0, 1, 10, half, and all-enabled account states; additive NQE and authoritative-manifest paths; multi-setup isolation; concurrent setup changes; rollback; and snapshot pinning. - Per-account External ID selection and CSV workflows from v2.3.0 remain supported. - Release assets remain available for Linux and macOS on amd64 and arm64 with SHA-256 checksums and GitHub build-provenance attestations. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 343a9d4..5ea1632 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,14 +25,45 @@ jobs: with: go-version-file: go.mod cache: true - - name: Check formatting - run: make fmt-check + - name: Check Go formatting + shell: bash + run: | + unformatted="$(gofmt -l .)" + if [[ -n "${unformatted}" ]]; then + echo "The following Go files are not gofmt-formatted:" >&2 + printf '%s\n' "${unformatted}" >&2 + exit 1 + fi - name: Vet - run: make vet + run: go vet ./... + - name: Guard phase 0 failure-test switches + shell: bash + run: | + guards=( + "internal/api/architecture_failure_test.go:runP0APIFailureTests" + "internal/app/architecture_failure_test.go:runP0ArchitectureFailureTests" + "internal/webhook/architecture_failure_test.go:runP0WebhookFailureTests" + ) + + status=0 + for entry in "${guards[@]}"; do + file="${entry%%:*}" + guard="${entry#*:}" + expected="^[[:space:]]*const[[:space:]]+${guard}([[:space:]]+bool)?[[:space:]]*=[[:space:]]*false[[:space:]]*(//.*)?$" + if ! grep -Eq "${expected}" "${file}"; then + actual="$(grep -En "^[[:space:]]*const[[:space:]]+${guard}([[:space:]]+bool)?[[:space:]]*=" "${file}" || true)" + printf '::error file=%s::%s must remain false; found: %s\n' \ + "${file}" "${guard}" "${actual:-}" + status=1 + fi + done + exit "${status}" - name: Test run: make test - - name: Race detector - run: make race + - name: Race detector (full suite) + run: go test -race ./... + - name: Race detector (webhook, 10 runs) + run: go test -race ./internal/webhook/ -count=10 - name: Vulnerability scan run: make vuln - name: Build diff --git a/README.md b/README.md index b8038ca..908b681 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Most operators should use `safe-sync`. It runs the safety checks, shows a short preview, and asks before changing Forward. It can add or re-enable accounts, but it cannot remove them. +Upgrading an existing deployment? Read [Upgrading `awssync`](docs/upgrading.md) before replacing the binary; this release intentionally breaks retired prune automation, applying webhook receivers without fixed authentication and network scope, and unattended destructive applies without an additional acknowledgement. + ## Routine Safe Sync ### 1. Download and verify @@ -59,6 +61,7 @@ For two AWS setups: 6. Otherwise, prompts for the word `apply`. 7. Confirms that the reviewed payload has not changed. 8. Writes a rollback file before PATCHing Forward. +9. Updates a durable per-setup result journal as the apply proceeds. Example preview: @@ -99,7 +102,7 @@ flowchart TD D -->|Yes| F[Forward Terraform provider] D -->|No or incomplete GovCloud inventory| G[Reviewed account manifest] B --> H[Preflight, preview, confirm, rollback, apply] - C --> I[Verify lifecycle outside Forward, then use explicit removal guards] + C --> I[Review a complete manifest, then use sync-accounts] ``` Use `safe-sync` for ordinary account additions and unchecked accounts. The remaining commands are expert workflows: @@ -108,7 +111,7 @@ Use `safe-sync` for ordinary account additions and unchecked accounts. The remai | --- | --- | | Routine existing-setup sync | `safe-sync` | | Scheduled or JSON automation | Standard `awssync` command | -| Independently verified account removal | Standard command with the reviewed removal workflow | +| Independently verified account removal | `sync-accounts` with a complete reviewed manifest | | New commercial AWS Organization | Forward Terraform provider; `discover-org` is the manual fallback | | No Organizations access | `onboard-accounts` or `sync-accounts` with a complete manifest | | GovCloud | [GovCloud workflow](docs/govcloud-workflow.md) | @@ -131,23 +134,15 @@ Fix the reported condition and run the same command again. Do not add removal ov ## Account Removal Is a Separate Expert Workflow -`safe-sync` has no removal switches. Removing an account requires an operator to confirm outside Forward that the AWS account was closed, retired, or removed from the intended Organization. - -The standard NQE workflow requires all of the following before a removal can be applied: +`safe-sync` and the standard NQE workflow cannot remove accounts. NQE reports observed snapshot inventory, which combines successfully collected accounts with accounts visible through Organizations metadata; absence is not proof of deletion. The recognized `--prune-missing` flag now fails with an explanation instead of producing a plan. -- `--prune-missing` -- `--allow-removals` -- a nonzero `--max-removals` -- a nonzero `--max-removal-percent` -- additional Organizations-evidence overrides when applicable +Use `sync-accounts` with a complete, human-reviewed manifest for lifecycle removals. Applying a manifest removal requires `--allow-removals` plus nonzero `--max-removals` and `--max-removal-percent` ceilings. A destructive run using `--yes`, CI, or another unattended context also requires `--allow-unattended-destructive`. Never remove an account only because its collection fails. -Prefer `sync-accounts` with a complete authoritative manifest for lifecycle removals. Never remove an account only because its collection fails. - -See [AWS account sync procedure](docs/aws-account-sync-procedure.md#apply-the-sync) for the reviewed removal commands and rollback procedure. +See [AWS account sync procedure](docs/aws-account-sync-procedure.md#reviewed-manifest-removal) for the reviewed removal commands and rollback procedure. ## Automation -For scheduled additive-only operation, use the standard command without any prune or removal flags: +For scheduled additive-only operation, use the standard command without removal flags: ```bash ./awssync-linux-amd64 \ @@ -158,17 +153,19 @@ For scheduled additive-only operation, use the standard command without any prun --apply --yes --json ``` -The standard command is additive by default, pins one processed snapshot, writes the payload before PATCH, verifies current setup state, and writes `.rollback.json`. +The standard command is additive by default, pins one processed snapshot, writes the payload before PATCH, verifies current setup state, and writes `.rollback.json`. Every apply also maintains `.result.json`, whose per-setup status distinguishes applied, conflicted, and failed work after a partial or ambiguous run. For event-driven operation, `serve-webhook` accepts Forward `SNAPSHOT_READY` events and serializes jobs through a bounded queue. +An applying receiver requires `--yes`, an explicit `--network-id`, and inbound Basic Auth credentials. Configure Forward to send the same credentials, and keep the receiver's durable state file on service-owned storage. Failed events are attempted at most five times and then remain dead-lettered for operator recovery. + Do not pass Forward or AWS secrets on command lines in shared process environments. Use protected environment injection or a service-manager secret facility. ## External IDs, Onboarding, and GovCloud These are separate from routine synchronization: -- [External ID procedure](docs/aws-account-sync-procedure.md#customer-defined-external-id-with-an-iam-user) +- [External ID procedure](docs/aws-account-sync-procedure.md#add-a-customer-defined-external-id-to-an-existing-setup) - [New AWS Organizations onboarding](docs/aws-account-sync-procedure.md#onboard-from-aws-organizations-directly) - [Account-manifest workflow](docs/architecture-flow.md) - [AWS GovCloud workflow](docs/govcloud-workflow.md) @@ -178,12 +175,15 @@ Existing per-account External IDs are preserved during ordinary synchronization. ## Safety Guarantees - Routine NQE synchronization is additive; accounts missing from NQE remain configured. +- NQE-derived plans cannot select `CompleteInventory` removal semantics; `--prune-missing` is retained only to return an actionable refusal. - `safe-sync` cannot remove accounts. - Human-readable output is the default; `--json` is for standard-command automation. - The latest processed snapshot is pinned before planning. -- Invalid NQE account-ID placeholders are ignored and reported. -- Every apply writes a complete pre-change rollback payload. +- Malformed NQE account IDs fail by default; `--allow-malformed-rows` skips and reports them only for incomplete additive runs. +- Every apply writes a pre-change rollback payload containing the complete `assumeRoleInfos` account list and the PATCHable setup fields (`type`, `name`, `regions`, `regionToProxyServerId`, and `proxyServerId`). It does not capture `collect`, `connectionTimeoutSeconds`, `requestTimeoutSeconds`, `numVirtualizedDevices`, or `useForwardAccountToAssumeRole`. Forward PATCH leaves absent top-level fields unchanged, so the artifact safely restores the fields `awssync` changes without overwriting those settings; it is not a full setup backup or a setup-creation payload. +- Every apply writes a durable per-setup result journal. - The reviewed target payload and current Forward setup are revalidated before PATCH. +- Forward exposes no atomic compare-and-swap token; unattended destructive applies require a separate explicit acknowledgement. - Generated payloads use atomic owner-only `0600` files. - Idempotent reads and full-state updates use bounded transient retries. @@ -191,6 +191,7 @@ Existing per-account External IDs are preserved during ordinary synchronization. | Guide | Use it for | | --- | --- | +| [Upgrade guide](docs/upgrading.md) | Breaking changes and migration steps for existing automation | | [Routine safe sync](docs/routine-safe-sync.md) | One-page operator handoff | | [Quick start](docs/quick-start.md) | Standard CLI examples and troubleshooting | | [AWS account sync procedure](docs/aws-account-sync-procedure.md) | IAM prerequisites, automation, removals, and rollback | diff --git a/cmd/awssync/main.go b/cmd/awssync/main.go index 0282dc0..98ec88a 100644 --- a/cmd/awssync/main.go +++ b/cmd/awssync/main.go @@ -31,6 +31,8 @@ var ( buildDate = "unknown" ) +const pruneMissingRefusal = "--prune-missing is no longer supported: the NQE result is observed inventory, not an account manifest, so an account's absence cannot prove it should be deleted; use sync-accounts with a reviewed manifest instead" + func main() { if err := newRootCommand().Execute(); err != nil { emitError(os.Stderr, err) @@ -57,6 +59,9 @@ func newRootCommand() *cobra.Command { SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, _ []string) error { + if err := refusePruneMissing(cmd, v); err != nil { + return err + } password, err := resolvePassword(v, os.Stdin, os.Stderr) if err != nil { return err @@ -72,6 +77,9 @@ func newRootCommand() *cobra.Command { apply := flagBool(cmd, v, "apply") yes := flagBool(cmd, v, "yes") snapshotID := flagString(cmd, v, "snapshot-id") + expectedPlanDigest := "" + planningInstant := time.Now().UTC() + policy := app.NewNQEReconcilePolicy(flagBool(cmd, v, "allow-no-org-evidence"), planningInstant) if apply && !yes && term.IsTerminal(int(os.Stdin.Fd())) { preview := app.Config{ Host: v.GetString("host"), @@ -92,10 +100,11 @@ func newRootCommand() *cobra.Command { MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), AllowNoCandidates: flagBool(cmd, v, "allow-no-candidates"), AllowNoOrgEvidence: flagBool(cmd, v, "allow-no-org-evidence"), - PruneMissing: flagBool(cmd, v, "prune-missing"), MaxSnapshotAge: flagDuration(cmd, v, "max-snapshot-age"), ExternalIDFile: flagString(cmd, v, "external-id-file"), + Policy: policy, PinSnapshot: true, + AllowMalformedRows: flagBool(cmd, v, "allow-malformed-rows"), } previewSummary, err := app.Run(cmd.Context(), preview) if err != nil { @@ -105,35 +114,46 @@ func newRootCommand() *cobra.Command { return err } snapshotID = previewSummary.SnapshotID + expectedPlanDigest = previewSummary.PlanDigest } else { if err := confirmApply(apply, yes, os.Stdin, os.Stderr); err != nil { return err } } cfg := app.Config{ - Host: v.GetString("host"), - Username: v.GetString("username"), - Password: password, - NetworkID: networkID, - SnapshotID: snapshotID, - QueryID: flagString(cmd, v, "query-id"), - QuerySetupParam: flagString(cmd, v, "query-setup-param"), - SetupIDs: setupIDs, - Output: flagString(cmd, v, "output"), - ManualOutput: flagString(cmd, v, "manual-output"), - APIPrefix: v.GetString("api-prefix"), - Insecure: v.GetBool("insecure"), - Timeout: v.GetDuration("timeout"), - Apply: apply, - AllowRemovals: flagBool(cmd, v, "allow-removals"), - MaxRemovals: flagInt(cmd, v, "max-removals"), - MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), - AllowNoCandidates: flagBool(cmd, v, "allow-no-candidates"), - AllowNoOrgEvidence: flagBool(cmd, v, "allow-no-org-evidence"), - PruneMissing: flagBool(cmd, v, "prune-missing"), - MaxSnapshotAge: flagDuration(cmd, v, "max-snapshot-age"), - ExternalIDFile: flagString(cmd, v, "external-id-file"), - PinSnapshot: true, + Host: v.GetString("host"), + Username: v.GetString("username"), + Password: password, + NetworkID: networkID, + SnapshotID: snapshotID, + QueryID: flagString(cmd, v, "query-id"), + QuerySetupParam: flagString(cmd, v, "query-setup-param"), + SetupIDs: setupIDs, + Output: flagString(cmd, v, "output"), + ManualOutput: flagString(cmd, v, "manual-output"), + APIPrefix: v.GetString("api-prefix"), + Insecure: v.GetBool("insecure"), + Timeout: v.GetDuration("timeout"), + Apply: apply, + AllowRemovals: flagBool(cmd, v, "allow-removals"), + MaxRemovals: flagInt(cmd, v, "max-removals"), + MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), + AllowNoCandidates: flagBool(cmd, v, "allow-no-candidates"), + AllowNoOrgEvidence: flagBool(cmd, v, "allow-no-org-evidence"), + MaxSnapshotAge: flagDuration(cmd, v, "max-snapshot-age"), + ExternalIDFile: flagString(cmd, v, "external-id-file"), + Policy: policy, + PinSnapshot: true, + ExpectedPlanDigest: expectedPlanDigest, + AllowMalformedRows: flagBool(cmd, v, "allow-malformed-rows"), + Unattended: yes, + AllowUnattendedDestructive: flagBool(cmd, v, "allow-unattended-destructive"), + AuthorizationActor: func() string { + if yes { + return "CLI --yes" + } + return "CLI interactive confirmation" + }(), } summary, err := app.Run(cmd.Context(), cfg) if err != nil { @@ -191,6 +211,7 @@ func newSafeSyncCommand(v *viper.Viper) *cobra.Command { return err } const maxSnapshotAge = 24 * time.Hour + planningInstant := time.Now().UTC() base := app.Config{ Host: v.GetString("host"), Username: v.GetString("username"), @@ -201,6 +222,7 @@ func newSafeSyncCommand(v *viper.Viper) *cobra.Command { Insecure: v.GetBool("insecure"), Timeout: v.GetDuration("timeout"), MaxSnapshotAge: maxSnapshotAge, + Policy: app.NewNQEReconcilePolicy(false, planningInstant), } preflight, err := app.Preflight(cmd.Context(), base) if err != nil { @@ -237,6 +259,13 @@ func newSafeSyncCommand(v *viper.Viper) *cobra.Command { base.Apply = true base.Output = preview.Output base.ExpectedPayloadSHA256 = preview.PayloadSHA256 + base.ExpectedPlanDigest = preview.PlanDigest + base.Unattended = flagBool(cmd, v, "yes") + if base.Unattended { + base.AuthorizationActor = "safe-sync --yes" + } else { + base.AuthorizationActor = "safe-sync interactive confirmation" + } result, err := app.Run(cmd.Context(), base) if err != nil { return err @@ -279,9 +308,6 @@ func newExternalIDCommand(v *viper.Viper) *cobra.Command { externalIDFile, _ := cmd.Flags().GetString("external-id-file") apply, _ := cmd.Flags().GetBool("apply") yes, _ := cmd.Flags().GetBool("yes") - if err := confirmApply(apply, yes, os.Stdin, os.Stderr); err != nil { - return err - } output, _ := cmd.Flags().GetString("output") summary, err := app.ChangeExternalID(cmd.Context(), app.ExternalIDConfig{ Host: v.GetString("host"), @@ -298,6 +324,17 @@ func newExternalIDCommand(v *viper.Viper) *cobra.Command { Insecure: v.GetBool("insecure"), Timeout: v.GetDuration("timeout"), Apply: apply, + Unattended: yes, + AuthorizationActor: func() string { + if yes { + return "CLI external-id --yes" + } + return "CLI external-id interactive confirmation" + }(), + ConfirmApply: func(planDigest string) error { + fmt.Fprintf(os.Stderr, "External ID apply intent SHA-256: %s\n", planDigest) + return confirmApply(true, yes, os.Stdin, os.Stderr) + }, }) if err != nil { return err @@ -353,11 +390,12 @@ func bindPreflightFlags(v *viper.Viper, flags *pflag.FlagSet) { flags.String("query-setup-param", "", "optional saved-query String parameter name to receive the single selected --setup-id") flags.StringSlice("setup-id", nil, "optional Forward AWS setup ID to sync; repeatable") flags.Bool("allow-no-org-evidence", false, "allow removals when no AWS Organizations evidence is visible in NQE") - flags.Bool("prune-missing", false, "plan removal of configured accounts missing from NQE; additive preservation is the default") + flags.Bool("prune-missing", false, "retired: NQE absence cannot prove deletion; use sync-accounts with a reviewed manifest") flags.Int("max-removals", 0, "required nonzero aggregate account-removal ceiling when removals are planned") flags.Float64("max-removal-percent", 0, "required nonzero per-setup removal-percentage ceiling when removals are planned") flags.Duration("max-snapshot-age", 0, "fail if latest processed snapshot is older than this duration; 0 disables the check") flags.String("external-id-file", "", "CSV file of explicit per-account External IDs for mixed-ID setups") + flags.Bool("allow-malformed-rows", false, "skip and report malformed NQE account rows; the observed result is marked incomplete") mustBind(v, flags, "snapshot-id") mustBind(v, flags, "query-id") mustBind(v, flags, "query-setup-param") @@ -368,6 +406,7 @@ func bindPreflightFlags(v *viper.Viper, flags *pflag.FlagSet) { mustBind(v, flags, "max-removal-percent") mustBind(v, flags, "max-snapshot-age") mustBind(v, flags, "external-id-file") + mustBind(v, flags, "allow-malformed-rows") } func bindProcessingFlags(v *viper.Viper, flags *pflag.FlagSet) { @@ -378,14 +417,16 @@ func bindProcessingFlags(v *viper.Viper, flags *pflag.FlagSet) { flags.String("manual-output", "", "optional JSON path for manual platform drag-and-drop payloads") flags.Bool("apply", false, "PATCH the generated setup payloads back into Forward") flags.Bool("yes", false, "skip apply confirmation prompt") + flags.Bool("allow-unattended-destructive", false, "allow --yes, webhook, or CI applies to remove or disable accounts despite the lack of atomic compare-and-swap") flags.Bool("allow-removals", false, "allow planned account removals during apply") flags.Int("max-removals", 0, "required nonzero aggregate account-removal ceiling when removals are planned") flags.Float64("max-removal-percent", 0, "required nonzero per-setup removal-percentage ceiling when removals are planned") flags.Bool("allow-no-candidates", false, "allow removals when no uncollected candidate accounts are visible") flags.Bool("allow-no-org-evidence", false, "allow removals when no AWS Organizations evidence is visible in NQE") - flags.Bool("prune-missing", false, "plan removal of configured accounts missing from NQE; additive preservation is the default") + flags.Bool("prune-missing", false, "retired: NQE absence cannot prove deletion; use sync-accounts with a reviewed manifest") flags.Duration("max-snapshot-age", 0, "fail if latest processed snapshot is older than this duration; 0 disables the check") flags.String("external-id-file", "", "CSV file of explicit per-account External IDs for mixed-ID setups") + flags.Bool("allow-malformed-rows", false, "skip and report malformed NQE account rows; the observed result is marked incomplete") mustBind(v, flags, "query-id") mustBind(v, flags, "query-setup-param") mustBind(v, flags, "setup-id") @@ -393,6 +434,7 @@ func bindProcessingFlags(v *viper.Viper, flags *pflag.FlagSet) { mustBind(v, flags, "manual-output") mustBind(v, flags, "apply") mustBind(v, flags, "yes") + mustBind(v, flags, "allow-unattended-destructive") mustBind(v, flags, "allow-removals") mustBind(v, flags, "max-removals") mustBind(v, flags, "max-removal-percent") @@ -401,6 +443,7 @@ func bindProcessingFlags(v *viper.Viper, flags *pflag.FlagSet) { mustBind(v, flags, "prune-missing") mustBind(v, flags, "max-snapshot-age") mustBind(v, flags, "external-id-file") + mustBind(v, flags, "allow-malformed-rows") } func newPreflightCommand(v *viper.Viper) *cobra.Command { @@ -410,6 +453,9 @@ func newPreflightCommand(v *viper.Viper) *cobra.Command { SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, _ []string) error { + if err := refusePruneMissing(cmd, v); err != nil { + return err + } password, err := resolvePassword(v, os.Stdin, os.Stderr) if err != nil { return err @@ -422,6 +468,7 @@ func newPreflightCommand(v *viper.Viper) *cobra.Command { if err != nil { return err } + planningInstant := time.Now().UTC() summary, err := app.Preflight(cmd.Context(), app.Config{ Host: v.GetString("host"), Username: v.GetString("username"), @@ -435,11 +482,12 @@ func newPreflightCommand(v *viper.Viper) *cobra.Command { Insecure: v.GetBool("insecure"), Timeout: v.GetDuration("timeout"), AllowNoOrgEvidence: flagBool(cmd, v, "allow-no-org-evidence"), - PruneMissing: flagBool(cmd, v, "prune-missing"), MaxRemovals: flagInt(cmd, v, "max-removals"), MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), MaxSnapshotAge: flagDuration(cmd, v, "max-snapshot-age"), ExternalIDFile: flagString(cmd, v, "external-id-file"), + AllowMalformedRows: flagBool(cmd, v, "allow-malformed-rows"), + Policy: app.NewNQEReconcilePolicy(flagBool(cmd, v, "allow-no-org-evidence"), planningInstant), }) if err != nil { return err @@ -504,17 +552,19 @@ func newApplyPlanCommand(v *viper.Viper) *cobra.Command { return err } summary, err := app.ApplyPlan(cmd.Context(), app.ApplyPlanConfig{ - Host: v.GetString("host"), - Username: v.GetString("username"), - Password: password, - NetworkID: networkID, - PlanPath: flagString(cmd, v, "plan"), - APIPrefix: v.GetString("api-prefix"), - Insecure: v.GetBool("insecure"), - Timeout: v.GetDuration("timeout"), - AllowRemovals: flagBool(cmd, v, "allow-removals"), - MaxRemovals: flagInt(cmd, v, "max-removals"), - MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), + Host: v.GetString("host"), + Username: v.GetString("username"), + Password: password, + NetworkID: networkID, + PlanPath: flagString(cmd, v, "plan"), + APIPrefix: v.GetString("api-prefix"), + Insecure: v.GetBool("insecure"), + Timeout: v.GetDuration("timeout"), + AllowRemovals: flagBool(cmd, v, "allow-removals"), + MaxRemovals: flagInt(cmd, v, "max-removals"), + MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), + AllowUnattendedDestructive: flagBool(cmd, v, "allow-unattended-destructive"), + AuthorizationActor: "CLI apply-plan --yes", }) if err != nil { return err @@ -525,14 +575,16 @@ func newApplyPlanCommand(v *viper.Viper) *cobra.Command { bindNetworkFlag(v, cmd.Flags()) cmd.Flags().String("plan", "aws_sync_payload.json", "reviewed payload file to apply") cmd.Flags().Bool("yes", false, "confirm applying the reviewed payload file") - cmd.Flags().Bool("allow-removals", false, "allow reviewed commercial-partition account removals; GovCloud removals must use their source workflow") - cmd.Flags().Int("max-removals", 0, "required nonzero aggregate account-removal ceiling when removals are planned") - cmd.Flags().Float64("max-removal-percent", 0, "required nonzero per-setup removal-percentage ceiling when removals are planned") + cmd.Flags().Bool("allow-removals", false, "allow reviewed commercial-partition account removals or disables; GovCloud destructive changes must use their source workflow") + cmd.Flags().Int("max-removals", 0, "required nonzero aggregate removal-or-disable ceiling when destructive changes are planned") + cmd.Flags().Float64("max-removal-percent", 0, "required nonzero per-setup removal-or-disable percentage ceiling when destructive changes are planned") + cmd.Flags().Bool("allow-unattended-destructive", false, "allow apply-plan --yes to remove or disable accounts despite the lack of atomic compare-and-swap") mustBind(v, cmd.Flags(), "plan") mustBind(v, cmd.Flags(), "yes") mustBind(v, cmd.Flags(), "allow-removals") mustBind(v, cmd.Flags(), "max-removals") mustBind(v, cmd.Flags(), "max-removal-percent") + mustBind(v, cmd.Flags(), "allow-unattended-destructive") return cmd } @@ -801,26 +853,49 @@ func newSyncAccountsCommand(v *viper.Viper) *cobra.Command { return err } apply := flagBool(cmd, v, "apply") - if err := confirmApply(apply, flagBool(cmd, v, "yes"), os.Stdin, os.Stderr); err != nil { + yes := flagBool(cmd, v, "yes") + planningInstant := time.Now().UTC() + cfg := app.Config{ + Host: v.GetString("host"), + Username: v.GetString("username"), + Password: password, + NetworkID: networkID, + SetupIDs: setupIDs, + Output: flagString(cmd, v, "output"), + ManualOutput: flagString(cmd, v, "manual-output"), + APIPrefix: v.GetString("api-prefix"), + Insecure: v.GetBool("insecure"), + Timeout: v.GetDuration("timeout"), + Apply: apply, + AllowRemovals: flagBool(cmd, v, "allow-removals"), + MaxRemovals: flagInt(cmd, v, "max-removals"), + MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), + ExternalIDFile: flagString(cmd, v, "external-id-file"), + Unattended: yes, + AllowUnattendedDestructive: flagBool(cmd, v, "allow-unattended-destructive"), + Policy: app.NewAuthoritativeManifestReconcilePolicy(planningInstant), + } + if yes { + cfg.AuthorizationActor = "sync-accounts --yes" + } else { + cfg.AuthorizationActor = "sync-accounts interactive confirmation" + } + if apply && !yes && term.IsTerminal(int(os.Stdin.Fd())) { + previewConfig := cfg + previewConfig.Apply = false + preview, err := app.SyncAWSAccountManifest(cmd.Context(), previewConfig, accounts) + if err != nil { + return err + } + if err := confirmApplyFromSummary(preview, os.Stdin, os.Stderr); err != nil { + return err + } + cfg.Output = preview.Output + cfg.ExpectedPlanDigest = preview.PlanDigest + } else if err := confirmApply(apply, yes, os.Stdin, os.Stderr); err != nil { return err } - summary, err := app.SyncAWSAccountManifest(cmd.Context(), app.Config{ - Host: v.GetString("host"), - Username: v.GetString("username"), - Password: password, - NetworkID: networkID, - SetupIDs: setupIDs, - Output: flagString(cmd, v, "output"), - ManualOutput: flagString(cmd, v, "manual-output"), - APIPrefix: v.GetString("api-prefix"), - Insecure: v.GetBool("insecure"), - Timeout: v.GetDuration("timeout"), - Apply: apply, - AllowRemovals: flagBool(cmd, v, "allow-removals"), - MaxRemovals: flagInt(cmd, v, "max-removals"), - MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), - ExternalIDFile: flagString(cmd, v, "external-id-file"), - }, accounts) + summary, err := app.SyncAWSAccountManifest(cmd.Context(), cfg, accounts) if err != nil { return err } @@ -834,11 +909,12 @@ func newSyncAccountsCommand(v *viper.Viper) *cobra.Command { cmd.Flags().String("manual-output", "", "optional UI-friendly account JSON output path") cmd.Flags().Bool("apply", false, "PATCH the generated setup payload into Forward") cmd.Flags().Bool("yes", false, "skip apply confirmation prompt") + cmd.Flags().Bool("allow-unattended-destructive", false, "allow --yes or CI applies to remove or disable accounts despite the lack of atomic compare-and-swap") cmd.Flags().Bool("allow-removals", false, "allow reviewed manifest entries to remove accounts from the setup") cmd.Flags().Int("max-removals", 0, "required nonzero aggregate account-removal ceiling when removals are planned") cmd.Flags().Float64("max-removal-percent", 0, "required nonzero removal-percentage ceiling when removals are planned") cmd.Flags().String("external-id-file", "", "CSV file of explicit per-account External IDs") - for _, name := range []string{"accounts-file", "setup-id", "output", "manual-output", "apply", "yes", "allow-removals", "max-removals", "max-removal-percent", "external-id-file"} { + for _, name := range []string{"accounts-file", "setup-id", "output", "manual-output", "apply", "yes", "allow-unattended-destructive", "allow-removals", "max-removals", "max-removal-percent", "external-id-file"} { mustBind(v, cmd.Flags(), name) } return cmd @@ -851,6 +927,9 @@ func newServeWebhookCommand(v *viper.Viper) *cobra.Command { SilenceErrors: true, Short: "Receive Forward SNAPSHOT_READY webhooks and run awssync for the exact snapshot", RunE: func(cmd *cobra.Command, _ []string) error { + if err := refusePruneMissing(cmd, v); err != nil { + return err + } password, err := resolvePassword(v, os.Stdin, os.Stderr) if err != nil { return err @@ -858,32 +937,39 @@ func newServeWebhookCommand(v *viper.Viper) *cobra.Command { if flagBool(cmd, v, "apply") && !flagBool(cmd, v, "yes") { return fmt.Errorf("serve-webhook with --apply requires --yes") } + policy := app.NewNQEReconcilePolicy(flagBool(cmd, v, "allow-no-org-evidence"), time.Time{}) srv, err := webhook.New(webhook.Config{ Listen: flagString(cmd, v, "listen"), Path: flagString(cmd, v, "path"), BasicUsername: flagString(cmd, v, "webhook-basic-username"), BasicPassword: flagString(cmd, v, "webhook-basic-password"), + StatePath: flagString(cmd, v, "webhook-state-file"), App: app.Config{ - Host: v.GetString("host"), - Username: v.GetString("username"), - Password: password, - QueryID: flagString(cmd, v, "query-id"), - QuerySetupParam: flagString(cmd, v, "query-setup-param"), - SetupIDs: flagStringSlice(cmd, v, "setup-id"), - Output: flagString(cmd, v, "output"), - ManualOutput: flagString(cmd, v, "manual-output"), - APIPrefix: v.GetString("api-prefix"), - Insecure: v.GetBool("insecure"), - Timeout: v.GetDuration("timeout"), - Apply: flagBool(cmd, v, "apply"), - AllowRemovals: flagBool(cmd, v, "allow-removals"), - MaxRemovals: flagInt(cmd, v, "max-removals"), - MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), - AllowNoCandidates: flagBool(cmd, v, "allow-no-candidates"), - AllowNoOrgEvidence: flagBool(cmd, v, "allow-no-org-evidence"), - PruneMissing: flagBool(cmd, v, "prune-missing"), - MaxSnapshotAge: flagDuration(cmd, v, "max-snapshot-age"), - ExternalIDFile: flagString(cmd, v, "external-id-file"), + Host: v.GetString("host"), + Username: v.GetString("username"), + Password: password, + NetworkID: flagString(cmd, v, "network-id"), + QueryID: flagString(cmd, v, "query-id"), + QuerySetupParam: flagString(cmd, v, "query-setup-param"), + SetupIDs: flagStringSlice(cmd, v, "setup-id"), + Output: flagString(cmd, v, "output"), + ManualOutput: flagString(cmd, v, "manual-output"), + APIPrefix: v.GetString("api-prefix"), + Insecure: v.GetBool("insecure"), + Timeout: v.GetDuration("timeout"), + Apply: flagBool(cmd, v, "apply"), + AllowRemovals: flagBool(cmd, v, "allow-removals"), + MaxRemovals: flagInt(cmd, v, "max-removals"), + MaxRemovalPercent: flagFloat64(cmd, v, "max-removal-percent"), + AllowNoCandidates: flagBool(cmd, v, "allow-no-candidates"), + AllowNoOrgEvidence: flagBool(cmd, v, "allow-no-org-evidence"), + MaxSnapshotAge: flagDuration(cmd, v, "max-snapshot-age"), + ExternalIDFile: flagString(cmd, v, "external-id-file"), + AllowMalformedRows: flagBool(cmd, v, "allow-malformed-rows"), + Policy: policy, + Unattended: true, + AllowUnattendedDestructive: flagBool(cmd, v, "allow-unattended-destructive"), + AuthorizationActor: "webhook", }, }) if err != nil { @@ -894,13 +980,16 @@ func newServeWebhookCommand(v *viper.Viper) *cobra.Command { } cmd.Flags().String("listen", ":8080", "listen address for the webhook receiver") cmd.Flags().String("path", "/forward/snapshot-ready", "HTTP path for webhook POST requests") - cmd.Flags().String("webhook-basic-username", "", "optional Basic Auth username required on incoming webhook requests") - cmd.Flags().String("webhook-basic-password", "", "optional Basic Auth password required on incoming webhook requests") + cmd.Flags().String("webhook-basic-username", "", "Basic Auth username required on incoming webhook requests when --apply is enabled") + cmd.Flags().String("webhook-basic-password", "", "Basic Auth password required on incoming webhook requests when --apply is enabled") + cmd.Flags().String("webhook-state-file", "", "durable pending, dead-letter, dedupe, and snapshot-watermark JSON file (defaults to the user config directory)") + bindNetworkFlag(v, cmd.Flags()) bindProcessingFlags(v, cmd.Flags()) mustBind(v, cmd.Flags(), "listen") mustBind(v, cmd.Flags(), "path") mustBind(v, cmd.Flags(), "webhook-basic-username") mustBind(v, cmd.Flags(), "webhook-basic-password") + mustBind(v, cmd.Flags(), "webhook-state-file") return cmd } @@ -1202,6 +1291,13 @@ func flagBool(cmd *cobra.Command, v *viper.Viper, name string) bool { return v.GetBool(name) } +func refusePruneMissing(cmd *cobra.Command, v *viper.Viper) error { + if flagBool(cmd, v, "prune-missing") { + return errors.New(pruneMissingRefusal) + } + return nil +} + func flagInt(cmd *cobra.Command, v *viper.Viper, name string) int { if flag := cmd.Flags().Lookup(name); flag != nil && flag.Changed { value, _ := cmd.Flags().GetInt(name) @@ -1319,6 +1415,9 @@ func emitResult(cmd *cobra.Command, v *viper.Viper, value any) error { func emitStatusHuman(result *monitor.StatusResult) error { fmt.Fprintln(os.Stdout, "Snapshot status") fmt.Fprintf(os.Stdout, " network: %s\n", result.NetworkID) + fmt.Fprintf(os.Stdout, " observation atomic: %t\n", result.ObservationAtomic) + fmt.Fprintf(os.Stdout, " latest/list consistent: %t\n", result.LatestListConsistent) + fmt.Fprintf(os.Stdout, " observation warning: %s\n", result.ObservationWarning) if result.LatestProcessedSnapshot != nil { fmt.Fprintf(os.Stdout, " latest processed: %s\n", result.LatestProcessedSnapshot.ID) } @@ -1346,8 +1445,13 @@ func emitApplyPlanHuman(summary *app.ApplyPlanSummary) error { fmt.Fprintf(os.Stdout, " network: %s\n", summary.NetworkID) fmt.Fprintf(os.Stdout, " plan: %s\n", summary.PlanPath) fmt.Fprintf(os.Stdout, " patched: %d (%s)\n", summary.PatchedSetupCount, strings.Join(summary.PatchedSetups, ", ")) - fmt.Fprintf(os.Stdout, " rollback: %s\n", summary.RollbackOutput) - fmt.Fprintf(os.Stdout, " rollback sha256: %s\n", summary.RollbackSHA256) + if summary.RollbackOutput != "" { + fmt.Fprintf(os.Stdout, " rollback: %s\n", summary.RollbackOutput) + fmt.Fprintf(os.Stdout, " rollback sha256: %s\n", summary.RollbackSHA256) + } + if summary.ResultJournalOutput != "" { + fmt.Fprintf(os.Stdout, " journal: %s\n", summary.ResultJournalOutput) + } return nil } @@ -1365,6 +1469,13 @@ func emitExternalIDHuman(summary *app.ExternalIDSummary) error { fmt.Fprintf(os.Stdout, " patched: %t\n", summary.Patched) fmt.Fprintf(os.Stdout, " output: %s\n", summary.Output) fmt.Fprintf(os.Stdout, " sha256: %s\n", summary.PayloadSHA256) + if summary.RollbackOutput != "" { + fmt.Fprintf(os.Stdout, " rollback: %s\n", summary.RollbackOutput) + fmt.Fprintf(os.Stdout, " rollback sha256: %s\n", summary.RollbackSHA256) + } + if summary.ResultJournalOutput != "" { + fmt.Fprintf(os.Stdout, " journal: %s\n", summary.ResultJournalOutput) + } return nil } @@ -1448,26 +1559,48 @@ func emitSummaryHuman(summary *app.Summary) error { } if summary.IgnoredNQEItemCount > 0 { fmt.Fprintf(os.Stdout, " ignored: %d invalid NQE account row(s)\n", summary.IgnoredNQEItemCount) + for _, row := range summary.SkippedNQERows { + setup := row.SetupID + if setup == "" { + setup = "" + } + accountID := row.AccountID + if accountID == "" { + accountID = "" + } + fmt.Fprintf(os.Stdout, " row %d setup=%s account_id=%s: %s\n", row.Row, setup, accountID, row.Reason) + } } fmt.Fprintf(os.Stdout, " planned: %d\n", summary.PlannedSetupCount) fmt.Fprintf(os.Stdout, " patched: %d\n", summary.PatchedSetupCount) + if summary.ResultJournalOutput != "" { + fmt.Fprintf(os.Stdout, " journal: %s\n", summary.ResultJournalOutput) + } if summary.RemovalBlocked { - fmt.Fprintln(os.Stdout, "\nApply blocked. Add --allow-removals, --allow-no-candidates, and --allow-no-org-evidence as needed.") + if summary.RemovalBlockReason != "" { + fmt.Fprintf(os.Stdout, "\nApply would be blocked: %s\n", summary.RemovalBlockReason) + } else { + fmt.Fprintln(os.Stdout, "\nApply blocked.") + } + fmt.Fprintln(os.Stdout, "For destructive apply, add --allow-removals, --max-removals, --max-removal-percent, and --allow-unattended-destructive as needed.") } fmt.Fprintln(os.Stdout, "\nSetups:") - addedTotal, reenabledTotal, removedTotal := 0, 0, 0 + addedTotal, reenabledTotal, disabledTotal, removedTotal := 0, 0, 0, 0 for _, setup := range summary.PlannedSetups { addedTotal += len(setup.AddedAccounts) reenabledTotal += len(setup.ReenabledAccounts) + disabledTotal += len(setup.DisabledAccounts) removedTotal += len(setup.RemovedAccounts) fmt.Fprintf( os.Stdout, - " - %s: add=%d reenable=%d remove=%d unchanged=%d\n", + " - %s: add=%d reenable=%d disable=%d remove=%d unchanged=%d status=%s\n", setup.SetupID, len(setup.AddedAccounts), len(setup.ReenabledAccounts), + len(setup.DisabledAccounts), len(setup.RemovedAccounts), setup.UnchangedAccountCount, + setup.ApplyStatus, ) if len(setup.AddedAccounts) > 0 { fmt.Fprintf(os.Stdout, " added: %s\n", accountSummaryIDs(setup.AddedAccounts)) @@ -1475,10 +1608,13 @@ func emitSummaryHuman(summary *app.Summary) error { if len(setup.RemovedAccounts) > 0 { fmt.Fprintf(os.Stdout, " removed: %s\n", accountSummaryIDs(setup.RemovedAccounts)) } + if len(setup.DisabledAccounts) > 0 { + fmt.Fprintf(os.Stdout, " disabled: %s\n", accountSummaryIDs(setup.DisabledAccounts)) + } fmt.Fprintf(os.Stdout, " %s\n", setup.OrganizationDiscoveryMessage) } fmt.Fprintln(os.Stdout, "\nSummary:") - fmt.Fprintf(os.Stdout, " total added=%d, total reenabled=%d, total removed=%d\n", addedTotal, reenabledTotal, removedTotal) + fmt.Fprintf(os.Stdout, " total added=%d, total reenabled=%d, total disabled=%d, total removed=%d\n", addedTotal, reenabledTotal, disabledTotal, removedTotal) return nil } @@ -1508,7 +1644,15 @@ func summaryChangeCounts(summary *app.Summary) (int, int, int) { func summaryRemovalCount(summary *app.Summary) int { _, _, removed := summaryChangeCounts(summary) - return removed + return removed + summaryDisableCount(summary) +} + +func summaryDisableCount(summary *app.Summary) int { + disabled := 0 + for _, setup := range summary.PlannedSetups { + disabled += len(setup.DisabledAccounts) + } + return disabled } func emitSafeSyncPreview(summary *app.Summary) { @@ -1635,14 +1779,15 @@ func confirmPost(post, yes bool, setupID string, stdin *os.File, stderr io.Write } func confirmApplyFromSummary(summary *app.Summary, stdin *os.File, stderr io.Writer) error { - addedTotal, removedTotal := 0, 0 + addedTotal, disabledTotal, removedTotal := 0, 0, 0 for _, setup := range summary.PlannedSetups { addedTotal += len(setup.AddedAccounts) + disabledTotal += len(setup.DisabledAccounts) removedTotal += len(setup.RemovedAccounts) } - fmt.Fprintf(stderr, "Planned changes: add=%d remove=%d.\n", addedTotal, removedTotal) - if removedTotal > 0 { - fmt.Fprintln(stderr, "Warning: removes are included. Review setup output carefully.") + fmt.Fprintf(stderr, "Planned changes: add=%d disable=%d remove=%d.\n", addedTotal, disabledTotal, removedTotal) + if disabledTotal+removedTotal > 0 { + fmt.Fprintln(stderr, "Warning: destructive changes are included. Review setup output carefully.") } fmt.Fprint(stderr, "Type 'apply' to continue: ") var response string diff --git a/cmd/awssync/main_test.go b/cmd/awssync/main_test.go index c07769b..47f2cbb 100644 --- a/cmd/awssync/main_test.go +++ b/cmd/awssync/main_test.go @@ -29,6 +29,27 @@ func TestRootCommandIncludesBuildMetadataInVersion(t *testing.T) { } } +func TestNQECommandsRefusePruneMissingAtCLIBoundary(t *testing.T) { + tests := []struct { + name string + args []string + }{ + {name: "root", args: []string{"--prune-missing"}}, + {name: "preflight", args: []string{"preflight", "--prune-missing"}}, + {name: "webhook", args: []string{"serve-webhook", "--prune-missing"}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := newRootCommand() + cmd.SetArgs(test.args) + if err := cmd.Execute(); err == nil || err.Error() != pruneMissingRefusal { + t.Fatalf("Execute() error = %v; want exactly %q", err, pruneMissingRefusal) + } + }) + } +} + func TestRootCommandHonorsLocalSnapshotAndOutputFlags(t *testing.T) { var seenNQEQuery string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -41,10 +62,10 @@ func TestRootCommandHonorsLocalSnapshotAndOutputFlags(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": seenNQEQuery = r.URL.RawQuery w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":false}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":false}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","externalId":"Org:99","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","externalId":"Org:99","enabled":true}]}]`)) default: w.WriteHeader(http.StatusNotFound) } @@ -99,6 +120,121 @@ func TestRootCommandHonorsLocalSnapshotAndOutputFlags(t *testing.T) { } } +func TestEmitSummaryHumanReportsSkippedNQERows(t *testing.T) { + stdout := captureStdout(t, func() { + err := emitSummaryHuman(&app.Summary{ + Host: "https://fwd.example", + NetworkID: "network-1", + Output: "payload.json", + FetchedItemCount: 2, + IgnoredNQEItemCount: 1, + SkippedNQERows: []app.MalformedNQERowSummary{{ + Row: 2, + SetupID: "setup-a", + AccountID: "bad-row", + Reason: "invalid AWS account ID", + }}, + }) + if err != nil { + t.Fatalf("emitSummaryHuman() error = %v", err) + } + }) + if !strings.Contains(stdout, "ignored: 1 invalid NQE account row(s)") || + !strings.Contains(stdout, "row 2 setup=setup-a account_id=bad-row: invalid AWS account ID") { + t.Fatalf("expected skipped row details in human output:\n%s", stdout) + } +} + +func TestSyncAccountsDryRunReportsUnattendedDestructiveGate(t *testing.T) { + patchCount := 0 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + _, _ = w.Write([]byte(`[{ + "type":"AWS", + "name":"setup-a", + "assumeRoleInfos":[ + {"accountId":"111111111111","accountName":"keep","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}, + {"accountId":"222222222222","accountName":"remove","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true} + ] + }]`)) + case r.Method == http.MethodPatch: + patchCount++ + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + dir := t.TempDir() + manifestPath := filepath.Join(dir, "accounts.json") + if err := os.WriteFile(manifestPath, []byte(`[{"id":"111111111111","name":"keep"}]`), 0o600); err != nil { + t.Fatal(err) + } + args := func(outputPath string, jsonOutput bool) []string { + result := []string{ + "sync-accounts", + "--host", server.URL, + "--username", "alice", + "--password", "secret", + "--network-id", "network-1", + "--accounts-file", manifestPath, + "--setup-id", "setup-a", + "--output", outputPath, + "--yes", + "--allow-removals", + "--max-removals", "1", + "--max-removal-percent", "100", + "--insecure", + } + if jsonOutput { + result = append(result, "--json") + } + return result + } + + jsonOutput := captureStdout(t, func() { + cmd := newRootCommand() + cmd.SetArgs(args(filepath.Join(dir, "json-payload.json"), true)) + if err := cmd.Execute(); err != nil { + t.Fatalf("JSON dry-run Execute() error = %v", err) + } + }) + var summary app.Summary + if err := json.Unmarshal([]byte(jsonOutput), &summary); err != nil { + t.Fatalf("decode JSON dry-run summary: %v\n%s", err, jsonOutput) + } + if !summary.RemovalBlocked || !strings.Contains(summary.RemovalBlockReason, "--allow-unattended-destructive") { + t.Fatalf("JSON dry-run did not report unattended destructive gate: %#v", summary) + } + + humanOutput := captureStdout(t, func() { + cmd := newRootCommand() + cmd.SetArgs(args(filepath.Join(dir, "human-payload.json"), false)) + if err := cmd.Execute(); err != nil { + t.Fatalf("human dry-run Execute() error = %v", err) + } + }) + for _, want := range []string{ + "Apply would be blocked:", + "--allow-unattended-destructive", + "--allow-removals", + "--max-removals", + "--max-removal-percent", + } { + if !strings.Contains(humanOutput, want) { + t.Fatalf("human dry-run output missing %q:\n%s", want, humanOutput) + } + } + if strings.Contains(humanOutput, "--allow-no-candidates") || strings.Contains(humanOutput, "--allow-no-org-evidence") { + t.Fatalf("human dry-run output lists retired NQE removal flags:\n%s", humanOutput) + } + if patchCount != 0 { + t.Fatalf("dry-run unexpectedly patched %d setup(s)", patchCount) + } +} + func TestApplyPlanCommandHonorsLocalYesFlag(t *testing.T) { patched := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -115,7 +251,9 @@ func TestApplyPlanCommandHonorsLocalYesFlag(t *testing.T) { defer server.Close() planPath := filepath.Join(t.TempDir(), "payload.json") - if err := os.WriteFile(planPath, []byte(`{"setup-a":{"type":"AWS","name":"setup-a","regionToProxyServerId":{},"assumeRoleInfos":[]}}`), 0o600); err != nil { + if err := os.WriteFile(planPath, []byte(`{"setup-a":{"type":"AWS","name":"setup-a","regionToProxyServerId":{},"assumeRoleInfos":[ + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true} + ]}}`), 0o600); err != nil { t.Fatal(err) } captureStdout(t, func() { @@ -140,6 +278,59 @@ func TestApplyPlanCommandHonorsLocalYesFlag(t *testing.T) { } } +func TestHumanRecoveryOutputIncludesArtifactPaths(t *testing.T) { + tests := []struct { + name string + emit func() error + want []string + }{ + { + name: "apply-plan", + emit: func() error { + return emitApplyPlanHuman(&app.ApplyPlanSummary{ + RollbackOutput: "/tmp/plan.rollback.json", + RollbackSHA256: "rollback-digest", + ResultJournalOutput: "/tmp/plan.result.json", + }) + }, + want: []string{ + "rollback: /tmp/plan.rollback.json", + "rollback sha256: rollback-digest", + "journal: /tmp/plan.result.json", + }, + }, + { + name: "external-id", + emit: func() error { + return emitExternalIDHuman(&app.ExternalIDSummary{ + RollbackOutput: "/tmp/external-id.rollback.json", + RollbackSHA256: "rollback-digest", + ResultJournalOutput: "/tmp/external-id.result.json", + }) + }, + want: []string{ + "rollback: /tmp/external-id.rollback.json", + "rollback sha256: rollback-digest", + "journal: /tmp/external-id.result.json", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + stdout := captureStdout(t, func() { + if err := test.emit(); err != nil { + t.Fatalf("emit human output: %v", err) + } + }) + for _, want := range test.want { + if !strings.Contains(stdout, want) { + t.Fatalf("human output missing %q:\n%s", want, stdout) + } + } + }) + } +} + func TestSafeSyncRunsPreflightPreviewAndAdditiveApply(t *testing.T) { enabled := false patched := false @@ -149,14 +340,16 @@ func TestSafeSyncRunsPreflightPreviewAndAdditiveApply(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots/latestProcessed": w.Header().Set("Content-Type", "application/json") _, _ = fmt.Fprintf(w, `{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}`, processedAt) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots": + _, _ = fmt.Fprintf(w, `{"snapshots":[{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}]}`, processedAt) case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":false}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":false}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") _, _ = fmt.Fprintf( w, - `[{"type":"AWS","name":"setup-a","regions":{"us-east-1":{"testInstant":123}},"assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":%t}]}]`, + `[{"type":"AWS","name":"setup-a","regions":{"us-east-1":{"testInstant":123}},"assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":%t}]}]`, enabled, ) case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/setup-a": @@ -220,15 +413,17 @@ func TestSafeSyncHandlesMultipleSetups(t *testing.T) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots/latestProcessed": _, _ = fmt.Fprintf(w, `{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}`, processedAt) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots": + _, _ = fmt.Fprintf(w, `{"snapshots":[{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}]}`, processedAt) case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": _, _ = w.Write([]byte(`{"items":[ - {"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Collected?":false}, - {"Cloud Setup ID":"setup-b","Cloud Account ID":"222","Collected?":false} + {"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Collected?":false}, + {"Cloud Setup ID":"setup-b","Cloud Account ID":"222222222222","Collected?":false} ]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": _, _ = w.Write([]byte(`[ - {"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":false}]}, - {"type":"AWS","name":"setup-b","assumeRoleInfos":[{"roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":false}]} + {"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":false}]}, + {"type":"AWS","name":"setup-b","assumeRoleInfos":[{"roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":false}]} ]`)) case r.Method == http.MethodPatch && strings.HasPrefix(r.URL.Path, "/api/networks/network-1/cloudAccounts/"): setupID := strings.TrimPrefix(r.URL.Path, "/api/networks/network-1/cloudAccounts/") @@ -276,10 +471,12 @@ func TestSafeSyncRequiresConfirmationOutsideAutomation(t *testing.T) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots/latestProcessed": _, _ = fmt.Fprintf(w, `{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}`, processedAt) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots": + _, _ = fmt.Fprintf(w, `{"snapshots":[{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}]}`, processedAt) case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": - _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Collected?":false}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Collected?":false}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": - _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":false}]}]`)) + _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":false}]}]`)) case r.Method == http.MethodPatch: patched = true _, _ = w.Write([]byte(`{}`)) @@ -318,10 +515,12 @@ func TestSafeSyncDoesNotPatchWhenNoChangesAreNeeded(t *testing.T) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots/latestProcessed": _, _ = fmt.Fprintf(w, `{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}`, processedAt) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots": + _, _ = fmt.Fprintf(w, `{"snapshots":[{"id":"snapshot-1","state":"PROCESSED","processedAt":%q}]}`, processedAt) case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": - _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Collected?":true}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Collected?":true}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": - _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = true _, _ = w.Write([]byte(`{}`)) @@ -359,7 +558,7 @@ func TestSafeSyncStopsWhenPreflightIsNotReady(t *testing.T) { server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": - _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/snapshots/latestProcessed": _, _ = w.Write([]byte(`{"id":"stale","state":"PROCESSED","processedAt":"2020-01-01T00:00:00Z"}`)) case r.Method == http.MethodPatch: diff --git a/docs/ARCHITECTURE_REVIEW.md b/docs/ARCHITECTURE_REVIEW.md new file mode 100644 index 0000000..d2fcc92 --- /dev/null +++ b/docs/ARCHITECTURE_REVIEW.md @@ -0,0 +1,422 @@ +# Architecture Review: `awssync` + +Review date: 2026-07-25 +Repository state reviewed: `0c0dbd5` (`forwardnetworks/aws-sync`) + +## Executive summary + +- **CRITICAL — CONFIRMED — FIXED (`1828278`, `e854787`):** the original review found independent existing-setup mutation paths. Existing AWS setup PATCHes now have one typed intent model and one test-enforced `GuardAndApply` chokepoint. Setup creation remains a separate POST operation because it does not replace an existing account list (`internal/app/apply_gateway.go`, `internal/app/patch_chokepoint_test.go`). +- **CRITICAL — CONFIRMED — FIXED (current uncommitted retirement):** NQE pruning equated “not present in this observed query result” with “remove from the setup.” The deeper error was treating `network.cloudAccounts` as a configured-account inventory at all: it is a union of successfully collected accounts and accounts merely visible through Organizations metadata, with routine partial results. `--prune-missing` now refuses and the NQE policy constructor can produce only `Additive`; planning also rejects `CompleteInventory` for an NQE source. +- **CRITICAL — CONFIRMED — FIXED (current uncommitted retirement):** A partial NQE result could remove most configured accounts when pruning and sufficiently broad ceilings were enabled. Live measurements on 2026-07-25 showed network `253234` at 978 configured versus 10 NQE rows (968 deletions, all enabled), and network `253236` at 565 configured versus 540 rows (27 deletions, all enabled). The removal path is now unreachable regardless of pagination completeness or evidence flags. +- **CRITICAL — CONFIRMED:** A truly empty NQE result is rejected, so zero rows do not directly become “delete everything”; this protection does not cover a one-row or otherwise partial result (`internal/app/run.go:1084-1102`). +- **CRITICAL — CONFIRMED:** The client reads a setup, constructs a complete `assumeRoleInfos` array, and PATCHes it without `ETag`, version, `If-Match`, or another atomic compare-and-swap token (`internal/api/client.go:76-105`, `internal/api/client.go:344-363`, `internal/api/client.go:432-440`). +- **CRITICAL — CONFIRMED:** The pre-PATCH re-read is only a time-of-check check; a Forward UI edit after that GET and before the PATCH can still be overwritten (`internal/app/run.go:337-356`, `internal/app/run.go:1851-1870`). +- **HIGH — CONFIRMED:** `apply-plan` classifies danger only by missing account IDs; a reviewed payload can keep every ID but set every account to `enabled:false` without `--allow-removals` or removal ceilings (`internal/app/apply_plan.go:58-79`, `internal/app/apply_plan.go:108-130`). +- **HIGH — CONFIRMED — FIXED (`e854787`):** External ID remains a specialized planning adapter, but mutation now uses the shared intent/gateway with rollback, final equality re-read, digest authorization, zero-diff suppression, and result journal. Atomic revision checking remains unavailable (`internal/app/external_id.go`, `internal/app/apply_gateway.go`). +- **HIGH — CONFIRMED:** Standard interactive sync previews and confirms one computation, then recomputes without passing the reviewed payload hash; only `safe-sync` binds apply to the preview digest (`cmd/awssync/main.go:75-138`, `cmd/awssync/main.go:213-240`). +- **HIGH — CONFIRMED:** Webhook jobs call `app.Run` directly, so they bypass the preflight command and any per-job confirmation; startup `--yes` is the only confirmation (`cmd/awssync/main.go:847-887`, `internal/webhook/server.go:167-193`). +- **HIGH — CONFIRMED — FIXED (current uncommitted webhook slice):** webhook apply mode requires Basic Auth, and event network/setup scope can only equal or narrow configured scope (`internal/webhook/server.go`). +- **HIGH — CONFIRMED:** Webhook deduplication records an event before queue admission and before successful processing; queue-full and failed-job retries can be acknowledged as duplicates and lost (`internal/webhook/server.go:139-148`, `internal/webhook/server.go:180-215`). +- **HIGH — HISTORICAL, NQE DESTRUCTIVE CONSEQUENCE FIXED:** Older delayed webhook events were especially dangerous when the daemon could prune. Webhook NQE work is now additive even if event ordering regresses; ordering still matters for correctness of additions/re-enables. +- **HIGH — CONFIRMED:** Multi-setup apply is a sequential PATCH loop with no transaction or durable progress record; failure on setup N leaves earlier setups changed and later setups untouched (`internal/app/run.go:851-863`, `internal/app/run.go:356-359`). +- **HIGH — CONFIRMED:** HTTP PATCH is automatically retried after transport and selected status failures, but no idempotency key or revision precondition is sent (`internal/api/client.go:402-450`, `internal/api/client.go:460-492`). +- **HIGH — CONFIRMED:** `safe-sync` is genuinely additive with respect to membership and refuses its own preview if it contains removals, but those guarantees live in its CLI orchestration rather than the mutation boundary (`cmd/awssync/main.go:165-240`, `internal/app/run.go:1063-1076`). +- **HIGH — CONFIRMED:** `sync-accounts` treats a reviewed manifest as authoritative and turns omission into removal; it bypasses NQE candidate and organization-evidence checks by setting `AuthoritativeInput` (`internal/app/account_manifest.go:71-101`, `internal/app/run.go:321-333`). +- **HIGH — CONFIRMED:** There is no domain distinction between “absent,” “suspended,” “closed,” “moved,” and “explicitly deprovisioned” in the reconciliation rows; the planner consumes raw string-keyed maps containing only ID/name/setup/evidence fields (`internal/app/run.go:20-39`, `internal/app/run.go:1263-1311`). +- **MEDIUM — CONFIRMED:** The NQE paginator stops on any short page and has no total-count, completeness marker, repeated-page detection, or maximum-page guard (`internal/api/client.go:227-277`). +- **MEDIUM — CONFIRMED:** In single-setup mode, local filtering is disabled and rows without setup identity are assigned to that setup, increasing the damage from a saved query or server-side filter that returns overbroad data (`internal/api/client.go:302-318`, `internal/app/run.go:1326-1347`). +- **MEDIUM — CONFIRMED — FIXED (`00b7e89`):** the original adapters silently accepted some first-wins duplicates. Typed adapters now reject duplicate/conflicting account identities consistently (`internal/app/adapters.go`, `internal/app/domain.go`). +- **MEDIUM — CHANGED (Phase 1):** The shared domain now validates exactly 12 digits in NQE parsing as the account-ID contract; this fails previously lenient inputs consistently and is a deliberate fail-closed availability tradeoff (`internal/app/run.go:1313-1324`, `internal/app/account_manifest.go:14-50`, `internal/app/external_id.go:142-153`). +- **MEDIUM — CONFIRMED:** Additive reconciliation re-enables every disabled account retained in the target, including configured accounts absent from the NQE result (`internal/app/run.go:1228-1261`, `internal/app/run.go:1666-1672`). +- **MEDIUM — CONFIRMED — FIXED (current uncommitted webhook slice):** explicit snapshot IDs, including webhook-supplied snapshots, are looked up and checked against `MaxSnapshotAge`; webhook watermarks also reject old-after-new delivery (`internal/app/run.go`, `internal/webhook/state.go`). +- **MEDIUM — CONFIRMED:** Generated payloads can be time-dependent because a zero region `TestInstant` is replaced with `time.Now()`, making preview/apply digest stability depend on current setup data (`internal/app/run.go:1765-1781`). +- **MEDIUM — CONFIRMED:** `status` performs non-atomic “latest” and “list” reads, while `wait` has no monotonicity, paginated snapshot listing, unknown-terminal-state, or missing-snapshot handling beyond polling until context cancellation (`internal/api/client.go:334-342`, `internal/monitor/monitor.go:25-51`, `internal/monitor/monitor.go:54-100`). +- **MEDIUM — CONFIRMED:** The test suite contains meaningful removal, GovCloud, bounds, hash, and pre-PATCH-change tests; it is not merely happy-path coverage (`internal/app/run_test.go:389-433`, `internal/app/run_test.go:826-863`, `internal/app/run_test.go:931-1302`, `internal/app/apply_plan_test.go:72-251`). +- **HIGH — CONFIRMED:** The highest-risk adversarial cases remain untested: an edit in the final GET/PATCH race window, partial multi-setup apply, incomplete nonempty inventory, disabling through `apply-plan`, webhook retry loss/order, and External ID concurrency (`internal/app/apply_plan_test.go:209-251`, `internal/webhook/server_test.go:18-185`, `internal/app/external_id_test.go:16-229`). +- **MEDIUM — CONFIRMED:** Documentation says every apply writes rollback data, but External ID apply writes only an audit payload and the procedure documents manual reversal instead (`README.md:178-188`, `internal/app/external_id.go:241-251`, `docs/aws-account-sync-procedure.md:438-458`). +- **HIGH — CONFIRMED — HISTORICAL, FIXED (`1828278`, `e854787`):** the original review found safety spread across individual seams. Phase 3 replaced the divergent mutation sites with one guarded engine; the historical finding is retained to explain the refactor. +- **TARGET:** All modes should produce one typed `DesiredSetup`, one typed field-level `ChangeSet`, and one immutable `ApplyIntent`; every PATCH must pass one guard/CAS/audit gateway (`internal/app/run.go:955-980`, `internal/app/apply_plan.go:41-159`, `internal/app/external_id.go:62-251`). +- **TARGET:** Absence must not mean deletion unless the source proves completeness and organization/setup identity, or supplies an explicit deprovision tombstone (`internal/api/client.go:227-277`, `internal/app/run.go:1129-1136`). +- **TARGET STATUS:** authorization, removal/disable ceilings, zero-diff skip, rollback/audit, last-moment conflict detection, PATCH, and result journaling now live in `GuardAndApply`. Atomic concurrency and idempotent retry remain blocked by the Forward API contract (`internal/app/apply_gateway.go`). + +## Corrections (2026-07-25) + +- **2026-07-25:** The review's central NQE assumption is corrected. `FQ_6d355dca…` queries the snapshot's observed `network.cloudAccounts`, not configured Forward account membership and not an authoritative AWS Organizations inventory. Forward constructs that data as the union of accounts that collected successfully and accounts visible through `organizations:ListAccounts` metadata (`CloudAccountUtils.java:46,73`). Collector authorization failures are ignored (`AwsApi.java:1489`), and service exceptions return the partial accumulated list (`AwsPipeline.java:2681`). Pagination completeness can prove only that this already-partial result terminated cleanly; it cannot make absence a deletion signal. +- **2026-07-25:** Live production measurements demonstrate the consequence: network `253234` had 978 configured accounts and 10 NQE rows, so pruning would delete 968 accounts, all 968 enabled; network `253236` had 565 configured accounts and 540 NQE rows, so pruning would delete 27 accounts, all 27 enabled. +- **2026-07-25:** NQE-based removal is retired. The CLI keeps `--prune-missing` recognized but refuses it with guidance to `sync-accounts`; NQE policy construction is additive-only and planning rejects `CompleteInventory` for NQE snapshots. `CompleteInventory` remains because a complete human-reviewed manifest is legitimately authoritative, and `sync-accounts` continues through `ComputeDesired` and `GuardAndApply`. +- **2026-07-25:** `SUSPECTED` finding at the Forward boundary on unmodeled field loss is corrected to `CONFIRMED` top-level merge semantics with preserved omissions, based on `UpdateCloudAccountRequest.applyTo` in `~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateCloudAccountRequest.java`. +- **2026-07-25:** `SUSPECTED` behavior for `assumeRoleInfos` merge-vs-replace was updated to **CONFIRMED** replace-when-present; the field is set from the parsed request array in `UpdateAwsAccountRequest.applyTo` (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateAwsAccountRequest.java:88-91`). +- **2026-07-25:** Concurrency findings were corrected: no client-visible ETag/version or `If-Match` contract exists on `PatchCloudAccount`, and Forward’s internal update path uses a `kvStore.getAndUpdate` retry loop that can deterministically reapply stale intent onto fresh state (`~/src/fwd/app/src/main/java/com/forwardnetworks/cv/sources/cloud/CloudAccountService.java:204-235`). Phase 4 is therefore not blocked by ambiguity; it is closed pending API contract change and policy controls. +- **2026-07-25:** Additional confirmed server-side behavior now recorded: duplicate `assumeRoleInfos` account IDs are rejected with `BadRequestException`, and single-account setups cannot be updated to multi-account. +- **2026-07-25:** Operational facts were added: network `253234` has `978` accounts against `PageLimit = 1000` (22 accounts of headroom before truncation becomes immediate), and setup identity is targeting-name based on both `run` and API route binding (`internal/api/client.go:19`, `~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/controller/CloudAccountController.java:196-204`). +- **2026-07-25:** Confirmed that `regionToProxyServerId` is currently preserved by omission and explicitly copied from current setup state before patch payload construction (`internal/app/run.go:1837`), matching observed behavior despite `collect`-field omissions. +- **2026-07-25:** Phase 1 deliberately made account-ID parsing fail-closed, and the tradeoff is recorded as explicit risk: one malformed NQE row now fails the whole plan instead of being silently skipped. Skipping rows is the mechanism by which a partial inventory becomes a deletion, so failing closed is the intended behavior — but on a large setup a single bad row is a full sync outage. +- **2026-07-25:** Phases 1 and 2 are complete. Commit `00b7e89` introduced typed domain adapters, `8cf4ef9` made absence-based removal fail closed without completeness proof, and `fbf78bb` centralized deterministic desired-state/diff computation. Findings below are retained and marked fixed rather than removed. +- **2026-07-25:** Phase 3 is complete. Commits `1828278` and `e854787` route planned sync, manifests, `apply-plan`, and External ID mutation through `GuardAndApply`. `internal/app/patch_chokepoint_test.go` enforces exactly one production caller of `api.PatchCloudAccount`. +- **2026-07-25:** The Phase 4 CAS proposal is closed by finding, not implemented: Forward exposes no revision token. Commit `1828278` shipped the compensating `--allow-unattended-destructive` policy, the last-moment equality re-read, and a durable per-setup result journal; neither is atomic CAS. +- **2026-07-25:** The current uncommitted webhook slice (left uncommitted by instruction) requires Basic Auth and an explicit configured network whenever apply is enabled, intersects event and configured scope, records successful scoped dedupe and snapshot watermarks in an atomic JSON state file, makes failed work redeliverable, rejects backward snapshot movement, and validates explicit snapshot age. Phase 0's guarded webhook characterization assertions pass unchanged when enabled; only per-test state-file isolation/cleanup scaffolding was added. +- **2026-07-25:** Phase 5 webhook durability is complete in the current uncommitted slice. Schema v2 adds pending and dead-letter event records to the existing atomic state file, persists admission before `202`, replays queued and in-flight work after restart, and bounds failures at five attempts with exponential backoff. Re-delivering a dead-lettered event starts a fresh bounded cycle so an operator can drain it after correcting the cause. Dedupe and watermarks are still written only in the atomic success transition. +- **2026-07-25:** Rollback artifacts are corrected from “complete setup” copies to pre-change PATCH payloads. They contain the complete `assumeRoleInfos` account list and the PATCHable setup fields `type`, `name`, `regions`, `regionToProxyServerId`, and `proxyServerId`. They do not capture `collect`, `connectionTimeoutSeconds`, `requestTimeoutSeconds`, `numVirtualizedDevices`, or `useForwardAccountToAssumeRole`. This is safe for restoring an `awssync` mutation because Forward's top-level PATCH merge leaves absent fields unchanged, but the artifact is not a full backup and cannot reconstruct a setup from scratch. +- **2026-07-25:** Live write validation at `a674b3f` ran against Forward workspace networks `253234` and `253236`; both were restored to their byte-identical baseline endpoint hashes. Verified against real Forward were: `sync-accounts` removal of one account and restoration; the removal-ceiling, missing-`--allow-removals`, and missing-`--allow-unattended-destructive` refusals, each with a `failed` journal entry and no PATCH; `apply-plan` mutation and restoration from its emitted rollback for one setup and two setups; approval-digest stability across separate processes; webhook authentication, scope rejection with `403`, and dedupe suppression of a replay; `status` reporting `observation_atomic=false`; `wait`; and zero-diff suppression with no PATCH. +- **2026-07-25:** Two areas remain untested live. The older-snapshot `409` watermark rejection is unit-tested only because each validation workspace had a single snapshot. GovCloud paths are also test-only for this validation because neither workspace had an AWS GovCloud partition setup. +- **2026-07-25:** Live restoration also exposed an ordering distinction. Restoring an account set through `sync-accounts` sorts the list, so the raw endpoint hash can differ from the original even when the account configuration is semantically identical. Applying the emitted rollback preserves the original list order and reproduced the original endpoint bytes on both workspace networks. This is expected and matters only when operators use raw hashes to verify recovery. + +## Review basis + +This review covered the requested production files, their corresponding tests, the four named documents plus `README.md`, and the diffs for `0c0dbd5`, `2794e14`, `ac15c6c`, `fe4baf4`, and `b159af6`. The unmodified tree passed `go test ./...` and `go test -race ./...` (121 tests in six packages). + +Severity is ranked as requested: **CRITICAL** means credible data loss or silent destructive overwrite; **HIGH** means a destructive bypass or serious correctness/operability failure; **MEDIUM** means a material modeling or resilience gap; **LOW** means localized maintainability or diagnostic debt. + +“CONFIRMED” means the behavior is directly implemented or asserted in this repository. “SUSPECTED” means the conclusion depends on Forward server behavior or an external operational assumption not present in this repository. + +--- + +## 1. Core model + +### Verdict + +#### CRITICAL — CONFIRMED: there is no single reconcile-and-apply model + +The shared core is now typed adapters → pure reconciliation/classification → immutable `ApplyIntent` → `GuardAndApply`. Legacy CLI booleans are mapped once into tagged reconciliation policy at the boundary; they do not create alternate mutation paths (`internal/app/domain.go`, `internal/app/reconcile.go`, `internal/app/apply_gateway.go`). + +`ApplyPlan` and `ChangeExternalID` independently reimplement setup lookup, current-state parsing, validation, audit, concurrency checking, and PATCH behavior (`internal/app/apply_plan.go:41-159`, `internal/app/external_id.go:67-251`). Direct AWS Organizations and manifest onboarding use a separate create-payload builder and POST path, and deliberately refuse to update an existing named setup (`internal/app/run.go:376-542`). + +### Every binary path that can mutate a Forward AWS setup + +| Path | Desired-state computation | Mutation | Semantics and agreement | +|---|---|---|---| +| Root `awssync --apply` | Typed NQE adapter and pure additive reconcile; `--prune-missing` is recognized only to refuse | Sequential per-setup execution through `GuardAndApply` | NQE absence cannot remove membership; shares typed diff, digest authorization, rollback/re-read/PATCH/journal with every writer. | +| `safe-sync` | Preflight, dry-run `app.Run`, then a second `app.Run`; no prune flag is exposed | Shared `GuardAndApply` gateway | Additive membership and preview removal rejection are adapter guarantees; digest authorization and zero-diff suppression are shared gateway guarantees. | +| `webhook --apply --yes` | Authenticated event selects an exact snapshot and narrows configured network/setup scope, then calls ordinary `app.Run` (`internal/webhook/server.go`) | Shared `GuardAndApply` gateway | No preflight or per-event interactive confirmation; launch-time automation policy and intent digest apply, with durable event dedupe/watermark state. | +| `sync-accounts` | Reviewed manifest enters through the typed manifest adapter with complete-inventory policy | Shared `GuardAndApply` gateway | Omission is removal; the human manifest is the asserted completeness proof rather than NQE candidate/org evidence. | +| `apply-plan --yes` | Accepts operator-authored JSON targets, adapts them to typed payloads, and classifies every field change against current state | Shared `GuardAndApply` gateway | Lacks NQE candidate/completeness evidence by format, but disable/removal classification, budgets, digest, rollback, re-read, PATCH, and journal are shared. | +| `external-id --apply` | Typed explicit operations modify selected External IDs against current state | Shared `GuardAndApply` gateway | Preserves membership/enabled values and shares digest authorization, rollback, final re-read, zero-diff suppression, PATCH, and journal. AWS trust-policy readiness is not verified. | +| `discover-org --post` | Direct AWS Organizations discovery produces a create payload (`internal/awsorg/discover.go:75-107`, `internal/app/run.go:376-487`) | POST creates a new setup (`internal/app/run.go:476-487`) | It cannot reconcile an existing setup: an existing name is rejected, and zero discovered accounts are rejected (`internal/app/run.go:413-435`). | +| `onboard-accounts --post` | Reviewed manifest goes through the new-setup builder (`internal/app/account_manifest.go:62-68`, `internal/app/run.go:376-542`) | POST creates a new setup (`internal/app/run.go:476-487`) | It shares direct-onboarding semantics, not existing-setup reconciliation; the manifest loader requires a nonempty, unique, exact-12-digit list (`internal/app/account_manifest.go:21-59`). | + +`configure-webhook` mutates Forward webhook configuration, not the AWS setup account list, while `status`, `wait`, and the monitor are read-only with respect to cloud setups (`cmd/awssync/main.go:907-1052`, `internal/monitor/monitor.go:25-100`). + +### Semantic disagreements + +- **HIGH — CONFIRMED:** The shared planner always emits `Enabled: true` for target accounts, so standard, safe, webhook, and manifest sync re-enable disabled entries; External ID rotation preserves their prior enabled flags, while `apply-plan` accepts either value (`internal/app/run.go:1244-1261`, `internal/app/run.go:1666-1672`, `internal/app/external_id.go:121-193`, `internal/app/apply_plan.go:58-79`). +- **HIGH — CONFIRMED — FIXED (current uncommitted retirement):** “Missing” formerly meant preserve in default NQE mode but remove in prune and manifest modes. NQE policy construction is now unconditionally `Additive`; only reviewed manifests construct `CompleteInventory`, and the planner rejects that policy when the snapshot source is NQE. +- **HIGH — CONFIRMED:** `apply-plan` recognizes only add/remove ID membership, while the main planner separately recognizes add/remove/re-enable and External ID state; neither has a general typed field-level diff (`internal/app/apply_plan.go:108-130`, `internal/app/run.go:1135-1158`). +- **MEDIUM — CONFIRMED:** Zero-change suppression is inconsistent: safe-sync exits only when aggregate additions and re-enables are zero, External ID exits when its selected field is unchanged, and the shared executor plus `apply-plan` otherwise PATCH their planned setups even when account membership is unchanged (`cmd/awssync/main.go:223-227`, `internal/app/external_id.go:241-242`, `internal/app/run.go:851-863`, `internal/app/apply_plan.go:144-147`). + +### What the five commits reveal + +- **CONFIRMED:** `b159af6` introduced a reusable limits helper but enforcement remained duplicated in main planned sync, preflight, and `apply-plan` (`internal/app/removal_limits.go:14-93`, `internal/app/run.go:307-320`, `internal/app/preflight.go:138-150`, `internal/app/apply_plan.go:118-130`). +- **CONFIRMED:** `fe4baf4` added per-account External IDs both inside the planner and through the pre-existing independent External ID writer, increasing the number of credential mutation semantics (`internal/app/run.go:1137-1158`, `internal/app/external_id.go:67-251`, `internal/app/external_id_file.go:13-96`). +- **CONFIRMED:** `ac15c6c` made NQE reconciliation additive through `PreserveMissing`, but retained authoritative omission-as-delete and made the shared builder re-enable every target account (`internal/app/run.go:1063-1076`, `internal/app/run.go:1228-1261`, `internal/app/run.go:1666-1672`). +- **CONFIRMED:** `2794e14` added a separate safe-sync orchestration layer around the same planner instead of adding a safety policy object and mutation gateway (`cmd/awssync/main.go:165-257`). +- **CONFIRMED:** `0c0dbd5` added the no-change exit to that CLI layer only, leaving the underlying executor unchanged (`cmd/awssync/main.go:223-227`, `internal/app/run.go:851-863`). + +--- + +## 2. Deletion semantics + +### All intentional and incidental removal/disable paths + +| Removal or disable path | Trigger | Guards actually applied | Empty, partial, or stale source behavior | +|---|---|---|---| +| NQE prune through root CLI | Retired. Passing recognized `--prune-missing` returns an actionable error before credentials, NQE, planning, or apply. | No override exists. NQE policy construction returns only `Additive`, and planning rejects `CompleteInventory` for source `nqe`. | Empty, partial, stale, failed-collection, or wrong-organization observations cannot remove configured membership. | +| NQE prune through webhook | Retired. `serve-webhook --prune-missing` returns the same startup refusal. | No event, evidence flag, removal authorization, or ceiling can enable NQE deletion. | Event snapshots remain observed inventory and are additive regardless of apparent completeness. | +| Authoritative manifest sync | Configured ID omitted from the reviewed manifest (`internal/app/account_manifest.go:71-101`, `internal/app/run.go:1063-1076`) | Generic confirmation/`--yes`; `--allow-removals`; both removal ceilings; pre-PATCH re-read. Candidate, org-evidence, and GovCloud NQE evidence checks are bypassed because the source is marked authoritative (`cmd/awssync/main.go:780-845`, `internal/app/run.go:307-350`) | Empty manifests and invalid/duplicate IDs fail before planning; a nonempty incomplete human-generated manifest is accepted as complete and removes omissions within bounds (`internal/app/account_manifest.go:21-59`). | +| `apply-plan` target omission | An account ID present in current state is missing from an arbitrary reviewed payload (`internal/app/apply_plan.go:58-79`, `internal/app/apply_plan.go:108-114`) | `--yes`; `--allow-removals`; both ceilings; GovCloud removal always blocked; rollback file and pre-PATCH re-read (`cmd/awssync/main.go:488-536`, `internal/app/apply_plan.go:118-147`) | An empty `assumeRoleInfos` array is structurally accepted and can remove all commercial accounts if the explicit ceilings permit; there is no source evidence or completeness check (`internal/app/apply_plan.go:58-79`, `internal/app/apply_plan.go:108-130`). | +| `apply-plan` disable | Account ID remains present but its `enabled` field changes false | Typed `Disable` classification; gateway destructive authorization and aggregate/per-setup budgets | Independent of inventory, but no longer a destructive-policy bypass (`internal/app/apply_plan.go`, `internal/app/apply_gateway.go`; fixed by `e854787`). | +| Stale read/modify/write overwrite | A concurrent actor adds/removes/edits accounts after the gateway equality re-read but before full-list PATCH (`internal/app/apply_gateway.go`) | Every writer performs the same non-atomic equality re-read; no path can send a revision precondition (`internal/api/client.go`) | Not inventory-dependent. A newly added concurrent account absent from the target can still be silently removed inside the final race window. | + +No code path intentionally deletes the Forward setup object itself; setup mutations are POST for creation and PATCH for replacement/update (`internal/api/client.go:355-368`). + +### Observed inventory, emptiness, and truncation + +#### CRITICAL — CONFIRMED — ROOT CAUSE FIXED BY FEATURE RETIREMENT + +The prior analysis treated pagination completeness as the missing safety proof. That was one real defect, but not the root cause. `network.cloudAccounts` is an observed snapshot view: collected accounts unioned with accounts visible through Organizations metadata. Authorization failures can be ignored and service failures can return a partial accumulated result. A clean final NQE page proves only that pagination over that observed view terminated; it provides no expected configured count, organization identity, collection-success contract, or account-lifecycle assertion. + +The production measurements make the distinction concrete: + +| network | configured | NQE rows | would have been deleted | enabled among them | +|---|---:|---:|---:|---:| +| 253234 | 978 | 10 | 968 | 968 | +| 253236 | 565 | 540 | 27 | 27 | + +Accordingly: + +- **CONFIRMED — FIXED:** NQE absence cannot authorize deletion at any cardinality. The root CLI, preflight, and webhook retain `--prune-missing` only to refuse it; the NQE policy constructor is additive-only; and an internal NQE snapshot paired with `CompleteInventory` is rejected. +- **CONFIRMED — RETAINED:** Phase 2a's pagination completeness characterization remains useful for detecting truncated observed results and reporting data quality. It is no longer a gate that can turn absence into removal. +- **CONFIRMED:** `sync-accounts` can remove omissions because its input is an explicit, complete, human-reviewed manifest. Empty manifests and invalid/duplicate IDs fail before planning; `ComputeDesired`, destructive authorization, both removal ceilings, rollback, re-read, and `GuardAndApply` remain in force. +- **CONFIRMED:** `apply-plan` can directly express an empty target list, subject to its explicit removal authorization and bounds for commercial setups. That explicit target payload is a separate reviewed-operation path, not an inference from NQE absence. + +### Absence versus explicit deprovisioning + +#### CRITICAL — CONFIRMED — FIXED FOR NQE; MANIFEST REMOVAL RETAINED + +The domain distinguishes additive NQE absence (`Preserve`) from reviewed-manifest omission (`Remove`). Completeness metadata alone no longer selects the latter: `CompleteInventory` remains a legitimate policy kind only for the authoritative manifest path, while NQE construction and source validation prevent it from being selected for observed inventory. Explicit lifecycle tombstones and complete source-organization identity are still not modeled; the human review of the manifest is the removal assertion (`internal/app/domain.go`, `internal/app/reconcile.go`, `internal/app/account_manifest.go`). + +The direct AWS discovery code does know active versus non-active status, but it is used for new setup creation rather than existing reconciliation; non-active accounts are skipped unless `includeSuspended` is set (`internal/awsorg/discover.go:84-107`, `internal/awsorg/discover.go:130-146`, `internal/app/run.go:376-542`). + +--- + +## 3. Read-modify-write safety + +### Is PATCH a full-list replacement? + +#### CONFIRMED in the client contract + +The production model serializes `assumeRoleInfos` as a complete array in `PatchPayload`; the planner rebuilds every target entry, and rollback also captures a complete array (`internal/api/client.go:89-105`, `internal/app/run.go:1141-1165`, `internal/app/run.go:1819-1849`). The architecture document explicitly calls the account list “full-state, not incremental,” and `apply-plan` detects omission as removal before sending the payload (`docs/architecture-flow.md:132-143`, `internal/app/apply_plan.go:108-130`). + +#### CONFIRMED at the Forward server boundary + +The Forward server is confirmed to apply incoming fields with tri-state merge semantics (`JsonProp`) and set `assumeRoleInfos` only when present. In that case, `builder.assumeRoleInfos(roleInfos)` replaces the array; all other fields remain unset by omission (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateAwsAccountRequest.java:88-91`, `~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateCloudAccountRequest.java:73-81`). Top-level patching starts from `account.toBuilder()`, so omitted keys preserve existing values (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateCloudAccountRequest.java:80-94`). + +### Optimistic concurrency + +#### CRITICAL — CONFIRMED: no client-visible CAS token; server replay makes contention deterministic + +`PatchCloudAccount` sends a plain PATCH; request construction adds content type, accept, and Basic Auth only (`internal/api/client.go:355-363`, `internal/api/client.go:432-440`). `CloudAccount` contains no revision/version field and the PATCH function accepts no ETag or If-Match (`internal/api/client.go:76-105`, `internal/api/client.go:344-363`). + +`CloudAccountService` updates accounts via `kvStore.getAndUpdate(...)` with a transform that may be retried (`~/src/fwd/app/src/main/java/com/forwardnetworks/cv/sources/cloud/CloudAccountService.java:204-235`). On contention, the service re-reads fresh state and faithfully re-applies the client's absolute full-list intent onto it. The clobber is therefore deterministic rather than probabilistic: a concurrent edit is not lost to unlucky timing, it is lost *because* the retry loop correctly replays stale intent over newer state. + +All writers now capture an immutable baseline and `GuardAndApply` immediately re-GETs each selected setup before its PATCH, including External ID rotation (`internal/app/apply_gateway.go`, `internal/app/external_id.go`; fixed by `1828278` and `e854787`). This detects a change before that GET completes, but cannot protect the interval from the successful GET to the subsequent PATCH. The characterization race tests intentionally remain failing because no client-visible CAS token exists. + +### Idempotency, retries, and partial failure + +- **HIGH — CONFIRMED:** Reapplying an identical complete target is logically idempotent if no concurrent writer exists, because each retry sends the same serialized body; no application-level idempotency key makes that guarantee explicit (`internal/api/client.go:416-450`). +- **HIGH — CONFIRMED:** PATCH is classified as retryable on transport errors and 429/502/503/504 responses; if the server committed but the response was lost, the client sends the same body again without a revision or operation key (`internal/api/client.go:402-450`, `internal/api/client.go:460-492`). +- **HIGH — CONFIRMED:** A multi-setup plan is not atomic. `GuardAndApply` stops at the first conflict/PATCH error after any earlier successes. Since `1828278`, the returned result and atomically rewritten journal preserve `planned`, `pending`, `applied`, `conflicted`, and `failed` per-setup outcomes (`internal/app/apply_gateway.go`). +- **HIGH — CONFIRMED:** The durable journal makes partial completion inspectable, but automatic resume and rollback remain absent; an operator must use the journal and rollback artifact deliberately (`internal/app/apply_gateway.go`). +- **MEDIUM — CONFIRMED:** Rollback artifacts are written before every changed gateway apply, but rollback remains manual and uses the same non-transactional `apply-plan` path (`internal/app/apply_gateway.go`, `internal/app/apply_plan.go`, `docs/aws-account-sync-procedure.md:600-608`). +- **HIGH — CONFIRMED — FIXED (`e854787`):** External ID mutation now constructs a full typed target and uses `GuardAndApply`, which writes the pre-change rollback artifact, applied audit artifact, and result journal before PATCH (`internal/app/external_id.go`, `internal/app/apply_gateway.go`). +- **MEDIUM — CONFIRMED — FIXED (`1828278`):** `GuardAndApply` returns after journaling when every `ChangeSet` is empty, so every adapter centrally suppresses zero-diff PATCHes (`internal/app/apply_gateway.go`). + +#### CONFIRMED: top-level omitted fields are preserved + +`UpdateCloudAccountRequest` starts from `account.toBuilder()` and applies each present field via `ifPresent` (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateCloudAccountRequest.java:73-81`, `~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateCloudAccountRequest.java:80-94`). Unmodeled server fields remain unless explicitly modified by the request; omission in awssync payload therefore preserves existing values on these keys. + +#### Server-side guard note + +`UpdateAwsAccountRequest` has duplicate `assumeRoleInfos` account-ID validation and rejects a duplicate with `BadRequestException` (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateAwsAccountRequest.java:63-69`), and it also rejects updating a single-account setup to multi-account (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/json/cloud/UpdateAwsAccountRequest.java:89`). + +--- + +## 4. Edge cases not handled + +### Inventory cardinality and completeness + +- **HIGH — CONFIRMED — zero accounts:** NQE, manifest, and direct-onboarding zero-account sources fail; there is no explicitly authorized “empty authoritative desired set” model, while `apply-plan` can express the same outcome as arbitrary JSON (`internal/app/run.go:1097-1102`, `internal/app/account_manifest.go:38-40`, `internal/app/run.go:413-416`, `internal/app/apply_plan.go:58-79`). +- **MEDIUM — CONFIRMED — current setup has zero accounts:** the planner cannot derive a role name and skips the setup; External ID mutation rejects it, so the tool cannot repair an empty existing setup through its normal paths (`internal/app/run.go:1120-1126`, `internal/app/external_id.go:117-119`). +- **MEDIUM — CONFIRMED — one setup:** local NQE filtering is disabled when zero or one setup is requested, and setup-less rows are assigned wholesale to the sole setup; a query/filter regression can import unrelated AWS rows (`internal/api/client.go:302-318`, `internal/app/run.go:1326-1347`). +- **CRITICAL — CONFIRMED — FIXED BY RETIREMENT — partial nonempty observed inventory:** there is no manifest contract or total configured-account proof, so the former prune behavior interpreted ordinary observation gaps as removals. NQE reconciliation is now source-enforced additive; partial data can miss additions but cannot delete membership. +- **MEDIUM — CONFIRMED — pagination pathologies:** the client has no repeated-page/cursor guard or advertised total; an API that repeats a full page loops indefinitely, and an API that silently caps below 1000 produces a false complete result (`internal/api/client.go:19`, `internal/api/client.go:242-277`). +- **LOW — CONFIRMED — direct AWS pagination errors:** Organizations discovery safely returns an error on any account or parent page failure rather than applying its accumulated prefix, but no test covers a later-page failure (`internal/awsorg/discover.go:84-107`, `internal/awsorg/discover.go:148-164`, `internal/awsorg/discover_test.go:49-96`). + +### Identity, duplication, and movement + +- **HIGH — CONFIRMED — account moved between organizations/setups:** the desired row contains no source organization identity or move operation; each setup is patched independently, so a move across two selected setups can partially complete and leave the account in both or neither (`internal/app/run.go:20-39`, `internal/app/run.go:1104-1201`, `internal/app/run.go:851-863`). +- **MEDIUM — CONFIRMED — FIXED (`00b7e89`):** duplicate/conflicting discovered IDs are rejected by typed adapters instead of silently first-wins (`internal/app/adapters.go`). +- **MEDIUM — CONFIRMED — FIXED (`00b7e89`):** duplicate configured account identities are rejected consistently at the current-setup adapter boundary (`internal/app/adapters.go`). +- **MEDIUM — CONFIRMED — duplicate setup names:** `SetupID` is derived from setup name (`internal/app/run.go:1481`) and the route key in the Forward API is `accountName` (`~/src/fwd/web/src/main/java/com/forwardnetworks/cv/web/controller/CloudAccountController.java:196-204`), so a later same-name setup overwrites the earlier one as a targeting collision (`internal/app/run.go:1474-1491`). +- **MEDIUM — CHANGED (Phase 1):** account IDs are now validated to exactly 12 digits across shared adapters (`internal/app/run.go:1313-1324`, `internal/app/account_manifest.go:14-50`, `internal/app/external_id.go:142-153`). This is a fail-closed change: malformed rows fail with operator-visible errors like `invalid AWS account ID "setup-a"; expected exactly 12 digits`, and a single malformed row can block a full sync on a large setup (`internal/app/run_test.go:748`). +- **MEDIUM — CONFIRMED — type/case/whitespace mismatch:** raw row extraction requires exact column keys and string values; numeric JSON IDs become empty, alternate key case is ignored, and setup matching is exact inside the planner even though interactive CLI selection canonicalizes case (`internal/app/run.go:1263-1289`, `internal/app/run.go:1412-1425`, `cmd/awssync/main.go:1685-1724`). +- **MEDIUM — CONFIRMED — FIXED (`00b7e89`):** when both account ID and role ARN are present, typed current-state adaptation asserts that their account components agree (`internal/app/adapters.go`, `internal/app/adapters_test.go`). +- **MEDIUM — CONFIRMED — FIXED (`fbf78bb`):** name drift is a typed `Rename` change and is no longer hidden behind membership-only no-op logic (`internal/app/reconcile.go`). + +### Lifecycle and state + +- **HIGH — CONFIRMED — suspended/closed accounts:** NQE planning has no lifecycle field and cannot distinguish suspension from query absence; direct AWS onboarding either omits non-active accounts or, with `includeSuspended`, creates them as ordinary enabled target entries (`internal/app/run.go:20-39`, `internal/awsorg/discover.go:84-107`, `internal/app/run.go:645-657`). +- **MEDIUM — CONFIRMED — disabled accounts absent from additive inventory:** because current entries are merged into the target and every emitted target is enabled, an additive run re-enables disabled accounts even when NQE did not return them (`internal/app/run.go:1228-1261`, `internal/app/run.go:1666-1672`). +- **MEDIUM — CONFIRMED — explicit disable intent:** only raw `apply-plan` can preserve or introduce `enabled:false` as desired state; the normal planner has no typed disable transition (`internal/app/apply_plan.go:58-79`, `internal/app/run.go:1666-1672`). + +### External ID drift and rotation + +- **HIGH — CONFIRMED:** External ID rotation changes Forward first/only; there is no verification that the matching AWS role trust policy already accepts the value and no coordinated two-phase rotation (`internal/app/external_id.go:161-251`). +- **HIGH — CONFIRMED — FIXED (`e854787`):** External ID now computes/classifies the target before gateway authorization and receives the common rollback artifact and concurrent-update recheck. AWS trust-policy readiness remains unverified (`internal/app/external_id.go`, `internal/app/apply_gateway.go`). +- **MEDIUM — CONFIRMED:** standard sync can also change External IDs from a CSV while its main diff reports membership/re-enable state rather than a typed per-account credential change, weakening review visibility (`internal/app/run.go:1137-1158`, `internal/app/run.go:1173-1201`). +- **MEDIUM — CONFIRMED:** mixed-ID setups require explicit assignments for new accounts, which safely fails closed, but there is no drift comparison to AWS or planned rotation window (`internal/app/run.go:1611-1664`). + +### Ordering, time, and monitor/webhook behavior + +- **HIGH — CONFIRMED — FIXED (current uncommitted webhook slice):** the worker resolves snapshot chronology and enforces an in-progress/applied watermark per network/setup both before admission and again before execution. Older events receive a non-2xx response or are discarded before `app.Run`; watermarks survive restart (`internal/webhook/server.go`, `internal/webhook/state.go`). +- **HIGH — CONFIRMED — FIXED (current uncommitted webhook slice):** queue-full rejection never creates an in-flight/dedupe record; a failed `app.Run` clears in-flight admission; only successful work enters the 24-hour completed-event map. Concurrent same-key deliveries wait for the first outcome, so failure is redeliverable without double-running successful work (`internal/webhook/server.go`, `internal/webhook/state.go`). +- **MEDIUM — CONFIRMED — FIXED (current uncommitted webhook slice):** dedupe keys contain type, network, snapshot, sorted setup scope, and event ID and are persisted across restart in the webhook state file (`internal/webhook/state.go`). +- **HIGH — CONFIRMED — FIXED (current uncommitted webhook slice):** a configured network must equal the event network, event setup IDs must be a subset of configured setup IDs, and an omitted event setup scope inherits the configured set. Apply-enabled servers refuse to start unless a network and both Basic Auth values are configured; requests must authenticate (`internal/webhook/server.go`, `cmd/awssync/main.go`). +- **MEDIUM — CONFIRMED — PARTIALLY FIXED (current uncommitted webhook slice):** explicit snapshot IDs now use the snapshot list to enforce `MaxSnapshotAge`; the pre-existing future-timestamp behavior remains because validation still only rejects age greater than the maximum (`internal/app/run.go`). +- **MEDIUM — CONFIRMED — FIXED (`fbf78bb`):** payload planning uses an injected policy planning instant, so preview/apply no longer derive region test time independently (`internal/app/domain.go`, `internal/app/reconcile.go`). +- **LOW — CONFIRMED — filename ordering:** default artifact names use second-level timestamps, so multiple runs in one second can address the same filename and the later atomic rename can replace the earlier artifact (`internal/app/run.go:739-751`, `internal/app/run.go:1872-1970`). +- **MEDIUM — CONFIRMED — monitor consistency:** `Status` fetches latest and the list in separate requests; `Wait` compares states case-sensitively, recognizes only `FAILED` and `ARCHIVED` as terminal, and polls forever for an absent snapshot until context cancellation (`internal/monitor/monitor.go:25-51`, `internal/monitor/monitor.go:54-100`). + +--- + +## 5. Guard placement + +### Guard matrix + +| Guard | Root NQE | `safe-sync` | Webhook | Manifest sync | `apply-plan` | External ID | +|---|---:|---:|---:|---:|---:|---:| +| Typed desired-state/change validation | Yes | Yes | Yes, through root NQE adapter | Yes, through manifest adapter | Yes, payload classified against typed current setup | Yes, typed External ID operations classified against current setup | +| Preflight required | No | Yes | No | No | No | No | +| Apply authorization bound to immutable intent digest | Yes | Yes | Yes, unattended actor | Yes | Yes | Yes | +| Removal/disable authorization and ceilings | Additive/no removal | Additive invariant | Additive/no removal | Yes | Yes | Applicable if classified destructive | +| Candidate/org/completeness evidence | Diagnostic only; never authorizes removal | Diagnostic/additive | Diagnostic only; never authorizes removal | Reviewed manifest completeness policy | Compatibility allowance for operator-authored file; GovCloud still blocked | Explicit operation, not absence-based | +| Rollback + applied audit artifact | Yes | Yes | Yes | Yes | Yes | Yes | +| Last-moment equality re-read | Yes | Yes | Yes | Yes | Yes | Yes | +| Atomic CAS/version | No (`internal/api/client.go:355-363`) | No | No | No | No | No | +| Zero-diff PATCH suppression | Gateway | Gateway | Gateway | Gateway | Gateway | Gateway | +| Durable per-setup result journal | Yes | Yes | Yes | Yes | Yes | Yes | + +### Bypasses + +1. **HIGH — CONFIRMED — PARTIALLY FIXED (`e854787`):** `apply-plan` still bypasses source inventory/candidate evidence by design, but its payload is now typed/classified and `Disable` and `Remove` share gateway authorization and budgets. The compatibility adapter explicitly records its lack of NQE evidence (`internal/app/apply_plan.go`, `internal/app/apply_gateway.go`). +2. **HIGH — CONFIRMED — FIXED (`e854787`):** External ID apply no longer bypasses common rollback, final equality re-read, intent digest, zero-diff suppression, or journaling (`internal/app/external_id.go`, `internal/app/apply_gateway.go`). +3. **HIGH — CONFIRMED — RESIDUAL:** webhook still bypasses preflight and interactive per-event confirmation. Apply mode requires the launch-time `--yes` automation decision, gateway authorization, and now authenticated requests; unattended destructive work additionally requires `--allow-unattended-destructive` (`cmd/awssync/main.go`, `internal/webhook/server.go`). +4. **HIGH — CONFIRMED — FIXED (current uncommitted webhook slice):** webhook events can narrow configured setup scope but cannot replace a configured network or expand a configured setup allowlist (`internal/webhook/server.go`). +5. **HIGH — CONFIRMED:** `sync-accounts` bypasses candidate and organization evidence by asserting a human manifest is authoritative; omission remains destructive (`internal/app/account_manifest.go:71-101`, `internal/app/run.go:321-333`). +6. **HIGH — CONFIRMED — MITIGATED (`1828278`):** noninteractive root/CI with `--yes` does not require preflight, but gateway policy blocks destructive work unless `--allow-unattended-destructive` is also explicit. Additive automation remains allowed (`cmd/awssync/main.go`, `internal/app/apply_gateway.go`). +7. **HIGH — CONFIRMED — FIXED (`1828278`):** every gateway authorization is checked against the immutable `ApplyIntent` digest; standard interactive preview copies its expected plan digest into apply (`cmd/awssync/main.go`, `internal/app/apply_gateway.go`). +8. **MEDIUM — CONFIRMED — FIXED (current uncommitted webhook slice):** explicit snapshots, including webhook event snapshots, are looked up and checked against `MaxSnapshotAge` (`internal/app/run.go`). +9. **MEDIUM — CONFIRMED — FIXED (`1828278`):** empty `ChangeSet` is centrally suppressed in `GuardAndApply` for every caller (`internal/app/apply_gateway.go`). +10. **MEDIUM — CONFIRMED:** a positive candidate or OU count bypasses the no-evidence block without proving inventory completeness (`internal/app/run.go:1427-1455`). + +Since `1828278`/`e854787`, `GuardAndApply` is a true mutation chokepoint: it contains the only production call to `api.PatchCloudAccount`, and `internal/app/patch_chokepoint_test.go` fails if another caller appears. Preflight remains an advisory adapter-specific layer; mutation authorization, destructive classification, ceilings, evidence, rollback, last-moment re-read, zero-diff suppression, PATCH, and result journaling are gateway responsibilities. + +--- + +## 6. Test coverage + +### What is covered well enough to be meaningful + +- **CONFIRMED:** Main planner tests exercise additive preservation, direct rejection of `CompleteInventory` for NQE snapshots, manifest completeness, and malformed IDs (`internal/app/run_test.go`, `internal/app/reconcile_test.go`). +- **CONFIRMED:** Apply tests cover rollback output and reviewed-payload hash mismatch; supported removal opt-in and both blast-radius dimensions are exercised through authoritative manifest and `apply-plan` paths. Former NQE removal/evidence cases are explicitly marked obsolete rather than silently deleted (`internal/app/account_manifest_test.go`, `internal/app/run_test.go`, `internal/app/apply_plan_test.go`). +- **CONFIRMED:** `apply-plan` tests cover GovCloud rejection, percentage/count bounds, and a setup change observed by the second GET before PATCH (`internal/app/apply_plan_test.go:72-251`). +- **CONFIRMED:** safe-sync tests cover preflight/preview/apply, multiple setups, noninteractive confirmation, zero-change skip, and failed preflight (`cmd/awssync/main_test.go:143-419`). +- **CONFIRMED:** External ID tests cover set/clear, selected-account scoping, CSV actions, preservation of other entries, and unsafe input rows (`internal/app/external_id_test.go:16-229`). +- **CONFIRMED:** API tests cover normal pagination, setup filtering, selected retries, and non-retry of create (`internal/api/client_test.go:13-155`, `internal/api/client_test.go:253-364`). +- **CONFIRMED:** Phase 0 characterization covers the final GET/PATCH race, partial multi-setup failure, disable classification, External ID recovery/concurrency, and webhook loss/order/scope. The incomplete-nonempty and undetectable-short-page NQE deletion premises are explicitly obsolete because NQE pruning is unreachable; Phase 2a completeness tests remain elsewhere. All Phase 0 guard constants remain `false` (`internal/*/architecture_failure_test.go`). +- **CONFIRMED:** Gateway tests cover zero-diff suppression, plan-digest authorization, destructive budgets/evidence, last-moment conflict, rollback, and durable partial journals. A source scan test enforces exactly one production `PatchCloudAccount` caller (`1828278`, `e854787`, `internal/app/apply_gateway_test.go`, `internal/app/patch_chokepoint_test.go`). +- **CONFIRMED:** Webhook tests cover apply-mode authentication and the guarded six-case delivery/scope contract; explicit snapshot freshness has direct fresh/stale tests (`internal/webhook/server_test.go`, `internal/webhook/architecture_failure_test.go`, `internal/app/snapshot_freshness_test.go`). + +The suite is therefore not “mostly happy paths.” It verifies many of the reactive safeguards. Its weakness is that it tests each safeguard in the path where it was added, not the system-level invariants across every writer. + +### Highest-value missing tests, in priority order + +1. **CRITICAL — unresolved CAS characterization:** the final GET/PATCH race tests now exist and must continue to demonstrate clobber until Forward exposes a revision token. They are evidence of a missing contract, not tests to make green by weakening the assertion (`internal/app/architecture_failure_test.go`, `internal/api/architecture_failure_test.go`). +2. **HIGH — ambiguous PATCH retry:** simulate server commit followed by connection loss and a concurrent edit before retry. No idempotency/revision contract currently prevents overwrite (`internal/api/client.go`). +3. **HIGH — cross-setup move:** fail either PATCH order for an account moving from setup A to B and require an explicit transaction/move invariant that prevents duplicate or missing final ownership (`internal/app/reconcile.go`, `internal/app/apply_gateway.go`). +4. **HIGH — partial-operation recovery:** durable per-setup results now exist, but explicit resume and verified rollback commands still need crash/restart tests (`internal/app/apply_gateway.go`). +5. **MEDIUM — account identity property/fuzz tests:** retain exact 12-digit, whitespace, numeric JSON, conflicting duplicate-page, account/ARN disagreement, and duplicate setup-name coverage beyond the current table tests (`internal/app/domain_test.go`, `internal/app/adapters_test.go`). +6. **MEDIUM — lifecycle tests:** active, suspended, closing, closed, moved, and unknown states need explicit typed decisions rather than absence-based pruning (`internal/awsorg/discover.go`, `internal/app/domain.go`). +7. **MEDIUM — snapshot/monitor tests:** future timestamps, missing explicit snapshots through the full `Run` path, lowercase/unknown terminal states, list pagination, and non-atomic latest/list changes remain (`internal/app/run.go`, `internal/monitor/monitor.go`). +8. **MEDIUM — Organizations pagination tests:** multiple account/parent pages, empty discovery, suspended inclusion, duplicate IDs, and second-page failure remain high-value (`internal/awsorg/discover.go`). + +--- + +## 7. Recommended target architecture + +### Design goal + +Replace mode-specific mutation logic with one domain pipeline: + +```text +Source adapter + -> typed InventorySnapshot + Provenance/Completeness + -> ComputeDesired(CurrentSetup, InventorySnapshot, ReconcilePolicy) + -> typed ChangeSet + immutable ApplyIntent + -> one GuardAndApply gateway + -> CAS-protected API write + durable per-setup result +``` + +Phases 1-3 implemented the typed adapter, pure diff, immutable intent, single gateway, and durable result portions. “CAS-protected API write” remains an unavailable target because Forward exposes no token; the shipped fallback is an immediate equality re-read plus a prohibition on unattended destructive work unless the operator adds `--allow-unattended-destructive`. + +### Layer 1: typed domain model + +Phase 1 (`00b7e89`) introduced the typed domain/adapters, and Phase 2a (`8cf4ef9`) added completeness provenance. Remaining lifecycle/server-revision gaps are called out explicitly: + +- `AccountID` validates exactly 12 digits once; `SetupID` has a canonical comparison form; `Partition` is an enum; `RoleARN` validates partition and asserts its account component matches `AccountID` (`internal/app/run.go:1313-1324`, `internal/app/account_manifest.go:14-50`, `internal/app/run.go:1708-1714`). +- `AccountLifecycle` is `Active | Suspended | Closing | Closed | Unknown`; `DesiredMembership` is `PresentEnabled | PresentDisabled | ExplicitlyRemove | Preserve`, so absence alone is not an action (`internal/awsorg/discover.go:130-146`, `internal/app/run.go:1063-1076`). +- `InventorySnapshot` includes source kind, network, snapshot ID/time, organization ID, selected setup scope, page/completeness proof, expected/observed counts, collection status, and lifecycle rows; the current NQE output lacks most of these fields (`internal/app/run.go:20-39`, `internal/api/client.go:227-277`). +- `CurrentSetup` includes a revision/ETag and preserves opaque server fields required for round-trip safety; current structs have neither (`internal/api/client.go:76-105`, `internal/api/client.go:355-363`). +- `ChangeSet` classifies `Add`, `Enable`, `Disable`, `Remove`, `Rename`, `RotateExternalID`, `ChangeRole`, and setup-metadata changes; current diffing is ID-only in `apply-plan` and membership/re-enable-only in the main planner (`internal/app/apply_plan.go:108-130`, `internal/app/run.go:1135-1158`). +- `ReconcilePolicy` remains tagged as `Additive`, `CompleteInventory`, or `ExplicitOperations`. `CompleteInventory` is retained for the legitimate reviewed-manifest caller; NQE uses a dedicated additive constructor and source validation rejects pairing NQE with `CompleteInventory`, preventing the retired boolean from being recreated (`internal/app/run.go`, `internal/app/account_manifest.go`). + +Raw NQE maps and JSON/CSV files should exist only inside adapters. They must normalize or reject duplicate/conflicting IDs and exact column/type errors before reaching the domain planner (`internal/app/run.go:1263-1324`, `internal/app/run.go:1457-1472`). + +### Layer 2: one desired-state and diff engine + +Phase 2b (`fbf78bb`) made desired-state and diff computation pure and deterministic: no API calls, file writes, or hidden `time.Now`; inputs include an explicit planning instant. Every mutation adapter uses it or typed payload classification: + +- NQE and manifests supply inventory adapters. +- `safe-sync` supplies `Additive` policy. +- root/webhook supply an explicitly selected policy. +- External ID rotation supplies explicit per-account credential operations against the same typed current state. +- `apply-plan` deserializes a versioned `ApplyIntent`, not an arbitrary patch map. + +`GuardAndApply` suppresses PATCH when every `ChangeSet` is empty. A transactional unique-ownership invariant for cross-setup moves remains future work. + +### Layer 3: one guard chokepoint + +Every account-list PATCH is now impossible except through `GuardAndApply(intent, authorization)`, test-enforced by `internal/app/patch_chokepoint_test.go`. The gateway enforces: + +1. Exact account/ARN/partition uniqueness and consistency for current and target (`internal/app/run.go:1611-1664`, `internal/app/run.go:1708-1714`). +2. A complete, scope-matched inventory proof before any absence-based removal; otherwise only explicit tombstones can remove (`internal/api/client.go:227-277`, `internal/app/run.go:1129-1136`). +3. Typed destructive classification covering both `Remove` and `Disable`, not ID omission only (`internal/app/apply_plan.go:108-130`). +4. Aggregate/per-setup ceilings and explicit destructive authorization for all writers (`internal/app/removal_limits.go:24-80`). +5. GovCloud/source-specific evidence rules as policy, not CLI conditionals (`internal/app/run.go:321-333`). +6. Plan digest bound to the available baseline state, source snapshot/completeness proof, policy, and target payload; authorization actor and policy are durably recorded. A server revision cannot be bound because none exists (`internal/app/apply_gateway.go`). +7. Central zero-diff exit, rollback capture, applied audit, and redacted credential reporting (`internal/app/apply_gateway.go`). +8. CAS with `If-Match`/version. If the Forward API cannot provide CAS, treat account-list PATCH as unsafe for unattended destructive use; a last-second GET/hash is only a documented weak fallback (`internal/api/client.go:355-363`, `internal/app/run.go:1851-1870`). +9. An idempotency key for retryable writes, or no automatic retry after ambiguous transport failure (`internal/api/client.go:402-450`). +10. A durable result journal recording `planned`, `applied`, `conflicted`, `failed`, and `pending` per setup (`internal/app/apply_gateway.go`). Automatic resume remains future work. + +Confirmation is now a user-interface adapter that issues `ApplyAuthorization` for an immutable intent. `--yes` is a recorded automation authorization; destructive unattended use additionally requires `--allow-unattended-destructive` (`cmd/awssync/main.go`, `internal/app/apply_gateway.go`). + +### Layer 4: safe event processing + +Webhook handling now keys successful dedupe by event ID plus network/snapshot/setup scope, persists completed keys and per-network/setup watermarks, intersects event scope with configured scope, makes queue rejection and failed jobs redeliverable, rejects older snapshots, and reaches mutation only through `app.Run` and the shared gateway (`internal/webhook/server.go`, `internal/webhook/state.go`). Bounded retry/dead-letter processing and a crash-recoverable on-disk pending queue remain future work; callers must redeliver after a failed accepted job. + +Monitor/status should consume the same snapshot ordering model, normalize states, expose missing/terminal outcomes, and avoid presenting separately fetched “latest” and “list” as one atomic observation (`internal/monitor/monitor.go:25-100`). + +### Required invariants + +The following should be executable assertions at domain and gateway boundaries: + +1. A target contains unique exact-12-digit account IDs, and each ID agrees with its role ARN (`internal/app/account_manifest.go:14-50`, `internal/app/run.go:1708-1714`). +2. Every account belongs to at most one selected setup after a multi-setup transaction (`internal/app/run.go:1104-1201`). +3. Absence never produces `Remove` without a complete, matching source proof or explicit tombstone (`internal/api/client.go:227-277`, `internal/app/run.go:1129-1136`). +4. `Disable` and `Remove` are both destructive and consume the same authorization/budget (`internal/app/apply_plan.go:108-130`). +5. Additive policy can only add, enable when explicitly requested by policy, or update separately authorized fields; it cannot infer deletion (`internal/app/run.go:1063-1076`, `internal/app/run.go:1228-1261`). +6. The applied target, baseline revision, evidence, and policy exactly match the approved intent (`cmd/awssync/main.go:75-138`, `cmd/awssync/main.go:237-240`). +7. Empty `ChangeSet` never makes a network mutation (`cmd/awssync/main.go:223-227`, `internal/app/run.go:851-863`). +8. Every mutation has a durable pre-state, target digest, authorization record, and per-setup result, including External ID; a server-confirmed post-state and automatic recovery remain absent (`internal/app/apply_gateway.go`, `internal/app/external_id.go`). +9. A write conflict never silently retries against a newer baseline (`internal/api/client.go:402-450`). +10. A webhook cannot expand configured network/setup scope or move a setup backward to an older snapshot (`internal/webhook/server.go`, `internal/webhook/state.go`). + +### Phased refactor plan + +| Phase | Work | Risk | Exit criterion | +|---|---|---|---| +| 0. Characterize destructive behavior — **DONE (`09d4d48`)** | Guarded race, partial inventory, disable, webhook, External ID, and partial-apply characterizations exist | **LOW** | Complete; impossible no-CAS expectations intentionally remain failing when enabled. | +| 1. Introduce domain types and adapters — **DONE (`00b7e89`)** | Typed IDs, lifecycle/provenance adapters, and conflict rejection | **MEDIUM** | Complete; malformed input now fails closed. | +| 2. Build pure desired-state/diff engine — **DONE (`8cf4ef9`, `fbf78bb`)** | Completeness-gated absence semantics, tagged policy, typed `ChangeSet`, injected planning time | **MEDIUM** | Complete. | +| 3. Create `GuardAndApply` gateway — **DONE (`1828278`, `e854787`)** | Central no-op, destructive policy, evidence, ceilings, digest authorization, rollback/audit, last re-read, PATCH, and journal | **HIGH** | Complete; exactly one production PATCH caller is test-enforced. | +| 4. Add concurrency/idempotency contract — **CLOSED BY FINDING** | Forward has no client-visible revision token; `If-Match` cannot be implemented. `1828278` shipped the weak last re-read plus `--allow-unattended-destructive` mitigation | **HIGH / external dependency** | Closed pending Forward API change; race characterizations must keep failing. | +| 5. Make multi-setup and webhook execution durable — **DONE (current uncommitted webhook slice)** | Per-setup result journal shipped in `1828278`; schema-v2 webhook state now adds pre-ack pending persistence, queued/in-flight restart recovery, bounded exponential retry, and visible dead-letter records to durable dedupe/watermarks, authentication, scope intersection, and monotonic ordering | **MEDIUM-HIGH** | Complete; crash/restart, retry exhaustion, v1 upgrade, admission-write failure, and unchanged guard-flipped webhook characterizations pass. | +| 6. Remove legacy paths and flags | Delete direct External ID/apply-plan writers, boolean combinations, and duplicate CLI safeguards after all callers use typed intents (`internal/app/run.go:48-76`, `cmd/awssync/main.go:373-403`) | **LOW-MEDIUM**: CLI compatibility | One planner, one guard gateway, one writer; deprecated flags map to explicit policy during a documented transition. | +| 7. Correct documentation and operating procedure | Align rollback, webhook auth/scope, completeness, CAS, and failure recovery claims with the implemented contract (`README.md:178-188`, `docs/aws-account-sync-procedure.md:438-458`) | **LOW** | No safety claim is broader than an enforced gateway invariant and its test. | + +--- + +## Prioritized action list + +1. **P0 / CRITICAL — MITIGATED (`8cf4ef9`, `1828278`):** absence-based pruning now requires completeness proof, and unattended destructive work additionally requires `--allow-unattended-destructive`. Candidate/OU evidence is still not a proof of source correctness (`internal/app/reconcile.go`, `internal/app/apply_gateway.go`). +2. **P0 / CLOSED:** Full-list PATCH has no client-visible `ETag`/version/If-Match path. Phase 4 is closed pending Forward API changes; preserve the failing race characterizations and the unattended-destructive policy (`internal/api/client.go`, `internal/app/apply_gateway.go`). +3. **P0 / HIGH — CLOSED (`e854787`):** `apply-plan` disable is typed destructive work and consumes the same authorization and budgets as removal (`internal/app/apply_plan.go`, `internal/app/apply_gateway.go`). +4. **P0 / HIGH — CLOSED (current uncommitted webhook slice):** apply-mode authentication, configured-scope intersection, failure redelivery, durable scoped dedupe/watermarks, monotonic snapshot ordering, and explicit-snapshot freshness are enforced (`internal/webhook/server.go`, `internal/webhook/state.go`, `internal/app/run.go`). +5. **P1 / HIGH — MOSTLY CLOSED (`09d4d48`):** the failure-injection characterizations exist. Add the still-missing ambiguous transport retry and do not make the no-CAS tests pass by weakening them (`internal/*/architecture_failure_test.go`). +6. **P1 / HIGH — CLOSED (`00b7e89`, `8cf4ef9`, `fbf78bb`):** typed IDs, provenance/completeness, desired membership, tagged policies, and field-level `ChangeSet` are the production pipeline. +7. **P1 / HIGH — CLOSED (`1828278`, `e854787`):** immutable `ApplyIntent` plus `GuardAndApply` is the test-enforced single mutation gateway (`internal/app/patch_chokepoint_test.go`). +8. **P1 / HIGH — PARTIALLY CLOSED (`e854787`):** External ID uses the gateway with digest authorization, rollback, re-read, and journal. Atomic CAS and AWS trust-policy readiness verification remain unavailable/unimplemented (`internal/app/external_id.go`). +9. **P1 / HIGH — PARTIALLY CLOSED (`1828278`):** per-setup partial outcomes are durable; explicit resume and verified rollback commands remain (`internal/app/apply_gateway.go`). +10. **P2 / MEDIUM — CLOSED (`fbf78bb`, `1828278`):** planning time is deterministic, zero-diff suppression is central, and authorization is bound to the available baseline/evidence/policy/target digest. A server revision cannot be included until Forward supplies one. +11. **P2 / MEDIUM:** Finish source-scope/pagination hardening and add cross-setup move invariants (`internal/api/client.go`, `internal/app/reconcile.go`). +12. **P2 / MEDIUM — CLOSED (current uncommitted webhook slice):** Phase 5 operations now include crash-recoverable pending webhook jobs, bounded retries/dead-letter status, and the existing durable per-setup result journal. Operator documentation covers authentication, state-file ownership, inspection, and dead-letter drain/discard procedures. diff --git a/docs/architecture-flow.md b/docs/architecture-flow.md index b115aa7..5a0cf14 100644 --- a/docs/architecture-flow.md +++ b/docs/architecture-flow.md @@ -21,7 +21,7 @@ flowchart LR preview["Show add / re-enable / remove\nremoval must equal zero"] confirm{"Operator types apply?"} verify["Recompute and verify\nreviewed payload SHA-256"] - rollback["Write complete rollback payload"] + rollback["Write rollback PATCH payload"] patch["PATCH selected Forward setups"] stop["STOP\nno Forward change"] @@ -46,40 +46,46 @@ flowchart LR ## GovCloud Inventory Decision -GovCloud resource collection and Organizations inventory are separate capabilities. Use the regular Forward snapshot/NQE path when Forward has positive GovCloud Organizations evidence. Use a complete, reviewed manifest when Organizations is unavailable or cannot be delegated. +GovCloud resource collection and Organizations inventory are separate capabilities. The regular Forward snapshot/NQE path is additive even when positive Organizations evidence is present. Every lifecycle removal uses a complete, reviewed manifest. ```mermaid flowchart TD start["GovCloud AWS setup\narn:aws-us-gov roles"] snapshot["Run connectivity test\nand fresh Forward snapshot"] + change{"Lifecycle removal?"} org_check{"Forward NQE shows positive\nOrganizations evidence?"} - nqe["Regular preflight + NQE plan"] + nqe["Regular preflight + additive NQE plan\nnever removes"] manifest["Authoritative account manifest\nonboard-accounts or sync-accounts"] removals{"Plan contains removals?"} review["Review exact account IDs"] approve["Explicit --allow-removals\nall removal paths"] blast{"Within --max-removals\nand --max-removal-percent?"} + unattended{"Unattended destructive apply?"} + acknowledge["Explicit\n--allow-unattended-destructive"] + gateway["Guarded account-list\napply gateway"] apply["PATCH Forward setup"] - block["BLOCK\nno empty/unproven inventory apply"] - - start --> snapshot --> org_check - org_check -- "yes" --> nqe --> removals - org_check -- "no / unavailable" --> manifest --> removals - removals -- "no" --> apply - removals -- "yes, NQE evidence present" --> review - removals -- "yes, authoritative manifest" --> review - review --> approve --> blast - blast -- "yes" --> apply + block["BLOCK\nno unreviewed or over-limit removal"] + + start --> change + change -- "no" --> snapshot --> org_check + change -- "yes" --> manifest --> removals + org_check -- "yes" --> nqe --> gateway + org_check -- "no / unavailable" --> manifest + removals -- "no" --> gateway + removals -- "yes" --> review --> approve --> blast + blast -- "yes" --> unattended blast -- "no" --> block - removals -- "yes, NQE evidence absent" --> block + unattended -- "no" --> gateway + unattended -- "yes" --> acknowledge --> gateway + gateway --> apply classDef neutral fill:#F1EFE8,stroke:#5F5E5A,color:#2C2C2A; classDef safe fill:#E1F5EE,stroke:#0F6E56,color:#04342C; classDef warn fill:#FAEEDA,stroke:#854F0B,color:#412402; classDef blocked fill:#FCEBEB,stroke:#A32D2D,color:#501313; - class start,snapshot,org_check,nqe,manifest,removals,review neutral; - class approve,blast,apply safe; + class start,snapshot,change,org_check,nqe,manifest,removals,review,unattended neutral; + class approve,blast,acknowledge,gateway,apply safe; class block blocked; ``` @@ -102,7 +108,9 @@ flowchart TB plan["plan / dry-run\nPOST /nqe + GET /cloudAccounts"] external_ids["Per-account External ID merge\npreserve existing values\nexplicit CSV for ambiguous additions"] disk["payload.json\nwritten to disk before any change"] - safety["Removal gates\nexplicit approval + count/% ceilings"] + gateway["Guarded apply gateway\napproval digest + current-state re-read"] + rollback["rollback.json\naccount list + PATCHable fields"] + journal["result.json\nper-setup durable disposition"] apply["--apply\nPATCH /cloudAccounts/{setupId}"] apply_plan["apply-plan\nreload current state + validate\nGovCloud removals refused"] end @@ -121,11 +129,10 @@ flowchart TB preflight -- "read-only" --> fwd plan --> external_ids --> disk - disk --> safety --> apply + disk --> gateway --> rollback --> apply --> journal disk --> apply_plan - apply_plan --> safety + apply_plan --> gateway apply --> patch_accts - apply_plan --> patch_accts plan --> nqe plan --> get_accts plan --> get_snap @@ -135,9 +142,9 @@ flowchart TB classDef artifact fill:#FAEEDA,stroke:#854F0B,color:#412402; class cli,cron,preflight neutral; - class plan,external_ids,safety,apply,apply_plan neutral; + class plan,external_ids,gateway,apply,apply_plan neutral; class nqe,get_accts,patch_accts,get_snap fwdnode; - class disk artifact; + class disk,rollback,journal artifact; ``` The account list is full-state, but External IDs are merged by AWS account ID. Existing mixed values are preserved. When a mixed-ID setup gains an account, planning stops unless `--external-id-file` explicitly supplies the new account's value; omitted existing accounts remain unchanged. @@ -211,7 +218,10 @@ flowchart TB sync["sync-accounts\nGET current setup"] diff["Print exact add/remove IDs\nwrite payload before change"] removal{"Any removals?"} - patch["--apply --yes\nPATCH /cloudAccounts/{setupId}"] + unattended{"--yes / CI?"} + acknowledge["--allow-unattended-destructive"] + gateway["guarded apply gateway"] + patch["--apply\nPATCH /cloudAccounts/{setupId}"] approved["--allow-removals\nexplicit approval"] blast["--max-removals\n--max-removal-percent"] end @@ -219,18 +229,21 @@ flowchart TB manifest --> validate validate --> onboard --> create_files --> post validate --> sync --> diff --> removal - removal -- "no" --> patch - removal -- "yes" --> approved --> blast --> patch + removal -- "no" --> gateway + removal -- "yes" --> approved --> blast --> unattended + unattended -- "no, interactive" --> gateway + unattended -- "yes" --> acknowledge --> gateway + gateway --> patch classDef neutral fill:#F1EFE8,stroke:#5F5E5A,color:#2C2C2A; classDef fwdnode fill:#E6F1FB,stroke:#185FA5,color:#042C53; classDef artifact fill:#FAEEDA,stroke:#854F0B,color:#412402; classDef safe fill:#E1F5EE,stroke:#0F6E56,color:#04342C; - class manifest,validate,onboard,sync,diff,removal neutral; + class manifest,validate,onboard,sync,diff,removal,unattended neutral; class create_files artifact; class post,patch fwdnode; - class approved,blast safe; + class approved,blast,acknowledge,gateway safe; ``` For GovCloud, use `--partition aws-us-gov` when onboarding. Existing-setup sync derives and preserves the partition from the current role ARNs. Mixed partitions or a mismatch between role ARNs and configured regions fail before a payload can be applied. @@ -321,8 +334,11 @@ flowchart TB end subgraph daemon["awssync serve-webhook (long-lived process)"] - recv["HTTP receiver\nlistens on configured port\nBasic Auth protected"] - sync["plan + PATCH\nsame as batch mode\nbut pinned to event snapshot ID"] + recv["HTTP receiver\nBasic Auth protected\nfixed --network-id scope"] + state["durable state file · 0600\npending + dedupe + watermarks"] + sync["plan + guarded PATCH\npinned to event snapshot ID"] + retry{"Job succeeded\nwithin 5 attempts?"} + dead["dead_letter_events\noperator recovery required"] end subgraph fwd["Forward platform"] @@ -333,7 +349,9 @@ flowchart TB cfg -- "HTTPS · Basic Auth\nFWD_USER / FWD_PASS" --> fwd webhook_out -- "inbound HTTP\nBasic Auth (shared secret)" --> recv - recv --> sync + recv --> state --> sync --> retry + retry -- "yes" --> state + retry -- "no" --> dead --> state sync --> nqe2 sync --> patch2 @@ -343,9 +361,9 @@ flowchart TB classDef fwdnode fill:#E6F1FB,stroke:#185FA5,color:#042C53; classDef warn fill:#FAEEDA,stroke:#854F0B,color:#412402; - class cfg,recv,sync neutral; + class cfg,recv,sync,retry neutral; class webhook_out,nqe2,patch2 fwdnode; - class note warn; + class state,dead,note warn; ``` --- @@ -433,6 +451,8 @@ flowchart LR | Listening port | Configurable (default example: `:8080`) | | Protocol | HTTP (TLS terminated at reverse proxy recommended for production) | | Authentication | HTTP Basic Auth — shared secret between Forward and receiver | +| Apply scope | Applying receivers require an explicitly configured Forward network ID | +| Recovery state | Durable owner-only file containing pending work, dedupe, watermarks, and dead letters | | Caller | Forward platform (SaaS: internet; on-prem: Forward app server) | --- @@ -449,17 +469,17 @@ flowchart LR - `discover-org` writes both onboarding JSON files before any optional `POST /cloudAccounts`. - `onboard-accounts` writes both onboarding JSON files before any optional `POST /cloudAccounts`. - Static-key collector secrets are only included in the create payload when explicitly supplied. Without the secret, the file contains a placeholder and is marked not POST-ready. -- Removals require explicit `--allow-removals` flag; `awssync` will not silently - remove accounts from a Forward setup. -- NQE sync preserves configured accounts by default. NQE-driven removal additionally requires explicit `--prune-missing`; authoritative manifest sync is preferred for lifecycle removal. +- Removals require explicit `--allow-removals`; unattended destructive applies additionally require `--allow-unattended-destructive` because Forward provides no atomic compare-and-swap token. +- NQE sync always preserves configured accounts absent from observed inventory. `--prune-missing` is retired and returns an actionable refusal; authoritative `sync-accounts` manifest reconciliation is the supported lifecycle-removal path. - Both nonzero `--max-removals` and `--max-removal-percent` ceilings are mandatory for any removal and are rechecked immediately before apply. - Existing disabled or failed `Collected? false` rows are not treated as AWS Organizations discovery candidates. -- CLI NQE plans pin one processed snapshot, and every apply writes a full pre-change rollback payload before the first PATCH. -- GovCloud NQE removals additionally require positive Organizations evidence. Generic no-evidence flags cannot override this gate. -- Manifest removals require an authoritative complete manifest plus `--allow-removals`. +- CLI NQE plans pin one processed snapshot. Snapshot timestamps more than five minutes ahead of the local clock are rejected. +- Every apply uses the same guarded gateway, writes a pre-change rollback PATCH payload containing the complete account list and PATCHable setup fields before the first PATCH, and atomically updates a per-setup result journal. The artifact omits `collect`, `connectionTimeoutSeconds`, `requestTimeoutSeconds`, `numVirtualizedDevices`, and `useForwardAccountToAssumeRole`; Forward PATCH leaves those absent fields unchanged, so rollback restoration is safe, but the artifact is not a full setup backup. +- Approval digests are stable across independent invocations for the same approval-relevant inputs. The immediate current-state re-read is a weak conflict detector, not atomic compare-and-swap. +- GovCloud NQE sync is additive. GovCloud lifecycle removals require an authoritative complete manifest plus `--allow-removals`; generic NQE evidence flags cannot substitute for that source. - `apply-plan` reloads current state and refuses GovCloud removals, so a saved payload cannot bypass the source workflow's safety checks. -- Webhook receiver is protected by HTTP Basic Auth with a shared secret - independent of Forward user credentials. +- Applying webhook receivers require HTTP Basic Auth with a shared secret independent of Forward user credentials and a fixed network scope. Accepted events are persisted before `202`; a job is attempted at most five times before it is dead-lettered. +- `status` reports `observation_atomic=false` because its latest-processed and snapshot-list values come from separate Forward API reads. For the full operational procedure see [AWS Account Sync Procedure](aws-account-sync-procedure.md) and diff --git a/docs/aws-account-sync-procedure.md b/docs/aws-account-sync-procedure.md index 40f9199..327335b 100644 --- a/docs/aws-account-sync-procedure.md +++ b/docs/aws-account-sync-procedure.md @@ -1,16 +1,31 @@ # AWS Account Sync Procedure -This guide explains how to keep a Forward AWS cloud setup aligned with AWS accounts that are added to or removed from an AWS Organization. +This guide explains how to keep a Forward AWS cloud setup aligned with an independently verified AWS account inventory. Forward NQE supports additive synchronization; lifecycle removals require a complete reviewed manifest. It is written for readers who may not work in AWS every day. It focuses on the practical setup, preflight checks, and safe use of `awssync`. +Upgrading from an earlier release? Complete [Upgrading `awssync`](upgrading.md) before using this runbook. + +## Operator Index + +| Situation | Go directly to | +| --- | --- | +| Routine human update; no removals | [Apply the sync](#apply-the-sync) and use `safe-sync` | +| Scheduled additive update | [Run a dry plan](#run-a-dry-plan), then [apply](#apply-the-sync) | +| Independently approved account removal | [Reviewed manifest removal](#reviewed-manifest-removal) | +| Failed, partial, or ambiguous apply | [Apply recovery](#apply-recovery) | +| Webhook deployment or failed event | [Webhook operation](#webhook-operation) and [webhook recovery](#webhook-recovery) | +| External ID migration or rollback | [Add a customer-defined External ID](#add-a-customer-defined-external-id-to-an-existing-setup) | +| New setup not yet in Forward | [Onboard from AWS Organizations directly](#onboard-from-aws-organizations-directly) | +| Command stopped during an incident | [Common failure modes](#common-failure-modes) | + ## Summary Forward collects AWS by using configured credentials to read AWS network metadata. In multi-account setups, Forward still assumes a role in each collected account. For many AWS accounts in the same AWS Organization, there are two separate requirements: -1. Forward must be able to discover the AWS account inventory from AWS Organizations. +1. Forward should be able to discover AWS accounts from AWS Organizations for additive onboarding, while operators must understand that the snapshot exposes observed inventory rather than a complete configured-account manifest. 2. Forward must be able to assume a collection role in every account that should be collected. `awssync` automates the Forward-side account list update for existing setups. It also has a separate `discover-org` onboarding mode for new setups that Forward has not collected yet. Neither mode creates IAM roles in AWS or grants Forward access to new accounts by itself. New accounts become collectable only after the expected IAM role exists in those accounts and trusts Forward. @@ -29,7 +44,8 @@ See the one-page [Routine AWS Safe Sync](routine-safe-sync.md) handoff. Use the Important separation: -- Use the default NQE sync path for an existing Forward AWS setup. That path uses Forward's collected data and can PATCH the setup after review. +- Use the default NQE sync path for additive updates to an existing Forward AWS setup. It can add or re-enable observed accounts but cannot remove an account because it is absent. +- Use `sync-accounts` with a complete reviewed manifest for existing-setup lifecycle removals. - Use `discover-org` only for initial onboarding. It calls AWS Organizations directly, writes files, and can POST a new Forward setup, but it does not PATCH an existing setup. ## AWS Terms @@ -45,7 +61,7 @@ Important separation: ## Required AWS Model -One AWS account must be available for Forward to use as the Organizations discovery point. This is usually the AWS Organizations management account. A delegated administrator account can also work if it has the required Organizations permissions. +For additive NQE discovery, one AWS account must be available for Forward to use as the Organizations discovery point. This is usually the AWS Organizations management account. A delegated administrator account can also work if it has the required Organizations permissions. The reviewed-manifest workflow does not require Forward to query AWS Organizations, but its manifest must come from independently authoritative lifecycle sources. That discovery account must allow Forward to call AWS Organizations read APIs, including account-listing APIs such as `organizations:ListAccounts`. Forward uses that visibility to learn which AWS accounts exist. @@ -72,15 +88,15 @@ A Forward cloud setup or snapshot can complete successfully even when collection ## Preflight Checklist -Complete these checks before running `awssync --apply`. +Complete these checks before running an additive NQE apply. For a manifest removal, use the independent inventory and review checks in [Reviewed manifest removal](#reviewed-manifest-removal) instead of treating NQE as authoritative. ### 1. Confirm Forward Is Collecting the AWS Organization Discovery Account In Forward, confirm the AWS setup includes the management account or the delegated discovery account. -This matters because account inventory comes from AWS Organizations. If Forward only collects a member account that cannot list the Organization, the script will not have the complete account list to sync. +This matters for discovering additions, but it does not make NQE authoritative. NQE combines accounts that collected successfully with accounts visible through Organizations metadata, and collection or authorization failures can leave either set partial. -Expected result: the latest processed Forward snapshot includes the AWS setup and shows AWS account inventory from the Organization. +Expected result: the latest processed Forward snapshot includes the AWS setup and shows the accounts Forward observed. Do not use missing rows as deletion evidence. ### 2. Confirm AWS Organizations Permissions @@ -118,7 +134,7 @@ Expected result: Forward setup/connectivity testing succeeds for the account and ### 5. Confirm the Platform Query Scope -`awssync` gets discovered AWS account rows from Forward NQE. +`awssync` gets observed AWS account rows from Forward NQE for additive synchronization. The tool defaults to an inline Forward NQE source query for AWS account discovery. That inline query returns `Cloud Setup ID` from `cloudAccount.cloudSetupId`, which is required when one network has multiple AWS setups. When exactly one `--setup-id` is selected, the inline query is parameterized with that setup ID so Forward can scope the query before returning rows. `--query-id` is optional and should only be used when support intentionally overrides that query. @@ -156,13 +172,7 @@ Use the Forward base URL for `FWD_HOST`; it can be SaaS or an on-prem Forward in Use this section when Forward has not collected the AWS Organization yet. The goal is to create onboarding files from AWS Organizations, not to update an existing Forward setup. -`discover-org` uses AWS credentials only for discovery. It uses the AWS SDK default credential chain, or the profile named by `--aws-profile`, and checks: - -- `organizations:DescribeOrganization` -- `organizations:ListAccounts` -- `organizations:ListParents` - -If any of those calls returns access denied, fix AWS Organizations access before continuing. The account list would otherwise be incomplete. +`discover-org` uses the AWS SDK default credential chain or `--aws-profile` to call `DescribeOrganization`, `ListAccounts`, and `ListParents`. If any call is denied, stop and fix Organizations access; continuing would create an incomplete onboarding inventory. Generate the Forward UI upload file and create-setup POST body: @@ -180,92 +190,15 @@ Outputs: - `fwd_accounts_data_.json`: flat account array for Forward's manual AWS account import step. - `aws_create_payload_.json`: body for `POST /api/networks/{networkId}/cloudAccounts`. -If Forward credentials are supplied, `discover-org` also resolves the network, verifies that the setup name does not already exist, and fetches the Forward-generated AWS external ID: +With Forward credentials, omitting `--external-id` also checks that the setup name is unused and fetches Forward's generated External ID. After reviewing both files, add `--post --yes` to create the setup. Static-key collection uses a separate collector credential; supply its secret through `AWSSYNC_COLLECTOR_SECRET_ACCESS_KEY`. Without that secret, the create payload contains a placeholder, reports `create_payload_ready: false`, and must not be POSTed. -```bash -AWS_PROFILE=org-readonly ./bin/awssync discover-org \ - --host "$FWD_HOST" \ - --username "$FWD_USER" \ - --password "$FWD_PASS" \ - --network-id "$FWD_NETWORK_ID" \ - --setup-id AWS-PROD \ - --role-name ForwardRole \ - --collect-region us-east-1 -``` +Do not use `discover-org` for a setup that already exists. Use the additive NQE sync path below so the existing Forward regions, proxy settings, and stored credentials are preserved; use `sync-accounts` when a reviewed manifest authorizes membership removal. -To create the new setup through the Forward API after writing both JSON files: - -```bash -AWS_PROFILE=org-readonly ./bin/awssync discover-org \ - --host "$FWD_HOST" \ - --username "$FWD_USER" \ - --password "$FWD_PASS" \ - --network-id "$FWD_NETWORK_ID" \ - --setup-id AWS-PROD \ - --role-name ForwardRole \ - --collect-region us-east-1 \ - --post \ - --yes -``` - -For static IAM key collection, do not assume the AWS discovery credentials are the collector credentials. Provide the collector key explicitly: - -```bash -export AWSSYNC_COLLECTOR_SECRET_ACCESS_KEY='collector-secret' - -AWS_PROFILE=org-readonly ./bin/awssync discover-org \ - --host "$FWD_HOST" \ - --username "$FWD_USER" \ - --password "$FWD_PASS" \ - --network-id "$FWD_NETWORK_ID" \ - --setup-id AWS-PROD \ - --role-name ForwardRole \ - --collect-region us-east-1 \ - --credential-mode static-keys \ - --collector-access-key-id AKIA... \ - --post \ - --yes -``` - -If `--credential-mode static-keys` is used without `AWSSYNC_COLLECTOR_SECRET_ACCESS_KEY` or `--collector-secret-access-key`, the create payload is still written, but it contains a placeholder password and `create_payload_ready` is `false`. That file is useful for review but should not be POSTed until the secret is supplied. - -Do not use `discover-org` for a setup that already exists. Use the NQE sync path below so Forward's collected data, regions, proxy settings, and stored credentials remain the source of truth. - -### Optional Terraform Bootstrap - -For new AWS Organizations onboarding, prefer the Forward Terraform provider as the native IaC workflow. The provider can fetch Forward's external ID, read AWS Organizations, and create or update the Forward AWS setup directly with `forward_aws_cloud_account`. It supports Forward assume-role, static-key, and collector instance-profile credential models. - -Use the `examples/terraform` bootstrap examples below when you need AWS-side prerequisites for either the provider workflow or the `awssync discover-org` CLI fallback: - -- `examples/terraform/aws-org-discovery-role`: creates an IAM role with Organizations read permissions for `discover-org`. -- `examples/terraform/forward-collection-role-stackset`: deploys the Forward collection role name into member accounts with CloudFormation StackSets. -- `examples/terraform/github-actions-discover-org`: creates a GitHub OIDC role so GitHub Actions can run `discover-org` without static AWS keys. - -Example: - -```bash -terraform -chdir=examples/terraform/aws-org-discovery-role init -terraform -chdir=examples/terraform/aws-org-discovery-role apply - -terraform -chdir=examples/terraform/forward-collection-role-stackset init -terraform -chdir=examples/terraform/forward-collection-role-stackset apply -``` - -Then use the StackSet role name with `discover-org`: - -```bash -AWS_PROFILE=org-readonly ./bin/awssync discover-org \ - --setup-id AWS-PROD \ - --role-name "$(terraform -chdir=examples/terraform/forward-collection-role-stackset output -raw role_name)" \ - --collect-region us-east-1 \ - --external-id Org:12345 -``` - -Static-key collection through Terraform requires protected encrypted state because Terraform stores sensitive values in state. If the collector secret must stay out of Terraform state, use `awssync discover-org` and pass `AWSSYNC_COLLECTOR_SECRET_ACCESS_KEY` from runtime secret storage. +For native IaC onboarding, prefer the Forward Terraform provider. The [quick start](quick-start.md#discover-before-onboarding) and `examples/terraform` cover provider and AWS-side bootstrap details without expanding this incident runbook. ## Run a Dry Plan -Start without `--apply`. This writes the planned PATCH payload to disk but does not update Forward. +Start without `--apply`. This writes the planned PATCH payload to disk but does not update Forward. NQE account IDs must contain exactly 12 digits; a malformed row fails by default. `--allow-malformed-rows` skips and reports malformed rows for an urgent additive run, marks the observed inventory incomplete, and cannot authorize removals. ```bash ./bin/awssync \ @@ -325,63 +258,34 @@ Human-readable output is the default. Use `--json` (or `--format json`) for mach ./bin/awssync --max-snapshot-age 24h --json ``` -Then review the summary and payload: - -- `selected_setup_ids`: setup IDs included in this run (useful when defaults are inferred). -- `planned_setups` (setup list sections): per-setup added/removed counts and OU visibility messages. - -Review the JSON summary printed by the command: - -- `fetched_item_count`: number of AWS account rows returned by NQE. -- `planned_setup_count`: number of Forward AWS setups that can be patched. -- `skipped_setup_count`: number of setups skipped because required metadata was missing. -- `configured_account_count`: number of accounts currently configured in Forward for a setup. -- `nqe_account_row_count`: number of AWS account rows returned by NQE for a setup. -- `nqe_candidate_row_count`: number of uncollected NQE accounts not already present in the setup. Existing disabled or failed accounts are not discovery candidates. -- `nqe_org_unit_row_count`: rows where NQE exposed AWS `organizationalUnitIds`. This is useful supporting evidence when present, but it can be zero for valid AWS Organizations where accounts are directly under the root. -- `planned_payload_account_count`: number of accounts planned for the PATCH payload. -- `added_accounts`: accounts that will be added to the Forward setup. -- `removed_accounts`: accounts that would be removed from the Forward setup. -- `reenabled_accounts`: currently configured disabled accounts that additive sync will retain and enable. -- `unchanged_account_count`: accounts already present and still discovered. -- `candidate_check`: whether uncollected candidate accounts were visible in the snapshot. If none are visible, verify the management or delegated discovery account before applying removals. -- `organization_discovery_signal`: whether an Organization-level signal was visible for the setup (`visible_candidates`, `visible_ou_ids`, `visible_candidates_and_ou_ids`, or `no_org_signal`). -- `role_name`: IAM role name that will be used in each generated role ARN. -- `external_id_configured`: whether the normal sync payload preserves an External ID from the existing setup. -- `payload_sha256`: fingerprint of the payload written to disk. -- `snapshot_id`: exact processed snapshot pinned for this plan. -- `ignored_nqe_item_count`: malformed NQE account rows excluded from the payload, such as a setup-name placeholder returned as an account ID. -- `rollback_output` and `rollback_sha256`: exact pre-apply setup payload and fingerprint, written before the first PATCH. -- `manual_output`: optional path of setup-keyed manual payload for UI drag-and-drop. -- `manual_payload_sha256`: fingerprint of the manual payload written to disk. -- `manual_payloads`: map keyed by setup ID containing the planned `assumeRoleInfo` entries for manual drag-and-drop workflows. -- `patched`: should be `false` in a dry plan. +Review the summary per setup: + +- `selected_setup_ids`, `snapshot_id`, and the configured, observed, and planned account counts identify the scope. +- `added_accounts`, `reenabled_accounts`, and `removed_accounts` are the change being approved. Standard NQE planning must show no removals. +- `nqe_candidate_row_count`, `nqe_org_unit_row_count`, `candidate_check`, and `organization_discovery_signal` diagnose discovery of additions; they never authorize removal. +- `ignored_nqe_item_count` is nonzero only with `--allow-malformed-rows`; without that flag, a malformed row stops planning. +- `role_name`, `external_id_configured`, regions, and proxy values show which existing setup metadata is preserved. +- `payload_sha256` fingerprints the generated file. `plan_digest` binds approval to the network, snapshot, policy, baseline, target, and classified changes. +- On apply, `rollback_output`, `rollback_sha256`, and `result_journal_output` identify the recovery artifacts. `patched` is `false` in a dry plan. Then review `aws_sync_payload.json`. Confirm: - Setup IDs are correct. -- Account IDs are expected 12-digit AWS account IDs. +- Every account ID contains exactly 12 digits. - Account names look correct. - Role ARNs use the intended role name. - External ID matches the existing setup. Use the separate `external-id` command below when intentionally adding, replacing, or clearing it. - Regions and proxy settings match the existing Forward setup. - The PATCH payload does not include access keys or secrets; those stored credentials remain unchanged in Forward. -- Removed accounts are expected. If removals are not expected, stop and inspect the Forward snapshot and NQE query before applying. - -If `--manual-output` is used, also confirm that manual payload file by opening it and verifying: +- `removed_accounts` is empty. Standard NQE planning is additive; use `sync-accounts` with a reviewed manifest when removal is intended. -- Setup keys match `selected_setup_ids`. -- Each setup value is an array of account records with generated role ARNs and external IDs (if configured). +If `--manual-output` is used, confirm its setup keys match `selected_setup_ids` and each array contains the same reviewed role ARNs and External IDs. ## Add a Customer-Defined External ID to an Existing Setup This is a separate, one-time hardening change, not a prerequisite for AWS Organizations discovery. It is supported for an existing IAM user/access-key setup: Forward keeps using the stored IAM user credentials, but includes the configured External ID when it calls `sts:AssumeRole` for each target account. -The simplest policy uses one customer-defined value per Forward AWS setup. Per-account values are also supported for staged testing or customer policy requirements. In every case, the value stored on an account's Forward `assumeRoleInfos` entry must exactly match that account's target-role trust policy. External IDs are not passwords, but use unguessable, customer-specific values and do not reuse them across unrelated customers. - -Use the dedicated `external-id` command rather than the normal NQE synchronization path for an isolated migration. It reads the existing Forward setup directly, preserves its account list, role ARNs, regions, and proxy settings, and changes only the selected External IDs. It does not depend on NQE account discovery or a new snapshot. - -First run a dry plan: +Use `external-id`, not NQE synchronization, for an isolated migration. It reads the setup directly, preserves account membership and setup metadata, and does not require a snapshot. First run a dry plan: ```bash ./bin/awssync external-id \ @@ -391,221 +295,154 @@ First run a dry plan: --format human ``` -Review the prior-state fields and confirm every entry in `aws_external_id_payload.json` contains the intended `externalId`. The command requires exactly one setup and either `--value VALUE`, `--clear`, or `--external-id-file FILE`; without `--apply`, it writes the payload but does not modify Forward. With no `--account-id`, `--value` and `--clear` apply to every account for backward compatibility. It does not change or expose the setup's stored IAM access key or secret. - -### Test one account or assign different values +Review the prior and target states and confirm every payload entry has the intended value. With no `--account-id`, `--value` and `--clear` affect every account; repeat `--account-id` for a test subset. Use `--external-id-file` for reviewed per-account set/clear assignments; duplicate, malformed, wrong-setup, and unknown rows fail before PATCH. The [quick start](quick-start.md#add-an-external-id-to-an-existing-iam-user-setup) contains the CSV format. -Repeat `--account-id` to apply one value or clear operation to a selected subset: +Apply the same reviewed inputs. The guarded gateway writes the pre-change account list and PATCHable setup fields to `.rollback.json` and maintains `.result.json`: ```bash ./bin/awssync external-id \ --setup-id AWS-PROD \ - --account-id 111111111111 \ - --value representative-test-value \ - --output aws_external_id_test_payload.json \ - --format human + --value customer-defined-value \ + --output aws_external_id_payload.json \ + --apply --yes ``` -For different values or mixed set/clear actions, create a reviewed CSV: +Stage the change in this order: apply the Forward value, run a snapshot and test a representative account, then require that identical `sts:ExternalId` in each target-role trust policy. Normal NQE, webhook, and manifest sync preserve existing per-account values. A mixed-ID setup that gains an account fails closed until `--external-id-file` assigns the new account explicitly. -```csv -setup_id,account_id,action,external_id -AWS-PROD,111111111111,set,representative-test-value -AWS-PROD,222222222222,set,account-two-value -AWS-PROD,333333333333,clear, -``` +Rollback order is the reverse dependency order: first relax or restore the affected AWS trust policies and verify role assumption, then restore Forward. Use the automatic rollback with [Apply recovery](#apply-recovery), or dry-run and apply `external-id --clear` or `--value PREVIOUS_VALUE` with the original `--account-id` scope. The human summary does not print the prior value; retrieve it from the owner-only rollback payload. -The shorter `account_id,action,external_id` header is accepted when the command or sync selects exactly one setup. An explicit `clear` action is mandatory; an empty cell never clears by implication. +## Run Preflight Checks -Dry-run and then apply the same file: +`preflight` performs read-only checks and prints a human-readable readiness report. Add `--json` for structured output. ```bash -./bin/awssync external-id \ - --setup-id AWS-PROD \ - --external-id-file external-ids.csv \ - --output aws_external_id_payload.json \ - --format human - -./bin/awssync external-id \ - --setup-id AWS-PROD \ - --external-id-file external-ids.csv \ - --output aws_external_id_payload.json \ - --apply --yes +./bin/awssync preflight \ + --max-snapshot-age 24h ``` -Omitted accounts are preserved. The command rejects duplicate, malformed, wrong-setup, and unknown account rows before writing or applying a PATCH. Review `selected_account_count`, `changed_account_count`, set/clear counts, and the per-account change list; values are visible only in the generated full-state payload. +Expected result: `ready` is `true`, `nqe_aws_accounts` passes, `patch_plan` passes, and `account_removals` passes because the NQE plan is additive. -For a representative-account test, record that account's prior value during change review. Rollback is a second scoped dry-run and apply using the same account ID: +If `management_account_discovery` fails, the snapshot did not show any genuinely new uncollected AWS account candidates. An already configured row with `Collected? false` does not satisfy this check; it may simply be disabled or failing collection. Treat this as an addition/discovery diagnostic, never as removal authorization. -```bash -# Restore the original null/no-External-ID state. -./bin/awssync external-id \ - --setup-id AWS-PROD \ - --account-id 111111111111 \ - --clear \ - --output aws_external_id_test_revert.json \ - --format human +`aws_organizations_evidence` reports if the observed rows include candidate visibility or OU ID visibility for each selected setup. In multi-setup runs, the check lists setup IDs that lack this signal. Treat both as supporting evidence only: -# Or restore a prior non-null value. -./bin/awssync external-id \ - --setup-id AWS-PROD \ - --account-id 111111111111 \ - --value PREVIOUS_VALUE \ - --output aws_external_id_test_revert.json \ - --format human -``` +- NQE absence never produces removals. +- `--prune-missing` is recognized only to return an actionable refusal. +- Manifest removals use `sync-accounts` and retain removal authorization and blast-radius ceilings. -Review the payload, then repeat only the applicable command with `--apply --yes`. Unselected accounts retain their current fields and order in the generated `assumeRoleInfos` list. The summary intentionally records configured/not-configured state rather than retaining the prior value as an automatic rollback artifact. Before clearing or replacing Forward's value, relax or restore the selected account's AWS trust-policy condition and verify role assumption so collection is not interrupted. +## Apply the Sync -Normal NQE sync, webhook sync, and `sync-accounts` now preserve each existing account's External ID instead of copying the first value across the setup. If a setup already has mixed values and discovery adds an account, the plan fails closed because no safe value can be inferred. Supply an assignment for every new account to preflight and the eventual dry-run/apply: +For a routine interactive sync, use the additive-only command: ```bash -./bin/awssync preflight \ - --setup-id AWS-PROD \ - --external-id-file external-ids.csv \ - --max-snapshot-age 24h \ - --format human +./awssync-linux-amd64 safe-sync \ + --setup-id AWS-PROD +``` + +The remaining commands in this section are the standard and expert workflow. For non-interactive automation, run with `--apply` after reviewing the dry plan. +```bash ./bin/awssync \ - --setup-id AWS-PROD \ - --external-id-file external-ids.csv \ --max-snapshot-age 24h \ --output aws_sync_payload.json \ - --format human + --apply \ + --yes ``` -Prepare the matching collection-role trust policy change for each target AWS account, but do not make the condition mandatory until the Forward payload has been applied and tested. For an IAM user in the connectivity account, the trust statement has this form: +Expected result: the command prints `patched_setup_count` greater than zero and each patched setup shows `patched: true`. -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "TrustForwardCollectorUser", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::user/forward-collector" - }, - "Action": "sts:AssumeRole", - "Condition": { - "StringEquals": { - "sts:ExternalId": "customer-defined-value" - } - } - } - ] -} -``` - -Apply the reviewed Forward payload before making the condition mandatory in AWS. AWS can receive an External ID on `AssumeRole` even while the existing trust statement does not yet require one, which makes it possible to stage the change without intentionally breaking collection: +NQE sync is always additive: configured accounts missing from NQE remain in the payload. This is required because disabled accounts, failed accounts, authorization failures, accounts in another Organization, and transient errors may all be absent from NQE. -```bash -./bin/awssync external-id \ - --setup-id AWS-PROD \ - --value customer-defined-value \ - --output aws_external_id_payload.json \ - --apply \ - --yes -``` +`--prune-missing` no longer creates a plan. It fails with: `--prune-missing is no longer supported: the NQE result is observed inventory, not an account manifest, so an account's absence cannot prove it should be deleted; use sync-accounts with a reviewed manifest instead`. -Run a Forward snapshot and verify one representative account still collects. Then roll out the matching trust-policy condition with the existing StackSet, Terraform module, or account-vending automation. Test the representative account again before enforcing it everywhere. After this one-time PATCH stores the new value, later normal syncs read and preserve it without rerunning `external-id`. +### Reviewed Manifest Removal -An `sts:AssumeRole` failure after the trust-policy rollout usually means the trust policy principal or `sts:ExternalId` value does not exactly match the Forward setup payload. +For an approved removal, build a complete manifest from sources that own account lifecycle: direct AWS Organizations inventory, the account-vending system or CMDB, approved standalone-account inventory, and explicit closure or transfer records. Start from the accounts currently configured in Forward and keep any account whose lifecycle is uncertain. Do not build the manifest from NQE or collection success. -To roll back intentionally, first remove the mandatory External ID condition from the affected role trust policies, then dry-run and apply the clear operation: +The file is a non-empty JSON array of unique, exactly 12-digit IDs and optional names. It must contain every account that should remain in exactly one setup: + +```json +[ + {"id": "111111111111", "name": "security"}, + {"id": "222222222222", "name": "production"} +] +``` + +Create a dry plan and inspect every `added_accounts` and `removed_accounts` entry: ```bash -./bin/awssync external-id \ +./bin/awssync sync-accounts \ + --network-id NETWORK_ID \ --setup-id AWS-PROD \ - --clear \ - --output aws_external_id_clear_payload.json + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json +``` -./bin/awssync external-id \ +Then apply with both blast-radius ceilings. `--max-removals` limits the count and `--max-removal-percent` limits the percentage of that setup's current configured accounts: + +```bash +./bin/awssync sync-accounts \ + --network-id NETWORK_ID \ --setup-id AWS-PROD \ - --clear \ - --output aws_external_id_clear_payload.json \ + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json \ --apply \ - --yes + --yes \ + --allow-removals \ + --max-removals 10 \ + --max-removal-percent 5 \ + --allow-unattended-destructive ``` -The clear payload omits `externalId` from every `assumeRoleInfos` entry, which stores it as null in Forward. Test a representative account again after the rollback. +Choose both nonzero limits from the reviewed plan, leaving enough room only for the approved account IDs. A removal is blocked if `--allow-removals` or either ceiling is omitted, or if either ceiling is exceeded. `--allow-unattended-destructive` is required because `--yes` skips the human confirmation and Forward provides no atomic compare-and-swap token. For an attended apply, keep the removal controls but omit `--yes` and `--allow-unattended-destructive`; type `apply` after reviewing the preview. All paths use the same guarded apply gateway. -## Run Preflight Checks - -`preflight` performs read-only checks and prints a human-readable readiness report. Add `--json` for structured output. +To apply an exact reviewed **non-destructive** payload file later without recomputing it: ```bash -./bin/awssync preflight \ - --max-snapshot-age 24h \ - --max-removals 10 \ - --max-removal-percent 5 +./bin/awssync apply-plan \ + --plan aws_sync_payload.json \ + --yes ``` -Expected result: `ready` is `true`, `nqe_aws_accounts` passes, `patch_plan` passes, and `account_removals` either passes or is understood and approved. +Do not use `apply-plan` as a substitute for the authoritative-manifest removal workflow. Run lifecycle removals through `sync-accounts` with the reviewed manifest and current removal ceilings. `apply-plan` remains useful for reviewed additive payloads and rollback recovery; it re-reads current state and routes through the same gateway. -If `management_account_discovery` fails, the snapshot did not show any genuinely new uncollected AWS account candidates. An already configured row with `Collected? false` does not satisfy this check; it may simply be disabled or failing collection. Do not apply account removals in that state unless AWS Organizations discovery has been independently verified and `--allow-no-candidates` is intentional. +Before the first PATCH, both normal apply and `apply-plan` write a pre-change PATCH payload beside the plan as `.rollback.json`. It contains the complete `assumeRoleInfos` account list plus `type`, `name`, `regions`, `regionToProxyServerId`, and `proxyServerId`. It does not capture the GET-returned fields `collect`, `connectionTimeoutSeconds`, `requestTimeoutSeconds`, `numVirtualizedDevices`, or `useForwardAccountToAssumeRole`. -`aws_organizations_evidence` reports if the plan has either candidate visibility or OU ID visibility for each selected setup. In multi-setup runs, the check lists the setup IDs that lack this signal. Treat both as supporting evidence only. The safer destructive-sync guard is: +This omission is safe for restoring an `awssync` change because Forward PATCH is a top-level merge: fields absent from the request body are left unchanged. The rollback restores the fields that `awssync` PATCHes without replacing those five settings. It is not a full backup of the setup and must not be used to reconstruct one from scratch. The apply summary includes its path and SHA-256. Use the result journal and recovery procedure below before applying the rollback. -- NQE absence cannot produce removals unless `--prune-missing` is explicit; `account_removals` then also requires `--allow-removals`. -- `removal_blast_radius` confirms the aggregate count and per-setup percentage remain within the operator-supplied ceilings. -- `management_account_discovery` fails: add `--allow-no-candidates` only after confirming discovery is complete. -- `aws_organizations_evidence` fails: add `--allow-no-org-evidence` only after independent verification that Forward has complete AWS Organizations visibility for that setup. +### Apply Recovery -## Apply the Sync - -For a routine interactive sync, use the additive-only command: +Every apply first creates `.result.json` and updates it atomically after each setup disposition. Inspect it before retrying a command that failed or lost its terminal output: ```bash -./awssync-linux-amd64 safe-sync \ - --setup-id AWS-PROD +jq '{plan_digest, network_id, setups}' aws_sync_payload.result.json ``` -The remaining commands in this section are the standard and expert workflow. For non-interactive automation, run with `--apply` after reviewing the dry plan. +The per-setup status is `planned`, `pending`, `applied`, `conflicted`, or `failed`. `applied` means the PATCH completed and was journaled. `conflicted` means the immediate pre-PATCH re-read detected a changed setup and no PATCH was sent for that setup. `pending` after a crash is ambiguous: read the actual Forward setup before deciding whether to retry or roll back. In a multi-setup run, treat each setup independently and do not assume all-or-nothing behavior. + +The pre-change PATCH payload is `.rollback.json`; its path and SHA-256 are also printed in the apply summary. After checking the journal and current Forward state, restore it with: ```bash -./bin/awssync \ - --max-snapshot-age 24h \ - --output aws_sync_payload.json \ - --apply \ +./bin/awssync apply-plan \ + --plan aws_sync_payload.rollback.json \ --yes ``` -Expected result: the command prints `patched_setup_count` greater than zero and each patched setup shows `patched: true`. - -NQE sync is additive by default: configured accounts missing from NQE remain in the payload. This is intentional because disabled accounts, failed accounts, and incomplete snapshots may be absent from NQE. - -To make NQE absence eligible for removal, add `--prune-missing`. Prefer `sync-accounts` with a complete authoritative manifest for actual account-lifecycle removals. If an explicit prune plan contains removals, `--apply` also fails unless `--allow-removals` is included. Use both flags only after reviewing `removed_accounts`. - -For an approved removal, also set both blast-radius ceilings. `--max-removals` applies to the total across every selected setup, while `--max-removal-percent` applies independently to each setup's current configured-account count: +If that rollback removes or disables accounts that were added by the original apply, it is itself an unattended destructive apply. Review the exact difference and add all four controls: ```bash -./bin/awssync \ - --max-snapshot-age 24h \ - --output aws_sync_payload.json \ - --apply \ +./bin/awssync apply-plan \ + --plan aws_sync_payload.rollback.json \ --yes \ - --prune-missing \ --allow-removals \ - --max-removals 10 \ - --max-removal-percent 5 + --max-removals APPROVED_COUNT \ + --max-removal-percent APPROVED_PERCENT \ + --allow-unattended-destructive ``` -Choose both nonzero limits from the reviewed plan, leaving enough room only for the approved account IDs. A removal is blocked if either option is omitted or either ceiling is exceeded. The same flags are enforced by `sync-accounts`, `apply-plan`, and webhook-driven apply runs. - -If an explicit prune includes account removals and no new uncollected candidate accounts are visible, `--apply` also requires `--allow-no-candidates`. Use that flag only after confirming the management or delegated discovery account is collected and AWS Organizations discovery is working. - -If an explicit prune includes account removals and the same setup has neither candidate rows nor OU rows visible, `--apply` also requires `--allow-no-org-evidence`. Use that flag only after an independent validation that the setup is collecting from the expected AWS Organization. - -To apply the exact reviewed payload file later without recomputing the plan: - -```bash -./bin/awssync apply-plan \ - --plan aws_sync_payload.json \ - --yes -``` +Account ordering matters only to byte-level comparisons. `sync-accounts` rebuilds the reviewed account set in sorted order, so using it to restore the same accounts can produce a different raw endpoint hash even though the configuration is semantically identical. In live validation, applying the emitted rollback preserved the original account order and reproduced the pre-change endpoint hash byte-for-byte. Use the rollback path when byte-identical restoration matters. -Before the first PATCH, both normal apply and `apply-plan` write the complete current setup state beside the plan as `.rollback.json`. The apply summary includes its path and SHA-256. Restore it with `apply-plan --plan .rollback.json --yes`; if the restore itself removes newly added accounts, supply `--allow-removals` and both narrowly reviewed removal bounds. +Do not blindly rerun an ambiguous apply: a PATCH may have succeeded before its result could be persisted. Keep the payload, rollback, and result files together until a new Forward snapshot confirms recovery. To recompute and apply for two selected setups after reviewing the expected changes: @@ -619,7 +456,7 @@ To recompute and apply for two selected setups after reviewing the expected chan --yes ``` -For NQE pruning, add `--prune-missing` and `--allow-removals` only when the reviewed plan contains expected removals. +NQE multi-setup runs remain additive. Run `sync-accounts` separately for each setup whose reviewed manifest authorizes removals. ## Validate After Apply @@ -641,36 +478,36 @@ Useful commands: --snapshot-id SNAPSHOT_ID ``` +`status --json` reports `observation_atomic: false`. It reads Forward's latest-processed endpoint and snapshot list separately, so even matching responses are an operational view rather than one atomic point-in-time observation. A mismatch is reported in `latest_list_consistent` and `observation_warning`. + +Planning rejects a processed snapshot timestamp more than five minutes ahead of the local clock. A smaller future offset is treated as ordinary clock skew and age zero. For a larger offset, correct NTP or the bad timestamp rather than bypassing freshness checks. + If a new account appears in the Forward setup but fails collection, the most likely cause is missing or incorrect IAM role setup in that AWS account. ## Ongoing Automation -Generated payload, rollback, manual, and applied-audit files are written atomically with owner-only `0600` permissions. Treat them as secrets when a workflow uses static AWS access keys, and configure retention accordingly. +Generated payload, rollback, result-journal, manual, and applied-audit files are written atomically with owner-only `0600` permissions. Treat them as secrets when a workflow uses static AWS access keys, and configure retention accordingly. The client retries transient `429`, `502`, `503`, and `504` failures only for idempotent reads, NQE reads, and full-state PATCH operations. It does not retry cloud-account or webhook creation POSTs. After an ambiguous create failure, inspect Forward for the requested object before retrying manually. -Run `awssync` on a schedule or after AWS account lifecycle events. - -The recommended automation policy is additive NQE sync without `--prune-missing`. Routine additions and re-enablement can proceed while NQE absence never deletes an account. Use an authoritative manifest for reviewed lifecycle removals. If NQE pruning is exceptionally required, an operator must verify the account lifecycle in AWS, review `removed_accounts`, and apply with `--prune-missing`, explicit removal approval, and narrow `--max-removals` and `--max-removal-percent` ceilings. +The automation policy is additive NQE sync. For a new account, create the Forward collection role, run a Forward snapshot that observes the account, run `awssync` against that processed snapshot, and validate the next collection. -Recommended sequence: +Lifecycle removal is a separate workflow. Confirm the closure, transfer, or retirement in an authoritative lifecycle system, update and review the complete manifest, then run `sync-accounts` with narrow removal ceilings. Do not remove the collection IAM role first and then interpret the resulting NQE absence as authorization. -1. AWS account is created or closed. -2. Automation creates or removes the Forward collection IAM role. -3. Forward runs a snapshot that can discover the updated Organization account inventory. -4. `awssync` runs against that processed snapshot or latest processed snapshot. -5. Forward runs the next collection with the updated account list. +### Webhook Operation -For event-driven workflows, `awssync serve-webhook` can receive Forward `SNAPSHOT_READY` events and run the sync against the exact snapshot from the event. +For event-driven additive workflows, `awssync serve-webhook` can receive Forward `SNAPSHOT_READY` events and run the sync against the exact snapshot from the event. It cannot remove accounts from NQE absence. Start the receiver: ```bash ./bin/awssync serve-webhook \ + --network-id NETWORK_ID \ --listen :8080 \ --path /forward/snapshot-ready \ --webhook-basic-username awssync \ --webhook-basic-password RECEIVER_SHARED_SECRET \ + --webhook-state-file /var/lib/awssync/webhook-state.json \ --apply \ --yes ``` @@ -679,13 +516,26 @@ Create the Forward webhook through the Forward API: ```bash ./bin/awssync configure-webhook \ + --network-id NETWORK_ID \ --webhook-url https://awssync.example.com/forward/snapshot-ready \ --webhook-basic-username awssync \ --webhook-basic-password RECEIVER_SHARED_SECRET \ --test-webhook ``` -Forward webhooks use Basic Auth credentials when credentials are configured. The `--webhook-basic-username` and `--webhook-basic-password` values on `configure-webhook` must match the receiver values on `serve-webhook`. +An applying receiver will not start without `--yes`, an explicit `--network-id`, and both Basic Auth values. The `--webhook-basic-username` and `--webhook-basic-password` values on `configure-webhook` must match the receiver values on `serve-webhook`; Forward includes them on delivery. + +### Webhook Recovery + +The receiver persists each accepted event before returning `202`. By default, queue state is `$UserConfigDir/awssync/webhook-state.json`; on Linux that is normally `$HOME/.config/awssync/webhook-state.json`. Services should use `--webhook-state-file` with an explicit service-owned path. Keep it on durable local storage, retain it across restarts, and do not share one state file between daemon processes because there is no interprocess lock. The file is atomically written with mode `0600`; keep its parent directory service-owned. `/healthz` reports `pendingDepth` and `deadLetterDepth` in addition to the in-memory `queueDepth`. + +Failed jobs run at most five times. The delays after failures are 1, 2, 4, and 8 seconds (the exponential delay is capped at 30 seconds). After the fifth failure, the full event, attempt timestamps, and last error remain under `dead_letter_events` in the state JSON. Inspect pending and dead-letter work with the service user, for example: + +```bash +jq '{pending_events, dead_letter_events}' /var/lib/awssync/webhook-state.json +``` + +Correct the underlying error before draining a dead-letter entry. Then POST the original event body to the configured receiver path with the normal Basic Auth credentials. Re-delivery removes that entry from `dead_letter_events`, persists it as a fresh pending job with a reset failure count, and returns `202`; the new cycle is bounded to five attempts again. Normal scope and snapshot-watermark checks still run, so an obsolete event may be rejected instead of requeued. To discard an obsolete entry, stop the receiver, remove only that exact record from `dead_letter_events`, preserve mode `0600`, and restart. Do not edit the state file while the receiver is running. `configure-webhook` is repeatable. It creates a missing webhook and updates the same named webhook if it already exists. If only specific AWS setups should sync from webhook events, add one or more `--setup-id` values. The tool adds those setup IDs to the receiver URL so the receiver can scope the run. Add `--webhook-per-setup` to create or update one webhook per setup ID. @@ -707,17 +557,19 @@ Recommended service practices: - Run as a dedicated low-privilege user such as `awssync`. - Store `FWD_HOST`, `FWD_USER`, `FWD_PASS`, `AWSSYNC_WEBHOOK_BASIC_USERNAME`, and `AWSSYNC_WEBHOOK_BASIC_PASSWORD` in a protected service environment file. +- Pin one `FWD_NETWORK_ID` or `--network-id` and reject events from every other network. +- Put `--webhook-state-file` on persistent service-owned storage and preserve its `0600` mode. - Start in dry-run mode first, without `--apply`, and confirm webhook delivery and payload generation. - Add `--apply --yes` only after dry-run output is reviewed. -- Use `--allow-removals` only after an operator reviews planned removals. - Use `--allow-no-candidates` only after confirming management or delegated discovery is working. - Use `--allow-no-org-evidence` only after independent verification that AWS Organizations discovery remains complete. - Send service logs to the normal log collection system. +- Alert when `/healthz` reports a nonzero `deadLetterDepth`, and retain the webhook state file across service restarts. Linux systemd command example: ```ini -ExecStart=/usr/local/bin/awssync serve-webhook --listen 0.0.0.0:8080 --apply --yes +ExecStart=/usr/local/bin/awssync serve-webhook --network-id NETWORK_ID --listen 0.0.0.0:8080 --webhook-state-file /var/lib/awssync/webhook-state.json --apply --yes EnvironmentFile=/etc/awssync/awssync.env Restart=on-failure RestartSec=10 @@ -738,6 +590,15 @@ Likely causes: Fix: verify the discovery account setup, run a new snapshot, and rerun the dry plan. +### Snapshot Has an Invalid Future Timestamp + +Likely causes: + +- the `awssync` host clock is behind Forward; +- Forward returned a bad `processedAt` or `createdAt` value. + +Fix: compare UTC time on both systems and correct NTP or the source timestamp. Planning tolerates up to five minutes of ordinary clock skew and rejects anything further ahead; changing `--max-snapshot-age` does not make a future timestamp valid. + ### discover-org AWS Organizations Access Denied Likely causes: @@ -767,7 +628,7 @@ Likely causes: - The discovery account role lacks AWS Organizations read permissions. - The query override does not include the `Collected?` column. -Fix: run `preflight`, verify `management_account_discovery`, and confirm the AWS Organizations access check. Do not approve removals from this state unless the account list is confirmed complete. +Fix: run `preflight`, verify `management_account_discovery`, and confirm the AWS Organizations access check. This affects discovery of additions. Make all removal decisions from a complete independently reviewed manifest, never from this NQE state. ### Webhook Does Not Trigger Sync @@ -776,9 +637,11 @@ Likely causes: - The Forward webhook URL is not reachable from the Forward app server. - Forward SaaS is pointed at a private or VPN-only receiver URL. - Basic Auth values in Forward do not match the receiver. -- The webhook is not scoped to the intended network. +- The applying receiver has no explicit `--network-id`, so it refuses to start. +- The event is outside the receiver's configured network scope. +- The event failed five times and is in `dead_letter_events`. -Fix: run `configure-webhook --test-webhook`, check receiver logs, and confirm `/healthz` is reachable from the same network path Forward will use. +Fix: run `configure-webhook --test-webhook`, check receiver logs and the durable state file, and confirm `/healthz` is reachable from the same network path Forward will use. Follow [Webhook recovery](#webhook-recovery) for a dead-lettered event. ### Missing Setup Metadata @@ -820,12 +683,3 @@ Use both AWS Organizations inventory and the per-account `sts:AssumeRole` result | Yes | Fails | The account is active and discoverable, but its collection role, trust policy, external ID, or permissions are incorrect. | Repair IAM in the member account; do not remove it from Forward. | | No | Succeeds | Forward can still reach the configured role, but the discovery account does not report the account. Organization membership or discovery scope may have changed. | Verify the management or delegated discovery account and the account's Organization membership; do not remove it based only on discovery. | | No | Fails | The account may be closed, removed, or moved, or its IAM configuration may also be broken. | Confirm the account lifecycle independently in AWS. Remove it only after that confirmation; otherwise repair discovery or IAM. | - -## Summary - -AWS account sync has two layers: - -1. AWS Organizations tells Forward which accounts exist. -2. IAM roles in each AWS account allow Forward to collect those accounts. - -`awssync` automates layer 1 into Forward's configured account list. Layer 2 is still required in AWS: every account must have the expected IAM role and trust policy. For IAM user/access-key setups, the stored credential must also be allowed to assume those roles. This is why the first setup step is verifying management-account or delegated-account Organizations visibility before running the script. diff --git a/docs/govcloud-workflow.md b/docs/govcloud-workflow.md index 70afbb0..67fcb12 100644 --- a/docs/govcloud-workflow.md +++ b/docs/govcloud-workflow.md @@ -4,10 +4,10 @@ Use this workflow for AWS GovCloud (US) accounts, including customers that canno AWS Organizations is available in GovCloud, but a GovCloud organization is independent from a commercial AWS organization. Its Organizations control plane is in `us-gov-west-1`. Forward's regular AWS collection pipeline can query Organizations using the GovCloud setup credentials and primary region. -There are two supported inventory paths: +There are two supported synchronization paths: -1. **GovCloud Organizations + Forward NQE** is preferred when the customer can grant read access to the GovCloud organization. -2. **Reviewed account manifest** is the fallback for standalone accounts or customers that cannot grant Organizations access. +1. **GovCloud Organizations + Forward NQE** can add and re-enable observed accounts when the customer grants read access to the GovCloud organization. It never removes accounts. +2. **Reviewed account manifest** is the authoritative path for lifecycle removals and the fallback for standalone accounts or customers that cannot grant Organizations access. Do not treat a successfully collected GovCloud region as proof that Organizations discovery succeeded. Resource collection and organization inventory are separate checks. @@ -41,26 +41,23 @@ Configure at least `us-gov-west-1` in the Forward AWS setup. Run a Forward conne --network-id NETWORK_ID \ --setup-id GOVCLOUD_SETUP \ --max-snapshot-age 24h \ - --max-removals 5 \ - --max-removal-percent 5 \ --format human ``` -Preflight must confirm all of the following before any removal: +Preflight should confirm all of the following before additive synchronization: - the setup's role ARNs consistently use `arn:aws-us-gov`; - the configured collection regions are GovCloud regions; - the current snapshot returns AWS accounts for the selected setup; -- Forward NQE exposes positive Organizations evidence, such as uncollected candidate accounts or Organizational Unit IDs; -- every proposed removed account ID has been reviewed. +- Forward NQE exposes the expected observed accounts and, when available, Organizations metadata such as Organizational Unit IDs. -An account directly under the organization root may have no OU ID. A missing OU ID alone does not prove failure, but a removal plan with neither candidate accounts nor OU evidence is unsafe. GovCloud removals from the NQE path are blocked in that state and cannot be forced with the generic no-evidence flags. +An account directly under the organization root may have no OU ID. More importantly, NQE is observed inventory rather than a configured-account manifest: authorization failures, collection failures, organization scope, and transient errors can all make an account absent. NQE absence therefore never produces a GovCloud removal. -If preflight is ready and the plan has no removals, generate the payload normally. If it proposes removals, review the exact IDs printed by the human report and the JSON payload before applying. +If preflight is ready, generate and review the additive payload normally. For any lifecycle removal, switch to Path B. ## Path B: Manual Account Manifest -Use this path when the customer has standalone GovCloud accounts, Organizations is unavailable by policy, or Forward cannot see the GovCloud organization. +Use this path for every lifecycle removal, and when the customer has standalone GovCloud accounts, Organizations is unavailable by policy, or Forward cannot see the GovCloud organization. Create a reviewed JSON file containing the complete authoritative account inventory: @@ -138,14 +135,15 @@ If removals are intentional, the apply is blocked unless the operator also suppl --allow-removals \ --max-removals 5 \ --max-removal-percent 5 \ + --allow-unattended-destructive \ --yes ``` -Set the ceilings to the reviewed change, not to the full account population. `--max-removals` limits the total removals in the run, and `--max-removal-percent` prevents a single setup from losing more than the approved percentage. Exceeding either value blocks before PATCH. +Set the ceilings to the reviewed change, not to the full account population. `--max-removals` limits the total removals in the run, and `--max-removal-percent` prevents a single setup from losing more than the approved percentage. Exceeding either value blocks before PATCH. `--allow-unattended-destructive` is also required here because `--yes` skips the human confirmation and Forward provides no atomic compare-and-swap token. For a human-attended removal, omit both `--yes` and `--allow-unattended-destructive` and type `apply` after reviewing the preview. After any update, run a Forward connectivity test for representative accounts, run a new snapshot, and inspect per-account collection errors. -Do not use `apply-plan` to bypass these source checks. `apply-plan` reloads the current Forward setup before patching and refuses GovCloud account removals; rerun the NQE or manifest workflow that produced the inventory instead. +Do not use `apply-plan` to bypass these source checks. `apply-plan` reloads the current Forward setup before patching and refuses GovCloud account removals; rerun `sync-accounts` with the reviewed manifest instead. ## When This Is a Forward Product Issue diff --git a/docs/quick-start.md b/docs/quick-start.md index 0fb2247..58ad605 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -1,5 +1,7 @@ # AWS Account Sync CLI Quick Start +If this replaces an earlier release, read [Upgrading `awssync`](upgrading.md) before changing an existing job or webhook service. + For routine updates of an existing Forward AWS setup, start with [`safe-sync`](routine-safe-sync.md). It performs preflight, previews the changes, refuses removals, prompts once, and writes rollback data: ```bash @@ -10,7 +12,7 @@ For routine updates of an existing Forward AWS setup, start with [`safe-sync`](r The rest of this guide covers the standard and expert commands for automation, onboarding, External IDs, GovCloud, and independently reviewed account removals. -Use the standard `awssync` command to update an existing Forward AWS setup when AWS Organization accounts are added or removed and Forward's collected NQE data is the source of truth. +Use the standard `awssync` command for additive updates to an existing Forward AWS setup. Forward's NQE rows are observed snapshot inventory, not a source of truth for configured membership; removals use `sync-accounts` with a reviewed manifest. For new AWS Organizations onboarding, prefer the Forward Terraform provider as the native IaC workflow. It supports Forward assume-role, static-key, and collector instance-profile credential models. Use `awssync discover-org` only when Forward has not onboarded that AWS Organization yet and you need manual JSON files, a break-glass create payload, or a static-key workflow that should stay outside Terraform state. @@ -21,7 +23,7 @@ For AWS GovCloud, use the dedicated [AWS GovCloud Account Workflow](govcloud-wor - Forward must collect the AWS management account or a delegated account that can list AWS Organizations accounts. - Each AWS account that Forward should collect must have the same Forward IAM role name. - Forward IAM role and IAM user/access-key multi-account setups are supported. -- Run a dry plan first. Do not apply removals until the account list is reviewed. +- Run a dry plan first. Use a complete reviewed manifest for removals. Grant AWS Organizations read permissions only to the management or delegated discovery account used for inventory. Do not grant organization-wide permissions to every member role. Member accounts need the Forward collection policy and trust policy for `sts:AssumeRole`. @@ -54,11 +56,7 @@ Expected result: `ready` is `true`. If `management_account_discovery` fails, confirm Forward is collecting the AWS management or delegated discovery account. -`nqe_org_unit_row_count` is helpful supporting evidence when it is nonzero, but it can be zero for valid AWS Organizations where accounts sit directly under the root. Do not use OU IDs as the only safety signal for removals. - -If both `nqe_candidate_row_count` and `nqe_org_unit_row_count` are zero and removals are planned, review `--allow-no-org-evidence` before applying. - -In multi-setup runs, `--allow-no-org-evidence` is required only for setup IDs that are missing both signals; preflight output shows the setup IDs in the failing check message. +`nqe_org_unit_row_count` and `nqe_candidate_row_count` are useful discovery diagnostics, but neither proves the NQE result is a complete account manifest. Missing accounts remain configured regardless of these counts. ## Create a Dry Plan @@ -81,9 +79,11 @@ If you need a manual fallback format for UI drag-and-drop, also review `aws_sync - each value is the list of `assumeRoleInfos` planned for that setup - you can paste a single setup block into the Forward UI or use it as a reference before apply -Stop if removed accounts are unexpected. +`removed_accounts` must be empty in a standard NQE plan. Stop without applying if it is not. -Generated payload, manual, and applied-audit files are atomically replaced with owner-only `0600` permissions. They can still contain sensitive credential material in static-key onboarding workflows, so store and dispose of them according to the customer's credential policy. +NQE account IDs must contain exactly 12 digits. Malformed rows fail the run by default. `--allow-malformed-rows` is an additive-only escape hatch: it skips and reports those rows, marks the observation incomplete, and cannot authorize removals. + +Generated payload, rollback, result-journal, manual, and applied-audit files are atomically replaced with owner-only `0600` permissions. They can still contain sensitive credential material in static-key onboarding workflows, so store and dispose of them according to the customer's credential policy. ## Add an External ID to an Existing IAM User Setup @@ -127,7 +127,7 @@ AWS-PROD,333333333333,clear, Duplicate, malformed, or unknown accounts fail before PATCH. Normal sync preserves mixed per-account values. If a mixed-ID setup gains a new account, pass the same `--external-id-file` to `preflight` and the normal dry-run/apply so the new account has an explicit value. -Scoped rollback uses the same `--account-id`: dry-run and apply `--clear` if the original value was null, or `--value PREVIOUS_VALUE` if it was non-null. Record the old non-null value before the test; the summary reports its configured state but does not save it as an automatic rollback value. Relax the selected account's AWS trust-policy condition before changing Forward back. All unselected accounts remain unchanged. +Every External ID apply writes the pre-change account list and PATCHable setup fields to `.rollback.json` and maintains `.result.json`. The rollback is a PATCH payload, not a full setup backup: it does not capture `collect`, `connectionTimeoutSeconds`, `requestTimeoutSeconds`, `numVirtualizedDevices`, or `useForwardAccountToAssumeRole`. Forward leaves those absent top-level fields unchanged when the rollback is applied. For a narrowly scoped manual revert, use the same `--account-id`: dry-run and apply `--clear` if the original value was null, or `--value PREVIOUS_VALUE` if it was non-null. The summary does not print the old value, so take it from the protected rollback payload if needed. Relax the selected account's AWS trust-policy condition before changing Forward back. All unselected accounts remain unchanged. Rollback order matters: first relax or remove the mandatory `sts:ExternalId` condition from the target-role trust policies and confirm a representative role can still be assumed. Then replace `--value VALUE` with `--clear`, review the dry run, apply it, and test collection again. Clearing Forward first while AWS still requires the External ID will interrupt collection. @@ -180,25 +180,31 @@ terraform -chdir=examples/terraform/forward-collection-role-stackset apply ./bin/awssync --max-snapshot-age 24h --output aws_sync_payload.json --apply --yes ``` -If removals are expected: +The standard command is additive: NQE absence never produces removal. If an account must be removed, prepare a complete reviewed manifest and dry-run the one affected setup: ```bash -./bin/awssync \ - --max-snapshot-age 24h \ - --output aws_sync_payload.json \ +./bin/awssync sync-accounts \ + --setup-id AWS-PROD \ + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json +``` + +After reviewing every added and removed ID, apply with narrow removal ceilings: + +```bash +./bin/awssync sync-accounts \ + --setup-id AWS-PROD \ + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json \ --apply \ --yes \ - --prune-missing \ --allow-removals \ --max-removals 10 \ - --max-removal-percent 5 + --max-removal-percent 5 \ + --allow-unattended-destructive ``` -NQE sync is additive by default. `--prune-missing` is required before an account absent from NQE can become a removal; prefer a complete authoritative manifest for lifecycle removals. Both limits are mandatory for any removal. `--max-removals` is the aggregate ceiling across selected setups. `--max-removal-percent` is evaluated separately for each setup against its current configured-account count. Preflight accepts the same limits and reports `removal_blast_radius` before anything is patched. - -If removals are expected and no uncollected candidate accounts are visible, also add `--allow-no-candidates` only after confirming AWS Organizations discovery is working. - -If removals are expected and there is no candidate signal and no OU signal for a setup, also add `--allow-no-org-evidence` only after independent verification that Forward’s discovery account is still collecting a complete AWS Organization account list. +`--prune-missing` is still recognized but always refuses: NQE is observed inventory, not an account manifest, so absence cannot prove deletion. Both limits remain mandatory for manifest removals. `--max-removals` is the aggregate ceiling and `--max-removal-percent` is evaluated against the setup's current configured-account count. The last flag is required because `--yes` makes this a destructive unattended apply; omit both `--yes` and `--allow-unattended-destructive` to use the interactive confirmation instead. ## Multiple AWS Setups @@ -234,13 +240,13 @@ Recompute and apply after reviewing the expected changes: --yes ``` -For one setup, pass a single `--setup-id AWS_SETUP_ID`. Repeat `--setup-id` for other setup combinations. Add `--prune-missing` and `--allow-removals` only after reviewing and confirming every proposed NQE-based removal. +For one setup, pass a single `--setup-id AWS_SETUP_ID`. Repeat `--setup-id` for additive NQE synchronization of other setup combinations. Removal is a separate, one-setup-at-a-time `sync-accounts` manifest workflow. When exactly one setup is selected, the default inline NQE query is parameterized by that setup ID to reduce returned rows. Multiple setup IDs are still separated by `Cloud Setup ID` in the NQE result. -For automation, omit `--prune-missing` and `--allow-removals`. Normal additions and re-enablement can proceed, while accounts absent from NQE remain configured. +Normal additions and re-enablement can proceed in automation, while accounts absent from NQE remain configured. Automate manifest removals only when the manifest itself has an independent human review and approval process. -Human-readable output is the default. Add `--json` for scripts. Every apply writes `.rollback.json` before the first PATCH; use that file with `apply-plan --plan` to restore the exact prior setup state. +Human-readable output is the default. Add `--json` for scripts. Every apply writes `.rollback.json` before the first PATCH and updates `.result.json` as each setup is applied, conflicted, or failed. Inspect the journal and current Forward state before recovering a partial run. A rollback that removes or disables accounts needs the normal removal limits and `--allow-unattended-destructive`; follow [Apply recovery](aws-account-sync-procedure.md#apply-recovery). An existing NQE row with `Collected? false` is not proof that AWS Organizations discovered a new account. It commonly represents an account that is configured but disabled or failing collection. Only uncollected IDs not already present in the setup count as discovery candidates. @@ -262,15 +268,29 @@ Forward API reads, NQE queries, and full-state PATCH operations use bounded retr For event-driven sync, run the receiver: ```bash -./bin/awssync serve-webhook --listen 0.0.0.0:8080 --webhook-basic-username awssync --webhook-basic-password RECEIVER_SECRET --apply --yes +./bin/awssync serve-webhook \ + --network-id NETWORK_ID \ + --listen 0.0.0.0:8080 \ + --webhook-basic-username awssync \ + --webhook-basic-password RECEIVER_SECRET \ + --webhook-state-file /var/lib/awssync/webhook-state.json \ + --apply \ + --yes ``` Then configure Forward: ```bash -./bin/awssync configure-webhook --webhook-url https://awssync.example.com/forward/snapshot-ready --webhook-basic-username awssync --webhook-basic-password RECEIVER_SECRET --test-webhook +./bin/awssync configure-webhook \ + --network-id NETWORK_ID \ + --webhook-url https://awssync.example.com/forward/snapshot-ready \ + --webhook-basic-username awssync \ + --webhook-basic-password RECEIVER_SECRET \ + --test-webhook ``` +The applying receiver will not start without the network and both Basic Auth values. Forward must send the same username and password. Retain the state file across restarts; after five failed attempts an event remains in `dead_letter_events` until an operator corrects the cause and redelivers or discards it. + For setup-scoped webhook sync, add `--setup-id SETUP_ID`. Repeat it for more than one setup, or add `--webhook-per-setup` to create one Forward webhook per setup. For Forward SaaS, the webhook URL must be reachable from the internet. diff --git a/docs/routine-safe-sync.md b/docs/routine-safe-sync.md index 91298d9..5c8e396 100644 --- a/docs/routine-safe-sync.md +++ b/docs/routine-safe-sync.md @@ -61,9 +61,10 @@ A successful run prints: - the number of patched setups; - the rollback file path; -- the rollback SHA-256. +- the rollback SHA-256; +- the result-journal path. -Keep the rollback file until the next successful collection confirms the expected account state. +Keep the rollback and result-journal files until the next successful collection confirms the expected account state. If the command fails after apply begins, inspect the journal and current Forward setup before retrying; see [Apply recovery](aws-account-sync-procedure.md#apply-recovery). If no changes were needed, the command instead confirms that no PATCH was sent; there is no rollback file because Forward was not changed. @@ -82,4 +83,4 @@ A collection failure does not mean an account should be removed. Repair IAM, rol ## Account Removal -Routine operators should not remove accounts with this tool. Escalate a removal to an operator who can independently verify the AWS account lifecycle and follow the reviewed removal procedure in [AWS account sync procedure](aws-account-sync-procedure.md#apply-the-sync). +Routine operators should not remove accounts with this tool. Escalate a removal to an operator who can independently verify the AWS account lifecycle and follow the [reviewed manifest removal procedure](aws-account-sync-procedure.md#reviewed-manifest-removal). diff --git a/docs/upgrading.md b/docs/upgrading.md new file mode 100644 index 0000000..d93d9af --- /dev/null +++ b/docs/upgrading.md @@ -0,0 +1,202 @@ +# Upgrading `awssync` + +This guide is for operators upgrading from the release that allowed NQE-based `--prune-missing`, allowed an applying webhook receiver without inbound credentials or a fixed network, and allowed unattended destructive applies without an additional acknowledgement. + +Read this before replacing the binary. Three existing automation patterns now fail closed. + +## Before the Upgrade + +1. Disable scheduled jobs that pass `--prune-missing`. +2. Record the configured account IDs in every Forward AWS setup from Forward, a recent payload, or a rollback artifact. A rollback is sufficient here for the account list, but it is not a full setup backup. This is the baseline for review; do not reconstruct it from NQE. +3. Back up the current webhook service definition and, if it exists, its webhook state file. +4. Identify every automation path that can remove or disable accounts, including `sync-accounts --yes` and destructive `apply-plan` runs. + +A rollback artifact contains the complete `assumeRoleInfos` account list and the PATCHable setup fields. It does not capture `collect`, `connectionTimeoutSeconds`, `requestTimeoutSeconds`, `numVirtualizedDevices`, or `useForwardAccountToAssumeRole`. Applying it is safe for restoration because Forward PATCH leaves absent top-level fields unchanged, but keep a separate full setup record if those settings must be backed up or the setup may need to be reconstructed. + +## 1. Replace `--prune-missing` With a Reviewed Manifest + +`--prune-missing` now exits with an error and never creates or applies a plan. Removing the flag makes the normal NQE workflow additive; it does **not** preserve the old removal behavior. + +Forward NQE returns accounts observed in a snapshot. It can combine successfully collected accounts with accounts visible through Organizations metadata, but collection failures, authorization failures, discovery scope, and transient errors can omit live accounts. It is not an account manifest. This is why NQE absence once caused live accounts to be deleted and is no longer accepted as removal evidence. + +`sync-accounts` with a complete, independently reviewed manifest is the only supported removal path. + +### Build the manifest from authoritative sources + +Start with the account IDs currently configured in the Forward setup. Then reconcile that baseline against sources that own account lifecycle, such as: + +- a direct AWS Organizations `ListAccounts` call made with management-account or authorized delegated credentials; +- the account-vending system or CMDB; +- the approved inventory for standalone accounts or accounts in another Organization; +- closure, transfer, or retirement records that identify the exact IDs approved for removal. + +Do not use NQE output, a failed collection, a missing IAM role, or `Collected? false` to decide that an account should be absent from the manifest. + +For one AWS Organization, a direct AWS CLI export can provide one input to the review: + +```bash +AWS_PROFILE=org-readonly aws organizations list-accounts \ + --query 'Accounts[?State==`ACTIVE`].{id:Id,name:Name}' \ + --output json > org-accounts.json +``` + +Run this against every relevant Organization. Add separately approved standalone accounts, and keep accounts whose lifecycle cannot be confirmed. If several reviewed JSON arrays must be combined, concatenate and sort them without hiding duplicates: + +```bash +jq -s 'add | sort_by(.id)' \ + org-accounts.json \ + approved-standalone-accounts.json \ + > reviewed-accounts.json +``` + +The manifest must be a non-empty JSON array and must contain every account that should remain in the one selected setup: + +```json +[ + { + "id": "111111111111", + "name": "security" + }, + { + "id": "222222222222", + "name": "production" + } +] +``` + +Every `id` must be a unique string containing exactly 12 digits. `sync-accounts` rejects unknown fields, duplicates, malformed IDs, and an empty manifest. + +### Dry-run, review, and apply + +Create a plan for exactly one setup: + +```bash +./bin/awssync sync-accounts \ + --network-id NETWORK_ID \ + --setup-id AWS-PROD \ + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json +``` + +Have an operator compare every `added_accounts` and `removed_accounts` ID with the current Forward setup and the lifecycle records. Stop if the removed set contains anything not independently approved. + +For a human-attended apply, omit `--yes` and type `apply` only after reviewing the preview: + +```bash +./bin/awssync sync-accounts \ + --network-id NETWORK_ID \ + --setup-id AWS-PROD \ + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json \ + --apply \ + --allow-removals \ + --max-removals APPROVED_COUNT \ + --max-removal-percent APPROVED_PERCENT +``` + +Set both nonzero ceilings just above the reviewed change, not to the full account population. An unattended equivalent also requires both `--yes` and the acknowledgement described below: + +```bash +./bin/awssync sync-accounts \ + --network-id NETWORK_ID \ + --setup-id AWS-PROD \ + --accounts-file reviewed-accounts.json \ + --output aws_manifest_plan.json \ + --apply \ + --yes \ + --allow-removals \ + --max-removals APPROVED_COUNT \ + --max-removal-percent APPROVED_PERCENT \ + --allow-unattended-destructive +``` + +## 2. Reconfigure Applying Webhook Receivers + +`serve-webhook --apply` will not start unless all of the following are configured: + +- `--yes`; +- a fixed `--network-id`; +- a non-empty inbound Basic Auth username and password. + +The fixed network is an authorization boundary: an event naming another network is rejected rather than causing the receiver to follow event-controlled scope. + +Use protected service environment variables for secrets. For example, an owner-readable service environment file can contain: + +```text +FWD_HOST=https://fwd.app +FWD_USER=awssync-service@example.com +FWD_PASS=FORWARD_SERVICE_PASSWORD +AWSSYNC_WEBHOOK_BASIC_USERNAME=awssync +AWSSYNC_WEBHOOK_BASIC_PASSWORD=RECEIVER_SHARED_SECRET +``` + +Start the applying receiver with an explicit network and durable service-owned state path: + +```bash +./bin/awssync serve-webhook \ + --network-id NETWORK_ID \ + --listen 0.0.0.0:8080 \ + --path /forward/snapshot-ready \ + --webhook-state-file /var/lib/awssync/webhook-state.json \ + --apply \ + --yes +``` + +Forward must send the same Basic Auth values. Create or update the named Forward webhook with matching credentials: + +```bash +./bin/awssync configure-webhook \ + --network-id NETWORK_ID \ + --webhook-url https://awssync.example.com/forward/snapshot-ready \ + --webhook-basic-username awssync \ + --webhook-basic-password RECEIVER_SHARED_SECRET \ + --test-webhook +``` + +`configure-webhook` updates the same named webhook when it already exists. Coordinate the Forward-side credential update with the receiver restart so the values match throughout the change. + +### Webhook state file + +Without `--webhook-state-file`, Go's user configuration directory is used: `$XDG_CONFIG_HOME/awssync/webhook-state.json`, normally `$HOME/.config/awssync/webhook-state.json` on Linux, and `$HOME/Library/Application Support/awssync/webhook-state.json` on macOS. Services should use an explicit path such as `/var/lib/awssync/webhook-state.json`. + +Create the parent directory as service-owned and inaccessible to other users. The state file is atomically written with mode `0600`; preserve that mode during backup or manual recovery. Keep it on durable local storage, retain it across restarts, and do not point two daemon processes at the same file because there is no interprocess lock. + +The file holds pending events, completed-event deduplication, snapshot watermarks, and dead-letter records. A failed event is attempted at most five times before moving to `dead_letter_events`. Alert on a nonzero `/healthz` `deadLetterDepth`; see [Webhook recovery](aws-account-sync-procedure.md#webhook-recovery) before redelivering or discarding an event. + +## 3. Review Unattended Destructive Applies + +An apply that removes or disables accounts is destructive. When it runs with `--yes`, from CI, or from another unattended context, it now refuses unless `--allow-unattended-destructive` is also present. + +Do not add the flag merely to silence the error. Forward exposes no ETag, version, or other compare-and-swap token for cloud-account setup updates. `awssync` re-reads the setup immediately before PATCH and detects a change that happened earlier, but it cannot make the following full-state PATCH atomic. A UI or automation edit made between that GET and PATCH is overwritten deterministically by the reviewed payload. + +Before authorizing unattended destruction: + +1. Ensure `sync-accounts` is using a complete, independently reviewed manifest. +2. Serialize all writers to the Forward setup, including UI, Terraform, other `awssync` jobs, and webhook daemons. +3. Use a maintenance window or another operational control that prevents concurrent edits. +4. Keep `--max-removals` and `--max-removal-percent` narrowly bounded. +5. Retain the rollback artifact and result journal, and verify the Forward setup immediately after apply. + +If those controls are not available, keep destructive applies interactive and omit `--yes`. + +## Other One-Time Changes + +### Approval digest changes once + +The approval digest format changed so the same approval-relevant plan now has the same digest across independent invocations. Old stored digests do not match the new format. After upgrading, discard any saved pre-upgrade digest, generate and review a fresh dry plan once, and store the new digest if external automation records it. Later changes to the network, snapshot, baseline, target, policy, or classified change counts still change the digest as intended. + +### Account IDs are strict + +NQE account IDs must now be exactly 12 digits. Rows that the previous version tolerated may stop preflight or planning. Fix the query or source data first. + +For an urgent additive NQE run, `--allow-malformed-rows` skips and reports malformed NQE rows, marks the observed inventory incomplete, and blocks using that inventory for removals. It does not relax `sync-accounts` manifest validation; reviewed manifests always require unique 12-digit IDs. + +## After the Upgrade + +1. Run an additive dry plan and confirm `removed_accounts` is empty. +2. Run `status --json`; expect `observation_atomic` to be `false` because the latest-processed and snapshot-list endpoints are separate reads. +3. Test the webhook through `configure-webhook --test-webhook`, then confirm `/healthz` reports the expected pending and dead-letter depths. +4. Check service logs and the apply result journal after the first apply. +5. Run a new Forward snapshot and verify representative accounts collect successfully. + +Snapshot timestamps more than five minutes ahead of the `awssync` host clock are rejected. If the first plan fails with an invalid future timestamp, correct NTP/clock configuration on the host or Forward side rather than increasing the snapshot-age limit. diff --git a/internal/api/architecture_failure_test.go b/internal/api/architecture_failure_test.go new file mode 100644 index 0000000..b408750 --- /dev/null +++ b/internal/api/architecture_failure_test.go @@ -0,0 +1,146 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +const runP0APIFailureTests = false + +func skipUntilP0APIFixed(t *testing.T, finding string) { + t.Helper() + if !runP0APIFailureTests { + t.Skip("P0 characterization disabled until fixed: " + finding) + } +} + +func TestP0AmbiguousPATCHRetryPreservesInterleavedEdit(t *testing.T) { + skipUntilP0APIFixed(t, "retryable PATCH has no idempotency key or revision precondition — docs/ARCHITECTURE_REVIEW.md §3, Idempotency, retries, and partial failure") + + const concurrentAccountID = "999999999999" + var ( + mu sync.Mutex + attempts int + applyCount int + version = `"version-1"` + stored []AssumeRoleInfo + idempotencyKeys []string + ifMatches []string + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPatch || r.URL.Path != "/api/networks/network-1/cloudAccounts/setup-a" { + http.NotFound(w, r) + return + } + var payload PatchPayload + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + mu.Lock() + attempts++ + attempt := attempts + key := r.Header.Get("Idempotency-Key") + ifMatch := r.Header.Get("If-Match") + idempotencyKeys = append(idempotencyKeys, key) + ifMatches = append(ifMatches, ifMatch) + + if attempt > 1 && key != "" && key == idempotencyKeys[0] { + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + return + } + if ifMatch != "" && ifMatch != version { + mu.Unlock() + http.Error(w, "revision conflict", http.StatusPreconditionFailed) + return + } + + stored = append([]AssumeRoleInfo(nil), payload.AssumeRoleInfos...) + applyCount++ + if attempt == 1 { + stored = append(stored, AssumeRoleInfo{ + AccountID: concurrentAccountID, + AccountName: "interleaved-ui-edit", + RoleArn: "arn:aws:iam::" + concurrentAccountID + ":role/ForwardRole", + Enabled: true, + }) + version = `"version-2"` + mu.Unlock() + + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Errorf("ResponseWriter does not implement http.Hijacker") + return + } + connection, _, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack committed response: %v", err) + return + } + _ = connection.Close() + return + } + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client, err := NewClient(server.URL, "/api", "alice", "secret", false, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + client.retryDelay = time.Millisecond + err = client.PatchCloudAccount(context.Background(), "network-1", "setup-a", PatchPayload{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []AssumeRoleInfo{{ + AccountID: "111111111111", + AccountName: "planned-account", + RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", + Enabled: true, + }}, + }) + + mu.Lock() + gotAttempts := attempts + gotApplyCount := applyCount + gotStored := append([]AssumeRoleInfo(nil), stored...) + gotKeys := append([]string(nil), idempotencyKeys...) + gotMatches := append([]string(nil), ifMatches...) + mu.Unlock() + + hasStableIdempotencyKey := len(gotKeys) >= 2 && gotKeys[0] != "" && gotKeys[0] == gotKeys[1] + hasStableRevision := len(gotMatches) >= 2 && gotMatches[0] != "" && gotMatches[0] == gotMatches[1] + if !hasStableIdempotencyKey && !hasStableRevision { + t.Errorf("PATCH retry headers idempotency=%q if-match=%q; want a stable idempotency key or revision precondition", gotKeys, gotMatches) + } + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "conflict") && + !strings.Contains(strings.ToLower(err.Error()), "precondition") && + !strings.Contains(strings.ToLower(err.Error()), "status 412") { + t.Errorf("PatchCloudAccount() error = %v; want success from idempotent replay or an explicit revision conflict", err) + } + if gotAttempts != 2 { + t.Errorf("PATCH attempts = %d; want 2 to exercise ambiguous committed-response retry", gotAttempts) + } + if gotApplyCount != 1 { + t.Errorf("server-side apply count = %d; want 1 after ambiguous retry", gotApplyCount) + } + hasConcurrentAccount := false + for _, info := range gotStored { + if info.AccountID == concurrentAccountID { + hasConcurrentAccount = true + break + } + } + if !hasConcurrentAccount { + t.Errorf("retry overwrote interleaved account %s; want concurrent edit preserved", concurrentAccountID) + } +} diff --git a/internal/api/client.go b/internal/api/client.go index 4da8d1f..46d86fb 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -38,6 +38,14 @@ type NQEResponse struct { Items []map[string]any `json:"items"` } +type QueryAWSAccountsResult struct { + Items []map[string]any + ObservedRowCount int + PageLimit int + CompletenessUnproven bool + CompletenessReason string +} + type QueryRequest struct { Query string `json:"query,omitempty"` QueryID string `json:"queryId,omitempty"` @@ -230,16 +238,32 @@ func (c *Client) QueryAWSAccounts( parameters map[string]any, setupIDs []string, ) ([]map[string]any, error) { + result, err := c.QueryAWSAccountsWithMetadata(ctx, networkID, snapshotID, query, queryID, parameters, setupIDs) + if err != nil { + return nil, err + } + return result.Items, nil +} + +func (c *Client) QueryAWSAccountsWithMetadata( + ctx context.Context, + networkID, snapshotID, query, queryID string, + parameters map[string]any, + setupIDs []string, +) (QueryAWSAccountsResult, error) { if strings.TrimSpace(networkID) == "" { - return nil, fmt.Errorf("network ID is required") + return QueryAWSAccountsResult{}, fmt.Errorf("network ID is required") } query = strings.TrimSpace(query) queryID = strings.TrimSpace(queryID) if query == "" && queryID == "" { - return nil, fmt.Errorf("query or query ID is required") + return QueryAWSAccountsResult{}, fmt.Errorf("query or query ID is required") } setupIDs = cleanSetupIDs(setupIDs) var allItems []map[string]any + var previousPageSignature string + var completenessUnproven bool + completenessReason := "NQE pagination returned a terminating short page" for offset := 0; ; offset += PageLimit { columnFilters := []ColumnFilter{{ ColumnName: "Cloud Type", @@ -267,14 +291,31 @@ func (c *Client) QueryAWSAccounts( endpointPath += fmt.Sprintf("&snapshotId=%s", url.QueryEscape(snapshotID)) } if err := c.doJSONRetryable(ctx, http.MethodPost, endpointPath, payload, &response); err != nil { - return nil, err + return QueryAWSAccountsResult{}, err + } + pageSignature := nqePageSignature(response.Items) + if len(response.Items) > 0 && pageSignature == previousPageSignature { + completenessUnproven = true + completenessReason = "NQE pagination returned a repeated page; the offset cursor did not advance the result window" + break } + previousPageSignature = pageSignature allItems = append(allItems, filterItemsBySetupID(response.Items, setupIDs)...) if len(response.Items) < PageLimit { break } } - return allItems, nil + if len(allItems) > 0 && len(allItems)%PageLimit == 0 && !completenessUnproven { + completenessUnproven = true + completenessReason = "NQE result count is an exact multiple of PageLimit, so truncation cannot be ruled out" + } + return QueryAWSAccountsResult{ + Items: allItems, + ObservedRowCount: len(allItems), + PageLimit: PageLimit, + CompletenessUnproven: completenessUnproven, + CompletenessReason: completenessReason, + }, nil } func (c *Client) Networks(ctx context.Context) ([]Network, error) { @@ -317,6 +358,17 @@ func filterItemsBySetupID(items []map[string]any, setupIDs []string) []map[strin return result } +func nqePageSignature(items []map[string]any) string { + if len(items) == 0 { + return "" + } + encoded, err := json.Marshal(items) + if err != nil { + return fmt.Sprintf("%#v", items) + } + return string(encoded) +} + func (c *Client) LatestProcessedSnapshot(ctx context.Context, networkID string) (*SnapshotInfo, error) { if strings.TrimSpace(networkID) == "" { return nil, fmt.Errorf("network ID is required") @@ -335,11 +387,59 @@ func (c *Client) ListSnapshots(ctx context.Context, networkID string) ([]Snapsho if strings.TrimSpace(networkID) == "" { return nil, fmt.Errorf("network ID is required") } - var snapshots NetworkSnapshots - if err := c.doJSON(ctx, http.MethodGet, fmt.Sprintf("/networks/%s/snapshots?includeArchived=true", networkID), nil, &snapshots); err != nil { - return nil, err + var allSnapshots []SnapshotInfo + seenSnapshotIDs := make(map[string]int) + var previousPage []SnapshotInfo + for offset := 0; ; offset += PageLimit { + var page NetworkSnapshots + endpointPath := fmt.Sprintf( + "/networks/%s/snapshots?includeArchived=true&offset=%d&limit=%d", + networkID, + offset, + PageLimit, + ) + if err := c.doJSON(ctx, http.MethodGet, endpointPath, nil, &page); err != nil { + return nil, err + } + if len(page.Snapshots) > PageLimit { + return nil, fmt.Errorf("list snapshots returned %d entries at offset %d, exceeding requested limit %d", len(page.Snapshots), offset, PageLimit) + } + if offset > 0 && sameSnapshotPage(previousPage, page.Snapshots) { + return nil, fmt.Errorf("list snapshots pagination repeated the page at offset %d", offset) + } + for _, snapshot := range page.Snapshots { + snapshotID := strings.TrimSpace(snapshot.ID) + if snapshotID == "" { + continue + } + if firstOffset, ok := seenSnapshotIDs[snapshotID]; ok { + return nil, fmt.Errorf( + "list snapshots pagination repeated snapshot %s at offset %d (first seen at offset %d)", + snapshotID, + offset, + firstOffset, + ) + } + seenSnapshotIDs[snapshotID] = offset + } + allSnapshots = append(allSnapshots, page.Snapshots...) + if len(page.Snapshots) < PageLimit { + return allSnapshots, nil + } + previousPage = append(previousPage[:0], page.Snapshots...) + } +} + +func sameSnapshotPage(left, right []SnapshotInfo) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } } - return snapshots.Snapshots, nil + return true } func (c *Client) CloudAccounts(ctx context.Context, networkID string) ([]CloudAccount, error) { if strings.TrimSpace(networkID) == "" { diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 9e1790b..dae6d39 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -6,6 +6,7 @@ import ( "errors" "net/http" "net/http/httptest" + "strings" "testing" "time" ) @@ -62,6 +63,73 @@ func TestQueryAWSAccountsPagesResults(t *testing.T) { } } +func TestQueryAWSAccountsMarksExactPageLimitMultipleUnproven(t *testing.T) { + var seenOffsets []int + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req QueryRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request: %v", err) + } + seenOffsets = append(seenOffsets, req.QueryOptions.Offset) + w.Header().Set("Content-Type", "application/json") + if req.QueryOptions.Offset == 0 { + items := make([]map[string]any, PageLimit) + for i := range items { + items[i] = map[string]any{"Cloud Account ID": "111111111111"} + } + _ = json.NewEncoder(w).Encode(NQEResponse{Items: items}) + return + } + _ = json.NewEncoder(w).Encode(NQEResponse{}) + })) + defer server.Close() + + client, err := NewClient(server.URL, "/api", "alice", "secret", true, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + result, err := client.QueryAWSAccountsWithMetadata(context.Background(), "network-1", "", "", "query-1", nil, nil) + if err != nil { + t.Fatalf("QueryAWSAccountsWithMetadata() error = %v", err) + } + if !result.CompletenessUnproven || !strings.Contains(result.CompletenessReason, "exact multiple of PageLimit") { + t.Fatalf("expected exact-multiple completeness warning, got %#v", result) + } + if result.ObservedRowCount != PageLimit || result.PageLimit != PageLimit { + t.Fatalf("unexpected counts: %#v", result) + } + if len(seenOffsets) != 2 || seenOffsets[0] != 0 || seenOffsets[1] != PageLimit { + t.Fatalf("unexpected offsets: %#v", seenOffsets) + } +} + +func TestQueryAWSAccountsMarksRepeatedPageUnproven(t *testing.T) { + items := make([]map[string]any, PageLimit) + for i := range items { + items[i] = map[string]any{"Cloud Account ID": "111111111111"} + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(NQEResponse{Items: items}) + })) + defer server.Close() + + client, err := NewClient(server.URL, "/api", "alice", "secret", true, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + result, err := client.QueryAWSAccountsWithMetadata(context.Background(), "network-1", "", "", "query-1", nil, nil) + if err != nil { + t.Fatalf("QueryAWSAccountsWithMetadata() error = %v", err) + } + if !result.CompletenessUnproven || !strings.Contains(result.CompletenessReason, "repeated page") { + t.Fatalf("expected repeated-page completeness warning, got %#v", result) + } + if result.ObservedRowCount != PageLimit { + t.Fatalf("expected only first page to be counted, got %#v", result) + } +} + func TestQueryAWSAccountsAddsSnapshotIDQueryParam(t *testing.T) { var rawQuery string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/app/account_manifest.go b/internal/app/account_manifest.go index e2e8cf8..0c33f75 100644 --- a/internal/app/account_manifest.go +++ b/internal/app/account_manifest.go @@ -5,14 +5,12 @@ import ( "encoding/json" "fmt" "os" - "regexp" "strings" + "time" "github.com/forwardnetworks/aws-sync/internal/api" ) -var awsAccountIDPattern = regexp.MustCompile(`^[0-9]{12}$`) - type AWSAccountManifestEntry struct { ID string `json:"id"` Name string `json:"name,omitempty"` @@ -43,7 +41,7 @@ func LoadAWSAccountManifest(path string) ([]AWSOrganizationAccount, error) { accounts := make([]AWSOrganizationAccount, 0, len(entries)) for index, entry := range entries { accountID := strings.TrimSpace(entry.ID) - if !awsAccountIDPattern.MatchString(accountID) { + if _, err := NewAccountID(accountID); err != nil { return nil, fmt.Errorf("accounts file entry %d has invalid AWS account ID %q; expected 12 digits", index+1, entry.ID) } if seen[accountID] { @@ -69,10 +67,19 @@ func RunAWSAccountManifest(ctx context.Context, cfg AWSOrganizationConfig, accou } func SyncAWSAccountManifest(ctx context.Context, cfg Config, accounts []AWSOrganizationAccount) (*Summary, error) { + cfg.AuthoritativeInput = true + if cfg.Policy.Kind == "" { + cfg.Policy = NewAuthoritativeManifestReconcilePolicy(time.Now().UTC()) + } else { + cfg.Policy.Kind = CompleteInventory + cfg.Policy.OrganizationEvidence = ReviewedAuthoritativeInventory + cfg = prepareReconcileConfig(cfg, time.Now().UTC()) + } setupIDs := cleanSetupIDs(cfg.SetupIDs) if len(setupIDs) != 1 { return nil, fmt.Errorf("account-manifest sync requires exactly one --setup-id") } + setupID := setupIDs[0] client, err := api.NewClient(cfg.Host, cfg.APIPrefix, cfg.Username, cfg.Password, cfg.Insecure, cfg.Timeout) if err != nil { return nil, err @@ -83,20 +90,26 @@ func SyncAWSAccountManifest(ctx context.Context, cfg Config, accounts []AWSOrgan } cfg.NetworkID = networkID cfg.Source = "account_manifest" - cfg.AuthoritativeInput = true cloudAccounts, err := client.CloudAccounts(ctx, networkID) if err != nil { return nil, err } - items := make([]map[string]any, 0, len(accounts)) - for _, account := range accounts { - items = append(items, map[string]any{ - "Cloud Setup ID": setupIDs[0], - "Cloud Account ID": account.ID, - "Cloud Account Name": account.Name, - "Collected?": false, - }) + discovered, err := adaptManifestAccountsToSetupRows(accounts, setupID) + if err != nil { + return nil, err + } + snapshot := &InventorySnapshot{ + Source: "account_manifest", + Completeness: InventoryCompletenessComplete, + NetworkID: networkID, + ObservedRowCount: len(discovered), + DiscoveredAccounts: discovered, + } + if len(discovered) == 0 { + snapshot.SelectedSetupIDs = []SetupID{} + } else { + snapshot.SelectedSetupIDs = []SetupID{SetupID(setupID)} } - return runPlannedSync(ctx, cfg, client, items, cloudAccounts) + return runPlannedSyncFromSnapshot(ctx, cfg, client, snapshot, cloudAccounts) } diff --git a/internal/app/account_manifest_test.go b/internal/app/account_manifest_test.go index af82e21..0d50ebc 100644 --- a/internal/app/account_manifest_test.go +++ b/internal/app/account_manifest_test.go @@ -6,9 +6,11 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "strings" "testing" + "time" "github.com/forwardnetworks/aws-sync/internal/api" ) @@ -106,6 +108,7 @@ func TestRunAWSAccountManifestRejectsPartitionRegionMismatch(t *testing.T) { func TestSyncAWSAccountManifestDryRunReportsRemovalAndApplyRequiresApproval(t *testing.T) { patchCount := 0 + var patchedPayload api.PatchPayload server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": @@ -114,6 +117,11 @@ func TestSyncAWSAccountManifestDryRunReportsRemovalAndApplyRequiresApproval(t *t {"accountId":"222222222222","accountName":"remove","roleArn":"arn:aws-us-gov:iam::222222222222:role/ForwardRole","enabled":true} ]}]`)) case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/gov-prod": + if err := json.NewDecoder(r.Body).Decode(&patchedPayload); err != nil { + t.Errorf("decode PATCH payload: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } patchCount++ w.WriteHeader(http.StatusNoContent) default: @@ -168,4 +176,96 @@ func TestSyncAWSAccountManifestDryRunReportsRemovalAndApplyRequiresApproval(t *t if patchCount != 1 { t.Fatalf("approved apply patch count = %d, want 1", patchCount) } + if len(patchedPayload.AssumeRoleInfos) != 1 || patchedPayload.AssumeRoleInfos[0].AccountID != "111111111111" { + t.Fatalf("approved manifest removal PATCH = %#v; want only reviewed account 111111111111", patchedPayload.AssumeRoleInfos) + } +} + +func TestApprovalDigestStableAcrossIndependentPlanningProcesses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/networks/network-1/cloudAccounts" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`[{ + "type":"AWS", + "name":"setup-a", + "regions":{"us-east-1":{"testInstant":0}}, + "assumeRoleInfos":[{ + "accountId":"111111111111", + "accountName":"keep", + "roleArn":"arn:aws:iam::111111111111:role/ForwardRole", + "enabled":true + }] + }]`)) + })) + defer server.Close() + + runPlanningProcess := func(name string, instant time.Time) Summary { + t.Helper() + dir := t.TempDir() + resultPath := filepath.Join(dir, "summary.json") + cmd := exec.Command(os.Args[0], "-test.run=^TestApprovalDigestPlanningProcess$") + cmd.Env = append(os.Environ(), + "AWSSYNC_PLAN_DIGEST_CHILD=1", + "AWSSYNC_PLAN_DIGEST_HOST="+server.URL, + "AWSSYNC_PLAN_DIGEST_INSTANT="+instant.Format(time.RFC3339Nano), + "AWSSYNC_PLAN_DIGEST_OUTPUT="+filepath.Join(dir, name+"-payload.json"), + "AWSSYNC_PLAN_DIGEST_RESULT="+resultPath, + ) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("planning process %s failed: %v\n%s", name, err, output) + } + data, err := os.ReadFile(resultPath) + if err != nil { + t.Fatalf("read planning result %s: %v", name, err) + } + var summary Summary + if err := json.Unmarshal(data, &summary); err != nil { + t.Fatalf("decode planning result %s: %v", name, err) + } + return summary + } + + first := runPlanningProcess("first", time.Unix(100, 0).UTC()) + second := runPlanningProcess("second", time.Unix(200, 0).UTC()) + if first.PlanDigest == "" || first.PlanDigest != second.PlanDigest { + t.Fatalf("approval digests differ across independent planning processes: %q != %q", first.PlanDigest, second.PlanDigest) + } + if first.PayloadSHA256 == second.PayloadSHA256 { + t.Fatalf("test setup did not produce distinct exact payloads: both hashes are %q", first.PayloadSHA256) + } +} + +func TestApprovalDigestPlanningProcess(t *testing.T) { + if os.Getenv("AWSSYNC_PLAN_DIGEST_CHILD") != "1" { + return + } + planningInstant, err := time.Parse(time.RFC3339Nano, os.Getenv("AWSSYNC_PLAN_DIGEST_INSTANT")) + if err != nil { + t.Fatalf("parse planning instant: %v", err) + } + summary, err := SyncAWSAccountManifest(context.Background(), Config{ + Host: os.Getenv("AWSSYNC_PLAN_DIGEST_HOST"), + Username: "user", + Password: "pass", + NetworkID: "network-1", + SetupIDs: []string{"setup-a"}, + APIPrefix: "/api", + Output: os.Getenv("AWSSYNC_PLAN_DIGEST_OUTPUT"), + Policy: NewAuthoritativeManifestReconcilePolicy(planningInstant), + }, []AWSOrganizationAccount{ + {ID: "111111111111", Name: "keep"}, + {ID: "222222222222", Name: "add"}, + }) + if err != nil { + t.Fatalf("plan account manifest: %v", err) + } + data, err := json.Marshal(summary) + if err != nil { + t.Fatalf("encode planning summary: %v", err) + } + if err := os.WriteFile(os.Getenv("AWSSYNC_PLAN_DIGEST_RESULT"), data, 0o600); err != nil { + t.Fatalf("write planning summary: %v", err) + } } diff --git a/internal/app/adapters.go b/internal/app/adapters.go new file mode 100644 index 0000000..4d4e04c --- /dev/null +++ b/internal/app/adapters.go @@ -0,0 +1,490 @@ +package app + +import ( + "fmt" + "sort" + "strings" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +var nqeSetupIDColumns = []string{"Cloud Setup ID", "Setup ID", "Cloud Account Setup ID", "Cloud Account Setup"} + +type externalIDBySetupAssignments map[SetupID]map[AccountID]string + +type parseNQESnapshotOptions struct { + AllowMalformedRows bool + Completeness InventoryCompleteness + CompletenessReason string + PageLimit int +} + +func adaptExternalIDAssignments(assignments externalIDAssignments) (externalIDBySetupAssignments, error) { + if len(assignments) == 0 { + return nil, nil + } + + converted := make(map[SetupID]map[AccountID]string, len(assignments)) + for setupIDRaw, byAccount := range assignments { + setupID, err := NewSetupID(setupIDRaw) + if err != nil { + return nil, fmt.Errorf("external ID assignment has invalid setup ID %q: %w", setupIDRaw, err) + } + if _, exists := converted[setupID]; exists { + return nil, fmt.Errorf("external ID file contains duplicate setup %q", setupID) + } + converted[setupID] = make(map[AccountID]string, len(byAccount)) + for rawAccountID, externalID := range byAccount { + accountID, err := NewAccountID(rawAccountID) + if err != nil { + return nil, fmt.Errorf("external ID assignment for setup %q has invalid AWS account ID %q: %w", setupID, rawAccountID, err) + } + if _, exists := converted[setupID][accountID]; exists { + return nil, fmt.Errorf("external ID file contains duplicate setup/account entry %s/%s", setupID, accountID) + } + converted[setupID][accountID] = strings.TrimSpace(externalID) + } + } + return converted, nil +} + +// parseNQESnapshotFromMaps adapts rows with no pagination metadata available. +// NQE cannot prove its result is complete, so completeness defaults to unknown +// and absence-based removal is refused downstream. Callers holding pagination +// metadata must use parseNQESnapshotFromMapsWithOptions instead. +func parseNQESnapshotFromMaps(items []map[string]any) (*InventorySnapshot, error) { + return parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + Completeness: InventoryCompletenessUnknown, + }) +} + +func parseNQESnapshotFromMapsWithOptions(items []map[string]any, options parseNQESnapshotOptions) (*InventorySnapshot, error) { + snapshot := &InventorySnapshot{ + Source: "nqe", + ObservedRowCount: len(items), + PageLimit: options.PageLimit, + Completeness: options.Completeness, + CompletenessReason: strings.TrimSpace(options.CompletenessReason), + } + seenBySetup := make(map[SetupID]map[AccountID]bool) + accountOwners := make(map[AccountID]SetupID) + selectedSetups := make(map[SetupID]struct{}) + for rowIndex, item := range items { + r := rowIndex + 1 + setupID, err := extractNQESetupID(item, r) + if err != nil { + return nil, err + } + accountID, err := extractNQEAccountID(item, r) + if err != nil { + if options.AllowMalformedRows && isMalformedNQEAccountIDError(err) { + snapshot.SkippedRows = append(snapshot.SkippedRows, MalformedNQERowSummary{ + Row: r, + SetupID: setupID.String(), + AccountID: rawNQEAccountID(item), + Reason: err.Error(), + }) + snapshot.IgnoredAccounts = append(snapshot.IgnoredAccounts, AccountSummary{AccountID: rawNQEAccountID(item)}) + snapshot.Completeness = InventoryCompletenessLikelyIncomplete + snapshot.CompletenessReason = "--allow-malformed-rows skipped malformed NQE rows, so the inventory is incomplete" + continue + } + return nil, err + } + accountName, err := extractOptionalString(item, "Cloud Account Name", r) + if err != nil { + return nil, err + } + if accountName == "" { + accountName = accountID.String() + } + if _, exists := accountOwners[accountID]; exists && accountOwners[accountID] != setupID { + return nil, fmt.Errorf("NQE row %d has account %s in setup %s but that account already appears in setup %s", r, accountID, setupID, accountOwners[accountID]) + } + if _, ok := seenBySetup[setupID]; !ok { + seenBySetup[setupID] = make(map[AccountID]bool) + } + if seenBySetup[setupID][accountID] { + return nil, fmt.Errorf("NQE row %d duplicates account %s in setup %s", r, accountID, setupID) + } + seenBySetup[setupID][accountID] = true + accountOwners[accountID] = setupID + if !setupID.IsZero() { + selectedSetups[setupID] = struct{}{} + } + + collectedSet := false + collected := false + if raw, ok := item["Collected?"]; ok { + collectedSet = true + parsed, err := parseCollectedFlag(raw) + if err != nil { + return nil, fmt.Errorf("NQE row %d has invalid Collected? value: %w", r, err) + } + collected = parsed + } + hasOrgIDs, err := parseHasOrgUnitIDs(item["Organizational Unit IDs"]) + if err != nil { + return nil, fmt.Errorf("NQE row %d has invalid Organizational Unit IDs: %w", r, err) + } + lifecycle, err := parseLifecycle(item["Account Lifecycle"], r) + if err != nil { + return nil, err + } + snapshot.DiscoveredAccounts = append(snapshot.DiscoveredAccounts, DiscoveredAccount{ + SetupID: setupID, + AccountID: accountID, + AccountName: accountName, + Lifecycle: lifecycle, + CollectedSet: collectedSet, + Collected: collected, + HasOrganizationalID: hasOrgIDs, + Membership: MembershipPreserve, + }) + } + if len(selectedSetups) > 0 { + snapshot.SelectedSetupIDs = make([]SetupID, 0, len(selectedSetups)) + for setupID := range selectedSetups { + snapshot.SelectedSetupIDs = append(snapshot.SelectedSetupIDs, setupID) + } + sort.Slice(snapshot.SelectedSetupIDs, func(i, j int) bool { + return snapshot.SelectedSetupIDs[i] < snapshot.SelectedSetupIDs[j] + }) + } + return snapshot, nil +} + +func isMalformedNQEAccountIDError(err error) bool { + if err == nil { + return false + } + message := err.Error() + return strings.Contains(message, "Cloud Account ID") || + strings.Contains(message, "invalid AWS account ID") +} + +func rawNQEAccountID(item map[string]any) string { + raw, ok := item["Cloud Account ID"] + if !ok || raw == nil { + return "" + } + if value, ok := raw.(string); ok { + return strings.TrimSpace(value) + } + return fmt.Sprintf("%v", raw) +} + +func extractNQESetupID(item map[string]any, row int) (SetupID, error) { + for _, key := range nqeSetupIDColumns { + raw, ok := item[key] + if !ok { + continue + } + value, err := extractString(raw) + if err != nil { + return "", fmt.Errorf("NQE row %d has non-string setup-id value in %s", row, key) + } + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "", nil + } + setupID, err := NewSetupID(trimmed) + if err != nil { + return "", fmt.Errorf("NQE row %d has invalid setup ID %q: %w", row, value, err) + } + return setupID, nil + } + return "", nil +} + +func extractNQEAccountID(item map[string]any, row int) (AccountID, error) { + raw, ok := item["Cloud Account ID"] + if !ok { + return "", fmt.Errorf("NQE row %d is missing Cloud Account ID", row) + } + text, err := extractString(raw) + if err != nil { + return "", fmt.Errorf("NQE row %d has non-string Cloud Account ID", row) + } + accountID, err := NewAccountID(text) + if err != nil { + return "", err + } + return accountID, nil +} + +func extractOptionalString(item map[string]any, column string, row int) (string, error) { + raw, ok := item[column] + if !ok { + return "", nil + } + value, err := extractString(raw) + if err != nil { + return "", fmt.Errorf("NQE row %d has non-string value for %s", row, column) + } + return strings.TrimSpace(value), nil +} + +func extractString(value any) (string, error) { + s, ok := value.(string) + if !ok { + return "", fmt.Errorf("value is not string") + } + return strings.TrimSpace(s), nil +} + +func parseCollectedFlag(value any) (bool, error) { + switch typed := value.(type) { + case bool: + return typed, nil + case string: + switch strings.ToLower(strings.TrimSpace(typed)) { + case "true", "yes", "1": + return true, nil + case "false", "no", "0": + return false, nil + } + } + return false, fmt.Errorf("expected boolean or true/false/yes/no") +} + +func parseHasOrgUnitIDs(value any) (bool, error) { + switch typed := value.(type) { + case nil: + return false, nil + case []any: + return len(typed) > 0, nil + case []string: + return len(typed) > 0, nil + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == "[]" { + return false, nil + } + return true, nil + default: + return false, fmt.Errorf("expected Organizational Unit IDs to be an array or string") + } +} + +func parseLifecycle(raw any, row int) (AccountLifecycle, error) { + if raw == nil { + return AccountLifecycleUnknown, nil + } + value, err := extractString(raw) + if err != nil { + return AccountLifecycleUnknown, fmt.Errorf("NQE row %d has non-string Account Lifecycle", row) + } + value = strings.TrimSpace(value) + if value == "" { + return AccountLifecycleUnknown, nil + } + switch AccountLifecycle(value) { + case AccountLifecycleActive, AccountLifecycleSuspended, AccountLifecycleClosing, AccountLifecycleClosed: + return AccountLifecycle(value), nil + default: + return AccountLifecycleUnknown, fmt.Errorf("NQE row %d has invalid lifecycle %q", row, value) + } +} + +func adaptManifestAccountsToSetupRows(accounts []AWSOrganizationAccount, setupID string) ([]DiscoveredAccount, error) { + canonicalSetup, err := NewSetupID(setupID) + if err != nil { + return nil, err + } + result := make([]DiscoveredAccount, 0, len(accounts)) + seen := make(map[AccountID]bool, len(accounts)) + for _, account := range accounts { + accountID, err := NewAccountID(account.ID) + if err != nil { + return nil, fmt.Errorf("accounts file entry for setup %s has invalid AWS account ID %q; expected exactly 12 digits", canonicalSetup, account.ID) + } + if seen[accountID] { + return nil, fmt.Errorf("accounts file contains duplicate AWS account ID %s", accountID) + } + seen[accountID] = true + name := strings.TrimSpace(account.Name) + if name == "" { + name = accountID.String() + } + result = append(result, DiscoveredAccount{SetupID: canonicalSetup, AccountID: accountID, AccountName: name}) + } + return result, nil +} + +func parseCloudSetupAccountInfo(info api.AssumeRoleInfo, setupID SetupID, row int) (AccountID, RoleARN, error) { + var accountID AccountID + var roleARN RoleARN + rawRole := strings.TrimSpace(info.RoleArn) + hasRoleARN := rawRole != "" + if rawRole != "" { + parsedRole, err := ParseRoleARN(rawRole) + if err != nil { + return "", RoleARN{}, fmt.Errorf("setup %s row %d has invalid role ARN %q: %w", setupID, row, rawRole, err) + } + roleARN = parsedRole + } + if strings.TrimSpace(info.AccountID) != "" { + id, err := NewAccountID(info.AccountID) + if err != nil { + return "", RoleARN{}, fmt.Errorf("setup %s row %d has invalid AWS account ID %q; expected exactly 12 digits", setupID, row, info.AccountID) + } + accountID = id + } + if accountID.IsZero() { + if !hasRoleARN { + return "", RoleARN{}, fmt.Errorf("setup %s row %d has no account identity", setupID, row) + } + accountID = roleARN.AccountID() + } + if hasRoleARN && strings.TrimSpace(info.AccountID) != "" { + if roleAccount := roleARN.AccountID().String(); roleAccount != accountID.String() { + return "", RoleARN{}, fmt.Errorf("setup %s row %d has account ID %s that disagrees with role ARN account %s", setupID, row, accountID, roleAccount) + } + } + return accountID, roleARN, nil +} + +type cloudSetupMetadata struct { + setupID SetupID + cloudType string + proxyServerID string + regionToProxyServer map[string]string + regions map[string]api.RegionMeta + assumeRoleInfos []api.AssumeRoleInfo +} + +func adaptCloudAccountsBySetupID(cloudAccounts []api.CloudAccount, setupIDs []string) (map[SetupID]cloudSetupMetadata, error) { + allowed := setupIDSet(setupIDs) + result := make(map[SetupID]cloudSetupMetadata, len(cloudAccounts)) + seenAccount := make(map[SetupID]map[AccountID]bool) + accountOwners := make(map[AccountID]SetupID) + for _, account := range cloudAccounts { + accountType := strings.ToUpper(strings.TrimSpace(account.Type)) + if accountType != "" && accountType != "AWS" { + continue + } + setupID, err := NewSetupID(account.Name) + if err != nil { + continue + } + if len(allowed) > 0 && !allowed[setupID.String()] { + continue + } + if _, ok := result[setupID]; ok { + return nil, fmt.Errorf("forward setup list contains duplicate setup-id %s", setupID) + } + if account.ProxyServerID != "" { + account.ProxyServerID = strings.TrimSpace(account.ProxyServerID) + } + normalizedRegions := map[string]string{} + for region, proxy := range account.RegionToProxyServerID { + region = strings.TrimSpace(region) + proxy = strings.TrimSpace(proxy) + if region != "" && proxy != "" { + normalizedRegions[region] = proxy + } + } + state := cloudSetupMetadata{ + setupID: setupID, + cloudType: strings.TrimSpace(account.Type), + proxyServerID: strings.TrimSpace(account.ProxyServerID), + regionToProxyServer: normalizedRegions, + regions: account.Regions, + } + seen := make(map[AccountID]bool, len(account.AssumeRoleInfos)) + for i, info := range account.AssumeRoleInfos { + accountID, _, err := parseCloudSetupAccountInfo(info, setupID, i+1) + if err != nil { + return nil, err + } + if owner, exists := accountOwners[accountID]; exists && owner != setupID { + return nil, fmt.Errorf("forward setup %s row %d has account %s also configured in setup %s", setupID, i+1, accountID, owner) + } + if seen[accountID] { + return nil, fmt.Errorf("setup %s has duplicate account %s", setupID, accountID) + } + seen[accountID] = true + accountOwners[accountID] = setupID + } + seenAccount[setupID] = seen + state.assumeRoleInfos = append(state.assumeRoleInfos, account.AssumeRoleInfos...) + result[setupID] = state + } + return result, nil +} + +func coalesce(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func SetupIDsFromSnapshot(snapshot *InventorySnapshot) []string { + result := make([]string, 0, len(snapshot.SelectedSetupIDs)) + for _, setupID := range snapshot.SelectedSetupIDs { + result = append(result, setupID.String()) + } + sort.Strings(result) + return result +} + +func adaptCurrentSetup(meta cloudSetupMetadata) (CurrentSetup, error) { + current := CurrentSetup{ + SetupID: meta.setupID, + Metadata: SetupMetadata{ + CloudType: strings.TrimSpace(meta.cloudType), + ProxyServerID: strings.TrimSpace(meta.proxyServerID), + RegionToProxyServer: stringMap(meta.regionToProxyServer), + Regions: make(map[string]int64, len(meta.regions)), + }, + Accounts: make([]SetupAccount, 0, len(meta.assumeRoleInfos)), + } + for region, regionMeta := range meta.regions { + current.Metadata.Regions[region] = regionMeta.TestInstant + } + for row, info := range meta.assumeRoleInfos { + accountID, roleARN, err := parseCloudSetupAccountInfo(info, meta.setupID, row+1) + if err != nil { + return CurrentSetup{}, err + } + accountName := strings.TrimSpace(info.AccountName) + if accountName == "" { + accountName = accountID.String() + } + current.Accounts = append(current.Accounts, SetupAccount{ + AccountID: accountID, + AccountName: accountName, + RoleARN: roleARN, + ExternalID: strings.TrimSpace(info.ExternalID), + Enabled: info.Enabled, + }) + } + return current, nil +} + +func patchPayloadFromDesired(desired DesiredSetup) api.PatchPayload { + payload := api.PatchPayload{ + Type: desired.Metadata.CloudType, + Name: desired.SetupID.String(), + Regions: cloneInt64Map(desired.Metadata.Regions), + RegionToProxyServerID: cloneStringMap(desired.Metadata.RegionToProxyServer), + AssumeRoleInfos: make([]api.AssumeRoleInfo, 0, len(desired.Accounts)), + } + if desired.Metadata.ProxyServerID != "" { + payload.ProxyServerID = desired.Metadata.ProxyServerID + } + for _, account := range desired.Accounts { + payload.AssumeRoleInfos = append(payload.AssumeRoleInfos, api.AssumeRoleInfo{ + AccountID: account.AccountID.String(), + AccountName: account.AccountName, + RoleArn: account.RoleARN.String(), + ExternalID: account.ExternalID, + Enabled: account.Enabled, + }) + } + return payload +} diff --git a/internal/app/adapters_test.go b/internal/app/adapters_test.go new file mode 100644 index 0000000..feb63fc --- /dev/null +++ b/internal/app/adapters_test.go @@ -0,0 +1,134 @@ +package app + +import ( + "strings" + "testing" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +func TestParseNQESnapshotFromMapsTrimsWhitespace(t *testing.T) { + items := []map[string]any{ + { + "Cloud Setup ID": " setup-a ", + "Cloud Account ID": " 111111111111 ", + "Cloud Account Name": " acct-a ", + "Collected?": " true ", + "Account Lifecycle": " Active ", + "Organizational Unit IDs": []string{"ou-root"}, + }, + } + snapshot, err := parseNQESnapshotFromMaps(items) + if err != nil { + t.Fatalf("parseNQESnapshotFromMaps() error = %v", err) + } + if got := snapshot.DiscoveredAccounts[0].AccountID.String(); got != "111111111111" { + t.Fatalf("account id = %q", got) + } + if got := snapshot.DiscoveredAccounts[0].AccountName; got != "acct-a" { + t.Fatalf("account name = %q", got) + } + if got := snapshot.DiscoveredAccounts[0].Lifecycle; got != AccountLifecycleActive { + t.Fatalf("lifecycle = %q", got) + } + if !snapshot.DiscoveredAccounts[0].Collected { + t.Fatalf("expected collected flag true") + } +} + +func TestParseNQESnapshotFromMapsRejectsNumericAccountID(t *testing.T) { + items := []map[string]any{{ + "Cloud Setup ID": "setup-a", + "Cloud Account ID": 111111111111, + "Cloud Account Name": "acct-a", + }} + if _, err := parseNQESnapshotFromMaps(items); err == nil || !strings.Contains(err.Error(), "non-string Cloud Account ID") { + t.Fatalf("expected numeric-ID type error, got %v", err) + } +} + +func TestParseNQESnapshotAllowsMalformedRowsAndMarksIncomplete(t *testing.T) { + items := []map[string]any{ + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "not-an-account", "Cloud Account Name": "bad"}, + } + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + AllowMalformedRows: true, + Completeness: InventoryCompletenessComplete, + PageLimit: 1000, + }) + if err != nil { + t.Fatalf("parseNQESnapshotFromMapsWithOptions() error = %v", err) + } + if len(snapshot.DiscoveredAccounts) != 1 { + t.Fatalf("expected one valid account, got %#v", snapshot.DiscoveredAccounts) + } + if snapshot.Completeness != InventoryCompletenessLikelyIncomplete { + t.Fatalf("skipped malformed row must make inventory incomplete, got %v", snapshot.Completeness) + } + if len(snapshot.SkippedRows) != 1 || snapshot.SkippedRows[0].Row != 2 || snapshot.SkippedRows[0].AccountID != "not-an-account" { + t.Fatalf("expected skipped row details, got %#v", snapshot.SkippedRows) + } +} + +func TestParseNQESnapshotMalformedRowsReplaceStaleCompletenessReason(t *testing.T) { + snapshot, err := parseNQESnapshotFromMapsWithOptions([]map[string]any{{ + "Cloud Setup ID": "setup-a", + "Cloud Account ID": "not-an-account", + }}, parseNQESnapshotOptions{ + AllowMalformedRows: true, + Completeness: InventoryCompletenessLikelyIncomplete, + CompletenessReason: "NQE pagination ended with a terminating short page", + PageLimit: 1000, + }) + if err != nil { + t.Fatalf("parseNQESnapshotFromMapsWithOptions() error = %v", err) + } + const want = "--allow-malformed-rows skipped malformed NQE rows, so the inventory is incomplete" + if snapshot.CompletenessReason != want { + t.Fatalf("completeness reason = %q, want %q", snapshot.CompletenessReason, want) + } +} + +func TestParseNQESnapshotFromMapsRejectsDuplicateAccountAcrossRows(t *testing.T) { + items := []map[string]any{ + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Cloud Setup ID": "setup-b", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-b"}, + } + if _, err := parseNQESnapshotFromMaps(items); err == nil || !strings.Contains(err.Error(), "already appears in setup setup-a") { + t.Fatalf("expected cross-setup duplicate error, got %v", err) + } +} + +func TestParseCloudSetupAccountInfoRejectsRoleARNAccountMismatch(t *testing.T) { + _, _, err := parseCloudSetupAccountInfo(api.AssumeRoleInfo{ + AccountID: "111111111111", + RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", + }, SetupID("setup-a"), 3) + if err == nil || !strings.Contains(err.Error(), "disagrees with role ARN account") { + t.Fatalf("expected account/ARN mismatch error, got %v", err) + } +} + +func TestAdaptCloudAccountsBySetupIDRejectsDuplicateSetupID(t *testing.T) { + _, err := adaptCloudAccountsBySetupID([]api.CloudAccount{ + {Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{AccountID: "111111111111", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole"}}}, + {Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{AccountID: "222222222222", RoleArn: "arn:aws:iam::222222222222:role/ForwardRole"}}}, + }, nil) + if err == nil || !strings.Contains(err.Error(), "forward setup list contains duplicate setup-id setup-a") { + t.Fatalf("expected duplicate setup error, got %v", err) + } +} + +func TestAdaptCloudAccountsBySetupIDAcceptsWhitespaceAccountIDs(t *testing.T) { + accounts, err := adaptCloudAccountsBySetupID([]api.CloudAccount{{ + Name: " setup-a ", + AssumeRoleInfos: []api.AssumeRoleInfo{{AccountID: " 111111111111 ", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole"}}, + }}, nil) + if err != nil { + t.Fatalf("adaptCloudAccountsBySetupID() error = %v", err) + } + if _, ok := accounts[SetupID("setup-a")]; !ok { + t.Fatalf("expected normalized setup-id key") + } +} diff --git a/internal/app/apply_gateway.go b/internal/app/apply_gateway.go new file mode 100644 index 0000000..0bfd6b1 --- /dev/null +++ b/internal/app/apply_gateway.go @@ -0,0 +1,680 @@ +package app + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "time" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +// ApplyStatus is the durable disposition of one setup in an apply journal. +type ApplyStatus string + +const ( + ApplyStatusPlanned ApplyStatus = "planned" + ApplyStatusPending ApplyStatus = "pending" + ApplyStatusApplied ApplyStatus = "applied" + ApplyStatusConflicted ApplyStatus = "conflicted" + ApplyStatusFailed ApplyStatus = "failed" +) + +// ApplyJournalEntry records the recoverable state of one planned setup. +type ApplyJournalEntry struct { + SetupID string `json:"setup_id"` + Status ApplyStatus `json:"status"` + History []ApplyStatus `json:"history"` + HasChanges bool `json:"has_changes"` + Error string `json:"error,omitempty"` +} + +// ApplyJournal is atomically rewritten after every setup disposition change. +type ApplyJournal struct { + PlanDigest string `json:"plan_digest"` + NetworkID string `json:"network_id"` + Authorization ApplyAuthorizationRecord `json:"authorization"` + UpdatedAt time.Time `json:"updated_at"` + Setups []ApplyJournalEntry `json:"setups"` +} + +// ApplyAuthorizationRecord is the audit-safe authorization persisted with the +// digest it approved. It contains no credentials. +type ApplyAuthorizationRecord struct { + PlanDigest string `json:"plan_digest"` + Actor string `json:"actor"` + Approved bool `json:"approved"` + AllowDestructive bool `json:"allow_destructive"` + MaxRemovals int `json:"max_removals"` + MaxRemovalPercent float64 `json:"max_removal_percent"` + AllowNoCandidates bool `json:"allow_no_candidates"` + Unattended bool `json:"unattended"` + AllowUnattendedDestructive bool `json:"allow_unattended_destructive"` +} + +// ApplyAuthorization is the single authorization record accepted by the +// account-list mutation gateway. +type ApplyAuthorization struct { + PlanDigest string + Actor string + Approved bool + AllowDestructive bool + MaxRemovals int + MaxRemovalPercent float64 + AllowNoCandidates bool + Unattended bool + AllowUnattendedDestructive bool +} + +// ApplyResult is returned even when an apply is partial or blocked. +type ApplyResult struct { + PatchedCount int + Blocked bool + RollbackOutput string + RollbackSHA256 string + JournalOutput string + Journal ApplyJournal +} + +// ApplyIntent is immutable after construction. Its state is private and every +// mutable input is cloned by newApplyIntent. +type ApplyIntent struct { + state *applyIntentState +} + +type applyIntentState struct { + networkID string + outputPath string + snapshot InventorySnapshot + policy ReconcilePolicy + baselines auditPayloads + targets auditPayloads + setups []applySetupIntent + digest string +} + +type applySetupIntent struct { + setupID string + baseline api.PatchPayload + target api.PatchPayload + changes ChangeSet + discoveredCandidateCount int + discoveredOrgUnitRowCount int +} + +type applyDigestMaterial struct { + Version int `json:"version"` + NetworkID string `json:"network_id"` + Baselines auditPayloads `json:"baselines"` + Snapshot InventorySnapshot `json:"snapshot"` + Policy applyDigestPolicy `json:"policy"` + Targets auditPayloads `json:"targets"` + Changes []applyDigestChangeSet `json:"changes"` +} + +type applyDigestPolicy struct { + Kind ReconcilePolicyKind `json:"kind"` + OrganizationEvidence OrganizationEvidencePolicy `json:"organization_evidence"` + DefaultRoleName string `json:"default_role_name"` + UniformExternalID *string `json:"uniform_external_id"` + ExternalIDByAccount map[AccountID]string `json:"external_id_by_account"` +} + +type applyDigestChangeSet struct { + SetupID string `json:"setup_id"` + Add int `json:"add"` + Enable int `json:"enable"` + Disable int `json:"disable"` + Remove int `json:"remove"` + Rename int `json:"rename"` + RotateExternalID int `json:"rotate_external_id"` + ChangeRole int `json:"change_role"` + SetupMetadata int `json:"setup_metadata"` +} + +func newApplyIntent( + cfg Config, + snapshot *InventorySnapshot, + cloudAccounts []api.CloudAccount, + plan *patchPlan, + outputPath string, +) (ApplyIntent, error) { + if snapshot == nil { + return ApplyIntent{}, fmt.Errorf("apply intent requires an inventory snapshot") + } + setupIDs := selectedSetupIDs(plan.Setups) + baselines, err := buildRollbackPayloads(cloudAccounts, setupIDs) + if err != nil { + return ApplyIntent{}, err + } + snapshotCopy := cloneInventorySnapshot(*snapshot) + if strings.TrimSpace(snapshotCopy.Source) == "" { + snapshotCopy.Source = strings.TrimSpace(cfg.Source) + } + snapshotCopy.NetworkID = strings.TrimSpace(cfg.NetworkID) + snapshotCopy.SnapshotID = strings.TrimSpace(cfg.SnapshotID) + + state := &applyIntentState{ + networkID: strings.TrimSpace(cfg.NetworkID), + outputPath: outputPath, + snapshot: snapshotCopy, + policy: cloneReconcilePolicy(cfg.Policy), + baselines: cloneAuditPayloads(baselines), + targets: cloneAuditPayloads(plan.Payloads), + setups: make([]applySetupIntent, 0, len(plan.Setups)), + } + for _, setup := range plan.Setups { + state.setups = append(state.setups, applySetupIntent{ + setupID: setup.SetupID, + baseline: clonePatchPayload(baselines[setup.SetupID]), + target: clonePatchPayload(setup.Payload), + changes: cloneChangeSet(setup.ChangeSet), + discoveredCandidateCount: setup.DiscoveredCandidateCount, + discoveredOrgUnitRowCount: setup.DiscoveredOrgUnitRowCount, + }) + } + sort.Slice(state.setups, func(i, j int) bool { + return state.setups[i].setupID < state.setups[j].setupID + }) + digest, err := computeApplyIntentDigest(state) + if err != nil { + return ApplyIntent{}, err + } + state.digest = digest + return ApplyIntent{state: state}, nil +} + +// newPayloadApplyIntent is the compatibility constructor for operator-authored +// payload workflows. Callers must classify every target through the typed +// reconciliation diff before constructing the immutable gateway intent. +func newPayloadApplyIntent( + networkID string, + outputPath string, + snapshot InventorySnapshot, + policy ReconcilePolicy, + cloudAccounts []api.CloudAccount, + targets auditPayloads, + changes map[string]ChangeSet, +) (ApplyIntent, error) { + if len(targets) == 0 { + return ApplyIntent{}, fmt.Errorf("apply intent requires at least one target payload") + } + setupIDs := make([]string, 0, len(targets)) + for setupID := range targets { + setupIDs = append(setupIDs, setupID) + if _, ok := changes[setupID]; !ok { + return ApplyIntent{}, fmt.Errorf("apply intent target %s has no classified change set", setupID) + } + } + sort.Strings(setupIDs) + baselines, err := buildRollbackPayloads(cloudAccounts, setupIDs) + if err != nil { + return ApplyIntent{}, err + } + snapshot.NetworkID = strings.TrimSpace(networkID) + + state := &applyIntentState{ + networkID: strings.TrimSpace(networkID), + outputPath: outputPath, + snapshot: cloneInventorySnapshot(snapshot), + policy: cloneReconcilePolicy(policy), + baselines: cloneAuditPayloads(baselines), + targets: cloneAuditPayloads(targets), + setups: make([]applySetupIntent, 0, len(setupIDs)), + } + for _, setupID := range setupIDs { + state.setups = append(state.setups, applySetupIntent{ + setupID: setupID, + baseline: clonePatchPayload(baselines[setupID]), + target: clonePatchPayload(targets[setupID]), + changes: cloneChangeSet(changes[setupID]), + }) + } + digest, err := computeApplyIntentDigest(state) + if err != nil { + return ApplyIntent{}, err + } + state.digest = digest + return ApplyIntent{state: state}, nil +} + +// Digest returns the SHA-256 approval binding for this immutable intent. +func (i ApplyIntent) Digest() string { + if i.state == nil { + return "" + } + return i.state.digest +} + +func computeApplyIntentDigest(state *applyIntentState) (string, error) { + changes := make([]applyDigestChangeSet, 0, len(state.setups)) + for _, setup := range state.setups { + changes = append(changes, applyDigestChangeSet{ + SetupID: setup.setupID, + Add: len(setup.changes.Add), + Enable: len(setup.changes.Enable), + Disable: len(setup.changes.Disable), + Remove: len(setup.changes.Remove), + Rename: len(setup.changes.Rename), + RotateExternalID: len(setup.changes.RotateExternalID), + ChangeRole: len(setup.changes.ChangeRole), + SetupMetadata: len(setup.changes.SetupMetadata), + }) + } + data, err := json.Marshal(applyDigestMaterial{ + Version: 2, + NetworkID: state.networkID, + Baselines: approvalDigestPayloads(state.baselines), + Snapshot: state.snapshot, + Policy: approvalDigestPolicy(state.policy), + Targets: approvalDigestPayloads(state.targets), + Changes: changes, + }) + if err != nil { + return "", fmt.Errorf("encode immutable apply intent: %w", err) + } + return fmt.Sprintf("%x", sha256.Sum256(data)), nil +} + +func approvalDigestPolicy(policy ReconcilePolicy) applyDigestPolicy { + // PlanningInstant is execution metadata, not a reconciliation decision. + // The exact bytes it produces remain covered by payload_sha256. + return applyDigestPolicy{ + Kind: policy.Kind, + OrganizationEvidence: policy.OrganizationEvidence, + DefaultRoleName: policy.DefaultRoleName, + UniformExternalID: policy.UniformExternalID, + ExternalIDByAccount: policy.ExternalIDByAccount, + } +} + +func approvalDigestPayloads(payloads auditPayloads) auditPayloads { + result := cloneAuditPayloads(payloads) + for setupID, payload := range result { + // Region membership is approval-relevant; volatile test instants are not. + for region := range payload.Regions { + payload.Regions[region] = 0 + } + result[setupID] = payload + } + return result +} + +// GuardAndApply is the sole Phase 3a account-list PATCH gateway. Forward does +// not expose an ETag or version, so the immediate re-read below is only a weak +// conflict detector. It cannot make the following full-list PATCH safe from a +// concurrent write in the GET/PATCH window and must never be described as CAS. +func GuardAndApply( + ctx context.Context, + client *api.Client, + intent ApplyIntent, + authorization ApplyAuthorization, +) (ApplyResult, error) { + if intent.state == nil { + return ApplyResult{}, fmt.Errorf("apply intent is required") + } + state := intent.state + result := ApplyResult{ + JournalOutput: resultJournalPath(state.outputPath), + Journal: ApplyJournal{ + PlanDigest: state.digest, + NetworkID: state.networkID, + Authorization: ApplyAuthorizationRecord{ + PlanDigest: strings.TrimSpace(authorization.PlanDigest), + Actor: strings.TrimSpace(authorization.Actor), + Approved: authorization.Approved, + AllowDestructive: authorization.AllowDestructive, + MaxRemovals: authorization.MaxRemovals, + MaxRemovalPercent: authorization.MaxRemovalPercent, + AllowNoCandidates: authorization.AllowNoCandidates, + Unattended: authorization.Unattended, + AllowUnattendedDestructive: authorization.AllowUnattendedDestructive, + }, + Setups: make([]ApplyJournalEntry, 0, len(state.setups)), + }, + } + for _, setup := range state.setups { + result.Journal.Setups = append(result.Journal.Setups, ApplyJournalEntry{ + SetupID: setup.setupID, + Status: ApplyStatusPlanned, + History: []ApplyStatus{ApplyStatusPlanned}, + HasChanges: !setup.changes.Empty(), + }) + } + if err := persistApplyJournal(&result); err != nil { + return result, err + } + if !intentHasChanges(state) { + return result, nil + } + if err := validateApplyAuthorization(state, authorization); err != nil { + result.Blocked = true + markChangedEntries(&result.Journal, ApplyStatusFailed, err.Error()) + _ = persistApplyJournal(&result) + return result, err + } + + for index := range result.Journal.Setups { + if !result.Journal.Setups[index].HasChanges { + continue + } + setJournalStatus(&result.Journal.Setups[index], ApplyStatusPending, "") + } + if err := persistApplyJournal(&result); err != nil { + return result, err + } + + result.RollbackOutput = rollbackPath(state.outputPath) + rollbackSHA256, err := writeAuditPayloads(result.RollbackOutput, state.baselines) + if err != nil { + return failPendingApply(result, "", fmt.Errorf("write pre-apply rollback payload: %w", err)) + } + result.RollbackSHA256 = rollbackSHA256 + if _, err := writeAuditPayloads(auditPath(state.outputPath), state.targets); err != nil { + return failPendingApply(result, "", err) + } + + for _, setup := range state.setups { + if setup.changes.Empty() { + continue + } + entry := journalEntry(&result.Journal, setup.setupID) + current, err := client.CloudAccounts(ctx, state.networkID) + if err != nil { + wrapped := fmt.Errorf("reload cloud setup %s immediately before apply: %w", setup.setupID, err) + return failPendingApply(result, setup.setupID, wrapped) + } + actual, err := buildRollbackPayloads(current, []string{setup.setupID}) + if err != nil { + setJournalStatus(entry, ApplyStatusConflicted, err.Error()) + _ = persistApplyJournal(&result) + return result, err + } + if !reflect.DeepEqual(setup.baseline, actual[setup.setupID]) { + conflict := fmt.Errorf( + "selected Forward cloud setup state changed after planning for setup %s; no PATCH was sent for that setup; rerun the dry plan (the last-second re-read is only a weak mitigation because Forward provides no atomic compare-and-swap)", + setup.setupID, + ) + setJournalStatus(entry, ApplyStatusConflicted, conflict.Error()) + _ = persistApplyJournal(&result) + return result, conflict + } + if err := client.PatchCloudAccount(ctx, state.networkID, setup.setupID, setup.target); err != nil { + wrapped := fmt.Errorf("patch setup %s: %w", setup.setupID, err) + return failPendingApply(result, setup.setupID, wrapped) + } + result.PatchedCount++ + setJournalStatus(entry, ApplyStatusApplied, "") + if err := persistApplyJournal(&result); err != nil { + return result, fmt.Errorf("setup %s was patched but its applied result could not be journaled: %w", setup.setupID, err) + } + } + return result, nil +} + +func validateApplyAuthorization(state *applyIntentState, authorization ApplyAuthorization) error { + if !authorization.Approved { + return fmt.Errorf("apply authorization is required") + } + if strings.TrimSpace(authorization.PlanDigest) == "" || + !strings.EqualFold(authorization.PlanDigest, state.digest) { + return fmt.Errorf( + "reviewed plan changed before apply: expected plan digest %s, got %s; no PATCH was sent", + strings.TrimSpace(authorization.PlanDigest), + state.digest, + ) + } + + stats := destructiveRemovalStats(state) + totalDestructive := 0 + totalRemoved := 0 + for _, setup := range state.setups { + totalDestructive += len(setup.changes.Remove) + len(setup.changes.Disable) + totalRemoved += len(setup.changes.Remove) + } + if totalDestructive == 0 { + return nil + } + if !authorization.AllowDestructive { + return fmt.Errorf("planned account removals or disables require --allow-removals") + } + if totalRemoved > 0 && strings.EqualFold(strings.TrimSpace(state.snapshot.Source), "nqe") { + return nqeCompleteInventoryError() + } + if totalRemoved > 0 && state.policy.Kind == CompleteInventory && !state.snapshot.Completeness.Proven() { + return incompleteInventoryPolicyError(state.snapshot) + } + if err := requireRemovalBounds(stats, authorization.MaxRemovals, authorization.MaxRemovalPercent); err != nil { + return err + } + if err := validateRemovalStats(stats, authorization.MaxRemovals, authorization.MaxRemovalPercent); err != nil { + return err + } + if err := validateDestructiveEvidence(state, authorization); err != nil { + return err + } + if err := unattendedDestructiveApplyError(state, authorization.Unattended, authorization.AllowUnattendedDestructive); err != nil { + return err + } + return nil +} + +func unattendedDestructiveApplyError(state *applyIntentState, unattended, allowed bool) error { + if !unattended || allowed { + return nil + } + totalDestructive := 0 + for _, setup := range state.setups { + totalDestructive += len(setup.changes.Remove) + len(setup.changes.Disable) + } + if totalDestructive == 0 { + return nil + } + return fmt.Errorf( + "refusing unattended destructive apply without --allow-unattended-destructive: plan removes or disables %d account(s); Forward provides no atomic compare-and-swap", + totalDestructive, + ) +} + +func validateDestructiveEvidence(state *applyIntentState, authorization ApplyAuthorization) error { + if state.policy.OrganizationEvidence == ReviewedAuthoritativeInventory { + return nil + } + missingEvidence := make([]string, 0) + hasDisable := false + for _, setup := range state.setups { + if len(setup.changes.Remove)+len(setup.changes.Disable) == 0 { + continue + } + hasDisable = hasDisable || len(setup.changes.Disable) > 0 + evidenceVisible := organizationDiscoveryVisible( + setup.discoveredCandidateCount, + setup.discoveredOrgUnitRowCount, + ) + if setup.discoveredCandidateCount == 0 && !authorization.AllowNoCandidates { + if len(setup.changes.Disable) == 0 { + return fmt.Errorf("planned removals with no uncollected candidate accounts visible require --allow-no-candidates") + } + return fmt.Errorf("planned removals or disables with no uncollected candidate accounts visible require --allow-no-candidates") + } + if !evidenceVisible && extractRolePartition(setup.baseline.AssumeRoleInfos) == "aws-us-gov" { + if len(setup.changes.Disable) == 0 { + return fmt.Errorf("GovCloud account removals require positive AWS Organizations evidence; use sync-accounts with an authoritative reviewed manifest when Organizations is unavailable") + } + return fmt.Errorf("GovCloud account removals or disables require positive AWS Organizations evidence; use sync-accounts with an authoritative reviewed manifest when Organizations is unavailable") + } + if !evidenceVisible && state.policy.OrganizationEvidence == RequireOrganizationEvidence { + missingEvidence = append(missingEvidence, setup.setupID) + } + } + if len(missingEvidence) > 0 { + sort.Strings(missingEvidence) + if !hasDisable { + return fmt.Errorf( + "planned removals with no AWS Organizations evidence in NQE for setup(s): %s require --allow-no-org-evidence", + strings.Join(missingEvidence, ", "), + ) + } + return fmt.Errorf( + "planned removals or disables with no AWS Organizations evidence in NQE for setup(s): %s require --allow-no-org-evidence", + strings.Join(missingEvidence, ", "), + ) + } + return nil +} + +func destructiveRemovalStats(state *applyIntentState) []removalStat { + stats := make([]removalStat, 0, len(state.setups)) + for _, setup := range state.setups { + stats = append(stats, removalStat{ + SetupID: setup.setupID, + ConfiguredCount: len(setup.baseline.AssumeRoleInfos), + RemovedCount: len(setup.changes.Remove) + len(setup.changes.Disable), + }) + } + return stats +} + +func intentHasChanges(state *applyIntentState) bool { + for _, setup := range state.setups { + if !setup.changes.Empty() { + return true + } + } + return false +} + +func failPendingApply(result ApplyResult, failedSetupID string, err error) (ApplyResult, error) { + if failedSetupID == "" { + markChangedEntries(&result.Journal, ApplyStatusFailed, err.Error()) + } else { + setJournalStatus(journalEntry(&result.Journal, failedSetupID), ApplyStatusFailed, err.Error()) + } + _ = persistApplyJournal(&result) + return result, err +} + +func markChangedEntries(journal *ApplyJournal, status ApplyStatus, message string) { + for index := range journal.Setups { + entry := &journal.Setups[index] + if !entry.HasChanges || entry.Status == ApplyStatusApplied { + continue + } + setJournalStatus(entry, status, message) + } +} + +func journalEntry(journal *ApplyJournal, setupID string) *ApplyJournalEntry { + for index := range journal.Setups { + if journal.Setups[index].SetupID == setupID { + return &journal.Setups[index] + } + } + return nil +} + +func setJournalStatus(entry *ApplyJournalEntry, status ApplyStatus, message string) { + if entry == nil { + return + } + entry.Status = status + entry.Error = message + if len(entry.History) == 0 || entry.History[len(entry.History)-1] != status { + entry.History = append(entry.History, status) + } +} + +func persistApplyJournal(result *ApplyResult) error { + result.Journal.UpdatedAt = time.Now().UTC() + data, err := json.MarshalIndent(result.Journal, "", " ") + if err != nil { + return fmt.Errorf("encode apply result journal: %w", err) + } + if err := writeFileAtomic0600(result.JournalOutput, data); err != nil { + return fmt.Errorf("write apply result journal: %w", err) + } + return nil +} + +func resultJournalPath(outputPath string) string { + ext := filepath.Ext(outputPath) + if ext == "" { + return outputPath + ".result" + } + return strings.TrimSuffix(outputPath, ext) + ".result" + ext +} + +func clonePatchPayload(payload api.PatchPayload) api.PatchPayload { + return api.PatchPayload{ + Type: payload.Type, + Name: payload.Name, + Regions: cloneInt64Map(payload.Regions), + RegionToProxyServerID: cloneStringMap(payload.RegionToProxyServerID), + ProxyServerID: payload.ProxyServerID, + AssumeRoleInfos: append([]api.AssumeRoleInfo(nil), payload.AssumeRoleInfos...), + } +} + +func cloneAuditPayloads(payloads auditPayloads) auditPayloads { + result := make(auditPayloads, len(payloads)) + for setupID, payload := range payloads { + result[setupID] = clonePatchPayload(payload) + } + return result +} + +func cloneInventorySnapshot(snapshot InventorySnapshot) InventorySnapshot { + result := snapshot + result.SelectedSetupIDs = append([]SetupID(nil), snapshot.SelectedSetupIDs...) + result.DiscoveredAccounts = append([]DiscoveredAccount(nil), snapshot.DiscoveredAccounts...) + result.IgnoredAccounts = append([]AccountSummary(nil), snapshot.IgnoredAccounts...) + result.SkippedRows = append([]MalformedNQERowSummary(nil), snapshot.SkippedRows...) + if snapshot.ExpectedRowCount != nil { + value := *snapshot.ExpectedRowCount + result.ExpectedRowCount = &value + } + if snapshot.SnapshotTime != nil { + value := *snapshot.SnapshotTime + result.SnapshotTime = &value + } + return result +} + +func cloneReconcilePolicy(policy ReconcilePolicy) ReconcilePolicy { + result := policy + result.ExternalIDByAccount = make(map[AccountID]string, len(policy.ExternalIDByAccount)) + for accountID, externalID := range policy.ExternalIDByAccount { + result.ExternalIDByAccount[accountID] = externalID + } + if policy.UniformExternalID != nil { + value := *policy.UniformExternalID + result.UniformExternalID = &value + } + return result +} + +func cloneChangeSet(changes ChangeSet) ChangeSet { + return ChangeSet{ + Add: cloneAccountChanges(changes.Add), + Enable: cloneAccountChanges(changes.Enable), + Disable: cloneAccountChanges(changes.Disable), + Remove: cloneAccountChanges(changes.Remove), + Rename: cloneAccountChanges(changes.Rename), + RotateExternalID: cloneAccountChanges(changes.RotateExternalID), + ChangeRole: cloneAccountChanges(changes.ChangeRole), + SetupMetadata: append([]SetupMetadataChange(nil), changes.SetupMetadata...), + } +} + +func cloneAccountChanges(changes []AccountChange) []AccountChange { + result := make([]AccountChange, 0, len(changes)) + for _, change := range changes { + result = append(result, accountChange(change.AccountID, change.Before, change.After)) + } + return result +} diff --git a/internal/app/apply_gateway_test.go b/internal/app/apply_gateway_test.go new file mode 100644 index 0000000..2fa8794 --- /dev/null +++ b/internal/app/apply_gateway_test.go @@ -0,0 +1,637 @@ +package app + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +func TestGuardAndApplyRejectsUnattendedDestructiveWithoutOverride(t *testing.T) { + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + gatewayAssumeRole("222222222222", true), + }, + target: []api.AssumeRoleInfo{gatewayAssumeRole("111111111111", true)}, + changes: ChangeSet{Remove: []AccountChange{{AccountID: AccountID("222222222222")}}}, + }}) + result, err := GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + AllowDestructive: true, + MaxRemovals: 1, + MaxRemovalPercent: 100, + AllowNoCandidates: true, + Unattended: true, + AllowUnattendedDestructive: false, + }) + const want = "refusing unattended destructive apply without --allow-unattended-destructive: plan removes or disables 1 account(s); Forward provides no atomic compare-and-swap" + if err == nil || err.Error() != want { + t.Fatalf("GuardAndApply() error = %v, want %q", err, want) + } + if !result.Blocked || result.PatchedCount != 0 { + t.Fatalf("unexpected blocked result: %+v", result) + } +} + +func TestGuardAndApplyRequiresCompletenessForAbsenceBasedRemoval(t *testing.T) { + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + gatewayAssumeRole("222222222222", true), + }, + target: []api.AssumeRoleInfo{gatewayAssumeRole("111111111111", true)}, + changes: ChangeSet{Remove: []AccountChange{{AccountID: AccountID("222222222222")}}}, + }}) + intent.state.snapshot.Completeness = InventoryCompletenessLikelyIncomplete + intent.state.snapshot.CompletenessReason = "test snapshot is truncated" + digest, err := computeApplyIntentDigest(intent.state) + if err != nil { + t.Fatal(err) + } + intent.state.digest = digest + _, err = GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + AllowDestructive: true, + MaxRemovals: 1, + MaxRemovalPercent: 100, + AllowNoCandidates: true, + AllowUnattendedDestructive: true, + }) + if err == nil || !strings.Contains(err.Error(), "inventory completeness is unproven: test snapshot is truncated") { + t.Fatalf("completeness error = %v", err) + } +} + +func TestGuardAndApplyRejectsNQEDerivedRemovalIntent(t *testing.T) { + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + gatewayAssumeRole("222222222222", true), + }, + target: []api.AssumeRoleInfo{gatewayAssumeRole("111111111111", true)}, + changes: ChangeSet{Remove: []AccountChange{{AccountID: AccountID("222222222222")}}}, + }}) + intent.state.snapshot.Source = "nqe" + digest, err := computeApplyIntentDigest(intent.state) + if err != nil { + t.Fatal(err) + } + intent.state.digest = digest + + _, err = GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + AllowDestructive: true, + MaxRemovals: 1, + MaxRemovalPercent: 100, + AllowNoCandidates: true, + AllowUnattendedDestructive: true, + }) + if err == nil || !strings.Contains(err.Error(), "refusing CompleteInventory reconciliation for NQE observed inventory") { + t.Fatalf("GuardAndApply() error = %v; want NQE removal refusal", err) + } +} + +func TestGuardAndApplyGovCloudRemovalUsesBaselinePartition(t *testing.T) { + govAccount := gatewayAssumeRole("111111111111", true) + govAccount.RoleArn = "arn:aws-us-gov:iam::111111111111:role/ForwardRole" + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-gov", + baseline: []api.AssumeRoleInfo{govAccount}, + target: nil, + changes: ChangeSet{Remove: []AccountChange{{AccountID: AccountID("111111111111")}}}, + }}) + intent.state.policy.OrganizationEvidence = AllowMissingOrganizationEvidence + digest, err := computeApplyIntentDigest(intent.state) + if err != nil { + t.Fatal(err) + } + intent.state.digest = digest + _, err = GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + AllowDestructive: true, + MaxRemovals: 1, + MaxRemovalPercent: 100, + AllowNoCandidates: true, + AllowUnattendedDestructive: true, + }) + if err == nil || !strings.Contains(err.Error(), "GovCloud account removals require positive AWS Organizations evidence") { + t.Fatalf("GovCloud baseline partition error = %v", err) + } +} + +func TestGuardAndApplyDisableUsesDestructiveAuthorizationAndRemovalBudget(t *testing.T) { + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + }, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", false), + }, + changes: ChangeSet{Disable: []AccountChange{{AccountID: AccountID("111111111111")}}}, + }}) + _, err := GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + }) + if err == nil || !strings.Contains(err.Error(), "--allow-removals") { + t.Fatalf("disable without destructive authorization error = %v", err) + } + + _, err = GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + AllowDestructive: true, + MaxRemovals: 1, + MaxRemovalPercent: 50, + AllowNoCandidates: true, + }) + if err == nil || !strings.Contains(err.Error(), "removes 1 of 1 accounts (100.00%)") { + t.Fatalf("disable per-setup budget error = %v", err) + } +} + +func TestApplyIntentDigestBindsBaselineSnapshotPolicyAndTarget(t *testing.T) { + setup := gatewayTestSetup{ + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + }, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + gatewayAssumeRole("222222222222", true), + }, + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("222222222222")}}}, + } + base := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{setup}) + digests := map[string]string{"base": base.Digest()} + + baseline := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{setup}) + baseline.state.baselines["setup-a"] = clonePatchPayload(baseline.state.baselines["setup-a"]) + payload := baseline.state.baselines["setup-a"] + payload.ProxyServerID = "changed-baseline" + baseline.state.baselines["setup-a"] = payload + baseline.state.setups[0].baseline = clonePatchPayload(payload) + digests["baseline"], _ = computeApplyIntentDigest(baseline.state) + + snapshot := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{setup}) + snapshot.state.snapshot.SnapshotID = "different-snapshot" + snapshot.state.snapshot.Completeness = InventoryCompletenessLikelyIncomplete + digests["snapshot"], _ = computeApplyIntentDigest(snapshot.state) + + policy := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{setup}) + policy.state.policy.Kind = Additive + digests["policy"], _ = computeApplyIntentDigest(policy.state) + + target := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{setup}) + payload = target.state.targets["setup-a"] + payload.ProxyServerID = "changed-target" + target.state.targets["setup-a"] = payload + target.state.setups[0].target = clonePatchPayload(payload) + digests["target"], _ = computeApplyIntentDigest(target.state) + + for name, digest := range digests { + if name == "base" { + continue + } + if digest == digests["base"] { + t.Errorf("%s change did not alter apply intent digest %s", name, digest) + } + } +} + +func TestApplyIntentDigestChangesWhenPlanMeaningfullyChanges(t *testing.T) { + baseline := []api.AssumeRoleInfo{gatewayAssumeRole("111111111111", true)} + first := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-a", + baseline: baseline, + target: append(append([]api.AssumeRoleInfo(nil), baseline...), + gatewayAssumeRole("222222222222", true)), + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("222222222222")}}}, + }}) + second := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{{ + setupID: "setup-a", + baseline: baseline, + target: append(append([]api.AssumeRoleInfo(nil), baseline...), + gatewayAssumeRole("333333333333", true)), + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("333333333333")}}}, + }}) + + if first.Digest() == "" || second.Digest() == "" { + t.Fatalf("meaningful plans produced empty digests: first=%q second=%q", first.Digest(), second.Digest()) + } + if first.Digest() == second.Digest() { + t.Fatalf("different target accounts produced the same digest %q", first.Digest()) + } +} + +func TestApplyIntentDigestZeroValueIsEmpty(t *testing.T) { + if got := (ApplyIntent{}).Digest(); got != "" { + t.Fatalf("zero-value ApplyIntent digest = %q, want empty", got) + } +} + +func TestValidateDestructiveEvidence(t *testing.T) { + type evidenceSetup struct { + setupID string + baseline api.AssumeRoleInfo + changes ChangeSet + candidateCount int + orgUnitCount int + } + govAccount := gatewayAssumeRole("111111111111", true) + govAccount.RoleArn = "arn:aws-us-gov:iam::111111111111:role/ForwardRole" + remove := ChangeSet{Remove: []AccountChange{{AccountID: AccountID("111111111111")}}} + disable := ChangeSet{Disable: []AccountChange{{AccountID: AccountID("111111111111")}}} + tests := []struct { + name string + evidence OrganizationEvidencePolicy + setups []evidenceSetup + allowNoCandidates bool + wantError string + }{ + { + name: "reviewed authoritative inventory bypasses discovery evidence", + evidence: ReviewedAuthoritativeInventory, + setups: []evidenceSetup{{setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: remove}}, + }, + { + name: "removal with no candidates requires override", + evidence: AllowMissingOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: remove}}, + wantError: "planned removals with no uncollected candidate accounts visible require --allow-no-candidates", + }, + { + name: "disable with no candidates requires override", + evidence: AllowMissingOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: disable}}, + wantError: "planned removals or disables with no uncollected candidate accounts visible require --allow-no-candidates", + }, + { + name: "GovCloud removal requires positive evidence", + evidence: AllowMissingOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-gov", baseline: govAccount, changes: remove}}, + allowNoCandidates: true, + wantError: "GovCloud account removals require positive AWS Organizations evidence; use sync-accounts with an authoritative reviewed manifest when Organizations is unavailable", + }, + { + name: "GovCloud disable requires positive evidence", + evidence: AllowMissingOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-gov", baseline: govAccount, changes: disable}}, + allowNoCandidates: true, + wantError: "GovCloud account removals or disables require positive AWS Organizations evidence; use sync-accounts with an authoritative reviewed manifest when Organizations is unavailable", + }, + { + name: "required evidence reports sorted removal setups", + evidence: RequireOrganizationEvidence, + setups: []evidenceSetup{ + {setupID: "setup-z", baseline: gatewayAssumeRole("111111111111", true), changes: remove}, + {setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: remove}, + }, + allowNoCandidates: true, + wantError: "planned removals with no AWS Organizations evidence in NQE for setup(s): setup-a, setup-z require --allow-no-org-evidence", + }, + { + name: "required evidence distinguishes disables", + evidence: RequireOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: disable}}, + allowNoCandidates: true, + wantError: "planned removals or disables with no AWS Organizations evidence in NQE for setup(s): setup-a require --allow-no-org-evidence", + }, + { + name: "allow missing evidence accepts explicit override", + evidence: AllowMissingOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: remove}}, + allowNoCandidates: true, + }, + { + name: "visible candidates satisfy required evidence", + evidence: RequireOrganizationEvidence, + setups: []evidenceSetup{{ + setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true), changes: remove, candidateCount: 1, + }}, + }, + { + name: "non-destructive setup needs no evidence", + evidence: RequireOrganizationEvidence, + setups: []evidenceSetup{{setupID: "setup-a", baseline: gatewayAssumeRole("111111111111", true)}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gatewaySetups := make([]gatewayTestSetup, 0, len(tt.setups)) + counts := make(map[string][2]int, len(tt.setups)) + for _, setup := range tt.setups { + gatewaySetups = append(gatewaySetups, gatewayTestSetup{ + setupID: setup.setupID, + baseline: []api.AssumeRoleInfo{setup.baseline}, + changes: setup.changes, + }) + counts[setup.setupID] = [2]int{setup.candidateCount, setup.orgUnitCount} + } + intent := gatewayTestIntent(t, t.TempDir(), gatewaySetups) + intent.state.policy.OrganizationEvidence = tt.evidence + for index := range intent.state.setups { + count := counts[intent.state.setups[index].setupID] + intent.state.setups[index].discoveredCandidateCount = count[0] + intent.state.setups[index].discoveredOrgUnitRowCount = count[1] + } + + err := validateDestructiveEvidence(intent.state, ApplyAuthorization{AllowNoCandidates: tt.allowNoCandidates}) + if tt.wantError == "" { + if err != nil { + t.Fatalf("validateDestructiveEvidence() error = %v", err) + } + return + } + if err == nil || err.Error() != tt.wantError { + t.Fatalf("validateDestructiveEvidence() error = %v, want %q", err, tt.wantError) + } + }) + } +} + +func TestGuardAndApplyPreApplyArtifactFailureFailsEveryPendingSetup(t *testing.T) { + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{ + { + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{gatewayAssumeRole("111111111111", true)}, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + gatewayAssumeRole("222222222222", true), + }, + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("222222222222")}}}, + }, + { + setupID: "setup-b", + baseline: []api.AssumeRoleInfo{gatewayAssumeRole("333333333333", true)}, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("333333333333", true), + gatewayAssumeRole("444444444444", true), + }, + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("444444444444")}}}, + }, + }) + if err := os.Mkdir(rollbackPath(intent.state.outputPath), 0o700); err != nil { + t.Fatal(err) + } + + result, err := GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + }) + if err == nil || !strings.Contains(err.Error(), "write pre-apply rollback payload") { + t.Fatalf("GuardAndApply() error = %v, want rollback artifact failure", err) + } + if result.PatchedCount != 0 { + t.Fatalf("patched count = %d, want 0", result.PatchedCount) + } + for _, setupID := range []string{"setup-a", "setup-b"} { + assertGatewayJournalEntry(t, result.Journal, setupID, ApplyStatusFailed, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending, ApplyStatusFailed}, + "write pre-apply rollback payload") + } + + persisted := readGatewayJournal(t, result.JournalOutput) + for _, setupID := range []string{"setup-a", "setup-b"} { + assertGatewayJournalEntry(t, persisted, setupID, ApplyStatusFailed, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending, ApplyStatusFailed}, + "write pre-apply rollback payload") + } +} + +func TestGuardAndApplyReturnsDurablePartialJournal(t *testing.T) { + var ( + mu sync.Mutex + state = map[string][]api.AssumeRoleInfo{ + "setup-a": {gatewayAssumeRole("111111111111", true)}, + "setup-b": {gatewayAssumeRole("222222222222", true)}, + "setup-c": {gatewayAssumeRole("333333333333", true)}, + } + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + mu.Lock() + accounts := []api.CloudAccount{ + {Name: "setup-a", Type: "AWS", AssumeRoleInfos: append([]api.AssumeRoleInfo(nil), state["setup-a"]...)}, + {Name: "setup-b", Type: "AWS", AssumeRoleInfos: append([]api.AssumeRoleInfo(nil), state["setup-b"]...)}, + {Name: "setup-c", Type: "AWS", AssumeRoleInfos: append([]api.AssumeRoleInfo(nil), state["setup-c"]...)}, + } + mu.Unlock() + _ = json.NewEncoder(w).Encode(accounts) + case r.Method == http.MethodPatch && strings.HasSuffix(r.URL.Path, "/setup-b"): + http.Error(w, "injected failure", http.StatusInternalServerError) + case r.Method == http.MethodPatch: + setupID := strings.TrimPrefix(r.URL.Path, "/api/networks/network-1/cloudAccounts/") + var payload api.PatchPayload + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("decode PATCH: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mu.Lock() + state[setupID] = append([]api.AssumeRoleInfo(nil), payload.AssumeRoleInfos...) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + client, err := api.NewClient(server.URL, "/api", "alice", "secret", false, time.Second) + if err != nil { + t.Fatal(err) + } + intent := gatewayTestIntent(t, t.TempDir(), []gatewayTestSetup{ + { + setupID: "setup-a", + baseline: []api.AssumeRoleInfo{gatewayAssumeRole("111111111111", true)}, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("111111111111", true), + gatewayAssumeRole("444444444444", true), + }, + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("444444444444")}}}, + }, + { + setupID: "setup-b", + baseline: []api.AssumeRoleInfo{gatewayAssumeRole("222222222222", true)}, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("222222222222", true), + gatewayAssumeRole("555555555555", true), + }, + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("555555555555")}}}, + }, + { + setupID: "setup-c", + baseline: []api.AssumeRoleInfo{gatewayAssumeRole("333333333333", true)}, + target: []api.AssumeRoleInfo{ + gatewayAssumeRole("333333333333", true), + gatewayAssumeRole("666666666666", true), + }, + changes: ChangeSet{Add: []AccountChange{{AccountID: AccountID("666666666666")}}}, + }, + }) + result, err := GuardAndApply(context.Background(), client, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + }) + if err == nil || !strings.Contains(err.Error(), "setup-b") { + t.Fatalf("GuardAndApply() error = %v, want setup-b failure", err) + } + if result.PatchedCount != 1 { + t.Fatalf("patched count = %d, want 1", result.PatchedCount) + } + assertGatewayJournalEntry(t, result.Journal, "setup-a", ApplyStatusApplied, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending, ApplyStatusApplied}, "") + assertGatewayJournalEntry(t, result.Journal, "setup-b", ApplyStatusFailed, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending, ApplyStatusFailed}, "patch setup setup-b") + assertGatewayJournalEntry(t, result.Journal, "setup-c", ApplyStatusPending, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending}, "") + + persisted := readGatewayJournal(t, result.JournalOutput) + if len(persisted.Setups) != 3 { + t.Fatalf("persisted journal = %#v", persisted) + } + assertGatewayJournalEntry(t, persisted, "setup-a", ApplyStatusApplied, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending, ApplyStatusApplied}, "") + assertGatewayJournalEntry(t, persisted, "setup-b", ApplyStatusFailed, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending, ApplyStatusFailed}, "patch setup setup-b") + assertGatewayJournalEntry(t, persisted, "setup-c", ApplyStatusPending, + []ApplyStatus{ApplyStatusPlanned, ApplyStatusPending}, "") +} + +func readGatewayJournal(t *testing.T, path string) ApplyJournal { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read durable journal: %v", err) + } + var journal ApplyJournal + if err := json.Unmarshal(data, &journal); err != nil { + t.Fatalf("decode durable journal: %v", err) + } + return journal +} + +func assertGatewayJournalEntry( + t *testing.T, + journal ApplyJournal, + setupID string, + wantStatus ApplyStatus, + wantHistory []ApplyStatus, + wantError string, +) { + t.Helper() + entry := journalEntry(&journal, setupID) + if entry == nil { + t.Fatalf("journal has no entry for %s: %#v", setupID, journal.Setups) + } + if entry.Status != wantStatus { + t.Fatalf("journal status for %s = %q, want %q", setupID, entry.Status, wantStatus) + } + if len(entry.History) != len(wantHistory) { + t.Fatalf("journal history for %s = %#v, want %#v", setupID, entry.History, wantHistory) + } + for index := range wantHistory { + if entry.History[index] != wantHistory[index] { + t.Fatalf("journal history for %s = %#v, want %#v", setupID, entry.History, wantHistory) + } + } + if wantError == "" { + if entry.Error != "" { + t.Fatalf("journal error for %s = %q, want empty", setupID, entry.Error) + } + return + } + if !strings.Contains(entry.Error, wantError) { + t.Fatalf("journal error for %s = %q, want substring %q", setupID, entry.Error, wantError) + } +} + +type gatewayTestSetup struct { + setupID string + baseline []api.AssumeRoleInfo + target []api.AssumeRoleInfo + changes ChangeSet +} + +func gatewayTestIntent(t *testing.T, dir string, setups []gatewayTestSetup) ApplyIntent { + t.Helper() + state := &applyIntentState{ + networkID: "network-1", + outputPath: filepath.Join(dir, "payload.json"), + snapshot: InventorySnapshot{ + Source: "test", + NetworkID: "network-1", + Completeness: InventoryCompletenessComplete, + }, + policy: ReconcilePolicy{ + Kind: CompleteInventory, + PlanningInstant: time.Unix(1, 0).UTC(), + OrganizationEvidence: ReviewedAuthoritativeInventory, + }, + baselines: make(auditPayloads), + targets: make(auditPayloads), + } + for _, setup := range setups { + baseline := api.PatchPayload{ + Type: "AWS", + Name: setup.setupID, + Regions: map[string]int64{}, + RegionToProxyServerID: map[string]string{}, + AssumeRoleInfos: setup.baseline, + } + target := api.PatchPayload{ + Type: "AWS", + Name: setup.setupID, + Regions: map[string]int64{}, + RegionToProxyServerID: map[string]string{}, + AssumeRoleInfos: setup.target, + } + state.baselines[setup.setupID] = clonePatchPayload(baseline) + state.targets[setup.setupID] = clonePatchPayload(target) + state.setups = append(state.setups, applySetupIntent{ + setupID: setup.setupID, + baseline: clonePatchPayload(baseline), + target: clonePatchPayload(target), + changes: cloneChangeSet(setup.changes), + }) + } + sort.Slice(state.setups, func(i, j int) bool { + return state.setups[i].setupID < state.setups[j].setupID + }) + digest, err := computeApplyIntentDigest(state) + if err != nil { + t.Fatal(err) + } + state.digest = digest + return ApplyIntent{state: state} +} + +func gatewayAssumeRole(accountID string, enabled bool) api.AssumeRoleInfo { + return api.AssumeRoleInfo{ + AccountID: accountID, + RoleArn: "arn:aws:iam::" + accountID + ":role/ForwardRole", + Enabled: enabled, + } +} diff --git a/internal/app/apply_plan.go b/internal/app/apply_plan.go index 673d05b..9472c9d 100644 --- a/internal/app/apply_plan.go +++ b/internal/app/apply_plan.go @@ -14,28 +14,32 @@ import ( ) type ApplyPlanConfig struct { - Host string - Username string - Password string - NetworkID string - PlanPath string - APIPrefix string - Insecure bool - Timeout time.Duration - AllowRemovals bool - MaxRemovals int - MaxRemovalPercent float64 + Host string + Username string + Password string + NetworkID string + PlanPath string + APIPrefix string + Insecure bool + Timeout time.Duration + AllowRemovals bool + MaxRemovals int + MaxRemovalPercent float64 + AllowUnattendedDestructive bool + AuthorizationActor string } type ApplyPlanSummary struct { - Host string `json:"host"` - NetworkID string `json:"network_id"` - PlanPath string `json:"plan_path"` - PayloadSHA256 string `json:"payload_sha256"` - RollbackOutput string `json:"rollback_output"` - RollbackSHA256 string `json:"rollback_sha256"` - PatchedSetupCount int `json:"patched_setup_count"` - PatchedSetups []string `json:"patched_setups"` + Host string `json:"host"` + NetworkID string `json:"network_id"` + PlanPath string `json:"plan_path"` + PayloadSHA256 string `json:"payload_sha256"` + PlanDigest string `json:"plan_digest"` + RollbackOutput string `json:"rollback_output,omitempty"` + RollbackSHA256 string `json:"rollback_sha256,omitempty"` + ResultJournalOutput string `json:"result_journal_output"` + PatchedSetupCount int `json:"patched_setup_count"` + PatchedSetups []string `json:"patched_setups"` } func ApplyPlan(ctx context.Context, cfg ApplyPlanConfig) (*ApplyPlanSummary, error) { @@ -77,6 +81,7 @@ func ApplyPlan(ctx context.Context, cfg ApplyPlanConfig) (*ApplyPlanSummary, err if len(setupIDs) == 0 { return nil, fmt.Errorf("plan contains no setup payloads") } + sort.Strings(setupIDs) cloudAccounts, err := client.CloudAccounts(ctx, cfg.NetworkID) if err != nil { return nil, fmt.Errorf("load current cloud setups before apply: %w", err) @@ -85,7 +90,9 @@ func ApplyPlan(ctx context.Context, cfg ApplyPlanConfig) (*ApplyPlanSummary, err for _, account := range cloudAccounts { currentByName[strings.TrimSpace(account.Name)] = account } - removalStats := make([]removalStat, 0, len(setupIDs)) + targets := make(auditPayloads, len(setupIDs)) + changeSets := make(map[string]ChangeSet, len(setupIDs)) + selectedSetupIDs := make([]SetupID, 0, len(setupIDs)) for _, setupID := range setupIDs { current, ok := currentByName[setupID] if !ok { @@ -105,55 +112,115 @@ func ApplyPlan(ctx context.Context, cfg ApplyPlanConfig) (*ApplyPlanSummary, err if err := validateCloudAccountPartition(planned); err != nil { return nil, fmt.Errorf("plan setup %s: %w", setupID, err) } - currentRows := currentAccounts(current.AssumeRoleInfos) - _, removed, _ := accountDiff(currentRows, currentAccounts(payload.AssumeRoleInfos)) - removalStats = append(removalStats, removalStat{ - SetupID: setupID, - ConfiguredCount: len(currentRows), - RemovedCount: len(removed), - }) - if len(removed) == 0 { - continue - } - if extractRolePartition(current.AssumeRoleInfos) == "aws-us-gov" { - return nil, fmt.Errorf("apply-plan cannot remove GovCloud accounts; rerun preflight/NQE with positive Organizations evidence or use sync-accounts with the authoritative manifest") + typedSetupID, err := NewSetupID(setupID) + if err != nil { + return nil, fmt.Errorf("plan contains invalid setup id %q: %w", setupID, err) } - if !cfg.AllowRemovals { - return nil, fmt.Errorf("plan removes %d account(s) from setup %s; apply-plan requires --allow-removals", len(removed), setupID) + changes, err := classifyPatchPayload(current, typedSetupID, payload) + if err != nil { + return nil, fmt.Errorf("classify plan setup %s: %w", setupID, err) } + targets[setupID] = clonePatchPayload(payload) + changeSets[setupID] = changes + selectedSetupIDs = append(selectedSetupIDs, typedSetupID) } - if err := requireRemovalBounds(removalStats, cfg.MaxRemovals, cfg.MaxRemovalPercent); err != nil { - return nil, err - } - if err := validateRemovalStats(removalStats, cfg.MaxRemovals, cfg.MaxRemovalPercent); err != nil { - return nil, err - } - sort.Strings(setupIDs) - rollbackPayloads, err := buildRollbackPayloads(cloudAccounts, setupIDs) + + policy := ReconcilePolicy{ + Kind: CompleteInventory, + PlanningInstant: time.Now().UTC(), + OrganizationEvidence: AllowMissingOrganizationEvidence, + } + intent, err := newPayloadApplyIntent( + cfg.NetworkID, + planPath, + InventorySnapshot{ + Source: "reviewed apply-plan payload", + SelectedSetupIDs: selectedSetupIDs, + Completeness: InventoryCompletenessComplete, + }, + policy, + cloudAccounts, + targets, + changeSets, + ) if err != nil { return nil, err } - rollbackOutput := rollbackPath(planPath) - rollbackSHA256, err := writeAuditPayloads(rollbackOutput, rollbackPayloads) - if err != nil { - return nil, fmt.Errorf("write pre-apply rollback payload: %w", err) - } - if err := verifyCloudAccountsUnchanged(ctx, client, cfg.NetworkID, setupIDs, rollbackPayloads); err != nil { - return nil, err + actor := strings.TrimSpace(cfg.AuthorizationActor) + if actor == "" { + actor = "apply-plan caller" + } + applyResult, applyErr := GuardAndApply(ctx, client, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Actor: actor, + Approved: true, + AllowDestructive: cfg.AllowRemovals, + MaxRemovals: cfg.MaxRemovals, + MaxRemovalPercent: cfg.MaxRemovalPercent, + // Legacy payload files contain no NQE candidate counts. Preserve that + // documented file-format limitation while the gateway still applies + // removal/disable budgets and its no-evidence GovCloud block. + AllowNoCandidates: true, + Unattended: true, + AllowUnattendedDestructive: cfg.AllowUnattendedDestructive, + }) + patchedSetups := make([]string, 0, applyResult.PatchedCount) + for _, entry := range applyResult.Journal.Setups { + if entry.Status == ApplyStatusApplied { + patchedSetups = append(patchedSetups, entry.SetupID) + } } - for _, setupID := range setupIDs { - if err := client.PatchCloudAccount(ctx, cfg.NetworkID, setupID, payloads[setupID]); err != nil { - return nil, fmt.Errorf("patch setup %s: %w", setupID, err) + summary := &ApplyPlanSummary{ + Host: cfg.Host, + NetworkID: cfg.NetworkID, + PlanPath: planPath, + PayloadSHA256: fmt.Sprintf("%x", sha256.Sum256(data)), + PlanDigest: intent.Digest(), + RollbackOutput: applyResult.RollbackOutput, + RollbackSHA256: applyResult.RollbackSHA256, + ResultJournalOutput: applyResult.JournalOutput, + PatchedSetupCount: applyResult.PatchedCount, + PatchedSetups: patchedSetups, + } + if applyErr != nil { + if applyResult.JournalOutput != "" { + return summary, fmt.Errorf("%w; apply result journal: %s", applyErr, applyResult.JournalOutput) } + return summary, applyErr + } + return summary, nil +} + +func classifyPatchPayload(current api.CloudAccount, setupID SetupID, target api.PatchPayload) (ChangeSet, error) { + currentSetup, err := adaptCurrentSetup(cloudSetupMetadata{ + setupID: setupID, + cloudType: current.Type, + proxyServerID: current.ProxyServerID, + regionToProxyServer: current.RegionToProxyServerID, + regions: current.Regions, + assumeRoleInfos: current.AssumeRoleInfos, + }) + if err != nil { + return ChangeSet{}, err + } + targetRegions := make(map[string]api.RegionMeta, len(target.Regions)) + for region, instant := range target.Regions { + targetRegions[region] = api.RegionMeta{TestInstant: instant} + } + targetSetup, err := adaptCurrentSetup(cloudSetupMetadata{ + setupID: setupID, + cloudType: target.Type, + proxyServerID: target.ProxyServerID, + regionToProxyServer: target.RegionToProxyServerID, + regions: targetRegions, + assumeRoleInfos: target.AssumeRoleInfos, + }) + if err != nil { + return ChangeSet{}, err } - return &ApplyPlanSummary{ - Host: cfg.Host, - NetworkID: cfg.NetworkID, - PlanPath: planPath, - PayloadSHA256: fmt.Sprintf("%x", sha256.Sum256(data)), - RollbackOutput: rollbackOutput, - RollbackSHA256: rollbackSHA256, - PatchedSetupCount: len(setupIDs), - PatchedSetups: setupIDs, - }, nil + return diffSetup(currentSetup, DesiredSetup{ + SetupID: targetSetup.SetupID, + Metadata: targetSetup.Metadata, + Accounts: targetSetup.Accounts, + }), nil } diff --git a/internal/app/apply_plan_test.go b/internal/app/apply_plan_test.go index 81f8a1d..7cb1aa8 100644 --- a/internal/app/apply_plan_test.go +++ b/internal/app/apply_plan_test.go @@ -1,13 +1,19 @@ package app import ( + "bytes" "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" + + "github.com/forwardnetworks/aws-sync/internal/api" ) func TestApplyPlanPatchesReviewedPayload(t *testing.T) { @@ -35,7 +41,9 @@ func TestApplyPlanPatchesReviewedPayload(t *testing.T) { planPath := filepath.Join(t.TempDir(), "payload.json") if err := os.WriteFile( planPath, - []byte(`{"setup-a":{"type":"AWS","name":"setup-a","regionToProxyServerId":{},"assumeRoleInfos":[]}}`), + []byte(`{"setup-a":{"type":"AWS","name":"setup-a","regionToProxyServerId":{},"assumeRoleInfos":[ + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true} + ]}}`), 0o644, ); err != nil { t.Fatalf("write plan: %v", err) @@ -69,6 +77,243 @@ func TestApplyPlanPatchesReviewedPayload(t *testing.T) { } } +func TestApplyPlanAcceptsPreBranchBinaryArtifacts(t *testing.T) { + tests := []struct { + name string + artifact string + baseline string + wantEnabled bool + }{ + { + name: "generated apply plan", + artifact: "pre_branch_apply_plan.json", + baseline: "pre_branch_apply_plan.rollback.json", + wantEnabled: true, + }, + { + name: "generated rollback", + artifact: "pre_branch_apply_plan.rollback.json", + baseline: "pre_branch_apply_plan.json", + wantEnabled: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + planPath, targetPayloads := materializePreBranchJSONArtifact(t, test.artifact) + _, baselinePayloads := materializePreBranchJSONArtifact(t, test.baseline) + baseline := baselinePayloads["setup-a"] + current := api.CloudAccount{ + Type: baseline.Type, + Name: baseline.Name, + ProxyServerID: baseline.ProxyServerID, + RegionToProxyServerID: baseline.RegionToProxyServerID, + Regions: make(map[string]api.RegionMeta, len(baseline.Regions)), + AssumeRoleInfos: baseline.AssumeRoleInfos, + } + for region, instant := range baseline.Regions { + current.Regions[region] = api.RegionMeta{TestInstant: instant} + } + + patchCount := 0 + var patched api.PatchPayload + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + _ = json.NewEncoder(w).Encode([]api.CloudAccount{current}) + case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/setup-a": + patchCount++ + if err := json.NewDecoder(r.Body).Decode(&patched); err != nil { + t.Fatalf("decode PATCH: %v", err) + } + _, _ = w.Write([]byte(`{}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + summary, err := ApplyPlan(context.Background(), ApplyPlanConfig{ + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + PlanPath: planPath, + APIPrefix: "/api", + AllowRemovals: true, + MaxRemovals: 2, + MaxRemovalPercent: 100, + AllowUnattendedDestructive: true, + }) + if err != nil { + t.Fatalf("ApplyPlan() with %s: %v", test.artifact, err) + } + if patchCount != 1 || summary.PatchedSetupCount != 1 { + t.Fatalf("patch count = %d, summary = %+v; want one PATCH", patchCount, summary) + } + if summary.ResultJournalOutput == "" { + t.Fatalf("accepted artifact did not produce a result journal: %+v", summary) + } + want := targetPayloads["setup-a"] + if len(patched.AssumeRoleInfos) != 2 || patched.AssumeRoleInfos[1].Enabled != test.wantEnabled { + t.Fatalf("old artifact account state was misread: %#v", patched.AssumeRoleInfos) + } + if patched.ProxyServerID != want.ProxyServerID || patched.Regions["us-east-1"] != 123 { + t.Fatalf("old artifact recovery fields were misread: %#v", patched) + } + }) + } +} + +func materializePreBranchJSONArtifact(t *testing.T, name string) (string, map[string]api.PatchPayload) { + t.Helper() + wantSHA256 := map[string]string{ + "pre_branch_apply_plan.json": "da2612db7cbd41071306e6a8d28404d36de74ae98ebb9dd9ecf2c28dfa63738e", + "pre_branch_apply_plan.rollback.json": "804a9a15d5aab5e5b65ff796990d61ad17bc64df9e59fcc3d5fbf385c94565b5", + }[name] + data, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read pre-branch artifact %s: %v", name, err) + } + data = bytes.TrimSuffix(data, []byte("\n")) + digest := sha256.Sum256(data) + if got := hex.EncodeToString(digest[:]); got != wantSHA256 { + t.Fatalf("pre-branch artifact %s SHA-256 = %s; want %s", name, got, wantSHA256) + } + var payloads map[string]api.PatchPayload + if err := json.Unmarshal(data, &payloads); err != nil { + t.Fatalf("decode pre-branch artifact %s in test setup: %v", name, err) + } + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("materialize pre-branch artifact %s: %v", name, err) + } + return path, payloads +} + +func TestApplyPlanSuppressesZeroDiffPatch(t *testing.T) { + patchCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + _, _ = w.Write([]byte(`[{"type":"AWS","name":"setup-a","assumeRoleInfos":[ + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true} + ]}]`)) + case r.Method == http.MethodPatch: + patchCount++ + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + planPath := filepath.Join(t.TempDir(), "same.json") + if err := os.WriteFile(planPath, []byte(`{"setup-a":{"type":"AWS","name":"setup-a","regionToProxyServerId":{},"assumeRoleInfos":[ + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true} + ]}}`), 0o600); err != nil { + t.Fatal(err) + } + summary, err := ApplyPlan(context.Background(), ApplyPlanConfig{ + Host: server.URL, + Username: "user", + Password: "pass", + NetworkID: "network-1", + PlanPath: planPath, + APIPrefix: "/api", + }) + if err != nil { + t.Fatalf("ApplyPlan() error = %v", err) + } + if patchCount != 0 || summary.PatchedSetupCount != 0 { + t.Fatalf("zero-diff apply = (patches=%d, summary=%+v), want no PATCH", patchCount, summary) + } + if summary.ResultJournalOutput == "" { + t.Fatalf("zero-diff apply did not persist a result journal: %+v", summary) + } + if summary.RollbackOutput != "" { + t.Fatalf("zero-diff apply unexpectedly wrote rollback output: %+v", summary) + } +} + +func TestApplyPlanDisableRequiresGatewayDestructiveAuthorization(t *testing.T) { + tests := []struct { + name string + allowRemovals bool + maxRemovals int + maxRemovalPercent float64 + allowUnattended bool + wantError string + }{ + {name: "no destructive authorization", wantError: "--allow-removals"}, + {name: "authorization without bounds", allowRemovals: true, wantError: "require both"}, + { + name: "unattended authorization required", + allowRemovals: true, + maxRemovals: 2, + maxRemovalPercent: 100, + wantError: "--allow-unattended-destructive", + }, + { + name: "fully authorized", + allowRemovals: true, + maxRemovals: 2, + maxRemovalPercent: 100, + allowUnattended: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + patchCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + _, _ = w.Write([]byte(`[{"type":"AWS","name":"prod","assumeRoleInfos":[ + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}, + {"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true} + ]}]`)) + case r.Method == http.MethodPatch: + patchCount++ + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + planPath := filepath.Join(t.TempDir(), "disable.json") + if err := os.WriteFile(planPath, []byte(`{"prod":{"type":"AWS","name":"prod","assumeRoleInfos":[ + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":false}, + {"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":false} + ]}}`), 0o600); err != nil { + t.Fatal(err) + } + _, err := ApplyPlan(context.Background(), ApplyPlanConfig{ + Host: server.URL, + Username: "user", + Password: "pass", + NetworkID: "network-1", + PlanPath: planPath, + APIPrefix: "/api", + AllowRemovals: test.allowRemovals, + MaxRemovals: test.maxRemovals, + MaxRemovalPercent: test.maxRemovalPercent, + AllowUnattendedDestructive: test.allowUnattended, + }) + if test.wantError == "" { + if err != nil || patchCount != 1 { + t.Fatalf("fully authorized disable = (patches=%d, err=%v), want one PATCH", patchCount, err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("ApplyPlan() error = %v, want %q", err, test.wantError) + } + if patchCount != 0 { + t.Fatalf("unauthorized disable PATCH count = %d, want 0", patchCount) + } + }) + } +} + func TestApplyPlanCannotBypassGovCloudRemovalSafety(t *testing.T) { patched := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -93,15 +338,18 @@ func TestApplyPlanCannotBypassGovCloudRemovalSafety(t *testing.T) { t.Fatal(err) } _, err := ApplyPlan(context.Background(), ApplyPlanConfig{ - Host: server.URL, - Username: "user", - Password: "pass", - NetworkID: "network-1", - PlanPath: planPath, - APIPrefix: "/api", - AllowRemovals: true, + Host: server.URL, + Username: "user", + Password: "pass", + NetworkID: "network-1", + PlanPath: planPath, + APIPrefix: "/api", + AllowRemovals: true, + MaxRemovals: 1, + MaxRemovalPercent: 100, + AllowUnattendedDestructive: true, }) - if err == nil || !strings.Contains(err.Error(), "cannot remove GovCloud accounts") { + if err == nil || !strings.Contains(err.Error(), "GovCloud account removals require positive AWS Organizations evidence") { t.Fatalf("expected GovCloud apply-plan block, got %v", err) } if patched { @@ -115,8 +363,8 @@ func TestApplyPlanBlocksRemovalPercentageAboveLimit(t *testing.T) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": _, _ = w.Write([]byte(`[{"type":"AWS","name":"prod","assumeRoleInfos":[ - {"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}, - {"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true} + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}, + {"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true} ]}]`)) case r.Method == http.MethodPatch: patched = true @@ -128,7 +376,7 @@ func TestApplyPlanBlocksRemovalPercentageAboveLimit(t *testing.T) { planPath := filepath.Join(t.TempDir(), "payload.json") if err := os.WriteFile(planPath, []byte(`{"prod":{"type":"AWS","name":"prod","assumeRoleInfos":[ - {"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true} + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true} ]}}`), 0o600); err != nil { t.Fatal(err) } @@ -168,8 +416,8 @@ func TestApplyPlanRequiresBothRemovalBounds(t *testing.T) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": _, _ = w.Write([]byte(`[{"type":"AWS","name":"prod","assumeRoleInfos":[ - {"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}, - {"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true} + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}, + {"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true} ]}]`)) case r.Method == http.MethodPatch: patched = true @@ -181,7 +429,7 @@ func TestApplyPlanRequiresBothRemovalBounds(t *testing.T) { planPath := filepath.Join(t.TempDir(), "payload.json") if err := os.WriteFile(planPath, []byte(`{"prod":{"type":"AWS","name":"prod","assumeRoleInfos":[ - {"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true} + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true} ]}}`), 0o600); err != nil { t.Fatal(err) } @@ -218,7 +466,7 @@ func TestApplyPlanBlocksConcurrentSetupChange(t *testing.T) { enabled = "false" } _, _ = w.Write([]byte(`[{"type":"AWS","name":"prod","assumeRoleInfos":[ - {"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":` + enabled + `} + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":` + enabled + `} ]}]`)) case r.Method == http.MethodPatch: patched = true @@ -230,7 +478,8 @@ func TestApplyPlanBlocksConcurrentSetupChange(t *testing.T) { planPath := filepath.Join(t.TempDir(), "payload.json") if err := os.WriteFile(planPath, []byte(`{"prod":{"type":"AWS","name":"prod","assumeRoleInfos":[ - {"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true} + {"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}, + {"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true} ]}}`), 0o600); err != nil { t.Fatal(err) } diff --git a/internal/app/architecture_failure_test.go b/internal/app/architecture_failure_test.go new file mode 100644 index 0000000..dccd4cc --- /dev/null +++ b/internal/app/architecture_failure_test.go @@ -0,0 +1,592 @@ +package app + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +const runP0ArchitectureFailureTests = false + +func skipUntilP0ArchitectureFixed(t *testing.T, finding string) { + t.Helper() + if !runP0ArchitectureFailureTests { + t.Skip("P0 characterization disabled until fixed: " + finding) + } +} + +func TestP0FinalGetPatchRaceRejectsConcurrentEdit(t *testing.T) { + skipUntilP0ArchitectureFixed(t, "no atomic CAS on full-list PATCH — docs/ARCHITECTURE_REVIEW.md §3, Optimistic concurrency") + + t.Run("main planned sync", func(t *testing.T) { + fake := newP0RaceForwardServer(t, 2, []map[string]any{ + { + "Cloud Setup ID": "setup-a", + "Cloud Account ID": "111111111111", + "Cloud Account Name": "existing", + "Collected?": true, + }, + { + "Cloud Setup ID": "setup-a", + "Cloud Account ID": "222222222222", + "Cloud Account Name": "planned-addition", + "Collected?": true, + }, + }) + defer fake.server.Close() + + _, err := Run(context.Background(), Config{ + Host: fake.server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + QueryID: "query-1", + Output: filepath.Join(t.TempDir(), "payload.json"), + APIPrefix: "/api", + Apply: true, + }) + assertP0ConcurrentEditRejected(t, "Run()", err, fake) + }) + + t.Run("apply-plan", func(t *testing.T) { + fake := newP0RaceForwardServer(t, 2, nil) + defer fake.server.Close() + + planPath := filepath.Join(t.TempDir(), "payload.json") + writeP0Plan(t, planPath, map[string]api.PatchPayload{ + "setup-a": { + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + p0AssumeRole("111111111111", "existing", true), + p0AssumeRole("222222222222", "planned-addition", true), + }, + }, + }) + _, err := ApplyPlan(context.Background(), ApplyPlanConfig{ + Host: fake.server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + PlanPath: planPath, + APIPrefix: "/api", + }) + assertP0ConcurrentEditRejected(t, "ApplyPlan()", err, fake) + }) + + t.Run("external ID", func(t *testing.T) { + fake := newP0RaceForwardServer(t, 2, nil) + defer fake.server.Close() + + _, err := ChangeExternalID(context.Background(), ExternalIDConfig{ + Host: fake.server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + SetupID: "setup-a", + ExternalID: "rotated-value", + Output: filepath.Join(t.TempDir(), "payload.json"), + APIPrefix: "/api", + Apply: true, + }) + assertP0ConcurrentEditRejected(t, "ChangeExternalID()", err, fake) + }) +} + +func TestP0IncompleteNonemptyNQEInventoryRequiresCompletenessProof(t *testing.T) { + t.Skip("obsolete Phase 0 premise: NQE-derived removal is retired; Phase 2a pagination completeness remains covered independently") + skipUntilP0ArchitectureFixed(t, "partial nonempty NQE inventory can become destructive intent — docs/ARCHITECTURE_REVIEW.md §2, Empty and truncated inventory") + + tests := []struct { + name string + repeatPage bool + wantOffsets []int + }{ + { + name: "exact multiple of PageLimit", + wantOffsets: []int{0, api.PageLimit}, + }, + { + name: "repeated page does not advance result window", + repeatPage: true, + wantOffsets: []int{0, api.PageLimit}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var ( + mu sync.Mutex + patchCount int + queryOffsets []int + ) + current := api.CloudAccount{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + p0AssumeRole("111111111111", "account-1", true), + p0AssumeRole("222222222222", "account-2", true), + p0AssumeRole("333333333333", "account-3", true), + p0AssumeRole("444444444444", "account-4", true), + p0AssumeRole("555555555555", "account-5", true), + }, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": + var request api.QueryRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Errorf("decode NQE request: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mu.Lock() + queryOffsets = append(queryOffsets, request.QueryOptions.Offset) + mu.Unlock() + count := api.PageLimit + if request.QueryOptions.Offset == api.PageLimit && !test.repeatPage { + count = 0 + } + items := make([]map[string]any, count) + for i := range items { + accountID := fmt.Sprintf("%012d", i+1) + items[i] = map[string]any{ + "Cloud Setup ID": "setup-a", + "Cloud Account ID": accountID, + "Cloud Account Name": "visible-account-" + accountID, + "Collected?": true, + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(api.NQEResponse{Items: items}) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]api.CloudAccount{current}) + case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/setup-a": + mu.Lock() + patchCount++ + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + _, err := Run(context.Background(), Config{ + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + QueryID: "query-1", + Output: filepath.Join(t.TempDir(), "payload.json"), + APIPrefix: "/api", + Apply: true, + AllowRemovals: true, + MaxRemovals: 10, + MaxRemovalPercent: 100, + AllowNoCandidates: true, + AllowNoOrgEvidence: true, + }) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "complete") { + t.Errorf("Run() error = %v; want inventory completeness error before destructive planning (removal ceilings deliberately allow 4 removals)", err) + } + if err != nil && strings.Contains(strings.ToLower(err.Error()), "blast-radius") { + t.Errorf("Run() failed on removal ceilings instead of inventory completeness: %v", err) + } + mu.Lock() + gotPatchCount := patchCount + gotOffsets := append([]int(nil), queryOffsets...) + mu.Unlock() + if gotPatchCount != 0 { + t.Errorf("PATCH count = %d; want 0 when inventory completeness is unproven", gotPatchCount) + } + if fmt.Sprint(gotOffsets) != fmt.Sprint(test.wantOffsets) { + t.Errorf("NQE offsets = %v; want %v", gotOffsets, test.wantOffsets) + } + }) + } +} + +func TestP0ShortFirstPageTruncationCannotBeDetectedClientSide(t *testing.T) { + t.Skip("obsolete Phase 0 premise: a plausible short NQE page still cannot prove completeness, but NQE absence-based pruning is now unreachable") +} + +func TestP0PartialMultiSetupApplyReturnsDispositionAndResumesSafely(t *testing.T) { + skipUntilP0ArchitectureFixed(t, "multi-setup PATCH has no durable partial result or safe resume — docs/ARCHITECTURE_REVIEW.md §3, Idempotency, retries, and partial failure") + + var ( + mu sync.Mutex + patchAttempts = map[string]int{} + state = map[string]api.CloudAccount{ + "setup-a": { + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{p0AssumeRole("111111111111", "a-existing", true)}, + }, + "setup-b": { + Type: "AWS", + Name: "setup-b", + AssumeRoleInfos: []api.AssumeRoleInfo{p0AssumeRole("333333333333", "b-existing", true)}, + }, + } + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": + _ = json.NewEncoder(w).Encode(api.NQEResponse{Items: []map[string]any{ + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "a-existing", "Collected?": true}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "222222222222", "Cloud Account Name": "a-addition", "Collected?": true}, + {"Cloud Setup ID": "setup-b", "Cloud Account ID": "333333333333", "Cloud Account Name": "b-existing", "Collected?": true}, + {"Cloud Setup ID": "setup-b", "Cloud Account ID": "444444444444", "Cloud Account Name": "b-addition", "Collected?": true}, + }}) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + mu.Lock() + accounts := []api.CloudAccount{ + cloneP0CloudAccount(state["setup-a"]), + cloneP0CloudAccount(state["setup-b"]), + } + mu.Unlock() + _ = json.NewEncoder(w).Encode(accounts) + case r.Method == http.MethodPatch && strings.HasPrefix(r.URL.Path, "/api/networks/network-1/cloudAccounts/"): + setupID := strings.TrimPrefix(r.URL.Path, "/api/networks/network-1/cloudAccounts/") + var payload api.PatchPayload + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("decode PATCH: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mu.Lock() + patchAttempts[setupID]++ + attempt := patchAttempts[setupID] + if setupID == "setup-b" && attempt == 1 { + mu.Unlock() + http.Error(w, "injected setup-b failure", http.StatusInternalServerError) + return + } + account := state[setupID] + account.AssumeRoleInfos = append([]api.AssumeRoleInfo(nil), payload.AssumeRoleInfos...) + state[setupID] = account + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + base := Config{ + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + QueryID: "query-1", + APIPrefix: "/api", + Apply: true, + } + firstConfig := base + firstConfig.Output = filepath.Join(t.TempDir(), "first.json") + first, firstErr := Run(context.Background(), firstConfig) + if firstErr == nil || !strings.Contains(firstErr.Error(), "setup-b") { + t.Errorf("first Run() error = %v; want injected setup-b failure", firstErr) + } + if first == nil { + t.Errorf("first Run() summary = nil; want applied=[setup-a] pending=[setup-b]") + } else { + applied, pending := p0SetupDisposition(first) + if fmt.Sprint(applied) != "[setup-a]" || fmt.Sprint(pending) != "[setup-b]" { + t.Errorf("first Run() disposition applied=%v pending=%v; want applied=[setup-a] pending=[setup-b]", applied, pending) + } + } + + secondConfig := base + secondConfig.Output = filepath.Join(t.TempDir(), "second.json") + second, secondErr := Run(context.Background(), secondConfig) + if secondErr != nil { + t.Fatalf("second Run() error = %v; want safe resume", secondErr) + } + mu.Lock() + attemptsA := patchAttempts["setup-a"] + attemptsB := patchAttempts["setup-b"] + mu.Unlock() + if attemptsA != 1 { + t.Errorf("setup-a PATCH attempts = %d; want 1 so rerun does not rewrite an already-applied setup", attemptsA) + } + if attemptsB != 2 { + t.Errorf("setup-b PATCH attempts = %d; want 2 (failed attempt plus resumed success)", attemptsB) + } + secondPatchedCount := -1 + if second != nil { + secondPatchedCount = second.PatchedSetupCount + } + if secondPatchedCount != 1 { + t.Errorf("second Run() patched_setup_count = %d; want 1 for the pending setup only", secondPatchedCount) + } +} + +func TestP0ApplyPlanDisableRequiresDestructiveAuthorization(t *testing.T) { + skipUntilP0ArchitectureFixed(t, "same-membership enabled=false bypasses destructive guards — docs/ARCHITECTURE_REVIEW.md §2, All intentional and incidental removal/disable paths") + + tests := []struct { + name string + allowRemovals bool + maxRemovals int + maxRemovalPercent float64 + allowUnattended bool + wantError string + }{ + { + name: "no destructive authorization", + wantError: "--allow-removals", + }, + { + name: "authorization without bounds", + allowRemovals: true, + wantError: "require both", + }, + { + name: "authorization and bounds", + allowRemovals: true, + maxRemovals: 2, + maxRemovalPercent: 100, + allowUnattended: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var patchCount int + current := api.CloudAccount{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + p0AssumeRole("111111111111", "account-1", true), + p0AssumeRole("222222222222", "account-2", true), + }, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + _ = json.NewEncoder(w).Encode([]api.CloudAccount{current}) + case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/setup-a": + patchCount++ + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + planPath := filepath.Join(t.TempDir(), "disable.json") + writeP0Plan(t, planPath, map[string]api.PatchPayload{ + "setup-a": { + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + p0AssumeRole("111111111111", "account-1", false), + p0AssumeRole("222222222222", "account-2", false), + }, + }, + }) + _, err := ApplyPlan(context.Background(), ApplyPlanConfig{ + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + PlanPath: planPath, + APIPrefix: "/api", + AllowRemovals: test.allowRemovals, + MaxRemovals: test.maxRemovals, + MaxRemovalPercent: test.maxRemovalPercent, + AllowUnattendedDestructive: test.allowUnattended, + }) + if test.wantError == "" { + if err != nil { + t.Errorf("ApplyPlan() error = %v; want authorized disable to proceed", err) + } + if patchCount != 1 { + t.Errorf("PATCH count = %d; want 1 for authorized disable", patchCount) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Errorf("ApplyPlan() error = %v; want destructive authorization error containing %q", err, test.wantError) + } + if patchCount != 0 { + t.Errorf("PATCH count = %d; want 0 without complete destructive authorization", patchCount) + } + }) + } +} + +type p0RaceForwardServer struct { + server *httptest.Server + + mu sync.Mutex + setup api.CloudAccount + version string + getCount int + mutateAfterGet int + patchCount int + concurrentID string + nqeItems []map[string]any + handlerAssertion error +} + +func newP0RaceForwardServer(t *testing.T, mutateAfterGet int, nqeItems []map[string]any) *p0RaceForwardServer { + t.Helper() + fake := &p0RaceForwardServer{ + setup: api.CloudAccount{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{p0AssumeRole("111111111111", "existing", true)}, + }, + version: `"version-1"`, + mutateAfterGet: mutateAfterGet, + concurrentID: "999999999999", + nqeItems: nqeItems, + } + fake.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(api.NQEResponse{Items: fake.nqeItems}) + case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": + fake.mu.Lock() + fake.getCount++ + getCount := fake.getCount + snapshot := cloneP0CloudAccount(fake.setup) + version := fake.version + fake.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", version) + _ = json.NewEncoder(w).Encode([]api.CloudAccount{snapshot}) + + if getCount == fake.mutateAfterGet { + fake.mu.Lock() + fake.setup.AssumeRoleInfos = append(fake.setup.AssumeRoleInfos, p0AssumeRole(fake.concurrentID, "concurrent-ui-addition", true)) + fake.version = `"version-2"` + fake.mu.Unlock() + } + case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/setup-a": + var payload struct { + AssumeRoleInfos []api.AssumeRoleInfo `json:"assumeRoleInfos"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + fake.mu.Lock() + fake.handlerAssertion = fmt.Errorf("decode PATCH: %w", err) + fake.mu.Unlock() + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + fake.mu.Lock() + ifMatch := r.Header.Get("If-Match") + if ifMatch != "" && ifMatch != fake.version { + fake.mu.Unlock() + http.Error(w, "revision conflict", http.StatusPreconditionFailed) + return + } + fake.setup.AssumeRoleInfos = append([]api.AssumeRoleInfo(nil), payload.AssumeRoleInfos...) + fake.patchCount++ + fake.version = `"version-3"` + fake.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + return fake +} + +func assertP0ConcurrentEditRejected(t *testing.T, operation string, err error, fake *p0RaceForwardServer) { + t.Helper() + lowerError := strings.ToLower(fmt.Sprint(err)) + conflict := err != nil && (strings.Contains(lowerError, "conflict") || + strings.Contains(lowerError, "precondition") || + strings.Contains(lowerError, "status 412") || + strings.Contains(lowerError, "changed after planning")) + if !conflict { + t.Errorf("%s error = %v; want atomic conflict after concurrent Forward edit", operation, err) + } + fake.mu.Lock() + patchCount := fake.patchCount + handlerAssertion := fake.handlerAssertion + hasConcurrent := false + for _, info := range fake.setup.AssumeRoleInfos { + if info.AccountID == fake.concurrentID { + hasConcurrent = true + break + } + } + fake.mu.Unlock() + if handlerAssertion != nil { + t.Errorf("fake Forward server assertion: %v", handlerAssertion) + } + if patchCount != 0 { + t.Errorf("%s committed PATCH count = %d; want 0 after concurrent edit", operation, patchCount) + } + if !hasConcurrent { + t.Errorf("%s clobbered concurrent account %s; want concurrent edit preserved", operation, fake.concurrentID) + } +} + +func p0AssumeRole(accountID, accountName string, enabled bool) api.AssumeRoleInfo { + return api.AssumeRoleInfo{ + AccountID: accountID, + AccountName: accountName, + RoleArn: "arn:aws:iam::" + accountID + ":role/ForwardRole", + Enabled: enabled, + } +} + +func cloneP0CloudAccount(account api.CloudAccount) api.CloudAccount { + account.AssumeRoleInfos = append([]api.AssumeRoleInfo(nil), account.AssumeRoleInfos...) + if account.Regions != nil { + regions := account.Regions + account.Regions = make(map[string]api.RegionMeta, len(account.Regions)) + for region, metadata := range regions { + account.Regions[region] = metadata + } + } + if account.RegionToProxyServerID != nil { + regionToProxy := account.RegionToProxyServerID + account.RegionToProxyServerID = make(map[string]string, len(account.RegionToProxyServerID)) + for region, proxy := range regionToProxy { + account.RegionToProxyServerID[region] = proxy + } + } + return account +} + +func writeP0Plan(t *testing.T, path string, payloads map[string]api.PatchPayload) { + t.Helper() + data, err := json.Marshal(payloads) + if err != nil { + t.Fatalf("encode plan: %v", err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } +} + +func p0SetupDisposition(summary *Summary) (applied, pending []string) { + for _, setup := range summary.PlannedSetups { + if setup.Patched { + applied = append(applied, setup.SetupID) + } else { + pending = append(pending, setup.SetupID) + } + } + sort.Strings(applied) + sort.Strings(pending) + return applied, pending +} diff --git a/internal/app/domain.go b/internal/app/domain.go new file mode 100644 index 0000000..4a62f4c --- /dev/null +++ b/internal/app/domain.go @@ -0,0 +1,334 @@ +package app + +import ( + "fmt" + "regexp" + "strings" + "time" +) + +var accountIDPattern = regexp.MustCompile(`^[0-9]{12}$`) + +// AccountID is a validated AWS account identifier. +type AccountID string + +func NewAccountID(value string) (AccountID, error) { + trimmed := strings.TrimSpace(value) + if !accountIDPattern.MatchString(trimmed) { + return "", fmt.Errorf("invalid AWS account ID %q; expected exactly 12 digits", value) + } + return AccountID(trimmed), nil +} + +func (id AccountID) String() string { + return string(id) +} + +func (id AccountID) IsZero() bool { + return id == "" +} + +// SetupID is a canonicalized setup identifier. +type SetupID string + +func NewSetupID(value string) (SetupID, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "", fmt.Errorf("setup ID is required") + } + return SetupID(trimmed), nil +} + +func (id SetupID) IsZero() bool { + return strings.TrimSpace(string(id)) == "" +} + +func SetupIDFrom(value string) SetupID { + return SetupID(strings.TrimSpace(value)) +} + +func (id SetupID) String() string { + return strings.TrimSpace(string(id)) +} + +// Partition enumerates AWS partition values used by IAM ARNs. +type Partition string + +const ( + PartitionAWS Partition = "aws" + PartitionAWSGov Partition = "aws-us-gov" + PartitionAWSCN Partition = "aws-cn" +) + +func NewPartition(value string) (Partition, error) { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed == "" { + return PartitionAWS, nil + } + switch trimmed { + case string(PartitionAWS), string(PartitionAWSGov), string(PartitionAWSCN): + return Partition(trimmed), nil + default: + return "", fmt.Errorf("invalid AWS partition %q; expected aws, aws-us-gov, or aws-cn", value) + } +} + +// RoleARN is a validated IAM role ARN. +type RoleARN struct { + value string + partition Partition + accountID AccountID + roleName string +} + +func ParseRoleARN(raw string) (RoleARN, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return RoleARN{}, fmt.Errorf("invalid IAM role ARN %q", raw) + } + parts := strings.Split(raw, ":") + if len(parts) < 6 || parts[0] != "arn" || parts[2] != "iam" { + return RoleARN{}, fmt.Errorf("invalid IAM role ARN %q", raw) + } + partition, err := NewPartition(parts[1]) + if err != nil { + return RoleARN{}, err + } + accountID, err := NewAccountID(parts[4]) + if err != nil { + return RoleARN{}, fmt.Errorf("invalid IAM role ARN %q account component: %w", raw, err) + } + rolePath := strings.Join(parts[5:], ":") + if !strings.HasPrefix(rolePath, "role/") { + return RoleARN{}, fmt.Errorf("invalid IAM role ARN %q", raw) + } + roleName := strings.TrimPrefix(rolePath, "role/") + roleName = strings.TrimSpace(roleName) + if roleName == "" { + return RoleARN{}, fmt.Errorf("invalid IAM role ARN %q", raw) + } + return RoleARN{value: raw, partition: partition, accountID: accountID, roleName: roleName}, nil +} + +func NewRoleARN(accountID AccountID, partition Partition, roleName string) (RoleARN, error) { + if accountID.IsZero() { + return RoleARN{}, fmt.Errorf("account ID is required") + } + validatedPartition, err := NewPartition(string(partition)) + if err != nil { + return RoleARN{}, err + } + r := strings.TrimSpace(roleName) + if r == "" { + return RoleARN{}, fmt.Errorf("role name is required") + } + value := fmt.Sprintf("arn:%s:iam::%s:role/%s", validatedPartition, accountID, r) + parsed, err := ParseRoleARN(value) + if err != nil { + return RoleARN{}, err + } + if parsed.AccountID() != accountID { + return RoleARN{}, fmt.Errorf("account ID mismatch in role ARN: %q vs %q", parsed.AccountID(), accountID) + } + return parsed, nil +} + +func (r RoleARN) String() string { + return strings.TrimSpace(r.value) +} + +func (r RoleARN) Partition() Partition { + return r.partition +} + +func (r RoleARN) AccountID() AccountID { + return r.accountID +} + +func (r RoleARN) RoleName() string { + return r.roleName +} + +// AccountLifecycle tracks the status of a known account row. +type AccountLifecycle string + +const ( + AccountLifecycleActive AccountLifecycle = "Active" + AccountLifecycleSuspended AccountLifecycle = "Suspended" + AccountLifecycleClosing AccountLifecycle = "Closing" + AccountLifecycleClosed AccountLifecycle = "Closed" + AccountLifecycleUnknown AccountLifecycle = "Unknown" +) + +// DesiredMembership captures expected membership in the target setup. +type DesiredMembership string + +const ( + MembershipPreserve DesiredMembership = "Preserve" + MembershipPresentEnabled DesiredMembership = "PresentEnabled" + MembershipPresentDisabled DesiredMembership = "PresentDisabled" + MembershipExplicitlyRemove DesiredMembership = "ExplicitlyRemove" +) + +// InventoryCompleteness marks source trust in row completeness. +type InventoryCompleteness int + +const ( + InventoryCompletenessUnknown InventoryCompleteness = iota + InventoryCompletenessLikelyIncomplete + InventoryCompletenessComplete +) + +func (c InventoryCompleteness) Proven() bool { + return c == InventoryCompletenessComplete +} + +// InventorySnapshot is a typed snapshot of discovered account inventory. +type InventorySnapshot struct { + Source string + NetworkID string + SnapshotID string + SnapshotTime *time.Time + OrganizationID string + SelectedSetupIDs []SetupID + ExpectedRowCount *int + ObservedRowCount int + PageLimit int + Completeness InventoryCompleteness + CompletenessReason string + DiscoveredAccounts []DiscoveredAccount + IgnoredAccounts []AccountSummary + SkippedRows []MalformedNQERowSummary + CompleteIndicator bool +} + +type MalformedNQERowSummary struct { + Row int `json:"row"` + SetupID string `json:"setup_id,omitempty"` + AccountID string `json:"account_id,omitempty"` + Reason string `json:"reason"` +} + +// DiscoveredAccount captures one discovered row in a typed inventory. +type DiscoveredAccount struct { + SetupID SetupID + AccountID AccountID + AccountName string + Lifecycle AccountLifecycle + CollectedSet bool + Collected bool + HasOrganizationalID bool + Membership DesiredMembership +} + +// SetupAccount is the typed account state used by the reconciliation engine. +type SetupAccount struct { + AccountID AccountID + AccountName string + RoleARN RoleARN + ExternalID string + Enabled bool +} + +// SetupMetadata contains the setup-level fields controlled by reconciliation. +type SetupMetadata struct { + CloudType string + ProxyServerID string + RegionToProxyServer map[string]string + Regions map[string]int64 +} + +// CurrentSetup is the typed Forward state supplied to ComputeDesired. +type CurrentSetup struct { + SetupID SetupID + Metadata SetupMetadata + Accounts []SetupAccount +} + +// DesiredSetup is the immutable target produced by ComputeDesired. +type DesiredSetup struct { + SetupID SetupID + Metadata SetupMetadata + Accounts []SetupAccount +} + +// ReconcilePolicyKind is the tag identifying how inventory affects membership. +type ReconcilePolicyKind string + +const ( + Additive ReconcilePolicyKind = "Additive" + CompleteInventory ReconcilePolicyKind = "CompleteInventory" +) + +// OrganizationEvidencePolicy records how a policy treats missing NQE +// Organizations evidence without reintroducing interacting CLI booleans. +type OrganizationEvidencePolicy string + +const ( + RequireOrganizationEvidence OrganizationEvidencePolicy = "RequireOrganizationEvidence" + AllowMissingOrganizationEvidence OrganizationEvidencePolicy = "AllowMissingOrganizationEvidence" + ReviewedAuthoritativeInventory OrganizationEvidencePolicy = "ReviewedAuthoritativeInventory" +) + +// ReconcilePolicy is a tagged reconciliation policy. PlanningInstant is +// mandatory: ComputeDesired never consults a clock or supplies a fallback. +type ReconcilePolicy struct { + Kind ReconcilePolicyKind + PlanningInstant time.Time + OrganizationEvidence OrganizationEvidencePolicy + DefaultRoleName string + UniformExternalID *string + ExternalIDByAccount map[AccountID]string +} + +// ChangeKind enumerates field-level changes emitted by ComputeDesired. +type ChangeKind string + +const ( + ChangeAdd ChangeKind = "Add" + ChangeEnable ChangeKind = "Enable" + ChangeDisable ChangeKind = "Disable" + ChangeRemove ChangeKind = "Remove" + ChangeRename ChangeKind = "Rename" + ChangeRotateExternalID ChangeKind = "RotateExternalID" + ChangeRole ChangeKind = "ChangeRole" +) + +// AccountChange contains the before/after account state for one field-level +// classification. Before is nil for Add and After is nil for Remove. +type AccountChange struct { + AccountID AccountID + Before *SetupAccount + After *SetupAccount +} + +// SetupMetadataChange classifies a setup-level field change. +type SetupMetadataChange struct { + Field string + Before any + After any +} + +// ChangeSet is the field-level diff between CurrentSetup and DesiredSetup. +// One account may appear in more than one field slice. +type ChangeSet struct { + Add []AccountChange + Enable []AccountChange + Disable []AccountChange + Remove []AccountChange + Rename []AccountChange + RotateExternalID []AccountChange + ChangeRole []AccountChange + SetupMetadata []SetupMetadataChange +} + +func (c ChangeSet) Empty() bool { + return len(c.Add) == 0 && + len(c.Enable) == 0 && + len(c.Disable) == 0 && + len(c.Remove) == 0 && + len(c.Rename) == 0 && + len(c.RotateExternalID) == 0 && + len(c.ChangeRole) == 0 && + len(c.SetupMetadata) == 0 +} diff --git a/internal/app/domain_test.go b/internal/app/domain_test.go new file mode 100644 index 0000000..9c1f2bf --- /dev/null +++ b/internal/app/domain_test.go @@ -0,0 +1,58 @@ +package app + +import ( + "testing" +) + +func TestNewAccountIDRejectsMalformedAndTrimsWhitespace(t *testing.T) { + got, err := NewAccountID(" 111111111111 ") + if err != nil { + t.Fatalf("NewAccountID() error = %v", err) + } + if got.String() != "111111111111" { + t.Fatalf("AccountID = %q", got) + } + + tests := []string{"", "123", "12345678901", "1234567890123", "123456789abc"} + for _, value := range tests { + if _, err := NewAccountID(value); err == nil { + t.Fatalf("expected malformed account-id error for %q", value) + } + } +} + +func TestNewSetupIDRejectsBlankAndTrims(t *testing.T) { + got, err := NewSetupID(" setup-a ") + if err != nil { + t.Fatalf("NewSetupID() error = %v", err) + } + if got != "setup-a" { + t.Fatalf("SetupID = %q", got) + } + + if _, err := NewSetupID(" "); err == nil { + t.Fatal("expected blank setup-id error") + } +} + +func TestNewPartitionRejectsInvalid(t *testing.T) { + if _, err := NewPartition("aws-bad"); err == nil { + t.Fatal("expected partition error") + } +} + +func TestParseRoleARNTrimsAndValidates(t *testing.T) { + role, err := ParseRoleARN(" arn:aws:iam::111111111111:role/ForwardRole ") + if err != nil { + t.Fatalf("ParseRoleARN() error = %v", err) + } + if role.String() == "" { + t.Fatal("role string is empty") + } + if role.AccountID().String() != "111111111111" { + t.Fatalf("role account = %q", role.AccountID()) + } + if role.RoleName() != "ForwardRole" { + t.Fatalf("role name = %q", role.RoleName()) + } +} diff --git a/internal/app/external_id.go b/internal/app/external_id.go index 4abcc7f..841409c 100644 --- a/internal/app/external_id.go +++ b/internal/app/external_id.go @@ -11,20 +11,23 @@ import ( ) type ExternalIDConfig struct { - Host string - Username string - Password string - NetworkID string - SetupID string - AccountIDs []string - ExternalID string - Clear bool - ExternalIDFile string - Output string - APIPrefix string - Insecure bool - Timeout time.Duration - Apply bool + Host string + Username string + Password string + NetworkID string + SetupID string + AccountIDs []string + ExternalID string + Clear bool + ExternalIDFile string + Output string + APIPrefix string + Insecure bool + Timeout time.Duration + Apply bool + ConfirmApply func(planDigest string) error + AuthorizationActor string + Unattended bool } type ExternalIDSummary struct { @@ -47,6 +50,10 @@ type ExternalIDSummary struct { Changes []ExternalIDChange `json:"changes"` Output string `json:"output"` PayloadSHA256 string `json:"payload_sha256"` + PlanDigest string `json:"plan_digest,omitempty"` + RollbackOutput string `json:"rollback_output,omitempty"` + RollbackSHA256 string `json:"rollback_sha256,omitempty"` + ResultJournalOutput string `json:"result_journal_output,omitempty"` Payload ExternalIDPatchPayload `json:"payload"` } @@ -143,7 +150,7 @@ func ChangeExternalID(ctx context.Context, cfg ExternalIDConfig) (*ExternalIDSum mode = "selected" for _, rawAccountID := range cfg.AccountIDs { accountID := strings.TrimSpace(rawAccountID) - if !awsAccountIDPattern.MatchString(accountID) { + if _, err := NewAccountID(accountID); err != nil { return nil, fmt.Errorf("invalid AWS account ID %q; expected 12 digits", rawAccountID) } if _, exists := selected[accountID]; exists { @@ -241,13 +248,67 @@ func ChangeExternalID(ctx context.Context, cfg ExternalIDConfig) (*ExternalIDSum if !cfg.Apply || changedCount == 0 { return summary, nil } - if _, _, err := writeJSONPayload(auditPath(output), payloads); err != nil { + + rollbackPayloads, err := buildRollbackPayloads(accounts, []string{setupID}) + if err != nil { + return nil, err + } + fullTarget := clonePatchPayload(rollbackPayloads[setupID]) + fullTarget.AssumeRoleInfos = append([]api.AssumeRoleInfo(nil), infos...) + typedSetupID, err := NewSetupID(setupID) + if err != nil { return nil, err } - if err := client.PatchCloudAccount(ctx, networkID, setupID, payload); err != nil { - return nil, fmt.Errorf("patch setup %s: %w", setupID, err) + changeSet, err := classifyPatchPayload(account, typedSetupID, fullTarget) + if err != nil { + return nil, fmt.Errorf("classify External ID change for setup %s: %w", setupID, err) + } + intent, err := newPayloadApplyIntent( + networkID, + output, + InventorySnapshot{ + Source: "external-id", + SelectedSetupIDs: []SetupID{typedSetupID}, + Completeness: InventoryCompletenessUnknown, + }, + ReconcilePolicy{ + Kind: Additive, + PlanningInstant: time.Now().UTC(), + OrganizationEvidence: AllowMissingOrganizationEvidence, + }, + accounts, + auditPayloads{setupID: fullTarget}, + map[string]ChangeSet{setupID: changeSet}, + ) + if err != nil { + return nil, err + } + summary.PlanDigest = intent.Digest() + if cfg.ConfirmApply != nil { + if err := cfg.ConfirmApply(intent.Digest()); err != nil { + return summary, err + } + } + actor := strings.TrimSpace(cfg.AuthorizationActor) + if actor == "" { + actor = "external-id caller" + } + applyResult, applyErr := GuardAndApply(ctx, client, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Actor: actor, + Approved: true, + Unattended: cfg.Unattended, + }) + summary.Patched = applyResult.PatchedCount > 0 + summary.RollbackOutput = applyResult.RollbackOutput + summary.RollbackSHA256 = applyResult.RollbackSHA256 + summary.ResultJournalOutput = applyResult.JournalOutput + if applyErr != nil { + if applyResult.JournalOutput != "" { + return summary, fmt.Errorf("%w; apply result journal: %s", applyErr, applyResult.JournalOutput) + } + return summary, applyErr } - summary.Patched = true return summary, nil } diff --git a/internal/app/external_id_file.go b/internal/app/external_id_file.go index 9bb7918..6dd29f3 100644 --- a/internal/app/external_id_file.go +++ b/internal/app/external_id_file.go @@ -65,13 +65,16 @@ func loadExternalIDAssignments(path, defaultSetupID string) (externalIDAssignmen setupID = record[0] offset = 1 } + if _, err := NewSetupID(setupID); err != nil { + return nil, fmt.Errorf("external ID file row %d has invalid setup_id %q: %w", row, setupID, err) + } accountID := record[offset] action := strings.ToLower(record[offset+1]) externalID := record[offset+2] if setupID == "" { return nil, fmt.Errorf("external ID file row %d has an empty setup_id", row) } - if !awsAccountIDPattern.MatchString(accountID) { + if _, err := NewAccountID(accountID); err != nil { return nil, fmt.Errorf("external ID file row %d has invalid AWS account ID %q; expected 12 digits", row, accountID) } switch action { diff --git a/internal/app/external_id_test.go b/internal/app/external_id_test.go index e900f75..5e89509 100644 --- a/internal/app/external_id_test.go +++ b/internal/app/external_id_test.go @@ -21,12 +21,12 @@ func TestChangeExternalIDSetsAndClearsWithoutNQE(t *testing.T) { "us-east-1": {TestInstant: 123}, }, AssumeRoleInfos: []api.AssumeRoleInfo{{ - AccountID: "111", + AccountID: "111111111111", AccountName: "acct-a", - RoleArn: "arn:aws:iam::111:role/ForwardRole", + RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true, }, { - AccountID: "222", + AccountID: "222222222222", AccountName: "failed-account", ErrorMsg: "role is not configured", Enabled: false, @@ -52,13 +52,18 @@ func TestChangeExternalIDSetsAndClearsWithoutNQE(t *testing.T) { if err := json.Unmarshal(data, &fields); err != nil { t.Fatalf("decode patch fields: %v", err) } - if len(fields) != 2 || fields["type"] == nil || fields["assumeRoleInfos"] == nil { - t.Fatalf("external ID PATCH changed unrelated fields: %s", string(data)) + for _, field := range []string{"type", "name", "regions", "regionToProxyServerId", "assumeRoleInfos"} { + if fields[field] == nil { + t.Fatalf("gateway External ID PATCH omitted preserved field %s: %s", field, string(data)) + } } var payload api.PatchPayload if err := json.Unmarshal(data, &payload); err != nil { t.Fatalf("decode patch: %v", err) } + if payload.Name != stored.Name || payload.Regions["us-east-1"] != 123 { + t.Fatalf("gateway External ID PATCH changed preserved setup metadata: %#v", payload) + } stored.AssumeRoleInfos = payload.AssumeRoleInfos patchCount++ w.Header().Set("Content-Type", "application/json") @@ -90,6 +95,9 @@ func TestChangeExternalIDSetsAndClearsWithoutNQE(t *testing.T) { if !setSummary.Patched || setSummary.PreviousExternalIDConfigured || !setSummary.TargetExternalIDConfigured { t.Fatalf("unexpected set summary: %#v", setSummary) } + if setSummary.PlanDigest == "" || setSummary.RollbackOutput == "" || setSummary.RollbackSHA256 == "" || setSummary.ResultJournalOutput == "" { + t.Fatalf("expected gateway digest and recovery artifacts: %#v", setSummary) + } if patchCount != 1 || stored.AssumeRoleInfos[0].ExternalID != "customer-value" || stored.AssumeRoleInfos[1].ExternalID != "customer-value" { t.Fatalf("expected set PATCH: count=%d stored=%#v", patchCount, stored.AssumeRoleInfos) } @@ -112,6 +120,69 @@ func TestChangeExternalIDSetsAndClearsWithoutNQE(t *testing.T) { } } +func TestChangeExternalIDConfirmsComputedDigestBeforeGatewayApply(t *testing.T) { + getCount := 0 + patchCount := 0 + stored := api.CloudAccount{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{{ + AccountID: "111111111111", + RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", + Enabled: true, + }}, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + getCount++ + _ = json.NewEncoder(w).Encode([]api.CloudAccount{stored}) + case http.MethodPatch: + patchCount++ + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + output := filepath.Join(t.TempDir(), "external-id.json") + confirmationCount := 0 + summary, err := ChangeExternalID(context.Background(), ExternalIDConfig{ + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + SetupID: "setup-a", + ExternalID: "rotated", + Output: output, + APIPrefix: "/api", + Apply: true, + ConfirmApply: func(planDigest string) error { + confirmationCount++ + if planDigest == "" { + t.Fatal("confirmation received an empty plan digest") + } + if _, err := os.Stat(output); err != nil { + t.Fatalf("target payload was not written before confirmation: %v", err) + } + if patchCount != 0 { + t.Fatal("PATCH occurred before digest confirmation") + } + return nil + }, + }) + if err != nil { + t.Fatalf("ChangeExternalID() error = %v", err) + } + if confirmationCount != 1 || getCount != 2 || patchCount != 1 { + t.Fatalf("confirmation/weak re-read/PATCH counts = %d/%d/%d, want 1/2/1", confirmationCount, getCount, patchCount) + } + if summary.PlanDigest == "" || summary.ResultJournalOutput == "" { + t.Fatalf("missing gateway digest or result journal: %#v", summary) + } +} + func TestChangeExternalIDRequiresOneAction(t *testing.T) { for _, cfg := range []ExternalIDConfig{ {SetupID: "setup-a"}, @@ -209,6 +280,54 @@ func TestChangeExternalIDUsesCSVSetAndClearActions(t *testing.T) { } } +func TestChangeExternalIDAcceptsPreBranchCSVArtifact(t *testing.T) { + stored := api.CloudAccount{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + { + AccountID: "111111111111", + RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", + ExternalID: "old-external-id", + Enabled: true, + }, + { + AccountID: "222222222222", + RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", + ExternalID: "remove-me", + Enabled: false, + }, + }, + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode([]api.CloudAccount{stored}) + })) + defer server.Close() + + summary, err := ChangeExternalID(context.Background(), ExternalIDConfig{ + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + SetupID: "setup-a", + ExternalIDFile: filepath.Join("testdata", "pre_branch_external_ids.csv"), + Output: filepath.Join(t.TempDir(), "payload.json"), + APIPrefix: "/api", + }) + if err != nil { + t.Fatalf("ChangeExternalID() with pre-branch CSV: %v", err) + } + if summary.Mode != "file" || summary.SetAccountCount != 1 || summary.ClearedAccountCount != 1 { + t.Fatalf("pre-branch CSV was misread: %#v", summary) + } + if got := summary.Payload.AssumeRoleInfos; got[0].ExternalID != "new-external-id" || got[1].ExternalID != "" { + t.Fatalf("pre-branch CSV values were misread: %#v", got) + } + if summary.PayloadSHA256 != "ac8741e262a2ee661839c6aee67e168557a30488b1b9f0ff37bf8c382ad69d05" { + t.Fatalf("current payload SHA-256 = %s; old binary produced ac8741e262a2ee661839c6aee67e168557a30488b1b9f0ff37bf8c382ad69d05", summary.PayloadSHA256) + } +} + func TestLoadExternalIDAssignmentsRejectsUnsafeRows(t *testing.T) { for name, contents := range map[string]string{ "blank set": "account_id,action,external_id\n111111111111,set,\n", diff --git a/internal/app/patch_chokepoint_test.go b/internal/app/patch_chokepoint_test.go new file mode 100644 index 0000000..1b0ca3f --- /dev/null +++ b/internal/app/patch_chokepoint_test.go @@ -0,0 +1,47 @@ +package app + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPatchCloudAccountProductionCallersAreChokepointed(t *testing.T) { + allowed := map[string]bool{ + filepath.Clean("internal/app/apply_gateway.go"): true, + } + counts := make(map[string]int) + err := filepath.Walk(filepath.Clean("../.."), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || filepath.Ext(path) != ".go" || strings.HasSuffix(path, "_test.go") { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + count := strings.Count(string(data), ".PatchCloudAccount(") + if count == 0 { + return nil + } + relative, err := filepath.Rel("../..", path) + if err != nil { + return err + } + relative = filepath.Clean(relative) + if !allowed[relative] { + t.Errorf("production caller of api.PatchCloudAccount outside gateway: %s", relative) + } + counts[relative] += count + return nil + }) + if err != nil { + t.Fatalf("scan production Go sources: %v", err) + } + if counts[filepath.Clean("internal/app/apply_gateway.go")] != 1 { + t.Fatalf("gateway PatchCloudAccount call count = %d, want exactly 1", counts[filepath.Clean("internal/app/apply_gateway.go")]) + } +} diff --git a/internal/app/preflight.go b/internal/app/preflight.go index bd8942d..48cc0b8 100644 --- a/internal/app/preflight.go +++ b/internal/app/preflight.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/forwardnetworks/aws-sync/internal/api" ) @@ -33,6 +34,7 @@ type PreflightCheck struct { } func Preflight(ctx context.Context, cfg Config) (*PreflightSummary, error) { + cfg = prepareReconcileConfig(cfg, time.Now().UTC()) if err := validateRemovalLimitValues(cfg.MaxRemovals, cfg.MaxRemovalPercent); err != nil { return nil, err } @@ -75,11 +77,12 @@ func Preflight(ctx context.Context, cfg Config) (*PreflightSummary, error) { return result, nil } query, queryID, parameters := queryInputs(cfg) - items, err := client.QueryAWSAccounts(ctx, cfg.NetworkID, cfg.SnapshotID, query, queryID, parameters, cfg.SetupIDs) + queryResult, err := client.QueryAWSAccountsWithMetadata(ctx, cfg.NetworkID, cfg.SnapshotID, query, queryID, parameters, cfg.SetupIDs) if err != nil { result.fail("nqe_aws_accounts", err.Error()) return result, nil } + items := queryResult.Items result.FetchedItemCount = len(items) if len(items) == 0 { result.fail("nqe_aws_accounts", "query returned no AWS account rows") @@ -92,16 +95,27 @@ func Preflight(ctx context.Context, cfg Config) (*PreflightSummary, error) { result.fail("forward_cloud_setups", err.Error()) return result, nil } + + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, nqeParseOptionsFromQueryResult(queryResult, cfg.AllowMalformedRows)) + if err != nil { + result.fail("patch_plan", err.Error()) + return result, nil + } + if len(cloudAccounts) == 0 { result.fail("forward_cloud_setups", "Forward returned no cloud account setups") } else { result.pass("forward_cloud_setups", fmt.Sprintf("Forward returned %d cloud account setups", len(cloudAccounts))) } - setupIDValues := nqeSetupIDValues(items) - awsSetups := cloudAccountMetaMap(cloudAccounts, cfg.SetupIDs) + setupIDValues := SetupIDsFromSnapshot(snapshot) + awsSetups, err := adaptCloudAccountsBySetupID(cloudAccounts, cfg.SetupIDs) + if err != nil { + result.fail("aws_account_setups", err.Error()) + return result, nil + } partitionIssues := make([]string, 0) for setupID, setup := range awsSetups { - if err := validateCloudAccountPartition(setup); err != nil { + if err := validateCloudAccountPartitionFromMetadata(setup); err != nil { partitionIssues = append(partitionIssues, fmt.Sprintf("%s: %s", setupID, err)) } } @@ -119,7 +133,12 @@ func Preflight(ctx context.Context, cfg Config) (*PreflightSummary, error) { result.pass("nqe_setup_id_differentiator", fmt.Sprintf("NQE rows include setup IDs: %s", strings.Join(setupIDValues, ", "))) } - plan, err := buildPlanForConfig(cfg, items, cloudAccounts) + planOptions, err := buildPlanOptionsFromConfig(cfg) + if err != nil { + result.fail("patch_plan", err.Error()) + return result, nil + } + plan, err := buildPlanFromSnapshot(snapshot, cloudAccounts, cfg.SetupIDs, planOptions) if err != nil { result.fail("patch_plan", err.Error()) return result, nil @@ -133,7 +152,7 @@ func Preflight(ctx context.Context, cfg Config) (*PreflightSummary, error) { if summary.IgnoredNQEItemCount > 0 { result.warn("nqe_account_id_validation", fmt.Sprintf("ignored %d NQE row(s) with invalid AWS account IDs", summary.IgnoredNQEItemCount)) } else { - result.pass("nqe_account_id_validation", "all NQE AWS account IDs are numeric") + result.pass("nqe_account_id_validation", "all NQE AWS account IDs are valid 12-digit IDs") } if plan.HasRemovals() { result.fail("account_removals", "planned account removals require review and --allow-removals for apply") @@ -204,18 +223,3 @@ func (s *PreflightSummary) fail(name, message string) { func (s *PreflightSummary) warn(name, message string) { s.Checks = append(s.Checks, PreflightCheck{Name: name, Status: "warn", Message: message}) } - -func nqeSetupIDValues(items []map[string]any) []string { - seen := make(map[string]bool) - result := make([]string, 0) - for _, item := range items { - setupID := itemSetupID(item) - if setupID == "" || seen[setupID] { - continue - } - seen[setupID] = true - result = append(result, setupID) - } - sort.Strings(result) - return result -} diff --git a/internal/app/preflight_test.go b/internal/app/preflight_test.go index dd836b2..9e8051a 100644 --- a/internal/app/preflight_test.go +++ b/internal/app/preflight_test.go @@ -9,6 +9,7 @@ import ( ) func TestPreflightReportsSetupSpecificOrgEvidenceFailures(t *testing.T) { + t.Skip("obsolete characterization: additive NQE preflight no longer produces removals, so removal-specific organization-evidence failures are unreachable") server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() if !ok || user != "alice" || pass != "secret" { @@ -19,14 +20,14 @@ func TestPreflightReportsSetupSpecificOrgEvidenceFailures(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"items":[ - {"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":false}, - {"Cloud Setup ID":"setup-b","Cloud Account ID":"222","Cloud Account Name":"acct-b","Collected?":true} + {"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":false}, + {"Cloud Setup ID":"setup-b","Cloud Account ID":"222222222222","Cloud Account Name":"acct-b","Collected?":true} ]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`[ - {"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}, - {"name":"setup-b","assumeRoleInfos":[{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true},{"accountId":"333","roleArn":"arn:aws:iam::333:role/ForwardRole","enabled":true}]} + {"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}, + {"name":"setup-b","assumeRoleInfos":[{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true},{"accountId":"333333333333","roleArn":"arn:aws:iam::333333333333:role/ForwardRole","enabled":true}]} ]`)) default: w.WriteHeader(http.StatusNotFound) @@ -46,7 +47,6 @@ func TestPreflightReportsSetupSpecificOrgEvidenceFailures(t *testing.T) { SetupIDs: nil, MaxSnapshotAge: 0, AllowNoOrgEvidence: false, - PruneMissing: true, MaxRemovals: 10, MaxRemovalPercent: 40, }) diff --git a/internal/app/reconcile.go b/internal/app/reconcile.go new file mode 100644 index 0000000..da01c6d --- /dev/null +++ b/internal/app/reconcile.go @@ -0,0 +1,402 @@ +package app + +import ( + "fmt" + "reflect" + "sort" + "strings" +) + +// ComputeDesired is the pure desired-state and diff engine. It has no clock or +// I/O fallback: callers must supply every input, including PlanningInstant. +func ComputeDesired(current CurrentSetup, snapshot InventorySnapshot, policy ReconcilePolicy) (DesiredSetup, ChangeSet, error) { + if current.SetupID.IsZero() { + return DesiredSetup{}, ChangeSet{}, fmt.Errorf("current setup ID is required") + } + if policy.PlanningInstant.IsZero() { + return DesiredSetup{}, ChangeSet{}, fmt.Errorf("reconcile policy planning instant is required") + } + switch policy.Kind { + case Additive, CompleteInventory: + default: + return DesiredSetup{}, ChangeSet{}, fmt.Errorf("invalid reconcile policy kind %q", policy.Kind) + } + if strings.EqualFold(strings.TrimSpace(snapshot.Source), "nqe") && policy.Kind == CompleteInventory { + return DesiredSetup{}, ChangeSet{}, nqeCompleteInventoryError() + } + if policy.Kind == CompleteInventory && !snapshot.Completeness.Proven() { + return DesiredSetup{}, ChangeSet{}, incompleteInventoryPolicyError(snapshot) + } + + currentByID, err := indexSetupAccounts(current.Accounts, "current setup") + if err != nil { + return DesiredSetup{}, ChangeSet{}, err + } + discoveredByID, err := discoveredAccountsForSetup(snapshot.DiscoveredAccounts, current.SetupID) + if err != nil { + return DesiredSetup{}, ChangeSet{}, err + } + + targetMembership := make(map[AccountID]DesiredMembership) + targetNames := make(map[AccountID]string) + switch policy.Kind { + case Additive: + for id, account := range discoveredByID { + targetMembership[id] = additiveMembership(account.Membership) + targetNames[id] = discoveredAccountName(account) + } + for id, account := range currentByID { + if _, exists := targetMembership[id]; exists { + continue + } + targetMembership[id] = MembershipPresentEnabled + targetNames[id] = accountName(account) + } + case CompleteInventory: + for id, account := range discoveredByID { + if account.Membership == MembershipExplicitlyRemove { + continue + } + targetMembership[id] = inventoryMembership(account.Membership) + targetNames[id] = discoveredAccountName(account) + } + } + + desiredAccounts, err := materializeDesiredAccounts(currentByID, targetMembership, targetNames, policy) + if err != nil { + return DesiredSetup{}, ChangeSet{}, err + } + desired := DesiredSetup{ + SetupID: current.SetupID, + Metadata: SetupMetadata{ + CloudType: "AWS", + ProxyServerID: strings.TrimSpace(current.Metadata.ProxyServerID), + RegionToProxyServer: cloneStringMap(current.Metadata.RegionToProxyServer), + Regions: desiredRegions(current.Metadata.Regions, policy.PlanningInstant.UnixMilli()), + }, + Accounts: sortedSetupAccounts(desiredAccounts), + } + changes := diffSetup(current, desired) + return desired, changes, nil +} + +func nqeCompleteInventoryError() error { + return fmt.Errorf("refusing CompleteInventory reconciliation for NQE observed inventory: absence cannot prove an account should be deleted; use sync-accounts with a reviewed manifest instead") +} + +func incompleteInventoryPolicyError(snapshot InventorySnapshot) error { + reason := strings.TrimSpace(snapshot.CompletenessReason) + if reason == "" { + reason = "inventory completeness is unproven" + } + return fmt.Errorf( + "refusing absence-based removals because inventory completeness is unproven: %s; observed_count=%d PageLimit=%d. Review and supply a complete authoritative manifest to sync-accounts; NQE absence cannot be used for removal", + reason, + snapshot.ObservedRowCount, + snapshot.PageLimit, + ) +} + +func indexSetupAccounts(accounts []SetupAccount, source string) (map[AccountID]SetupAccount, error) { + result := make(map[AccountID]SetupAccount, len(accounts)) + for _, account := range accounts { + if account.AccountID.IsZero() { + return nil, fmt.Errorf("%s contains an account with no ID", source) + } + if !account.RoleARN.AccountID().IsZero() && account.RoleARN.AccountID() != account.AccountID { + return nil, fmt.Errorf("%s account %s disagrees with role ARN account %s", source, account.AccountID, account.RoleARN.AccountID()) + } + if _, exists := result[account.AccountID]; exists { + return nil, fmt.Errorf("%s contains duplicate account %s", source, account.AccountID) + } + result[account.AccountID] = cloneSetupAccount(account) + } + return result, nil +} + +func discoveredAccountsForSetup(accounts []DiscoveredAccount, setupID SetupID) (map[AccountID]DiscoveredAccount, error) { + result := make(map[AccountID]DiscoveredAccount) + for _, account := range accounts { + if !account.SetupID.IsZero() && account.SetupID != setupID { + continue + } + if account.AccountID.IsZero() { + return nil, fmt.Errorf("inventory for setup %s contains an account with no ID", setupID) + } + if _, exists := result[account.AccountID]; exists { + return nil, fmt.Errorf("inventory for setup %s contains duplicate account %s", setupID, account.AccountID) + } + result[account.AccountID] = account + } + return result, nil +} + +func additiveMembership(membership DesiredMembership) DesiredMembership { + return MembershipPresentEnabled +} + +func inventoryMembership(membership DesiredMembership) DesiredMembership { + if membership == MembershipPresentDisabled { + return MembershipPresentDisabled + } + return MembershipPresentEnabled +} + +func discoveredAccountName(account DiscoveredAccount) string { + name := strings.TrimSpace(account.AccountName) + if name == "" { + return account.AccountID.String() + } + return name +} + +func accountName(account SetupAccount) string { + name := strings.TrimSpace(account.AccountName) + if name == "" { + return account.AccountID.String() + } + return name +} + +func materializeDesiredAccounts( + current map[AccountID]SetupAccount, + membership map[AccountID]DesiredMembership, + names map[AccountID]string, + policy ReconcilePolicy, +) (map[AccountID]SetupAccount, error) { + roleName := strings.TrimSpace(policy.DefaultRoleName) + partition, err := currentPartition(current) + if err != nil { + return nil, err + } + if len(membership) > 0 && roleName == "" { + return nil, fmt.Errorf("unable to determine role ARN name") + } + + currentExternalIDs := make(map[AccountID]string, len(current)) + currentExternalID := "" + currentExternalIDConsistent := true + firstExternalID := true + for id, account := range current { + value := strings.TrimSpace(account.ExternalID) + currentExternalIDs[id] = value + if firstExternalID { + currentExternalID = value + firstExternalID = false + } else if value != currentExternalID { + currentExternalIDConsistent = false + } + } + + for id := range policy.ExternalIDByAccount { + if _, exists := membership[id]; !exists { + return nil, fmt.Errorf("external ID file contains account(s) not present in the discovered inventory: %s", id) + } + } + + result := make(map[AccountID]SetupAccount, len(membership)) + missingAssignments := make([]string, 0) + for id, desiredMembership := range membership { + roleARN, err := NewRoleARN(id, partition, roleName) + if err != nil { + return nil, err + } + externalID := "" + switch { + case policy.UniformExternalID != nil: + externalID = strings.TrimSpace(*policy.UniformExternalID) + case hasExternalIDAssignment(policy.ExternalIDByAccount, id): + externalID = strings.TrimSpace(policy.ExternalIDByAccount[id]) + case currentExternalIDs[id] != "": + externalID = currentExternalIDs[id] + case currentAccountHasExternalID(current, id): + externalID = "" + case currentExternalIDConsistent: + externalID = currentExternalID + default: + missingAssignments = append(missingAssignments, id.String()) + } + result[id] = SetupAccount{ + AccountID: id, + AccountName: strings.TrimSpace(names[id]), + RoleARN: roleARN, + ExternalID: externalID, + Enabled: desiredMembership != MembershipPresentDisabled, + } + } + if len(missingAssignments) > 0 { + sort.Strings(missingAssignments) + return nil, fmt.Errorf( + "existing accounts use mixed External IDs; provide --external-id-file assignments for each new account: %s", + strings.Join(missingAssignments, ", "), + ) + } + return result, nil +} + +func currentPartition(current map[AccountID]SetupAccount) (Partition, error) { + var partition Partition + for _, account := range current { + if account.RoleARN.String() == "" { + continue + } + if partition == "" { + partition = account.RoleARN.Partition() + continue + } + if partition != account.RoleARN.Partition() { + return "", fmt.Errorf("current setup contains mixed role ARN partitions") + } + } + if partition == "" { + return PartitionAWS, nil + } + return partition, nil +} + +func hasExternalIDAssignment(assignments map[AccountID]string, id AccountID) bool { + _, ok := assignments[id] + return ok +} + +func currentAccountHasExternalID(current map[AccountID]SetupAccount, id AccountID) bool { + _, ok := current[id] + return ok +} + +func desiredRegions(current map[string]int64, planningInstant int64) map[string]int64 { + result := make(map[string]int64, len(current)) + for region, testInstant := range current { + if testInstant == 0 { + result[region] = planningInstant + continue + } + result[region] = testInstant + } + return result +} + +func sortedSetupAccounts(accounts map[AccountID]SetupAccount) []SetupAccount { + ids := make([]AccountID, 0, len(accounts)) + for id := range accounts { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + result := make([]SetupAccount, 0, len(ids)) + for _, id := range ids { + result = append(result, cloneSetupAccount(accounts[id])) + } + return result +} + +func diffSetup(current CurrentSetup, desired DesiredSetup) ChangeSet { + currentByID, _ := indexSetupAccounts(current.Accounts, "current setup") + desiredByID, _ := indexSetupAccounts(desired.Accounts, "desired setup") + ids := make([]AccountID, 0, len(currentByID)+len(desiredByID)) + seen := make(map[AccountID]bool, len(currentByID)+len(desiredByID)) + for id := range currentByID { + seen[id] = true + ids = append(ids, id) + } + for id := range desiredByID { + if !seen[id] { + ids = append(ids, id) + } + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + + var changes ChangeSet + for _, id := range ids { + before, beforeOK := currentByID[id] + after, afterOK := desiredByID[id] + switch { + case !beforeOK: + changes.Add = append(changes.Add, accountChange(id, nil, &after)) + case !afterOK: + changes.Remove = append(changes.Remove, accountChange(id, &before, nil)) + default: + if !before.Enabled && after.Enabled { + changes.Enable = append(changes.Enable, accountChange(id, &before, &after)) + } + if before.Enabled && !after.Enabled { + changes.Disable = append(changes.Disable, accountChange(id, &before, &after)) + } + if strings.TrimSpace(before.AccountName) != strings.TrimSpace(after.AccountName) { + changes.Rename = append(changes.Rename, accountChange(id, &before, &after)) + } + if strings.TrimSpace(before.ExternalID) != strings.TrimSpace(after.ExternalID) { + changes.RotateExternalID = append(changes.RotateExternalID, accountChange(id, &before, &after)) + } + if before.RoleARN.String() != after.RoleARN.String() { + changes.ChangeRole = append(changes.ChangeRole, accountChange(id, &before, &after)) + } + } + } + + if strings.TrimSpace(current.Metadata.CloudType) != strings.TrimSpace(desired.Metadata.CloudType) { + changes.SetupMetadata = append(changes.SetupMetadata, SetupMetadataChange{ + Field: "cloudType", Before: current.Metadata.CloudType, After: desired.Metadata.CloudType, + }) + } + if strings.TrimSpace(current.Metadata.ProxyServerID) != strings.TrimSpace(desired.Metadata.ProxyServerID) { + changes.SetupMetadata = append(changes.SetupMetadata, SetupMetadataChange{ + Field: "proxyServerId", Before: current.Metadata.ProxyServerID, After: desired.Metadata.ProxyServerID, + }) + } + if !reflect.DeepEqual(current.Metadata.RegionToProxyServer, desired.Metadata.RegionToProxyServer) { + changes.SetupMetadata = append(changes.SetupMetadata, SetupMetadataChange{ + Field: "regionToProxyServerId", + Before: cloneStringMap(current.Metadata.RegionToProxyServer), + After: cloneStringMap(desired.Metadata.RegionToProxyServer), + }) + } + if !reflect.DeepEqual(current.Metadata.Regions, desired.Metadata.Regions) { + changes.SetupMetadata = append(changes.SetupMetadata, SetupMetadataChange{ + Field: "regions", Before: cloneInt64Map(current.Metadata.Regions), After: cloneInt64Map(desired.Metadata.Regions), + }) + } + return changes +} + +func accountChange(id AccountID, before, after *SetupAccount) AccountChange { + change := AccountChange{AccountID: id} + if before != nil { + value := cloneSetupAccount(*before) + change.Before = &value + } + if after != nil { + value := cloneSetupAccount(*after) + change.After = &value + } + return change +} + +func cloneSetupAccount(account SetupAccount) SetupAccount { + return account +} + +func cloneStringMap(values map[string]string) map[string]string { + if values == nil { + return nil + } + result := make(map[string]string, len(values)) + for key, value := range values { + result[key] = value + } + return result +} + +func cloneInt64Map(values map[string]int64) map[string]int64 { + if values == nil { + return nil + } + result := make(map[string]int64, len(values)) + for key, value := range values { + result[key] = value + } + return result +} diff --git a/internal/app/reconcile_test.go b/internal/app/reconcile_test.go new file mode 100644 index 0000000..a7de992 --- /dev/null +++ b/internal/app/reconcile_test.go @@ -0,0 +1,246 @@ +package app + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +func TestComputeDesiredIsDeterministicAndClassifiesFieldChanges(t *testing.T) { + planningInstant := time.Date(2026, time.July, 25, 12, 34, 56, 0, time.UTC) + current := CurrentSetup{ + SetupID: SetupID("setup-a"), + Metadata: SetupMetadata{ + CloudType: "AWS", + ProxyServerID: "proxy-a", + RegionToProxyServer: map[string]string{"us-east-1": "proxy-a"}, + Regions: map[string]int64{"us-east-1": 0}, + }, + Accounts: []SetupAccount{ + reconcileTestAccount(t, "111111111111", "old-name", "OldRole", "old-external", false), + reconcileTestAccount(t, "333333333333", "remove-me", "OldRole", "old-external", true), + }, + } + snapshot := InventorySnapshot{ + Completeness: InventoryCompletenessComplete, + DiscoveredAccounts: []DiscoveredAccount{ + {SetupID: SetupID("setup-a"), AccountID: AccountID("111111111111"), AccountName: "new-name"}, + {SetupID: SetupID("setup-a"), AccountID: AccountID("222222222222"), AccountName: "add-me"}, + }, + } + uniformExternalID := "new-external" + policy := ReconcilePolicy{ + Kind: CompleteInventory, + PlanningInstant: planningInstant, + DefaultRoleName: "NewRole", + UniformExternalID: &uniformExternalID, + } + + firstDesired, firstChanges, err := ComputeDesired(current, snapshot, policy) + if err != nil { + t.Fatalf("ComputeDesired() error = %v", err) + } + secondDesired, secondChanges, err := ComputeDesired(current, snapshot, policy) + if err != nil { + t.Fatalf("second ComputeDesired() error = %v", err) + } + if !reflect.DeepEqual(firstDesired, secondDesired) || !reflect.DeepEqual(firstChanges, secondChanges) { + t.Fatalf("identical inputs produced different output:\nfirst=%#v %#v\nsecond=%#v %#v", firstDesired, firstChanges, secondDesired, secondChanges) + } + if got := current.Metadata.Regions["us-east-1"]; got != 0 { + t.Fatalf("ComputeDesired mutated current regions: %d", got) + } + if got := firstDesired.Metadata.Regions["us-east-1"]; got != planningInstant.UnixMilli() { + t.Fatalf("desired planning instant = %d, want %d", got, planningInstant.UnixMilli()) + } + if len(firstChanges.Add) != 1 || + len(firstChanges.Enable) != 1 || + len(firstChanges.Remove) != 1 || + len(firstChanges.Rename) != 1 || + len(firstChanges.RotateExternalID) != 1 || + len(firstChanges.ChangeRole) != 1 || + len(firstChanges.SetupMetadata) != 1 { + t.Fatalf("unexpected field-level changes: %#v", firstChanges) + } + if len(firstChanges.Disable) != 0 { + t.Fatalf("unexpected disable changes: %#v", firstChanges.Disable) + } +} + +func TestComputeDesiredCompletenessInvariantByPolicy(t *testing.T) { + current := CurrentSetup{ + SetupID: SetupID("setup-a"), + Metadata: SetupMetadata{ + CloudType: "AWS", + }, + Accounts: []SetupAccount{ + reconcileTestAccount(t, "111111111111", "keep", "ForwardRole", "", true), + reconcileTestAccount(t, "222222222222", "missing", "ForwardRole", "", true), + }, + } + snapshot := InventorySnapshot{ + Completeness: InventoryCompletenessLikelyIncomplete, + CompletenessReason: "repeated page", + ObservedRowCount: api.PageLimit, + PageLimit: api.PageLimit, + DiscoveredAccounts: []DiscoveredAccount{{ + SetupID: SetupID("setup-a"), AccountID: AccountID("111111111111"), AccountName: "keep", + }}, + } + base := ReconcilePolicy{ + PlanningInstant: time.Unix(123, 0).UTC(), + DefaultRoleName: "ForwardRole", + } + + complete := base + complete.Kind = CompleteInventory + if _, _, err := ComputeDesired(current, snapshot, complete); err == nil || !strings.Contains(err.Error(), "inventory completeness is unproven") { + t.Fatalf("CompleteInventory error = %v; want completeness refusal", err) + } + + additive := base + additive.Kind = Additive + desired, changes, err := ComputeDesired(current, snapshot, additive) + if err != nil { + t.Fatalf("Additive error = %v", err) + } + if len(desired.Accounts) != 2 || len(changes.Remove) != 0 { + t.Fatalf("Additive removed missing accounts: desired=%#v changes=%#v", desired, changes) + } +} + +func TestComputeDesiredRejectsCompleteInventoryForNQESource(t *testing.T) { + current := CurrentSetup{ + SetupID: SetupID("setup-a"), + Metadata: SetupMetadata{CloudType: "AWS"}, + Accounts: []SetupAccount{ + reconcileTestAccount(t, "111111111111", "keep", "ForwardRole", "", true), + reconcileTestAccount(t, "222222222222", "would-be-removed", "ForwardRole", "", true), + }, + } + snapshot := InventorySnapshot{ + Source: "nqe", + Completeness: InventoryCompletenessComplete, + DiscoveredAccounts: []DiscoveredAccount{{ + SetupID: SetupID("setup-a"), AccountID: AccountID("111111111111"), AccountName: "keep", + }}, + } + policy := NewAuthoritativeManifestReconcilePolicy(time.Unix(123, 0).UTC()) + + _, _, err := ComputeDesired(current, snapshot, policy) + if err == nil || !strings.Contains(err.Error(), "refusing CompleteInventory reconciliation for NQE observed inventory") { + t.Fatalf("ComputeDesired() error = %v; want NQE CompleteInventory refusal", err) + } +} + +func TestReconcilePolicyConstructorsKeepNQEAdditiveAndManifestComplete(t *testing.T) { + instant := time.Unix(123, 0).UTC() + + for _, allowNoOrg := range []bool{false, true} { + policy := NewNQEReconcilePolicy(allowNoOrg, instant) + wantEvidence := RequireOrganizationEvidence + if allowNoOrg { + wantEvidence = AllowMissingOrganizationEvidence + } + if policy.Kind != Additive || policy.OrganizationEvidence != wantEvidence || !policy.PlanningInstant.Equal(instant) { + t.Fatalf("unexpected NQE policy: %#v", policy) + } + } + + manifest := NewAuthoritativeManifestReconcilePolicy(instant) + if manifest.Kind != CompleteInventory || manifest.OrganizationEvidence != ReviewedAuthoritativeInventory || !manifest.PlanningInstant.Equal(instant) { + t.Fatalf("unexpected manifest policy: %#v", manifest) + } +} + +func TestBuildPlanRejectsCrossSetupMoveUntilApplyIsAtomic(t *testing.T) { + items := []map[string]any{ + {"Cloud Setup ID": "setup-b", "Cloud Account ID": "111111111111", "Cloud Account Name": "move-me"}, + {"Cloud Setup ID": "setup-b", "Cloud Account ID": "222222222222", "Cloud Account Name": "stay"}, + } + cloudAccounts := []api.CloudAccount{ + { + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{{ + AccountID: "111111111111", AccountName: "move-me", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true, + }}, + }, + { + Type: "AWS", + Name: "setup-b", + AssumeRoleInfos: []api.AssumeRoleInfo{{ + AccountID: "222222222222", AccountName: "stay", RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: true, + }}, + }, + } + if _, err := buildPlan(items, cloudAccounts, "", nil); err == nil || !strings.Contains(err.Error(), "refusing cross-setup move") { + t.Fatalf("buildPlan() error = %v; want non-atomic move refusal", err) + } +} + +func TestBuildPlanOmitsPayloadForEmptyChangeSet(t *testing.T) { + items := []map[string]any{{ + "Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "account-a", + }} + cloudAccounts := []api.CloudAccount{{ + Type: "AWS", + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{{ + AccountID: "111111111111", AccountName: "account-a", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true, + }}, + }} + plan, err := buildPlan(items, cloudAccounts, "", nil) + if err != nil { + t.Fatalf("buildPlan() error = %v", err) + } + if len(plan.Setups) != 1 || !plan.Setups[0].ChangeSet.Empty() { + t.Fatalf("unexpected no-op plan: %#v", plan) + } + if len(plan.Payloads) != 0 { + t.Fatalf("empty ChangeSet emitted payload: %#v", plan.Payloads) + } + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + Completeness: InventoryCompletenessComplete, + }) + if err != nil { + t.Fatalf("parse snapshot: %v", err) + } + cfg := Config{ + NetworkID: "network-1", + Policy: ReconcilePolicy{ + Kind: CompleteInventory, + PlanningInstant: time.Unix(1, 0).UTC(), + }, + } + intent, err := newApplyIntent(cfg, snapshot, cloudAccounts, plan, t.TempDir()+"/payload.json") + if err != nil { + t.Fatalf("newApplyIntent() error = %v", err) + } + result, err := GuardAndApply(context.Background(), nil, intent, ApplyAuthorization{ + PlanDigest: intent.Digest(), + Approved: true, + }) + if err != nil || result.PatchedCount != 0 { + t.Fatalf("empty ChangeSet apply = (%d, %v), want no mutation", result.PatchedCount, err) + } +} + +func reconcileTestAccount(t *testing.T, accountID, name, roleName, externalID string, enabled bool) SetupAccount { + t.Helper() + id, err := NewAccountID(accountID) + if err != nil { + t.Fatal(err) + } + roleARN, err := NewRoleARN(id, PartitionAWS, roleName) + if err != nil { + t.Fatal(err) + } + return SetupAccount{ + AccountID: id, AccountName: name, RoleARN: roleARN, ExternalID: externalID, Enabled: enabled, + } +} diff --git a/internal/app/removal_limits_test.go b/internal/app/removal_limits_test.go index 4da14e6..d6eead6 100644 --- a/internal/app/removal_limits_test.go +++ b/internal/app/removal_limits_test.go @@ -44,8 +44,8 @@ func TestPatchPlanRemovalStatsUseCurrentConfiguredCounts(t *testing.T) { plan := &patchPlan{Setups: []plannedSetup{ { SetupID: "setup-a", - CurrentAccounts: []accountRow{{AccountID: "111"}, {AccountID: "222"}}, - RemovedAccounts: []accountRow{{AccountID: "222"}}, + CurrentAccounts: []accountRow{{AccountID: "111111111111"}, {AccountID: "222222222222"}}, + RemovedAccounts: []accountRow{{AccountID: "222222222222"}}, }, }} stats := plan.removalStats() diff --git a/internal/app/run.go b/internal/app/run.go index 08035f3..d3571aa 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -46,33 +46,87 @@ const ( ) type Config struct { - Host string - Username string - Password string - NetworkID string - SnapshotID string - Query string - QueryID string - QuerySetupParam string - SetupIDs []string - Output string - ManualOutput string - APIPrefix string - Insecure bool - Timeout time.Duration - Apply bool - AllowRemovals bool - MaxRemovals int - MaxRemovalPercent float64 - AllowNoCandidates bool - AllowNoOrgEvidence bool - PruneMissing bool - MaxSnapshotAge time.Duration - ExternalIDFile string - Source string - AuthoritativeInput bool - PinSnapshot bool - ExpectedPayloadSHA256 string + Host string + Username string + Password string + NetworkID string + SnapshotID string + Query string + QueryID string + QuerySetupParam string + SetupIDs []string + Output string + ManualOutput string + APIPrefix string + Insecure bool + Timeout time.Duration + Apply bool + AllowRemovals bool + MaxRemovals int + MaxRemovalPercent float64 + AllowNoCandidates bool + AllowNoOrgEvidence bool + MaxSnapshotAge time.Duration + ExternalIDFile string + Source string + AuthoritativeInput bool + Policy ReconcilePolicy + PinSnapshot bool + ExpectedPayloadSHA256 string + ExpectedPlanDigest string + AllowMalformedRows bool + Unattended bool + AllowUnattendedDestructive bool + AuthorizationActor string +} + +// NewNQEReconcilePolicy returns the only policy permitted for observed NQE +// inventory. NQE absence is not an account-lifecycle signal. +func NewNQEReconcilePolicy(allowNoOrgEvidence bool, planningInstant time.Time) ReconcilePolicy { + evidence := RequireOrganizationEvidence + if allowNoOrgEvidence { + evidence = AllowMissingOrganizationEvidence + } + return ReconcilePolicy{ + Kind: Additive, + PlanningInstant: planningInstant, + OrganizationEvidence: evidence, + } +} + +// NewAuthoritativeManifestReconcilePolicy returns complete-inventory semantics +// for an explicitly reviewed account manifest. +func NewAuthoritativeManifestReconcilePolicy(planningInstant time.Time) ReconcilePolicy { + return ReconcilePolicy{ + Kind: CompleteInventory, + PlanningInstant: planningInstant, + OrganizationEvidence: ReviewedAuthoritativeInventory, + } +} + +func prepareReconcileConfig(cfg Config, planningInstant time.Time) Config { + if cfg.Policy.Kind == "" { + if cfg.AuthoritativeInput { + cfg.Policy = NewAuthoritativeManifestReconcilePolicy(planningInstant) + } else { + cfg.Policy = NewNQEReconcilePolicy(cfg.AllowNoOrgEvidence, planningInstant) + } + } else { + if cfg.Policy.PlanningInstant.IsZero() { + cfg.Policy.PlanningInstant = planningInstant + } + if cfg.Policy.OrganizationEvidence == "" { + cfg.Policy.OrganizationEvidence = RequireOrganizationEvidence + if cfg.AuthoritativeInput { + cfg.Policy.OrganizationEvidence = ReviewedAuthoritativeInventory + } else if cfg.AllowNoOrgEvidence { + cfg.Policy.OrganizationEvidence = AllowMissingOrganizationEvidence + } + } else if cfg.AllowNoOrgEvidence && cfg.Policy.OrganizationEvidence == RequireOrganizationEvidence { + cfg.Policy.OrganizationEvidence = AllowMissingOrganizationEvidence + } + } + return cfg } type Summary struct { @@ -102,10 +156,14 @@ type Summary struct { ManualPayloads map[string][]api.AssumeRoleInfo `json:"manual_payloads,omitempty"` RollbackOutput string `json:"rollback_output,omitempty"` RollbackSHA256 string `json:"rollback_sha256,omitempty"` + PlanDigest string `json:"plan_digest,omitempty"` + ResultJournalOutput string `json:"result_journal_output,omitempty"` + ApplyJournal *ApplyJournal `json:"apply_journal,omitempty"` Apply bool `json:"apply"` FetchedItemCount int `json:"fetched_item_count"` IgnoredNQEItemCount int `json:"ignored_nqe_item_count,omitempty"` IgnoredNQEAccounts []AccountSummary `json:"ignored_nqe_accounts,omitempty"` + SkippedNQERows []MalformedNQERowSummary `json:"skipped_nqe_rows,omitempty"` PlannedSetupCount int `json:"planned_setup_count"` PatchedSetupCount int `json:"patched_setup_count"` SkippedSetupCount int `json:"skipped_setup_count"` @@ -113,6 +171,7 @@ type Summary struct { SkippedSetups []SkipSummary `json:"skipped_setups,omitempty"` CandidateCheck []CandidateCheck `json:"candidate_check,omitempty"` RemovalBlocked bool `json:"removal_blocked,omitempty"` + RemovalBlockReason string `json:"removal_block_reason,omitempty"` } type CandidateCheck struct { @@ -147,8 +206,11 @@ type SetupSummary struct { AddedAccounts []AccountSummary `json:"added_accounts,omitempty"` RemovedAccounts []AccountSummary `json:"removed_accounts,omitempty"` ReenabledAccounts []AccountSummary `json:"reenabled_accounts,omitempty"` + DisabledAccounts []AccountSummary `json:"disabled_accounts,omitempty"` UnchangedAccountCount int `json:"unchanged_account_count"` Patched bool `json:"patched"` + ApplyStatus ApplyStatus `json:"apply_status"` + ApplyError string `json:"apply_error,omitempty"` } type AccountSummary struct { @@ -213,6 +275,7 @@ type AWSOrganizationConfig struct { } func Run(ctx context.Context, cfg Config) (*Summary, error) { + cfg = prepareReconcileConfig(cfg, time.Now().UTC()) if err := validateRemovalLimitValues(cfg.MaxRemovals, cfg.MaxRemovalPercent); err != nil { return nil, err } @@ -236,7 +299,7 @@ func Run(ctx context.Context, cfg Config) (*Summary, error) { return nil, err } query, queryID, parameters := queryInputs(cfg) - items, err := client.QueryAWSAccounts(ctx, cfg.NetworkID, cfg.SnapshotID, query, queryID, parameters, cfg.SetupIDs) + queryResult, err := client.QueryAWSAccountsWithMetadata(ctx, cfg.NetworkID, cfg.SnapshotID, query, queryID, parameters, cfg.SetupIDs) if err != nil { return nil, err } @@ -244,7 +307,14 @@ func Run(ctx context.Context, cfg Config) (*Summary, error) { if err != nil { return nil, err } - return runPlannedSync(ctx, cfg, client, items, cloudAccounts) + snapshot, err := parseNQESnapshotFromMapsWithOptions(queryResult.Items, nqeParseOptionsFromQueryResult(queryResult, cfg.AllowMalformedRows)) + if err != nil { + return nil, err + } + snapshot.Source = "nqe" + snapshot.NetworkID = cfg.NetworkID + snapshot.SnapshotID = cfg.SnapshotID + return runPlannedSyncFromSnapshot(ctx, cfg, client, snapshot, cloudAccounts) } func runPlannedSync( @@ -254,7 +324,32 @@ func runPlannedSync( items []map[string]any, cloudAccounts []api.CloudAccount, ) (*Summary, error) { - plan, err := buildPlanForConfig(cfg, items, cloudAccounts) + cfg = prepareReconcileConfig(cfg, time.Now().UTC()) + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + AllowMalformedRows: cfg.AllowMalformedRows, + Completeness: InventoryCompletenessComplete, + }) + if err != nil { + return nil, err + } + snapshot.Source = "nqe" + snapshot.NetworkID = cfg.NetworkID + snapshot.SnapshotID = cfg.SnapshotID + return runPlannedSyncFromSnapshot(ctx, cfg, client, snapshot, cloudAccounts) +} + +func runPlannedSyncFromSnapshot( + ctx context.Context, + cfg Config, + client *api.Client, + snapshot *InventorySnapshot, + cloudAccounts []api.CloudAccount, +) (*Summary, error) { + planOptions, err := buildPlanOptionsFromConfig(cfg) + if err != nil { + return nil, err + } + plan, err := buildPlanFromSnapshot(snapshot, cloudAccounts, cfg.SetupIDs, planOptions) if err != nil { return nil, err } @@ -292,6 +387,10 @@ func runPlannedSync( } manualPayloadsForSummary = manualPayloads } + intent, err := newApplyIntent(cfg, snapshot, cloudAccounts, plan, outputPath) + if err != nil { + return nil, err + } summary := buildSummary( cfg, @@ -300,77 +399,46 @@ func runPlannedSync( manualOutputPath, manualPayloadSHA256, manualPayloadsForSummary, - len(items), + snapshot.ObservedRowCount, plan, 0, ) - if cfg.Apply && plan.HasRemovals() && !cfg.AllowRemovals { - summary.RemovalBlocked = true - return summary, fmt.Errorf("planned account removals require --allow-removals") - } - if cfg.Apply { - if err := requireRemovalBounds(plan.removalStats(), cfg.MaxRemovals, cfg.MaxRemovalPercent); err != nil { - summary.RemovalBlocked = true - return summary, err - } - if err := validateRemovalStats(plan.removalStats(), cfg.MaxRemovals, cfg.MaxRemovalPercent); err != nil { + summary.PlanDigest = intent.Digest() + if !cfg.Apply { + if blockErr := unattendedDestructiveApplyError(intent.state, cfg.Unattended, cfg.AllowUnattendedDestructive); blockErr != nil { summary.RemovalBlocked = true - return summary, err + summary.RemovalBlockReason = blockErr.Error() } + return summary, nil } - if cfg.Apply && !cfg.AuthoritativeInput && plan.HasCandidateRemovalRisk() && !cfg.AllowNoCandidates { - summary.RemovalBlocked = true - return summary, fmt.Errorf("planned removals with no uncollected candidate accounts visible require --allow-no-candidates") - } - if cfg.Apply && !cfg.AuthoritativeInput && plan.HasGovCloudRemovalsWithoutOrganizationEvidence() { - summary.RemovalBlocked = true - return summary, fmt.Errorf("GovCloud account removals require positive AWS Organizations evidence; use sync-accounts with an authoritative reviewed manifest when Organizations is unavailable") + approvedDigest := intent.Digest() + if strings.TrimSpace(cfg.ExpectedPlanDigest) != "" { + approvedDigest = strings.TrimSpace(cfg.ExpectedPlanDigest) } - if cfg.Apply && !cfg.AuthoritativeInput && plan.HasNoOrganizationEvidenceForRemovals() && !cfg.AllowNoOrgEvidence { - missingSetups := strings.Join(plan.setupsWithoutOrganizationEvidenceForRemovals(), ", ") - summary.RemovalBlocked = true - return summary, fmt.Errorf("planned removals with no AWS Organizations evidence in NQE for setup(s): %s require --allow-no-org-evidence", missingSetups) - } - - rollbackOutputPath := "" - rollbackSHA256 := "" - if cfg.Apply { - rollbackPayloads, err := buildRollbackPayloads(cloudAccounts, selectedSetupIDs(plan.Setups)) - if err != nil { - return summary, err - } - rollbackOutputPath = rollbackPath(outputPath) - rollbackSHA256, err = writeAuditPayloads(rollbackOutputPath, rollbackPayloads) - if err != nil { - return summary, fmt.Errorf("write pre-apply rollback payload: %w", err) - } - summary.RollbackOutput = rollbackOutputPath - summary.RollbackSHA256 = rollbackSHA256 - if err := verifyCloudAccountsUnchanged(ctx, client, cfg.NetworkID, selectedSetupIDs(plan.Setups), rollbackPayloads); err != nil { - return summary, err - } - if _, err := writeAuditPayloads(auditPath(outputPath), plan.Payloads); err != nil { - return nil, err - } - } - patchedCount, err := applyPlan(ctx, cfg, client, plan) - if err != nil { - return nil, err + actor := strings.TrimSpace(cfg.AuthorizationActor) + if actor == "" { + if cfg.Unattended { + actor = "unattended app caller" + } else { + actor = "attended app caller" + } + } + applyResult, applyErr := GuardAndApply(ctx, client, intent, ApplyAuthorization{ + PlanDigest: approvedDigest, + Actor: actor, + Approved: true, + AllowDestructive: cfg.AllowRemovals, + MaxRemovals: cfg.MaxRemovals, + MaxRemovalPercent: cfg.MaxRemovalPercent, + AllowNoCandidates: cfg.AllowNoCandidates, + Unattended: cfg.Unattended, + AllowUnattendedDestructive: cfg.AllowUnattendedDestructive, + }) + applyResultToSummary(summary, applyResult) + if applyErr != nil && applyResult.JournalOutput != "" { + return summary, fmt.Errorf("%w; apply result journal: %s", applyErr, applyResult.JournalOutput) } - result := buildSummary( - cfg, - outputPath, - payloadSHA256, - manualOutputPath, - manualPayloadSHA256, - manualPayloadsForSummary, - len(items), - plan, - patchedCount, - ) - result.RollbackOutput = rollbackOutputPath - result.RollbackSHA256 = rollbackSHA256 - return result, nil + return summary, applyErr } func RunAWSOrganizations(ctx context.Context, cfg AWSOrganizationConfig, source AWSOrganizationSource) (*Summary, error) { @@ -752,24 +820,42 @@ func defaultManualOutputPath() string { } func validateSnapshotFreshness(ctx context.Context, client *api.Client, cfg Config) error { - if cfg.MaxSnapshotAge <= 0 || strings.TrimSpace(cfg.SnapshotID) != "" { + if cfg.MaxSnapshotAge <= 0 { return nil } + explicitSnapshotID := strings.TrimSpace(cfg.SnapshotID) + if explicitSnapshotID != "" { + snapshots, err := client.ListSnapshots(ctx, cfg.NetworkID) + if err != nil { + return fmt.Errorf("check explicit snapshot freshness: %w", err) + } + for _, snapshot := range snapshots { + if strings.TrimSpace(snapshot.ID) != explicitSnapshotID { + continue + } + return validateSnapshotAge(snapshot, cfg.MaxSnapshotAge, "explicit") + } + return fmt.Errorf("check explicit snapshot freshness: snapshot %s was not found in network %s", explicitSnapshotID, cfg.NetworkID) + } latest, err := client.LatestProcessedSnapshot(ctx, cfg.NetworkID) if err != nil { return fmt.Errorf("check latest processed snapshot freshness: %w", err) } - snapshotTime, err := snapshotTimestamp(*latest) + return validateSnapshotAge(*latest, cfg.MaxSnapshotAge, "latest processed") +} + +func validateSnapshotAge(snapshot api.SnapshotInfo, maxAge time.Duration, description string) error { + age, err := checkedSnapshotAge(snapshot, description) if err != nil { - return fmt.Errorf("check latest processed snapshot freshness: %w", err) + return err } - age := time.Since(snapshotTime) - if age > cfg.MaxSnapshotAge { + if age > maxAge { return fmt.Errorf( - "latest processed snapshot %s is stale: age %s exceeds max %s; pass --snapshot-id or increase --max-snapshot-age", - latest.ID, + "%s snapshot %s is stale: age %s exceeds max %s", + description, + snapshot.ID, age.Round(time.Second), - cfg.MaxSnapshotAge, + maxAge, ) } return nil @@ -781,11 +867,10 @@ func pinLatestProcessedSnapshot(ctx context.Context, client *api.Client, cfg *Co return fmt.Errorf("pin latest processed snapshot: %w", err) } if cfg.MaxSnapshotAge > 0 { - snapshotTime, err := snapshotTimestamp(*latest) + age, err := checkedSnapshotAge(*latest, "latest processed") if err != nil { - return fmt.Errorf("check latest processed snapshot freshness: %w", err) + return err } - age := time.Since(snapshotTime) if age > cfg.MaxSnapshotAge { return fmt.Errorf( "latest processed snapshot %s is stale: age %s exceeds max %s; pass --snapshot-id or increase --max-snapshot-age", @@ -799,6 +884,36 @@ func pinLatestProcessedSnapshot(ctx context.Context, client *api.Client, cfg *Co return nil } +// snapshotClockSkewTolerance is how far ahead of the local clock a snapshot +// timestamp may be before it is treated as bad data rather than clock drift. +// Forward and this host are different machines, so small skew is expected and +// must not fail a run; anything beyond this indicates a real problem. +const snapshotClockSkewTolerance = 5 * time.Minute + +func checkedSnapshotAge(snapshot api.SnapshotInfo, description string) (time.Duration, error) { + snapshotTime, err := snapshotTimestamp(snapshot) + if err != nil { + return 0, fmt.Errorf("check %s snapshot freshness: %w", description, err) + } + age := time.Since(snapshotTime) + if age < -snapshotClockSkewTolerance { + return 0, fmt.Errorf( + "%s snapshot %s has invalid future timestamp %s (%s ahead of the local clock, tolerance %s)", + description, + snapshot.ID, + snapshotTime.Format(time.RFC3339), + (-age).Round(time.Second), + snapshotClockSkewTolerance, + ) + } + if age < 0 { + // Within tolerance: ordinary NTP drift between this host and Forward. + // Treat as freshly processed rather than failing the run. + age = 0 + } + return age, nil +} + func snapshotTimestamp(snapshot api.SnapshotInfo) (time.Time, error) { for _, value := range []string{snapshot.ProcessedAt, snapshot.CreatedAt} { value = strings.TrimSpace(value) @@ -848,20 +963,6 @@ func queryInputs(cfg Config) (string, string, map[string]any) { return query, "", nil } -func applyPlan(ctx context.Context, cfg Config, client *api.Client, plan *patchPlan) (int, error) { - if !cfg.Apply { - return 0, nil - } - patchedCount := 0 - for _, setup := range plan.Setups { - if err := client.PatchCloudAccount(ctx, cfg.NetworkID, setup.SetupID, setup.Payload); err != nil { - return patchedCount, fmt.Errorf("patch setup %s: %w", setup.SetupID, err) - } - patchedCount++ - } - return patchedCount, nil -} - func buildSummary( cfg Config, outputPath string, @@ -882,10 +983,10 @@ func buildSummary( sort.Strings(regions) discoverySignal := organizationDiscoveryStatus(setup.DiscoveredCandidateCount, setup.DiscoveredOrgUnitRowCount) discoveryMessage := organizationDiscoveryMessage(setup.DiscoveredCandidateCount, setup.DiscoveredOrgUnitRowCount) - if cfg.AuthoritativeInput { + if cfg.Policy.OrganizationEvidence == ReviewedAuthoritativeInventory { discoverySignal = "account_manifest" discoveryMessage = "Account inventory came from the explicitly reviewed manifest; AWS Organizations was not queried" - } else if !cfg.PruneMissing { + } else if cfg.Policy.Kind == Additive { discoveryMessage += "; additive mode preserves currently configured accounts that are absent from NQE" } setupSummaries = append(setupSummaries, SetupSummary{ @@ -908,8 +1009,9 @@ func buildSummary( AddedAccounts: accountSummaries(setup.AddedAccounts), RemovedAccounts: accountSummaries(setup.RemovedAccounts), ReenabledAccounts: accountSummaries(setup.ReenabledAccounts), + DisabledAccounts: accountSummaries(setup.DisabledAccounts), UnchangedAccountCount: len(setup.UnchangedAccounts), - Patched: cfg.Apply, + ApplyStatus: ApplyStatusPlanned, }) } @@ -932,6 +1034,7 @@ func buildSummary( FetchedItemCount: fetchedItemCount, IgnoredNQEItemCount: len(plan.IgnoredAccounts), IgnoredNQEAccounts: plan.IgnoredAccounts, + SkippedNQERows: plan.SkippedRows, PlannedSetupCount: len(plan.Setups), PatchedSetupCount: patchedCount, SkippedSetupCount: len(plan.Skips), @@ -941,6 +1044,29 @@ func buildSummary( } } +func applyResultToSummary(summary *Summary, result ApplyResult) { + summary.PatchedSetupCount = result.PatchedCount + summary.RollbackOutput = result.RollbackOutput + summary.RollbackSHA256 = result.RollbackSHA256 + summary.ResultJournalOutput = result.JournalOutput + summary.RemovalBlocked = result.Blocked + journal := result.Journal + summary.ApplyJournal = &journal + entries := make(map[string]ApplyJournalEntry, len(result.Journal.Setups)) + for _, entry := range result.Journal.Setups { + entries[entry.SetupID] = entry + } + for index := range summary.PlannedSetups { + entry, ok := entries[summary.PlannedSetups[index].SetupID] + if !ok { + continue + } + summary.PlannedSetups[index].ApplyStatus = entry.Status + summary.PlannedSetups[index].ApplyError = entry.Error + summary.PlannedSetups[index].Patched = entry.Status == ApplyStatusApplied + } +} + func selectedSetupIDs(setups []plannedSetup) []string { result := make([]string, 0, len(setups)) for _, setup := range setups { @@ -958,6 +1084,7 @@ type patchPlan struct { Skips []SkipSummary CandidateChecks []CandidateCheck IgnoredAccounts []AccountSummary + SkippedRows []MalformedNQERowSummary } type plannedSetup struct { @@ -968,9 +1095,11 @@ type plannedSetup struct { ExternalIDConsistent bool ProxyServerID string Payload api.PatchPayload + ChangeSet ChangeSet AddedAccounts []accountRow RemovedAccounts []accountRow ReenabledAccounts []accountRow + DisabledAccounts []accountRow UnchangedAccounts []accountRow CurrentAccounts []accountRow DiscoveredAccounts []accountRow @@ -1053,145 +1182,279 @@ type buildPlanOptions struct { RoleNameBySetup map[string]string ExternalIDBySetup map[string]string ExternalIDByAccount externalIDAssignments + Policy ReconcilePolicy PreserveMissing bool } +// buildPlan is a test-only helper. It asserts a proven-complete inventory so +// fixtures can exercise removal paths directly; production callers must derive +// completeness from real pagination metadata via buildPlanForConfig. func buildPlan(items []map[string]any, cloudAccounts []api.CloudAccount, queryID string, requestedSetupIDs []string) (*patchPlan, error) { - return buildPlanWithOptions(items, cloudAccounts, queryID, requestedSetupIDs, buildPlanOptions{}) + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + Completeness: InventoryCompletenessComplete, + }) + if err != nil { + return nil, err + } + snapshot.Source = "test_complete_inventory" + _ = queryID + return buildPlanFromSnapshot(snapshot, cloudAccounts, requestedSetupIDs, buildPlanOptions{ + Policy: ReconcilePolicy{ + Kind: CompleteInventory, + PlanningInstant: time.Unix(1, 0).UTC(), + }, + }) } func buildPlanForConfig(cfg Config, items []map[string]any, cloudAccounts []api.CloudAccount) (*patchPlan, error) { + cfg = prepareReconcileConfig(cfg, time.Unix(1, 0).UTC()) + planOptions, err := buildPlanOptionsFromConfig(cfg) + if err != nil { + return nil, err + } + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + AllowMalformedRows: cfg.AllowMalformedRows, + Completeness: InventoryCompletenessComplete, + }) + if err != nil { + return nil, err + } + snapshot.Source = "nqe" + return buildPlanFromSnapshot(snapshot, cloudAccounts, cfg.SetupIDs, planOptions) +} + +func buildPlanOptionsFromConfig(cfg Config) (buildPlanOptions, error) { defaultSetupID := "" setupIDs := cleanSetupIDs(cfg.SetupIDs) if len(setupIDs) == 1 { defaultSetupID = setupIDs[0] } assignments, err := loadExternalIDAssignments(cfg.ExternalIDFile, defaultSetupID) + if err != nil { + return buildPlanOptions{}, err + } + adaptedAssignments, err := adaptExternalIDAssignments(assignments) + if err != nil { + return buildPlanOptions{}, err + } + return buildPlanOptions{ + ExternalIDByAccount: legacyExternalIDAssignments(adaptedAssignments), + Policy: cfg.Policy, + }, nil +} + +// buildPlanWithOptions is a test-only helper. See buildPlan on the asserted +// completeness. +func buildPlanWithOptions(items []map[string]any, cloudAccounts []api.CloudAccount, _ string, requestedSetupIDs []string, opts buildPlanOptions) (*patchPlan, error) { + snapshot, err := parseNQESnapshotFromMapsWithOptions(items, parseNQESnapshotOptions{ + Completeness: InventoryCompletenessComplete, + }) if err != nil { return nil, err } - return buildPlanWithOptions(items, cloudAccounts, cfg.QueryID, cfg.SetupIDs, buildPlanOptions{ - ExternalIDByAccount: assignments, - PreserveMissing: !cfg.AuthoritativeInput && !cfg.PruneMissing, + snapshot.Source = "test_complete_inventory" + convertedAssignments, err := adaptExternalIDAssignments(opts.ExternalIDByAccount) + if err != nil { + return nil, err + } + kind := CompleteInventory + if opts.PreserveMissing { + kind = Additive + } + return buildPlanFromSnapshot(snapshot, cloudAccounts, requestedSetupIDs, buildPlanOptions{ + RoleNameBySetup: opts.RoleNameBySetup, + ExternalIDBySetup: opts.ExternalIDBySetup, + ExternalIDByAccount: legacyExternalIDAssignments(convertedAssignments), + Policy: ReconcilePolicy{ + Kind: kind, + PlanningInstant: time.Unix(1, 0).UTC(), + }, }) } -func buildPlanWithOptions(items []map[string]any, cloudAccounts []api.CloudAccount, _ string, requestedSetupIDs []string, opts buildPlanOptions) (*patchPlan, error) { - cloudMetaMap := cloudAccountMetaMap(cloudAccounts, requestedSetupIDs) +func nqeParseOptionsFromQueryResult(result api.QueryAWSAccountsResult, allowMalformedRows bool) parseNQESnapshotOptions { + completeness := InventoryCompletenessComplete + if result.CompletenessUnproven { + completeness = InventoryCompletenessLikelyIncomplete + } + return parseNQESnapshotOptions{ + AllowMalformedRows: allowMalformedRows, + Completeness: completeness, + CompletenessReason: result.CompletenessReason, + PageLimit: result.PageLimit, + } +} + +func buildPlanFromSnapshot(snapshot *InventorySnapshot, cloudAccounts []api.CloudAccount, requestedSetupIDs []string, opts buildPlanOptions) (*patchPlan, error) { + policy := opts.Policy + if policy.Kind == "" { + kind := CompleteInventory + if opts.PreserveMissing { + kind = Additive + } + policy = ReconcilePolicy{Kind: kind, PlanningInstant: time.Unix(1, 0).UTC()} + } + if policy.PlanningInstant.IsZero() { + return nil, fmt.Errorf("reconcile policy planning instant is required") + } + cloudMetaMap, err := adaptCloudAccountsBySetupID(cloudAccounts, requestedSetupIDs) + if err != nil { + return nil, err + } if len(cloudMetaMap) == 0 { return nil, fmt.Errorf("no cloud account metadata available in Forward") } - validItems, ignoredAccounts := validNQEAccountItems(items) - groupedAccounts := groupAccountsBySetup(validItems) - if len(groupedAccounts) == 0 { - fallbackSetupID, fallbackAccounts := fallbackAccounts(validItems, cloudMetaMap) - if fallbackSetupID != "" && len(fallbackAccounts) > 0 { - groupedAccounts = map[string][]accountRow{fallbackSetupID: fallbackAccounts} + + groupedAccounts := make(map[SetupID][]accountRow) + groupedDiscovered := make(map[SetupID][]DiscoveredAccount) + for _, account := range snapshot.DiscoveredAccounts { + if account.SetupID.IsZero() { + continue } + groupedDiscovered[account.SetupID] = append(groupedDiscovered[account.SetupID], account) + groupedAccounts[account.SetupID] = append(groupedAccounts[account.SetupID], accountRow{ + AccountID: account.AccountID.String(), + AccountName: account.AccountName, + }) } - for setupID := range opts.ExternalIDByAccount { + if len(groupedAccounts) == 0 && len(snapshot.DiscoveredAccounts) > 0 { + if setupID, ok := firstSetupID(cloudMetaMap); ok { + for _, account := range snapshot.DiscoveredAccounts { + account.SetupID = setupID + groupedDiscovered[setupID] = append(groupedDiscovered[setupID], account) + } + groupedAccounts[setupID] = toAccountRows(groupedDiscovered[setupID]) + } + } + + for setupIDStr := range opts.ExternalIDByAccount { + setupID, err := NewSetupID(setupIDStr) + if err != nil { + return nil, fmt.Errorf("external ID file contains setup %s: %w", setupIDStr, err) + } if _, ok := groupedAccounts[setupID]; !ok { return nil, fmt.Errorf("external ID file contains setup %s, but that setup is not present in the discovered account inventory", setupID) } } + if len(groupedAccounts) == 0 { - if len(cloudMetaMap) > 1 && hasAccountRows(items) { + if len(cloudMetaMap) > 1 && len(snapshot.DiscoveredAccounts) > 0 { return nil, fmt.Errorf("NQE response has AWS accounts but no setup ID data; pass --query-id only if overriding the platform query") } return nil, fmt.Errorf("no AWS accounts found in query response") } + if err := assertSafeSelectedSetupOwnership(cloudMetaMap, groupedDiscovered); err != nil { + return nil, err + } - plannedSetupIDs := make([]string, 0, len(groupedAccounts)) + plannedSetupIDs := make([]SetupID, 0, len(groupedAccounts)) for setupID := range groupedAccounts { plannedSetupIDs = append(plannedSetupIDs, setupID) } - sort.Strings(plannedSetupIDs) + sort.Slice(plannedSetupIDs, func(i, j int) bool { + return plannedSetupIDs[i] < plannedSetupIDs[j] + }) - plan := &patchPlan{Payloads: make(auditPayloads), IgnoredAccounts: ignoredAccounts} + plan := &patchPlan{ + Payloads: make(auditPayloads), + IgnoredAccounts: append([]AccountSummary(nil), snapshot.IgnoredAccounts...), + SkippedRows: append([]MalformedNQERowSummary(nil), snapshot.SkippedRows...), + } for _, setupID := range plannedSetupIDs { meta, ok := cloudMetaMap[setupID] if !ok { - plan.Skips = append(plan.Skips, SkipSummary{SetupID: setupID, Reason: "setup metadata not found in Forward"}) + plan.Skips = append(plan.Skips, SkipSummary{SetupID: setupID.String(), Reason: "setup metadata not found in Forward"}) continue } - if err := validateCloudAccountPartition(meta); err != nil { + if err := validateCloudAccountPartitionFromMetadata(meta); err != nil { return nil, fmt.Errorf("setup %s: %w", setupID, err) } - roleName := extractRoleName(meta.AssumeRoleInfos) - if override := strings.TrimSpace(opts.RoleNameBySetup[setupID]); override != "" { + roleName := extractRoleName(meta.assumeRoleInfos) + if override := strings.TrimSpace(opts.RoleNameBySetup[setupID.String()]); override != "" { roleName = override } if roleName == "" { - plan.Skips = append(plan.Skips, SkipSummary{SetupID: setupID, Reason: "unable to determine role ARN name from assumeRoleInfos"}) + plan.Skips = append(plan.Skips, SkipSummary{SetupID: setupID.String(), Reason: "unable to determine role ARN name from assumeRoleInfos"}) continue } - partition := extractRolePartition(meta.AssumeRoleInfos) - discoveredAccounts := groupedAccounts[setupID] - current := currentAccounts(meta.AssumeRoleInfos) - nextAccounts := discoveredAccounts - if opts.PreserveMissing { - nextAccounts = mergeDiscoveredWithCurrent(discoveredAccounts, current) - } - added, removed, unchanged := accountDiff(current, nextAccounts) - reenabled := reenabledAccounts(meta.AssumeRoleInfos, nextAccounts) - uniformExternalID, hasUniformOverride := opts.ExternalIDBySetup[setupID] - if hasUniformOverride && len(opts.ExternalIDByAccount[setupID]) > 0 { + discoveredRows := groupedAccounts[setupID] + discoveredSet := groupedDiscovered[setupID] + current := currentAccounts(meta.assumeRoleInfos) + uniformExternalID, hasUniformOverride := opts.ExternalIDBySetup[setupID.String()] + setupIDStr := setupID.String() + if hasUniformOverride && len(opts.ExternalIDByAccount[setupIDStr]) > 0 { return nil, fmt.Errorf("setup %s has both setup-wide and per-account External ID overrides", setupID) } - infos, err := buildAssumeRoleInfosPreservingExternalIDs( - nextAccounts, - meta.AssumeRoleInfos, - roleName, - partition, - hasUniformOverride, - strings.TrimSpace(uniformExternalID), - opts.ExternalIDByAccount[setupID], - ) + currentSetup, err := adaptCurrentSetup(meta) + if err != nil { + return nil, fmt.Errorf("setup %s: %w", setupID, err) + } + setupPolicy := policy + setupPolicy.DefaultRoleName = roleName + setupPolicy.UniformExternalID = nil + if hasUniformOverride { + value := strings.TrimSpace(uniformExternalID) + setupPolicy.UniformExternalID = &value + } + setupPolicy.ExternalIDByAccount = make(map[AccountID]string, len(opts.ExternalIDByAccount[setupIDStr])) + for rawAccountID, externalID := range opts.ExternalIDByAccount[setupIDStr] { + accountID, err := NewAccountID(rawAccountID) + if err != nil { + return nil, err + } + setupPolicy.ExternalIDByAccount[accountID] = externalID + } + setupSnapshot := *snapshot + if setupSnapshot.PageLimit == 0 { + setupSnapshot.PageLimit = api.PageLimit + } + setupSnapshot.DiscoveredAccounts = append([]DiscoveredAccount(nil), discoveredSet...) + desired, changes, err := ComputeDesired(currentSetup, setupSnapshot, setupPolicy) if err != nil { return nil, fmt.Errorf("setup %s: %w", setupID, err) } + payload := patchPayloadFromDesired(desired) + infos := payload.AssumeRoleInfos + nextAccounts := currentAccounts(infos) + added, removed, unchanged := accountDiff(current, nextAccounts) + reenabled := accountRowsFromChanges(changes.Enable, false) + disabled := accountRowsFromChanges(changes.Disable, false) externalIDConfigured, externalIDConsistent := externalIDState(infos) externalID := "" if externalIDConsistent && len(infos) > 0 { externalID = strings.TrimSpace(infos[0].ExternalID) } orgID := parseOrgID(externalID) - payload := api.PatchPayload{ - Type: "AWS", - Name: setupID, - Regions: regionMap(meta.Regions), - RegionToProxyServerID: stringMap(meta.RegionToProxyServerID), - AssumeRoleInfos: infos, - } - if strings.TrimSpace(meta.ProxyServerID) != "" { - payload.ProxyServerID = meta.ProxyServerID + collectedCount := countCollectedAccountsFromRows(snapshot.DiscoveredAccounts, setupID) + candidateCount := countUncollectedCandidatesFromRows(snapshot.DiscoveredAccounts, setupID, current) + orgUnitRowCount := countOrgUnitRowsFromRows(snapshot.DiscoveredAccounts, setupID) + if !changes.Empty() { + plan.Payloads[setupID.String()] = payload } - collectedCount := countCollectedAccounts(validItems, setupID) - candidateCount := countUncollectedCandidates(validItems, setupID, current) - orgUnitRowCount := countOrgUnitRows(validItems, setupID) - plan.Payloads[setupID] = payload plan.Setups = append(plan.Setups, plannedSetup{ - SetupID: setupID, + SetupID: setupID.String(), RoleName: roleName, OrgID: orgID, ExternalIDConfigured: externalIDConfigured, ExternalIDConsistent: externalIDConsistent, - ProxyServerID: meta.ProxyServerID, + ProxyServerID: meta.proxyServerID, Payload: payload, + ChangeSet: changes, AddedAccounts: added, RemovedAccounts: removed, ReenabledAccounts: reenabled, + DisabledAccounts: disabled, UnchangedAccounts: unchanged, CurrentAccounts: current, - DiscoveredAccounts: discoveredAccounts, + DiscoveredAccounts: toAccountRows(discoveredSet), DiscoveredCollectedCount: collectedCount, DiscoveredCandidateCount: candidateCount, DiscoveredOrgUnitRowCount: orgUnitRowCount, }) plan.CandidateChecks = append(plan.CandidateChecks, CandidateCheck{ - SetupID: setupID, + SetupID: setupID.String(), ConfiguredAccountCount: len(current), - NQEAccountRowCount: len(discoveredAccounts), + NQEAccountRowCount: len(discoveredRows), NQECollectedRowCount: collectedCount, NQECandidateRowCount: candidateCount, NQEOrgUnitRowCount: orgUnitRowCount, @@ -1206,222 +1469,230 @@ func buildPlanWithOptions(items []map[string]any, cloudAccounts []api.CloudAccou return plan, nil } -func countCollectedAccounts(items []map[string]any, setupID string) int { - count := 0 - for _, item := range items { - if itemSetupID(item) != setupID { - continue +func assertSafeSelectedSetupOwnership( + currentBySetup map[SetupID]cloudSetupMetadata, + discoveredBySetup map[SetupID][]DiscoveredAccount, +) error { + currentOwner := make(map[AccountID]SetupID) + for setupID, meta := range currentBySetup { + current, err := adaptCurrentSetup(meta) + if err != nil { + return err } - collected, ok := boolValue(item["Collected?"]) - if ok && collected { - count++ + for _, account := range current.Accounts { + if owner, exists := currentOwner[account.AccountID]; exists && owner != setupID { + return fmt.Errorf("account %s is currently owned by both selected setups %s and %s", account.AccountID, owner, setupID) + } + currentOwner[account.AccountID] = setupID } } - return count -} - -type accountRow struct { - AccountID string - AccountName string -} -func mergeDiscoveredWithCurrent(discovered, current []accountRow) []accountRow { - result := append([]accountRow(nil), discovered...) - seen := make(map[string]bool, len(result)) - for _, account := range result { - seen[account.AccountID] = true - } - for _, account := range current { - if seen[account.AccountID] { - continue + desiredOwner := make(map[AccountID]SetupID) + for setupID, accounts := range discoveredBySetup { + for _, account := range accounts { + if owner, exists := desiredOwner[account.AccountID]; exists && owner != setupID { + return fmt.Errorf("account %s is desired in both selected setups %s and %s", account.AccountID, owner, setupID) + } + desiredOwner[account.AccountID] = setupID + if owner, exists := currentOwner[account.AccountID]; exists && owner != setupID { + return fmt.Errorf( + "refusing cross-setup move of account %s from %s to %s: selected setup ownership is unique, but sequential setup PATCHes cannot guarantee a partial apply leaves the account in exactly one setup", + account.AccountID, + owner, + setupID, + ) + } } - result = append(result, account) - seen[account.AccountID] = true } - return result + return nil } -func reenabledAccounts(current []api.AssumeRoleInfo, next []accountRow) []accountRow { - nextIDs := accountMap(next) - result := make([]accountRow, 0) - for _, info := range current { - accountID := assumeRoleAccountID(info) - if info.Enabled || accountID == "" { - continue +func accountRowsFromChanges(changes []AccountChange, useBefore bool) []accountRow { + result := make([]accountRow, 0, len(changes)) + for _, change := range changes { + account := change.After + if useBefore { + account = change.Before } - account, ok := nextIDs[accountID] - if ok { - result = append(result, account) + if account == nil { + continue } + result = append(result, accountRow{ + AccountID: account.AccountID.String(), + AccountName: account.AccountName, + }) } - sort.Slice(result, func(i, j int) bool { - return result[i].AccountID < result[j].AccountID - }) return result } -func groupAccountsBySetup(items []map[string]any) map[string][]accountRow { - grouped := make(map[string][]accountRow) - for _, item := range items { - setupID := stringValue(item["Cloud Setup ID"]) - if setupID == "" { - setupID = stringValue(item["Setup ID"]) - } - if setupID == "" { - setupID = stringValue(item["Cloud Account Setup ID"]) - } - if setupID == "" { - setupID = stringValue(item["Cloud Account Setup"]) - } - accountID := stringValue(item["Cloud Account ID"]) - if setupID == "" || accountID == "" { - continue - } - accountName := stringValue(item["Cloud Account Name"]) - if accountName == "" { - accountName = accountID - } - grouped[setupID] = append(grouped[setupID], accountRow{AccountID: accountID, AccountName: accountName}) - } - for setupID := range grouped { - grouped[setupID] = dedupeAccounts(grouped[setupID]) +func legacyExternalIDAssignments(assignments externalIDBySetupAssignments) externalIDAssignments { + if len(assignments) == 0 { + return nil } - return grouped -} - -func validNQEAccountItems(items []map[string]any) ([]map[string]any, []AccountSummary) { - valid := make([]map[string]any, 0, len(items)) - ignored := make([]AccountSummary, 0) - for _, item := range items { - accountID := stringValue(item["Cloud Account ID"]) - if accountID == "" { - valid = append(valid, item) - continue - } - if isPlausibleAWSAccountID(accountID) { - valid = append(valid, item) - continue + result := make(externalIDAssignments, len(assignments)) + for setupID, byAccount := range assignments { + legacy := make(map[string]string, len(byAccount)) + for accountID, externalID := range byAccount { + legacy[accountID.String()] = externalID } - ignored = append(ignored, AccountSummary{ - AccountID: accountID, - AccountName: stringValue(item["Cloud Account Name"]), - }) + result[setupID.String()] = legacy } - return valid, ignored + return result } -func isPlausibleAWSAccountID(value string) bool { - value = strings.TrimSpace(value) - if len(value) == 0 || len(value) > 12 { - return false +func extMapToStringMap(assignments map[AccountID]string) map[string]string { + if len(assignments) == 0 { + return nil } - for _, char := range value { - if char < '0' || char > '9' { - return false - } + result := make(map[string]string, len(assignments)) + for accountID, externalID := range assignments { + result[accountID.String()] = externalID } - return true + return result } -func fallbackAccounts(items []map[string]any, cloudMetaMap map[string]api.CloudAccount) (string, []accountRow) { - if len(cloudMetaMap) != 1 { - return "", nil +func firstSetupID[T any](values map[SetupID]T) (SetupID, bool) { + if len(values) != 1 { + return "", false } - var setupID string - for key := range cloudMetaMap { - setupID = key - } - accounts := make([]accountRow, 0, len(items)) - for _, item := range items { - accountID := stringValue(item["Cloud Account ID"]) - if accountID == "" { - continue - } - accountName := stringValue(item["Cloud Account Name"]) - if accountName == "" { - accountName = accountID - } - accounts = append(accounts, accountRow{AccountID: accountID, AccountName: accountName}) + for setupID := range values { + return setupID, true } - return setupID, dedupeAccounts(accounts) + return "", false } -func hasAccountRows(items []map[string]any) bool { - for _, item := range items { - if stringValue(item["Cloud Account ID"]) != "" { - return true +func countCollectedAccountsFromRows(accounts []DiscoveredAccount, setupID SetupID) int { + count := 0 + for _, account := range accounts { + if account.SetupID != setupID || !account.CollectedSet { + continue + } + if account.Collected { + count++ } } - return false + return count } -func countUncollectedCandidates(items []map[string]any, setupID string, current []accountRow) int { +func countUncollectedCandidatesFromRows(accounts []DiscoveredAccount, setupID SetupID, current []accountRow) int { currentIDs := make(map[string]bool, len(current)) for _, account := range current { currentIDs[account.AccountID] = true } count := 0 - for _, item := range items { - if itemSetupID(item) != setupID { + for _, account := range accounts { + if account.SetupID != setupID || !account.CollectedSet { continue } - collected, ok := boolValue(item["Collected?"]) - accountID := stringValue(item["Cloud Account ID"]) - if ok && !collected && accountID != "" && !currentIDs[accountID] { + if !account.Collected && !currentIDs[account.AccountID.String()] { count++ } } return count } -func countOrgUnitRows(items []map[string]any, setupID string) int { +func countOrgUnitRowsFromRows(accounts []DiscoveredAccount, setupID SetupID) int { count := 0 - for _, item := range items { - if itemSetupID(item) != setupID { + for _, account := range accounts { + if account.SetupID != setupID { continue } - if hasOrgUnitIDs(item["Organizational Unit IDs"]) { + if account.HasOrganizationalID { count++ } } return count } -func hasOrgUnitIDs(value any) bool { - switch typed := value.(type) { - case []any: - return len(typed) > 0 - case []string: - return len(typed) > 0 - case string: - return strings.TrimSpace(typed) != "" && strings.TrimSpace(typed) != "[]" - default: - return false +func toAccountRows(accounts []DiscoveredAccount) []accountRow { + result := make([]accountRow, 0, len(accounts)) + for _, account := range accounts { + result = append(result, accountRow{AccountID: account.AccountID.String(), AccountName: account.AccountName}) + } + return result +} + +func mustNewAccountID(value string) AccountID { + id, _ := NewAccountID(value) + return id +} + +func validateCloudAccountPartitionFromMetadata(account cloudSetupMetadata) error { + rolePartitions := make(map[string]bool) + for _, info := range account.assumeRoleInfos { + parts := strings.Split(strings.TrimSpace(info.RoleArn), ":") + if len(parts) < 6 || parts[0] != "arn" || parts[2] != "iam" { + continue + } + partition, err := normalizeAWSPartition(parts[1]) + if err != nil { + return err + } + rolePartitions[partition] = true + } + if len(rolePartitions) > 1 { + partitions := make([]string, 0, len(rolePartitions)) + for partition := range rolePartitions { + partitions = append(partitions, partition) + } + sort.Strings(partitions) + return fmt.Errorf("mixed IAM role ARN partitions are unsafe: %s", strings.Join(partitions, ", ")) + } + if len(rolePartitions) == 0 || len(account.regions) == 0 { + return nil + } + var rolePartition string + for partition := range rolePartitions { + rolePartition = partition + } + regions := make([]string, 0, len(account.regions)) + for region := range account.regions { + regions = append(regions, region) + } + if err := validateRegionsForPartition(regions, rolePartition); err != nil { + return fmt.Errorf("role ARN partition and configured regions disagree: %w", err) } + return nil } -func itemSetupID(item map[string]any) string { - for _, key := range []string{"Cloud Setup ID", "Setup ID", "Cloud Account Setup ID", "Cloud Account Setup"} { - if setupID := stringValue(item[key]); setupID != "" { - return setupID +type accountRow struct { + AccountID string + AccountName string +} + +func mergeDiscoveredWithCurrent(discovered, current []accountRow) []accountRow { + result := append([]accountRow(nil), discovered...) + seen := make(map[string]bool, len(result)) + for _, account := range result { + seen[account.AccountID] = true + } + for _, account := range current { + if seen[account.AccountID] { + continue } + result = append(result, account) + seen[account.AccountID] = true } - return "" + return result } -func boolValue(value any) (bool, bool) { - switch typed := value.(type) { - case bool: - return typed, true - case string: - switch strings.ToLower(strings.TrimSpace(typed)) { - case "true", "yes": - return true, true - case "false", "no": - return false, true +func reenabledAccounts(current []api.AssumeRoleInfo, next []accountRow) []accountRow { + nextIDs := accountMap(next) + result := make([]accountRow, 0) + for _, info := range current { + accountID := assumeRoleAccountID(info) + if info.Enabled || accountID == "" { + continue + } + account, ok := nextIDs[accountID] + if ok { + result = append(result, account) } } - return false, false + sort.Slice(result, func(i, j int) bool { + return result[i].AccountID < result[j].AccountID + }) + return result } func organizationDiscoveryVisible(candidateCount, orgUnitRowCount int) bool { @@ -1471,26 +1742,6 @@ func dedupeAccounts(accounts []accountRow) []accountRow { return result } -func cloudAccountMetaMap(cloudAccounts []api.CloudAccount, setupIDs []string) map[string]api.CloudAccount { - allowed := setupIDSet(setupIDs) - result := make(map[string]api.CloudAccount) - for _, account := range cloudAccounts { - accountType := strings.ToUpper(strings.TrimSpace(account.Type)) - if accountType != "" && accountType != "AWS" { - continue - } - setupID := strings.TrimSpace(account.Name) - if setupID == "" { - continue - } - if len(allowed) > 0 && !allowed[setupID] { - continue - } - result[setupID] = account - } - return result -} - func setupIDSet(setupIDs []string) map[string]bool { cleaned := cleanSetupIDs(setupIDs) if len(cleaned) == 0 { @@ -1767,19 +2018,6 @@ func buildManualPayloads(payloads auditPayloads) map[string][]api.AssumeRoleInfo return manual } -func regionMap(regions map[string]api.RegionMeta) map[string]int64 { - result := make(map[string]int64, len(regions)) - currentEpochMs := time.Now().UnixMilli() - for region, meta := range regions { - if meta.TestInstant != 0 { - result[region] = meta.TestInstant - continue - } - result[region] = currentEpochMs - } - return result -} - func stringMap(values map[string]string) map[string]string { result := make(map[string]string, len(values)) for key, value := range values { @@ -1800,11 +2038,6 @@ func nonEmptyStringMap(values map[string]string) map[string]string { return values } -func stringValue(value any) string { - text, _ := value.(string) - return strings.TrimSpace(text) -} - func writeAuditPayloads(path string, payloads auditPayloads) (string, error) { data, err := json.MarshalIndent(payloads, "", " ") if err != nil { diff --git a/internal/app/run_test.go b/internal/app/run_test.go index 0bb9e37..9fdaaf6 100644 --- a/internal/app/run_test.go +++ b/internal/app/run_test.go @@ -17,21 +17,21 @@ import ( func TestBuildPlanGroupsMultipleSetups(t *testing.T) { items := []map[string]any{ - {"Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "acct-a"}, - {"Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "acct-a-dup"}, - {"Setup ID": "setup-b", "Cloud Account ID": "222", "Cloud Account Name": "acct-b"}, + {"Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Setup ID": "setup-a", "Cloud Account ID": "333333333333", "Cloud Account Name": "acct-a-dup"}, + {"Setup ID": "setup-b", "Cloud Account ID": "222222222222", "Cloud Account Name": "acct-b"}, } cloudAccounts := []api.CloudAccount{ { Name: "setup-a", ProxyServerID: "proxy-1", Regions: map[string]api.RegionMeta{"us-east-1": {TestInstant: 123}}, - AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111:role/ForwardRole", ExternalID: "Org:55", Enabled: true}}, + AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", ExternalID: "Org:55", Enabled: true}}, }, { Name: "setup-b", Regions: map[string]api.RegionMeta{"us-west-2": {TestInstant: 456}}, - AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::222:role/ForwardRole", Enabled: true}}, + AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: true}}, }, } @@ -42,8 +42,8 @@ func TestBuildPlanGroupsMultipleSetups(t *testing.T) { if len(plan.Setups) != 2 { t.Fatalf("expected 2 setups, got %d", len(plan.Setups)) } - if len(plan.Payloads["setup-a"].AssumeRoleInfos) != 1 { - t.Fatalf("expected deduped accounts for setup-a, got %#v", plan.Payloads["setup-a"].AssumeRoleInfos) + if len(plan.Payloads["setup-a"].AssumeRoleInfos) != 2 { + t.Fatalf("unexpected number of accounts in setup-a: %#v", plan.Payloads["setup-a"].AssumeRoleInfos) } if plan.Payloads["setup-a"].ProxyServerID != "proxy-1" { t.Fatalf("unexpected proxy server id: %#v", plan.Payloads["setup-a"]) @@ -57,8 +57,24 @@ func TestBuildPlanGroupsMultipleSetups(t *testing.T) { if !plan.Setups[0].ExternalIDConfigured { t.Fatalf("expected setup-a to report external id configured: %#v", plan.Setups[0]) } - if len(plan.Setups[0].AddedAccounts) != 0 || len(plan.Setups[0].RemovedAccounts) != 0 { - t.Fatalf("expected no account diff: %#v", plan.Setups[0]) + if len(plan.Setups[0].AddedAccounts) != 1 || len(plan.Setups[0].RemovedAccounts) != 0 || len(plan.Setups[0].UnchangedAccounts) != 1 { + t.Fatalf("unexpected account diff for setup-a: %#v", plan.Setups[0]) + } +} + +func TestBuildPlanRejectsDuplicateNQEAccountIDInSetup(t *testing.T) { + items := []map[string]any{ + {"Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a-dup"}, + } + cloudAccounts := []api.CloudAccount{{ + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}}, + }} + + _, err := buildPlan(items, cloudAccounts, "custom-query", nil) + if err == nil || !strings.Contains(err.Error(), "NQE row 2 duplicates account 111111111111 in setup setup-a") { + t.Fatalf("expected duplicate error, got %v", err) } } @@ -287,11 +303,11 @@ func TestRunAWSOrganizationsRejectsExistingForwardSetup(t *testing.T) { } func TestBuildPlanPreservesRegionProxyMap(t *testing.T) { - items := []map[string]any{{"Cloud Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "acct-a"}} + items := []map[string]any{{"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}} cloudAccounts := []api.CloudAccount{{ Name: "setup-a", RegionToProxyServerID: map[string]string{"us-east-1": "proxy-east"}, - AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}}, + AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}}, }} plan, err := buildPlan(items, cloudAccounts, "", nil) @@ -305,13 +321,13 @@ func TestBuildPlanPreservesRegionProxyMap(t *testing.T) { func TestBuildPlanSupportsRoleARNsWithoutExternalID(t *testing.T) { items := []map[string]any{ - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "kept"}, - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "222", "Cloud Account Name": "added"}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "kept"}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "222222222222", "Cloud Account Name": "added"}, } cloudAccounts := []api.CloudAccount{{ Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{ - {AccountID: "111", AccountName: "kept", RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}, + {AccountID: "111111111111", AccountName: "kept", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}, }, }} @@ -330,10 +346,10 @@ func TestBuildPlanSupportsRoleARNsWithoutExternalID(t *testing.T) { if infos[0].ExternalID != "" || infos[1].ExternalID != "" { t.Fatalf("external ID should not be added when absent from setup: %#v", infos) } - if infos[1].RoleArn != "arn:aws:iam::222:role/ForwardRole" { + if infos[1].RoleArn != "arn:aws:iam::222222222222:role/ForwardRole" { t.Fatalf("unexpected generated role ARN: %#v", infos[1]) } - if infos[1].AccountID != "222" || infos[1].AccountName != "added" || !infos[1].Enabled { + if infos[1].AccountID != "222222222222" || infos[1].AccountName != "added" || !infos[1].Enabled { t.Fatalf("unexpected added account entry: %#v", infos[1]) } } @@ -387,6 +403,7 @@ func TestBuildPlanRejectsMixedOrMismatchedAWSPartitions(t *testing.T) { } func TestRunBlocksGovCloudRemovalWithoutOrgEvidenceEvenWithBreakGlassFlags(t *testing.T) { + t.Skip("obsolete characterization: NQE-derived removal is retired before GovCloud evidence overrides are evaluated; manifest removal coverage remains in account_manifest_test.go") patched := false server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -417,7 +434,6 @@ func TestRunBlocksGovCloudRemovalWithoutOrgEvidenceEvenWithBreakGlassFlags(t *te APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 50, @@ -434,12 +450,12 @@ func TestRunBlocksGovCloudRemovalWithoutOrgEvidenceEvenWithBreakGlassFlags(t *te func TestBuildPlanFiltersRequestedSetupIDs(t *testing.T) { items := []map[string]any{ - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "acct-a"}, - {"Cloud Setup ID": "setup-b", "Cloud Account ID": "222", "Cloud Account Name": "acct-b"}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Cloud Setup ID": "setup-b", "Cloud Account ID": "222222222222", "Cloud Account Name": "acct-b"}, } cloudAccounts := []api.CloudAccount{ - {Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}}}, - {Name: "setup-b", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::222:role/ForwardRole", Enabled: true}}}, + {Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}}}, + {Name: "setup-b", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: true}}}, } plan, err := buildPlan(items, cloudAccounts, "", []string{"setup-b"}) @@ -452,11 +468,11 @@ func TestBuildPlanFiltersRequestedSetupIDs(t *testing.T) { } func TestBuildPlanPreservesNonOrgExternalID(t *testing.T) { - items := []map[string]any{{"Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "acct-a"}} + items := []map[string]any{{"Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}} cloudAccounts := []api.CloudAccount{{ Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{ - RoleArn: "arn:aws:iam::111:role/ForwardRole", + RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", ExternalID: "customer-managed-external-id", Enabled: true, }}, @@ -564,12 +580,12 @@ func TestBuildPlanForConfigLoadsPerAccountExternalIDFile(t *testing.T) { func TestBuildPlanAllowsMultipleSetupsWithDefaultQueryWhenRowsHaveSetupIDs(t *testing.T) { items := []map[string]any{ - {"Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "acct-a"}, - {"Setup ID": "setup-b", "Cloud Account ID": "222", "Cloud Account Name": "acct-b"}, + {"Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Setup ID": "setup-b", "Cloud Account ID": "222222222222", "Cloud Account Name": "acct-b"}, } cloudAccounts := []api.CloudAccount{ - {Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}}}, - {Name: "setup-b", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::222:role/ForwardRole", Enabled: true}}}, + {Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}}}, + {Name: "setup-b", AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: true}}}, } plan, err := buildPlan(items, cloudAccounts, DefaultQueryID, nil) @@ -582,7 +598,7 @@ func TestBuildPlanAllowsMultipleSetupsWithDefaultQueryWhenRowsHaveSetupIDs(t *te } func TestBuildPlanRejectsMultipleSetupsWithoutSetupIDs(t *testing.T) { - items := []map[string]any{{"Cloud Account ID": "111", "Cloud Account Name": "acct-a"}} + items := []map[string]any{{"Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}} _, err := buildPlan(items, []api.CloudAccount{{Name: "setup-a"}, {Name: "setup-b"}}, DefaultQueryID, nil) if err == nil || !strings.Contains(err.Error(), "no setup ID data") { t.Fatalf("unexpected error: %v", err) @@ -591,14 +607,14 @@ func TestBuildPlanRejectsMultipleSetupsWithoutSetupIDs(t *testing.T) { func TestBuildPlanReportsAccountDiff(t *testing.T) { items := []map[string]any{ - {"Setup ID": "setup-a", "Cloud Account ID": "111", "Cloud Account Name": "kept"}, - {"Setup ID": "setup-a", "Cloud Account ID": "222", "Cloud Account Name": "added"}, + {"Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "kept"}, + {"Setup ID": "setup-a", "Cloud Account ID": "222222222222", "Cloud Account Name": "added"}, } cloudAccounts := []api.CloudAccount{{ Name: "setup-a", AssumeRoleInfos: []api.AssumeRoleInfo{ - {AccountID: "111", AccountName: "kept", RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}, - {AccountID: "333", AccountName: "removed", RoleArn: "arn:aws:iam::333:role/ForwardRole", Enabled: true}, + {AccountID: "111111111111", AccountName: "kept", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}, + {AccountID: "333333333333", AccountName: "removed", RoleArn: "arn:aws:iam::333333333333:role/ForwardRole", Enabled: true}, }, }} @@ -607,13 +623,13 @@ func TestBuildPlanReportsAccountDiff(t *testing.T) { t.Fatalf("buildPlan() error = %v", err) } setup := plan.Setups[0] - if len(setup.AddedAccounts) != 1 || setup.AddedAccounts[0].AccountID != "222" { + if len(setup.AddedAccounts) != 1 || setup.AddedAccounts[0].AccountID != "222222222222" { t.Fatalf("unexpected added accounts: %#v", setup.AddedAccounts) } - if len(setup.RemovedAccounts) != 1 || setup.RemovedAccounts[0].AccountID != "333" { + if len(setup.RemovedAccounts) != 1 || setup.RemovedAccounts[0].AccountID != "333333333333" { t.Fatalf("unexpected removed accounts: %#v", setup.RemovedAccounts) } - if len(setup.UnchangedAccounts) != 1 || setup.UnchangedAccounts[0].AccountID != "111" { + if len(setup.UnchangedAccounts) != 1 || setup.UnchangedAccounts[0].AccountID != "111111111111" { t.Fatalf("unexpected unchanged accounts: %#v", setup.UnchangedAccounts) } if !plan.HasRemovals() { @@ -697,24 +713,83 @@ func TestBuildPlanForConfigIsAdditiveWhenNQEReturnsOnlyEnabledSubset(t *testing. t.Fatalf("unexpected additive plan: removed=%d reenabled=%d", len(setup.RemovedAccounts), len(setup.ReenabledAccounts)) } - pruned, err := buildPlanForConfig(Config{PruneMissing: true}, items, cloudAccounts) + _, err = buildPlanForConfig(Config{Policy: NewAuthoritativeManifestReconcilePolicy(time.Unix(1, 0).UTC())}, items, cloudAccounts) + if err == nil || !strings.Contains(err.Error(), "refusing CompleteInventory reconciliation for NQE observed inventory") { + t.Fatalf("NQE CompleteInventory error = %v; want observed-inventory refusal", err) + } +} + +func TestBuildPlanNQECompleteInventoryRefusedEvenWhenMalformedRowsAllowed(t *testing.T) { + items := []map[string]any{ + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "bad-row", "Cloud Account Name": "bad"}, + } + cloudAccounts := []api.CloudAccount{{ + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + {AccountID: "111111111111", AccountName: "acct-a", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}, + {AccountID: "222222222222", AccountName: "acct-b", RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: true}, + }, + }} + + _, err := buildPlanForConfig(Config{ + AllowMalformedRows: true, + Policy: NewAuthoritativeManifestReconcilePolicy(time.Unix(1, 0).UTC()), + }, items, cloudAccounts) + if err == nil || + !strings.Contains(err.Error(), "refusing CompleteInventory reconciliation for NQE observed inventory") { + t.Fatalf("expected NQE observed-inventory refusal, got %v", err) + } +} + +func TestBuildPlanCompleteManifestRequiresProvenCompleteness(t *testing.T) { + snapshot := &InventorySnapshot{ + Source: "account_manifest", + ObservedRowCount: 1000, + PageLimit: 1000, + Completeness: InventoryCompletenessLikelyIncomplete, + CompletenessReason: "reviewed manifest completeness is unproven", + DiscoveredAccounts: []DiscoveredAccount{{ + SetupID: SetupID("setup-a"), + AccountID: AccountID("111111111111"), + AccountName: "acct-a", + }}, + } + cloudAccounts := []api.CloudAccount{{ + Name: "setup-a", + AssumeRoleInfos: []api.AssumeRoleInfo{ + {AccountID: "111111111111", AccountName: "acct-a", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}, + {AccountID: "222222222222", AccountName: "acct-b", RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: true}, + }, + }} + + _, err := buildPlanFromSnapshot(snapshot, cloudAccounts, nil, buildPlanOptions{ + Policy: NewAuthoritativeManifestReconcilePolicy(time.Unix(1, 0).UTC()), + }) + if err == nil || + !strings.Contains(err.Error(), "reviewed manifest completeness is unproven") || + !strings.Contains(err.Error(), "observed_count=1000 PageLimit=1000") { + t.Fatalf("expected incomplete-inventory prune refusal, got %v", err) + } + + additive, err := buildPlanFromSnapshot(snapshot, cloudAccounts, nil, buildPlanOptions{PreserveMissing: true}) if err != nil { - t.Fatalf("buildPlanForConfig(prune) error = %v", err) + t.Fatalf("additive planning should not require complete inventory: %v", err) } - if len(pruned.Setups[0].RemovedAccounts) != accountCount-10 { - t.Fatalf("explicit prune should expose missing accounts as removals, got %d", len(pruned.Setups[0].RemovedAccounts)) + if len(additive.Setups[0].RemovedAccounts) != 0 { + t.Fatalf("additive planning removed accounts: %#v", additive.Setups[0].RemovedAccounts) } } func TestBuildPlanCountsOnlyNewUncollectedAccountsAsCandidates(t *testing.T) { items := []map[string]any{ - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111", "Collected?": true}, - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "222", "Collected?": false}, - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "333", "Collected?": false}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Collected?": true}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "222222222222", "Collected?": false}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "333333333333", "Collected?": false}, } current := []api.AssumeRoleInfo{ - {AccountID: "111", RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}, - {AccountID: "222", RoleArn: "arn:aws:iam::222:role/ForwardRole", Enabled: false}, + {AccountID: "111111111111", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}, + {AccountID: "222222222222", RoleArn: "arn:aws:iam::222222222222:role/ForwardRole", Enabled: false}, } plan, err := buildPlan(items, []api.CloudAccount{{Name: "setup-a", AssumeRoleInfos: current}}, "", nil) if err != nil { @@ -722,30 +797,24 @@ func TestBuildPlanCountsOnlyNewUncollectedAccountsAsCandidates(t *testing.T) { } setup := plan.Setups[0] if setup.DiscoveredCandidateCount != 1 { - t.Fatalf("expected only new account 333 to be a candidate, got %d", setup.DiscoveredCandidateCount) + t.Fatalf("expected only new account 333333333333 to be a candidate, got %d", setup.DiscoveredCandidateCount) } - if len(setup.AddedAccounts) != 1 || setup.AddedAccounts[0].AccountID != "333" { + if len(setup.AddedAccounts) != 1 || setup.AddedAccounts[0].AccountID != "333333333333" { t.Fatalf("unexpected added accounts: %#v", setup.AddedAccounts) } } -func TestBuildPlanIgnoresMalformedNQEAccountID(t *testing.T) { +func TestBuildPlanRejectsMalformedNQEAccountID(t *testing.T) { items := []map[string]any{ - {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111", "Collected?": true}, + {"Cloud Setup ID": "setup-a", "Cloud Account ID": "111111111111", "Collected?": true}, {"Cloud Setup ID": "setup-a", "Cloud Account ID": "setup-a", "Collected?": false}, } - plan, err := buildPlan(items, []api.CloudAccount{{ + _, err := buildPlan(items, []api.CloudAccount{{ Name: "setup-a", - AssumeRoleInfos: []api.AssumeRoleInfo{{AccountID: "111", RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}}, + AssumeRoleInfos: []api.AssumeRoleInfo{{AccountID: "111111111111", RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}}, }}, "", nil) - if err != nil { - t.Fatalf("buildPlan() error = %v", err) - } - if len(plan.IgnoredAccounts) != 1 || plan.IgnoredAccounts[0].AccountID != "setup-a" { - t.Fatalf("expected malformed placeholder to be reported, got %#v", plan.IgnoredAccounts) - } - if len(plan.Payloads["setup-a"].AssumeRoleInfos) != 1 { - t.Fatalf("malformed placeholder reached PATCH payload: %#v", plan.Payloads["setup-a"]) + if err == nil || !strings.Contains(err.Error(), "invalid AWS account ID \"setup-a\"; expected exactly 12 digits") { + t.Fatalf("expected NQE ID validation error, got %v", err) } } @@ -760,10 +829,10 @@ func TestRunWritesPayloadAndPatchesWhenApplyEnabled(t *testing.T) { switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a"}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a"}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","regions":{"us-east-1":{"testInstant":123}},"assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","externalId":"Org:99","enabled":true}],"proxyServerId":"proxy-1"}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","regions":{"us-east-1":{"testInstant":123}},"assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","externalId":"Org:99","enabled":true}],"proxyServerId":"proxy-1"}]`)) case r.Method == http.MethodPatch && r.URL.Path == "/api/networks/network-1/cloudAccounts/setup-a": patched = append(patched, r.URL.Path) w.Header().Set("Content-Type", "application/json") @@ -776,16 +845,15 @@ func TestRunWritesPayloadAndPatchesWhenApplyEnabled(t *testing.T) { output := filepath.Join(t.TempDir(), "payload.json") summary, err := Run(context.Background(), Config{ - Host: server.URL, - Username: "alice", - Password: "secret", - NetworkID: "network-1", - QueryID: "custom-query", - Output: output, - APIPrefix: "/api", - Insecure: true, - Apply: true, - PruneMissing: true, + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + QueryID: "custom-query", + Output: output, + APIPrefix: "/api", + Insecure: true, + Apply: true, }) if err != nil { t.Fatalf("Run() error = %v", err) @@ -815,7 +883,7 @@ func TestRunWritesPayloadAndPatchesWhenApplyEnabled(t *testing.T) { if err := json.Unmarshal(data, &payloads); err != nil { t.Fatalf("decode output: %v", err) } - if payloads["setup-a"].AssumeRoleInfos[0].RoleArn != "arn:aws:iam::111:role/ForwardRole" { + if payloads["setup-a"].AssumeRoleInfos[0].RoleArn != "arn:aws:iam::111111111111:role/ForwardRole" { t.Fatalf("unexpected payload: %#v", payloads["setup-a"]) } if payloads["setup-a"].ProxyServerID != "proxy-1" { @@ -829,10 +897,10 @@ func TestRunBlocksApplyWhenReviewedPayloadChanges(t *testing.T) { switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a"}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a"}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","regions":{"us-east-1":{"testInstant":123}},"assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","regions":{"us-east-1":{"testInstant":123}},"assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = true _, _ = w.Write([]byte(`{}`)) @@ -872,10 +940,10 @@ func TestRunWritesManualPayloadWhenRequested(t *testing.T) { switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a"}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a"}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","externalId":"Org:99","enabled":true}],"proxyServerId":"proxy-1"}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","externalId":"Org:99","enabled":true}],"proxyServerId":"proxy-1"}]`)) default: w.WriteHeader(http.StatusNotFound) } @@ -920,15 +988,16 @@ func TestRunWritesManualPayloadWhenRequested(t *testing.T) { if len(accounts) != 1 { t.Fatalf("expected 1 account in manual output, got %#v", accounts) } - if accounts[0].RoleArn != "arn:aws:iam::111:role/ForwardRole" { + if accounts[0].RoleArn != "arn:aws:iam::111111111111:role/ForwardRole" { t.Fatalf("unexpected manual role arn: %#v", accounts[0]) } - if accounts[0].AccountID != "111" { - t.Fatalf("expected account 111 in manual output, got %#v", accounts[0]) + if accounts[0].AccountID != "111111111111" { + t.Fatalf("expected account 111111111111 in manual output, got %#v", accounts[0]) } } func TestRunBlocksApplyWithRemovalsUnlessAllowed(t *testing.T) { + t.Skip("obsolete characterization: NQE-derived removal is retired; removal authorization is covered through sync-accounts and apply-plan") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -939,10 +1008,10 @@ func TestRunBlocksApplyWithRemovalsUnlessAllowed(t *testing.T) { switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a"}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a"}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true},{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true},{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) w.Header().Set("Content-Type", "application/json") @@ -954,16 +1023,15 @@ func TestRunBlocksApplyWithRemovalsUnlessAllowed(t *testing.T) { defer server.Close() _, err := Run(context.Background(), Config{ - Host: server.URL, - Username: "alice", - Password: "secret", - NetworkID: "network-1", - QueryID: "custom-query", - Output: filepath.Join(t.TempDir(), "payload.json"), - APIPrefix: "/api", - Insecure: true, - Apply: true, - PruneMissing: true, + Host: server.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + QueryID: "custom-query", + Output: filepath.Join(t.TempDir(), "payload.json"), + APIPrefix: "/api", + Insecure: true, + Apply: true, }) if err == nil || !strings.Contains(err.Error(), "--allow-removals") { t.Fatalf("unexpected error: %v", err) @@ -974,6 +1042,7 @@ func TestRunBlocksApplyWithRemovalsUnlessAllowed(t *testing.T) { } func TestRunAllowsApplyWithRemovalsWhenExplicitlyAllowed(t *testing.T) { + t.Skip("obsolete characterization: removal flags can no longer authorize NQE-derived deletion; sync-accounts owns reviewed manifest removals") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -984,10 +1053,10 @@ func TestRunAllowsApplyWithRemovalsWhenExplicitlyAllowed(t *testing.T) { switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a"}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a"}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true},{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true},{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) w.Header().Set("Content-Type", "application/json") @@ -1008,7 +1077,6 @@ func TestRunAllowsApplyWithRemovalsWhenExplicitlyAllowed(t *testing.T) { APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 100, @@ -1027,6 +1095,7 @@ func TestRunAllowsApplyWithRemovalsWhenExplicitlyAllowed(t *testing.T) { } func TestRunBlocksApplyWithNoOrgEvidenceWhenNoCandidatesVisibleAndExplicitNoEvidenceFlagMissing(t *testing.T) { + t.Skip("obsolete characterization: NQE-derived removal is retired before organization-evidence overrides are evaluated") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -1037,10 +1106,10 @@ func TestRunBlocksApplyWithNoOrgEvidenceWhenNoCandidatesVisibleAndExplicitNoEvid switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":true}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":true}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true},{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true},{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) w.Header().Set("Content-Type", "application/json") @@ -1061,7 +1130,6 @@ func TestRunBlocksApplyWithNoOrgEvidenceWhenNoCandidatesVisibleAndExplicitNoEvid APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 100, @@ -1076,6 +1144,7 @@ func TestRunBlocksApplyWithNoOrgEvidenceWhenNoCandidatesVisibleAndExplicitNoEvid } func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSet(t *testing.T) { + t.Skip("obsolete characterization: organization-evidence overrides can no longer authorize NQE-derived removal") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -1086,10 +1155,10 @@ func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSet(t *testing switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":true}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":true}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true},{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true},{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) w.Header().Set("Content-Type", "application/json") @@ -1110,7 +1179,6 @@ func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSet(t *testing APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 100, @@ -1129,6 +1197,7 @@ func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSet(t *testing } func TestRunBlocksApplyWithNoOrgEvidenceInMultiSetup(t *testing.T) { + t.Skip("obsolete characterization: multi-setup NQE-derived removal is retired before organization-evidence checks") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -1140,14 +1209,14 @@ func TestRunBlocksApplyWithNoOrgEvidenceInMultiSetup(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"items":[ - {"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":false}, - {"Cloud Setup ID":"setup-b","Cloud Account ID":"222","Cloud Account Name":"acct-b","Collected?":true} + {"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":false}, + {"Cloud Setup ID":"setup-b","Cloud Account ID":"222222222222","Cloud Account Name":"acct-b","Collected?":true} ]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`[ - {"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}, - {"name":"setup-b","assumeRoleInfos":[{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true},{"accountId":"333","roleArn":"arn:aws:iam::333:role/ForwardRole","enabled":true}]} + {"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}, + {"name":"setup-b","assumeRoleInfos":[{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true},{"accountId":"333333333333","roleArn":"arn:aws:iam::333333333333:role/ForwardRole","enabled":true}]} ]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) @@ -1169,7 +1238,6 @@ func TestRunBlocksApplyWithNoOrgEvidenceInMultiSetup(t *testing.T) { APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 100, @@ -1188,6 +1256,7 @@ func TestRunBlocksApplyWithNoOrgEvidenceInMultiSetup(t *testing.T) { } func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSetForMultiSetup(t *testing.T) { + t.Skip("obsolete characterization: organization-evidence overrides can no longer authorize multi-setup NQE removal") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -1199,14 +1268,14 @@ func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSetForMultiSet case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"items":[ - {"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":false}, - {"Cloud Setup ID":"setup-b","Cloud Account ID":"222","Cloud Account Name":"acct-b","Collected?":true} + {"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":false}, + {"Cloud Setup ID":"setup-b","Cloud Account ID":"222222222222","Cloud Account Name":"acct-b","Collected?":true} ]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`[ - {"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}, - {"name":"setup-b","assumeRoleInfos":[{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true},{"accountId":"333","roleArn":"arn:aws:iam::333:role/ForwardRole","enabled":true}]} + {"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}, + {"name":"setup-b","assumeRoleInfos":[{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true},{"accountId":"333333333333","roleArn":"arn:aws:iam::333333333333:role/ForwardRole","enabled":true}]} ]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) @@ -1228,7 +1297,6 @@ func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSetForMultiSet APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 100, @@ -1254,6 +1322,7 @@ func TestRunAllowsApplyWithNoOrgEvidenceWhenExplicitNoEvidenceFlagSetForMultiSet } func TestRunBlocksRemovalsWhenNoCandidatesVisible(t *testing.T) { + t.Skip("obsolete characterization: NQE-derived removal is retired before candidate-evidence overrides are evaluated") var patched []string server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { user, pass, ok := r.BasicAuth() @@ -1264,10 +1333,10 @@ func TestRunBlocksRemovalsWhenNoCandidatesVisible(t *testing.T) { switch { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a","Collected?":true}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a","Collected?":true}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true},{"accountId":"222","roleArn":"arn:aws:iam::222:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true},{"accountId":"222222222222","roleArn":"arn:aws:iam::222222222222:role/ForwardRole","enabled":true}]}]`)) case r.Method == http.MethodPatch: patched = append(patched, r.URL.Path) w.Header().Set("Content-Type", "application/json") @@ -1288,7 +1357,6 @@ func TestRunBlocksRemovalsWhenNoCandidatesVisible(t *testing.T) { APIPrefix: "/api", Insecure: true, Apply: true, - PruneMissing: true, AllowRemovals: true, MaxRemovals: 1, MaxRemovalPercent: 100, @@ -1313,10 +1381,10 @@ func TestRunUsesExplicitSnapshotIDForNQE(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": seenQuery = r.URL.RawQuery w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111","Cloud Account Name":"acct-a"}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Setup ID":"setup-a","Cloud Account ID":"111111111111","Cloud Account Name":"acct-a"}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}]`)) default: w.WriteHeader(http.StatusNotFound) } @@ -1352,10 +1420,10 @@ func TestRunPinsLatestProcessedSnapshotForCLI(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == "/api/nqe": seenQuery = r.URL.RawQuery w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111","Collected?":true}]}`)) + _, _ = w.Write([]byte(`{"items":[{"Cloud Setup ID":"setup-a","Cloud Account ID":"111111111111","Collected?":true}]}`)) case r.Method == http.MethodGet && r.URL.Path == "/api/networks/network-1/cloudAccounts": w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111","roleArn":"arn:aws:iam::111:role/ForwardRole","enabled":true}]}]`)) + _, _ = w.Write([]byte(`[{"name":"setup-a","assumeRoleInfos":[{"accountId":"111111111111","roleArn":"arn:aws:iam::111111111111:role/ForwardRole","enabled":true}]}]`)) default: http.NotFound(w, r) } @@ -1413,10 +1481,10 @@ func TestRunRejectsStaleLatestProcessedSnapshot(t *testing.T) { } func TestRunFallsBackToSingleSetupWhenQueryLacksSetupID(t *testing.T) { - items := []map[string]any{{"Cloud Account ID": "111", "Cloud Account Name": "acct-a"}} + items := []map[string]any{{"Cloud Account ID": "111111111111", "Cloud Account Name": "acct-a"}} cloudAccounts := []api.CloudAccount{{ Name: "setup-only", - AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111:role/ForwardRole", Enabled: true}}, + AssumeRoleInfos: []api.AssumeRoleInfo{{RoleArn: "arn:aws:iam::111111111111:role/ForwardRole", Enabled: true}}, }} plan, err := buildPlan(items, cloudAccounts, DefaultQueryID+"-customized", nil) if err != nil { diff --git a/internal/app/snapshot_freshness_test.go b/internal/app/snapshot_freshness_test.go new file mode 100644 index 0000000..dc22139 --- /dev/null +++ b/internal/app/snapshot_freshness_test.go @@ -0,0 +1,115 @@ +package app + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +func TestValidateSnapshotFreshnessChecksExplicitSnapshot(t *testing.T) { + staleAt := time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/networks/network-1/snapshots" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, `{"snapshots":[{"id":"snapshot-stale","processedAt":"`+staleAt+`","state":"PROCESSED"}]}`) + })) + defer server.Close() + + client, err := api.NewClient(server.URL, "/api", "alice", "secret", false, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + err = validateSnapshotFreshness(context.Background(), client, Config{ + NetworkID: "network-1", + SnapshotID: "snapshot-stale", + MaxSnapshotAge: time.Hour, + }) + if err == nil || !strings.Contains(err.Error(), "explicit snapshot snapshot-stale is stale") { + t.Fatalf("validateSnapshotFreshness() error = %v; want explicit stale-snapshot rejection", err) + } +} + +func TestValidateSnapshotFreshnessAcceptsFreshExplicitSnapshot(t *testing.T) { + freshAt := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/networks/network-1/snapshots" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, `{"snapshots":[{"id":"snapshot-fresh","processedAt":"`+freshAt+`","state":"PROCESSED"}]}`) + })) + defer server.Close() + + client, err := api.NewClient(server.URL, "/api", "alice", "secret", false, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + if err := validateSnapshotFreshness(context.Background(), client, Config{ + NetworkID: "network-1", + SnapshotID: "snapshot-fresh", + MaxSnapshotAge: time.Hour, + }); err != nil { + t.Fatalf("validateSnapshotFreshness() error = %v", err) + } +} + +func TestValidateSnapshotFreshnessRejectsFutureExplicitSnapshot(t *testing.T) { + futureAt := time.Now().UTC().Add(time.Hour).Format(time.RFC3339) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/networks/network-1/snapshots" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, `{"snapshots":[{"id":"snapshot-future","processedAt":"`+futureAt+`","state":"PROCESSED"}]}`) + })) + defer server.Close() + + client, err := api.NewClient(server.URL, "/api", "alice", "secret", false, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + err = validateSnapshotFreshness(context.Background(), client, Config{ + NetworkID: "network-1", + SnapshotID: "snapshot-future", + MaxSnapshotAge: time.Hour, + }) + if err == nil || !strings.Contains(err.Error(), "invalid future timestamp") { + t.Fatalf("validateSnapshotFreshness() error = %v; want future-timestamp rejection", err) + } +} + +func TestPinLatestProcessedSnapshotRejectsFutureSnapshot(t *testing.T) { + futureAt := time.Now().UTC().Add(time.Hour).Format(time.RFC3339) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"id":"snapshot-future","processedAt":"`+futureAt+`","state":"PROCESSED"}`) + })) + defer server.Close() + + client, err := api.NewClient(server.URL, "/api", "alice", "secret", false, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + cfg := Config{NetworkID: "network-1", MaxSnapshotAge: time.Hour} + if err := pinLatestProcessedSnapshot(context.Background(), client, &cfg); err == nil || !strings.Contains(err.Error(), "invalid future timestamp") { + t.Fatalf("pinLatestProcessedSnapshot() error = %v; want future-timestamp rejection", err) + } +} + +func TestValidateSnapshotFreshnessToleratesSmallClockSkew(t *testing.T) { + skewedAt := time.Now().UTC().Add(30 * time.Second).Format(time.RFC3339) + if _, err := checkedSnapshotAge(api.SnapshotInfo{ID: "snapshot-skewed", ProcessedAt: skewedAt}, "explicit"); err != nil { + t.Fatalf("checkedSnapshotAge() rejected ordinary clock skew: %v", err) + } + aheadAt := time.Now().UTC().Add(snapshotClockSkewTolerance + time.Minute).Format(time.RFC3339) + if _, err := checkedSnapshotAge(api.SnapshotInfo{ID: "snapshot-future", ProcessedAt: aheadAt}, "explicit"); err == nil { + t.Fatal("checkedSnapshotAge() accepted a timestamp beyond the skew tolerance") + } +} diff --git a/internal/app/testdata/README.md b/internal/app/testdata/README.md new file mode 100644 index 0000000..2f40e9f --- /dev/null +++ b/internal/app/testdata/README.md @@ -0,0 +1,21 @@ +# Pre-branch compatibility artifacts + +These fixtures were exercised with the binary built from merge base +`0c0dbd5dfb41c8e713a33896b63b969225dcfd50` against a local fake Forward API. + +- `pre_branch_apply_plan.json` was emitted by a dry-run of the old binary. Its + original SHA-256 is + `da2612db7cbd41071306e6a8d28404d36de74ae98ebb9dd9ecf2c28dfa63738e`. +- `pre_branch_apply_plan.rollback.json` was emitted when that plan was applied + by the old binary. Its original SHA-256 is + `804a9a15d5aab5e5b65ff796990d61ad17bc64df9e59fcc3d5fbf385c94565b5`. +- `pre_branch_external_ids.csv` is not an output format: the old binary only + consumes External ID CSV files. This exact input was accepted by the old + binary and is retained to verify the historical input contract. + +The old JSON writer did not append a final newline. The compatibility test +removes the repository-added final newline before checking the original hash +and invoking the current reader. + +The pre-branch webhook server did not persist state, so that binary could not +produce an old webhook-state fixture. diff --git a/internal/app/testdata/pre_branch_apply_plan.json b/internal/app/testdata/pre_branch_apply_plan.json new file mode 100644 index 0000000..c77be4b --- /dev/null +++ b/internal/app/testdata/pre_branch_apply_plan.json @@ -0,0 +1,27 @@ +{ + "setup-a": { + "type": "AWS", + "name": "setup-a", + "regions": { + "us-east-1": 123 + }, + "regionToProxyServerId": {}, + "proxyServerId": "proxy-1", + "assumeRoleInfos": [ + { + "accountId": "111111111111", + "accountName": "account-one", + "roleArn": "arn:aws:iam::111111111111:role/ForwardRole", + "externalId": "old-external-id", + "enabled": true + }, + { + "accountId": "222222222222", + "accountName": "222222222222", + "roleArn": "arn:aws:iam::222222222222:role/ForwardRole", + "externalId": "remove-me", + "enabled": true + } + ] + } +} diff --git a/internal/app/testdata/pre_branch_apply_plan.rollback.json b/internal/app/testdata/pre_branch_apply_plan.rollback.json new file mode 100644 index 0000000..2e1fa69 --- /dev/null +++ b/internal/app/testdata/pre_branch_apply_plan.rollback.json @@ -0,0 +1,25 @@ +{ + "setup-a": { + "type": "AWS", + "name": "setup-a", + "regions": { + "us-east-1": 123 + }, + "regionToProxyServerId": {}, + "proxyServerId": "proxy-1", + "assumeRoleInfos": [ + { + "accountId": "111111111111", + "roleArn": "arn:aws:iam::111111111111:role/ForwardRole", + "externalId": "old-external-id", + "enabled": true + }, + { + "accountId": "222222222222", + "roleArn": "arn:aws:iam::222222222222:role/ForwardRole", + "externalId": "remove-me", + "enabled": false + } + ] + } +} diff --git a/internal/app/testdata/pre_branch_external_ids.csv b/internal/app/testdata/pre_branch_external_ids.csv new file mode 100644 index 0000000..2decfa0 --- /dev/null +++ b/internal/app/testdata/pre_branch_external_ids.csv @@ -0,0 +1,3 @@ +account_id,action,external_id +111111111111,set,new-external-id +222222222222,clear, diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 9dfd6e2..c214363 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -3,7 +3,7 @@ package monitor import ( "context" "fmt" - "slices" + "strings" "time" "github.com/forwardnetworks/aws-sync/internal/api" @@ -13,6 +13,9 @@ type StatusResult struct { NetworkID string `json:"network_id"` LatestProcessedSnapshot *api.SnapshotInfo `json:"latest_processed_snapshot,omitempty"` Snapshots []api.SnapshotInfo `json:"snapshots"` + ObservationAtomic bool `json:"observation_atomic"` + LatestListConsistent bool `json:"latest_list_consistent"` + ObservationWarning string `json:"observation_warning"` } type WaitResult struct { @@ -31,6 +34,7 @@ func Status(ctx context.Context, client *api.Client, networkID, snapshotID strin if err != nil { return nil, err } + latestListConsistent, observationWarning := compareLatestAndList(latest, snapshots) if snapshotID != "" { filtered := make([]api.SnapshotInfo, 0, 1) for _, snapshot := range snapshots { @@ -48,9 +52,33 @@ func Status(ctx context.Context, client *api.Client, networkID, snapshotID strin NetworkID: networkID, LatestProcessedSnapshot: latest, Snapshots: snapshots, + ObservationAtomic: false, + LatestListConsistent: latestListConsistent, + ObservationWarning: observationWarning, }, nil } +func compareLatestAndList(latest *api.SnapshotInfo, snapshots []api.SnapshotInfo) (bool, string) { + for _, snapshot := range snapshots { + if snapshot.ID != latest.ID { + continue + } + if snapshot.State != "" && latest.State != "" && !strings.EqualFold(strings.TrimSpace(snapshot.State), strings.TrimSpace(latest.State)) { + return false, fmt.Sprintf( + "latest processed endpoint reported snapshot %s in state %s, while the snapshot list reported state %s; the endpoints are separate reads and may have observed different points in time", + latest.ID, + latest.State, + snapshot.State, + ) + } + return true, "latest processed and snapshot list are separate API reads; matching responses do not guarantee a point-in-time atomic observation" + } + return false, fmt.Sprintf( + "latest processed endpoint reported snapshot %s, but it was absent from the snapshot list; the endpoints are separate reads and may have observed different points in time", + latest.ID, + ) +} + func Wait( ctx context.Context, client *api.Client, @@ -60,9 +88,16 @@ func Wait( if snapshotID == "" { return nil, fmt.Errorf("snapshot id is required") } + desiredState = normalizeSnapshotState(desiredState) if desiredState == "" { desiredState = "PROCESSED" } + if !recognizedSnapshotState(desiredState) { + return nil, fmt.Errorf( + "desired snapshot state %q is unrecognized; recognized states are PROCESSING, PROCESSED, FAILED, and ARCHIVED", + desiredState, + ) + } if pollInterval <= 0 { pollInterval = 10 * time.Second } @@ -72,22 +107,37 @@ func Wait( if err != nil { return nil, err } + found := false for _, snapshot := range snapshots { if snapshot.ID != snapshotID { continue } - if snapshot.State == desiredState { + found = true + state := normalizeSnapshotState(snapshot.State) + if !recognizedSnapshotState(state) { + return nil, fmt.Errorf( + "snapshot %s has unrecognized state %q; recognized states are PROCESSING, PROCESSED, FAILED, and ARCHIVED", + snapshotID, + snapshot.State, + ) + } + if state == desiredState { return &WaitResult{ NetworkID: networkID, Snapshot: snapshot, DesiredState: desiredState, }, nil } - if slices.Contains([]string{"FAILED", "ARCHIVED"}, snapshot.State) { + switch state { + case "FAILED", "ARCHIVED": return nil, fmt.Errorf("snapshot %s entered terminal state %s before reaching %s", snapshotID, snapshot.State, desiredState) + case "PROCESSING", "PROCESSED": } break } + if !found { + return nil, fmt.Errorf("snapshot %s not found in network %s", snapshotID, networkID) + } timer := time.NewTimer(pollInterval) select { @@ -98,3 +148,16 @@ func Wait( } } } + +func normalizeSnapshotState(state string) string { + return strings.ToUpper(strings.TrimSpace(state)) +} + +func recognizedSnapshotState(state string) bool { + switch state { + case "PROCESSING", "PROCESSED", "FAILED", "ARCHIVED": + return true + default: + return false + } +} diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 66f6f29..0533e45 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -2,8 +2,11 @@ package monitor import ( "context" + "encoding/json" + "fmt" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -29,6 +32,75 @@ func TestStatusFiltersBySnapshotID(t *testing.T) { } } +func TestStatusReportsDisagreeingLatestAndListResponses(t *testing.T) { + client, server := newTestClient(t, []string{ + `{"id":"latest","state":"PROCESSED"}`, + `{"snapshots":[{"id":"other","state":"PROCESSED"}]}`, + }) + defer server.Close() + + result, err := Status(context.Background(), client, "n1", "") + if err != nil { + t.Fatalf("Status() error = %v", err) + } + if result.ObservationAtomic { + t.Fatal("Status() reported an atomic observation from separate API requests") + } + if result.LatestListConsistent { + t.Fatal("Status() reported disagreeing latest/list responses as consistent") + } + if !strings.Contains(result.ObservationWarning, "latest") || !strings.Contains(result.ObservationWarning, "absent") { + t.Fatalf("unexpected observation warning: %q", result.ObservationWarning) + } +} + +func TestStatusFindsSnapshotBeyondFirstPage(t *testing.T) { + firstPage := make([]api.SnapshotInfo, api.PageLimit) + for index := range firstPage { + firstPage[index] = api.SnapshotInfo{ID: fmt.Sprintf("snapshot-%04d", index), State: "PROCESSED"} + } + firstPageJSON := mustSnapshotsJSON(t, firstPage) + secondPageJSON := mustSnapshotsJSON(t, []api.SnapshotInfo{{ID: "target", State: "PROCESSED"}}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/networks/n1/snapshots/latestProcessed": + _, _ = w.Write([]byte(`{"id":"target","state":"PROCESSED"}`)) + case "/api/networks/n1/snapshots": + if r.URL.Query().Get("includeArchived") != "true" || r.URL.Query().Get("limit") != fmt.Sprint(api.PageLimit) { + t.Errorf("unexpected snapshot query: %s", r.URL.RawQuery) + } + switch r.URL.Query().Get("offset") { + case "0": + _, _ = w.Write([]byte(firstPageJSON)) + case fmt.Sprint(api.PageLimit): + _, _ = w.Write([]byte(secondPageJSON)) + default: + t.Errorf("unexpected snapshot offset: %s", r.URL.Query().Get("offset")) + http.Error(w, "unexpected offset", http.StatusBadRequest) + } + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client, err := api.NewClient(server.URL, "/api", "u", "p", false, time.Second) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + result, err := Status(context.Background(), client, "n1", "target") + if err != nil { + t.Fatalf("Status() error = %v", err) + } + if len(result.Snapshots) != 1 || result.Snapshots[0].ID != "target" { + t.Fatalf("unexpected snapshots: %#v", result.Snapshots) + } + if !result.LatestListConsistent { + t.Fatalf("expected paginated list to contain latest snapshot: %#v", result) + } +} + func TestWaitReturnsWhenDesiredStateReached(t *testing.T) { client, server := newTestClient(t, []string{ `{"snapshots":[{"id":"s1","state":"PROCESSING"}]}`, @@ -48,6 +120,67 @@ func TestWaitReturnsWhenDesiredStateReached(t *testing.T) { } } +func TestWaitRejectsUnknownState(t *testing.T) { + client, server := newTestClient(t, []string{ + `{"snapshots":[{"id":"s1","state":"SOMETHING_NEW"}]}`, + }) + defer server.Close() + + _, err := Wait(context.Background(), client, "n1", "s1", "PROCESSED", time.Millisecond) + if err == nil || !strings.Contains(err.Error(), `unrecognized state "SOMETHING_NEW"`) { + t.Fatalf("Wait() error = %v; want unrecognized-state error", err) + } +} + +func TestWaitRejectsUnknownDesiredState(t *testing.T) { + client, server := newTestClient(t, nil) + defer server.Close() + + _, err := Wait(context.Background(), client, "n1", "s1", "SOMETHING_NEW", time.Millisecond) + if err == nil || !strings.Contains(err.Error(), `desired snapshot state "SOMETHING_NEW" is unrecognized`) { + t.Fatalf("Wait() error = %v; want unrecognized desired-state error", err) + } +} + +func TestWaitRecognizesMixedCaseTerminalStates(t *testing.T) { + for _, state := range []string{"failed", "ArChIvEd"} { + t.Run(state, func(t *testing.T) { + client, server := newTestClient(t, []string{ + fmt.Sprintf(`{"snapshots":[{"id":"s1","state":%q}]}`, state), + }) + defer server.Close() + + _, err := Wait(context.Background(), client, "n1", "s1", "processed", time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "terminal state "+state) { + t.Fatalf("Wait() error = %v; want terminal-state error", err) + } + }) + } +} + +func TestWaitFailsFastWhenSnapshotIsMissing(t *testing.T) { + client, server := newTestClient(t, []string{ + `{"snapshots":[{"id":"other","state":"PROCESSING"}]}`, + }) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err := Wait(ctx, client, "n1", "missing", "PROCESSED", 100*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "snapshot missing not found in network n1") { + t.Fatalf("Wait() error = %v; want missing-snapshot error", err) + } +} + +func mustSnapshotsJSON(t *testing.T, snapshots []api.SnapshotInfo) string { + t.Helper() + encoded, err := json.Marshal(api.NetworkSnapshots{Snapshots: snapshots}) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + return string(encoded) +} + func newTestClient(t *testing.T, responses []string) (*api.Client, *httptest.Server) { t.Helper() index := 0 diff --git a/internal/webhook/architecture_failure_test.go b/internal/webhook/architecture_failure_test.go new file mode 100644 index 0000000..ef4f552 --- /dev/null +++ b/internal/webhook/architecture_failure_test.go @@ -0,0 +1,376 @@ +package webhook + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/http/httptest" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/forwardnetworks/aws-sync/internal/app" +) + +const runP0WebhookFailureTests = false + +func skipUntilP0WebhookFixed(t *testing.T, finding string) { + t.Helper() + if !runP0WebhookFailureTests { + t.Skip("P0 characterization disabled until fixed: " + finding) + } +} + +func TestP0WebhookDeliveryAndScopeSafety(t *testing.T) { + skipUntilP0WebhookFixed(t, "webhook dedupe/order/scope are not durable or monotonic — docs/ARCHITECTURE_REVIEW.md §4, Ordering, time, and monitor/webhook behavior; §5, Bypasses") + + t.Run("queue full does not poison dedupe", func(t *testing.T) { + server := newP0WebhookServer(t, Config{}) + for i := 0; i < cap(server.jobs); i++ { + server.jobs <- Event{ID: fmt.Sprintf("filler-%d", i)} + } + event := Event{ + ID: "evt-queue-full", + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: "snapshot-1", + } + firstStatus, _ := p0HandleWebhookEvent(t, server, event) + if firstStatus != http.StatusServiceUnavailable { + t.Fatalf("first status = %d; want %d for full queue", firstStatus, http.StatusServiceUnavailable) + } + <-server.jobs + + secondStatus, secondBody := p0HandleWebhookEvent(t, server, event) + if secondStatus != http.StatusAccepted { + t.Errorf("retry status = %d; want %d after queue space becomes available", secondStatus, http.StatusAccepted) + } + if duplicate, _ := secondBody["duplicate"].(bool); duplicate { + t.Errorf("retry body duplicate = true; want false because the first delivery was never admitted") + } + if depth := len(server.jobs); depth != cap(server.jobs) { + t.Errorf("queue depth after retry = %d; want %d so the previously rejected event is not lost", depth, cap(server.jobs)) + } + }) + + t.Run("failed run is redeliverable", func(t *testing.T) { + var attempts atomic.Int32 + attemptCh := make(chan int, 2) + server := newP0WebhookServer(t, Config{ + Run: func(_ context.Context, cfg app.Config) (*app.Summary, error) { + attempt := int(attempts.Add(1)) + attemptCh <- attempt + if attempt == 1 { + return nil, errors.New("injected reconciliation failure") + } + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go server.worker(ctx) + + event := Event{ + ID: "evt-redelivery", + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: "snapshot-1", + } + firstStatus, _ := p0HandleWebhookEvent(t, server, event) + if firstStatus != http.StatusAccepted { + t.Fatalf("first status = %d; want %d", firstStatus, http.StatusAccepted) + } + p0WaitForAttempt(t, attemptCh, 1) + + secondStatus, _ := p0HandleWebhookEvent(t, server, event) + if secondStatus != http.StatusAccepted { + t.Errorf("redelivery status = %d; want %d", secondStatus, http.StatusAccepted) + } + select { + case attempt := <-attemptCh: + if attempt != 2 { + t.Errorf("redelivery attempt = %d; want 2", attempt) + } + case <-time.After(250 * time.Millisecond): + t.Errorf("redelivery was silently suppressed after failed run; attempts=%d want=2", attempts.Load()) + } + }) + + t.Run("restart retains successful dedupe", func(t *testing.T) { + var attempts atomic.Int32 + attemptCh := make(chan int, 2) + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + cfg := Config{ + StatePath: statePath, + Run: func(_ context.Context, cfg app.Config) (*app.Summary, error) { + attempt := int(attempts.Add(1)) + attemptCh <- attempt + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }, + } + event := Event{ + ID: "evt-persisted", + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: "snapshot-1", + } + + first := newP0WebhookServer(t, cfg) + firstCtx, firstCancel := context.WithCancel(context.Background()) + go first.worker(firstCtx) + firstStatus, _ := p0HandleWebhookEvent(t, first, event) + if firstStatus != http.StatusAccepted { + firstCancel() + t.Fatalf("first status = %d; want %d", firstStatus, http.StatusAccepted) + } + p0WaitForAttempt(t, attemptCh, 1) + firstCancel() + first.waitForWorker() + + restarted := newP0WebhookServer(t, cfg) + secondCtx, secondCancel := context.WithCancel(context.Background()) + defer secondCancel() + go restarted.worker(secondCtx) + secondStatus, secondBody := p0HandleWebhookEvent(t, restarted, event) + if secondStatus != http.StatusAccepted { + t.Errorf("post-restart duplicate status = %d; want %d", secondStatus, http.StatusAccepted) + } + if duplicate, _ := secondBody["duplicate"].(bool); !duplicate { + t.Errorf("post-restart duplicate = false; want durable duplicate recognition") + } + select { + case attempt := <-attemptCh: + t.Errorf("post-restart duplicate executed as attempt %d; want total attempts=1", attempt) + case <-time.After(100 * time.Millisecond): + } + }) + + t.Run("event ID collision does not suppress different scope", func(t *testing.T) { + server := newP0WebhookServer(t, Config{}) + first := Event{ + ID: "shared-event-id", + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: "snapshot-1", + } + second := Event{ + ID: "shared-event-id", + Type: "SNAPSHOT_READY", + NetworkID: "network-2", + SnapshotID: "snapshot-2", + } + firstStatus, _ := p0HandleWebhookEvent(t, server, first) + if firstStatus != http.StatusAccepted { + t.Fatalf("first status = %d; want %d", firstStatus, http.StatusAccepted) + } + secondStatus, secondBody := p0HandleWebhookEvent(t, server, second) + if secondStatus != http.StatusAccepted { + t.Errorf("second status = %d; want %d", secondStatus, http.StatusAccepted) + } + if duplicate, _ := secondBody["duplicate"].(bool); duplicate { + t.Errorf("second event duplicate = true; want false when network/snapshot scope differs") + } + if depth := len(server.jobs); depth != 2 { + t.Errorf("queue depth = %d; want 2 distinct scoped events", depth) + } + }) + + t.Run("older snapshot cannot follow newer snapshot", func(t *testing.T) { + const ( + newerSnapshotID = "snapshot-new" + olderSnapshotID = "snapshot-old" + ) + forward := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/networks/network-1/snapshots": + _, _ = io.WriteString(w, `{"snapshots":[ + {"id":"snapshot-new","createdAt":"2026-07-25T12:00:00Z","processedAt":"2026-07-25T12:05:00Z","state":"PROCESSED"}, + {"id":"snapshot-old","createdAt":"2026-07-25T11:00:00Z","processedAt":"2026-07-25T11:05:00Z","state":"PROCESSED"} + ]}`) + case "/api/networks/network-1/snapshots/latestProcessed": + _, _ = io.WriteString(w, `{"id":"snapshot-new","createdAt":"2026-07-25T12:00:00Z","processedAt":"2026-07-25T12:05:00Z","state":"PROCESSED"}`) + default: + http.NotFound(w, r) + } + })) + defer forward.Close() + + var ( + mu sync.Mutex + calls []string + ) + callCh := make(chan string, 2) + server := newP0WebhookServer(t, Config{ + App: app.Config{ + Host: forward.URL, + Username: "alice", + Password: "secret", + NetworkID: "network-1", + APIPrefix: "/api", + }, + Run: func(_ context.Context, cfg app.Config) (*app.Summary, error) { + mu.Lock() + calls = append(calls, cfg.SnapshotID) + mu.Unlock() + callCh <- cfg.SnapshotID + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go server.worker(ctx) + + newer := Event{ + ID: "evt-newer", + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: newerSnapshotID, + } + older := Event{ + ID: "evt-older", + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: olderSnapshotID, + } + newerStatus, _ := p0HandleWebhookEvent(t, server, newer) + if newerStatus != http.StatusAccepted { + t.Fatalf("newer status = %d; want %d", newerStatus, http.StatusAccepted) + } + p0WaitForSnapshot(t, callCh, newer.SnapshotID) + + olderStatus, _ := p0HandleWebhookEvent(t, server, older) + if olderStatus == http.StatusAccepted { + t.Errorf("older snapshot status = %d; want rejection after newer snapshot completed", olderStatus) + } + select { + case snapshotID := <-callCh: + t.Errorf("older snapshot %s executed after newer snapshot; want monotonic per-network watermark", snapshotID) + case <-time.After(100 * time.Millisecond): + } + mu.Lock() + gotCalls := append([]string(nil), calls...) + mu.Unlock() + if len(gotCalls) != 1 || gotCalls[0] != newer.SnapshotID { + t.Errorf("snapshot calls = %v; want [%s]", gotCalls, newer.SnapshotID) + } + }) + + t.Run("event cannot expand configured scope", func(t *testing.T) { + tests := []struct { + name string + event Event + }{ + { + name: "network", + event: Event{ + ID: "evt-network-expansion", + Type: "SNAPSHOT_READY", + NetworkID: "other-network", + SnapshotID: "snapshot-1", + SetupIDs: []string{"allowed-setup"}, + }, + }, + { + name: "setup", + event: Event{ + ID: "evt-setup-expansion", + Type: "SNAPSHOT_READY", + NetworkID: "allowed-network", + SnapshotID: "snapshot-1", + SetupIDs: []string{"other-setup"}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := newP0WebhookServer(t, Config{ + App: app.Config{ + Host: "https://fwd.example", + Username: "alice", + Password: "secret", + NetworkID: "allowed-network", + SetupIDs: []string{"allowed-setup"}, + }, + }) + status, _ := p0HandleWebhookEvent(t, server, test.event) + if status >= 200 && status < 300 { + t.Errorf("scope-expansion status = %d; want non-2xx rejection for configured scope", status) + } + if depth := len(server.jobs); depth != 0 { + t.Errorf("queue depth = %d; want 0 after scope-expansion attempt", depth) + } + }) + } + }) +} + +func newP0WebhookServer(t *testing.T, cfg Config) *Server { + t.Helper() + if cfg.App.Host == "" { + cfg.App = app.Config{ + Host: "https://fwd.example", + Username: "alice", + Password: "secret", + } + } + if cfg.StatePath == "" { + cfg.StatePath = filepath.Join(t.TempDir(), "webhook-state.json") + } + cfg.Logger = log.New(io.Discard, "", 0) + server, err := New(cfg) + if err != nil { + t.Fatalf("New() error = %v", err) + } + t.Cleanup(func() { + waitForWebhookStateIdle(t, server) + }) + return server +} + +func p0HandleWebhookEvent(t *testing.T, server *Server, event Event) (int, map[string]any) { + t.Helper() + data, err := json.Marshal(event) + if err != nil { + t.Fatalf("encode event: %v", err) + } + request := httptest.NewRequest(http.MethodPost, server.cfg.Path, bytes.NewReader(data)) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + server.handleEvent(recorder, request) + response := recorder.Result() + defer response.Body.Close() + body := make(map[string]any) + _ = json.NewDecoder(response.Body).Decode(&body) + return response.StatusCode, body +} + +func p0WaitForAttempt(t *testing.T, attempts <-chan int, want int) { + t.Helper() + select { + case got := <-attempts: + if got != want { + t.Fatalf("run attempt = %d; want %d", got, want) + } + case <-time.After(time.Second): + t.Fatalf("timed out waiting for run attempt %d", want) + } +} + +func p0WaitForSnapshot(t *testing.T, snapshots <-chan string, want string) { + t.Helper() + select { + case got := <-snapshots: + if got != want { + t.Fatalf("snapshot run = %q; want %q", got, want) + } + case <-time.After(time.Second): + t.Fatalf("timed out waiting for snapshot %q", want) + } +} diff --git a/internal/webhook/durable_queue_test.go b/internal/webhook/durable_queue_test.go new file mode 100644 index 0000000..5a43c7f --- /dev/null +++ b/internal/webhook/durable_queue_test.go @@ -0,0 +1,415 @@ +package webhook + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/forwardnetworks/aws-sync/internal/app" +) + +func TestAcceptedEventSurvivesCrashBeforeCompletion(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + event := durableTestEvent("evt-admitted") + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + + status, _ := handleDurableTestEvent(t, server, event) + if status != http.StatusAccepted { + t.Fatalf("admission status = %d; want %d", status, http.StatusAccepted) + } + state := mustLoadDurableTestState(t, statePath) + job, exists := state.PendingEvents[eventDedupeKey(event)] + if !exists { + t.Fatal("accepted event is absent from durable pending state") + } + if job.Status != webhookJobQueued || job.Attempts != 0 { + t.Fatalf("persisted job = %#v; want queued with zero attempts", job) + } + if _, completed := state.CompletedEvents[eventDedupeKey(event)]; completed { + t.Fatal("accepted but incomplete event was recorded as completed") + } + + // A real crash discards process-local admission signals and the channel. + finishProcessAdmission(statePath, eventDedupeKey(event)) + var recoveredRuns atomic.Int32 + restarted := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + recoveredRuns.Add(1) + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go restarted.worker(ctx) + + waitForDurableTestState(t, statePath, func(state webhookState) bool { + _, pending := state.PendingEvents[eventDedupeKey(event)] + _, completed := state.CompletedEvents[eventDedupeKey(event)] + return !pending && completed + }) + if got := recoveredRuns.Load(); got != 1 { + t.Fatalf("recovered run count = %d; want 1", got) + } +} + +func TestInFlightEventIsReplayedAfterCrash(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + event := durableTestEvent("evt-in-flight") + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + status, _ := handleDurableTestEvent(t, server, event) + if status != http.StatusAccepted { + t.Fatalf("admission status = %d; want %d", status, http.StatusAccepted) + } + queued := <-server.jobs + if _, exists, err := server.markJobInFlight(queued); err != nil || !exists { + t.Fatalf("markJobInFlight() exists=%v error=%v", exists, err) + } + finishProcessAdmission(statePath, eventDedupeKey(event)) + + restartedRuns := make(chan struct{}, 1) + restarted := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + restartedRuns <- struct{}{} + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + state := mustLoadDurableTestState(t, statePath) + if got := state.PendingEvents[eventDedupeKey(event)].Status; got != webhookJobQueued { + t.Fatalf("recovered job status = %q; want %q", got, webhookJobQueued) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go restarted.worker(ctx) + select { + case <-restartedRuns: + case <-time.After(2 * time.Second): + t.Fatal("in-flight event was not replayed after restart") + } + waitForDurableTestState(t, statePath, func(state webhookState) bool { + _, pending := state.PendingEvents[eventDedupeKey(event)] + _, completed := state.CompletedEvents[eventDedupeKey(event)] + return !pending && completed + }) +} + +func TestFinalInFlightAttemptIsDeadLetteredAfterCrash(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + event := durableTestEvent("evt-final-crash") + state := newWebhookState() + now := time.Now().UTC() + state.PendingEvents[eventDedupeKey(event)] = pendingWebhookEvent{ + Event: event, + Status: webhookJobInFlight, + Attempts: webhookMaxAttempts, + AcceptedAt: now.Add(-time.Minute), + LastAttemptAt: &now, + } + if err := persistWebhookState(statePath, state); err != nil { + t.Fatalf("persist in-flight state: %v", err) + } + + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + t.Fatal("final crashed attempt must remain visible for operator review instead of running forever") + return nil, nil + }) + if _, pending := server.state.PendingEvents[eventDedupeKey(event)]; pending { + t.Fatal("exhausted in-flight event remains pending after restart") + } + deadLetter, exists := server.state.DeadLetterEvents[eventDedupeKey(event)] + if !exists { + t.Fatal("exhausted in-flight event was not dead-lettered after restart") + } + if deadLetter.Attempts != webhookMaxAttempts || !strings.Contains(deadLetter.LastError, "completion is ambiguous") { + t.Fatalf("dead-letter record = %#v; want bounded ambiguous-crash record", deadLetter) + } +} + +func TestRetryExhaustionMovesEventToDeadLetterAndRedeliveryDrainsIt(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + event := durableTestEvent("evt-dead-letter") + var ( + fail atomic.Bool + runs atomic.Int32 + ) + fail.Store(true) + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + runs.Add(1) + if fail.Load() { + return nil, errors.New("permanent test failure") + } + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + server.maxAttempts = 3 + server.retryBaseDelay = time.Millisecond + server.retryMaxDelay = 2 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go server.worker(ctx) + + status, _ := handleDurableTestEvent(t, server, event) + if status != http.StatusAccepted { + t.Fatalf("admission status = %d; want %d", status, http.StatusAccepted) + } + waitForDurableTestState(t, statePath, func(state webhookState) bool { + _, deadLettered := state.DeadLetterEvents[eventDedupeKey(event)] + return deadLettered + }) + state := mustLoadDurableTestState(t, statePath) + deadLetter := state.DeadLetterEvents[eventDedupeKey(event)] + if deadLetter.Attempts != server.maxAttempts { + t.Fatalf("dead-letter attempts = %d; want %d", deadLetter.Attempts, server.maxAttempts) + } + if _, pending := state.PendingEvents[eventDedupeKey(event)]; pending { + t.Fatal("dead-lettered event remains pending") + } + if _, completed := state.CompletedEvents[eventDedupeKey(event)]; completed { + t.Fatal("dead-lettered event was recorded as completed") + } + time.Sleep(10 * time.Millisecond) + if got := runs.Load(); got != int32(server.maxAttempts) { + t.Fatalf("run count after retry exhaustion = %d; want bounded count %d", got, server.maxAttempts) + } + + // Re-delivering the original authenticated payload is the operator drain action. + fail.Store(false) + status, body := handleDurableTestEvent(t, server, event) + if status != http.StatusAccepted { + t.Fatalf("dead-letter redelivery status = %d; want %d", status, http.StatusAccepted) + } + if duplicate, _ := body["duplicate"].(bool); duplicate { + t.Fatal("dead-letter redelivery reported duplicate instead of starting a fresh retry cycle") + } + waitForDurableTestState(t, statePath, func(state webhookState) bool { + _, deadLettered := state.DeadLetterEvents[eventDedupeKey(event)] + _, completed := state.CompletedEvents[eventDedupeKey(event)] + return !deadLettered && completed + }) +} + +func TestWebhookStateV1IsUpgraded(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + completedAt := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC) + mark := snapshotWatermark{ + NetworkID: "network-1", + SetupID: "setup-1", + SnapshotID: "snapshot-1", + SnapshotAt: completedAt.Add(-time.Minute), + CompletedAt: completedAt, + } + v1 := struct { + Version int `json:"version"` + CompletedEvents map[string]time.Time `json:"completed_events"` + Watermarks map[string]snapshotWatermark `json:"snapshot_watermarks"` + }{ + Version: previousWebhookStateVersion, + CompletedEvents: map[string]time.Time{"completed-key": completedAt}, + Watermarks: map[string]snapshotWatermark{watermarkKey("network-1", "setup-1"): mark}, + } + data, err := json.Marshal(v1) + if err != nil { + t.Fatalf("marshal v1 state: %v", err) + } + if err := os.WriteFile(statePath, data, 0o600); err != nil { + t.Fatalf("write v1 state: %v", err) + } + + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + if server.state.Version != webhookStateVersion { + t.Fatalf("loaded state version = %d; want %d", server.state.Version, webhookStateVersion) + } + upgraded := mustLoadDurableTestState(t, statePath) + if upgraded.CompletedEvents["completed-key"] != completedAt { + t.Fatal("v1 completed-event record was not preserved") + } + if got := upgraded.Watermarks[watermarkKey("network-1", "setup-1")]; got != mark { + t.Fatalf("v1 watermark = %#v; want %#v", got, mark) + } + if upgraded.PendingEvents == nil || upgraded.DeadLetterEvents == nil { + t.Fatal("v2 queue maps were not initialized") + } + info, err := os.Stat(statePath) + if err != nil { + t.Fatalf("stat upgraded state: %v", err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("upgraded state mode = %o; want 600", got) + } +} + +func TestAdmissionPersistenceFailureIsNotAcknowledged(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + server.persistState = func(string, webhookState) error { + return errors.New("injected state write failure") + } + event := durableTestEvent("evt-persist-failure") + + status, body := handleDurableTestEvent(t, server, event) + if status != http.StatusServiceUnavailable { + t.Fatalf("admission status = %d; want %d", status, http.StatusServiceUnavailable) + } + if message, _ := body["error"].(string); !strings.Contains(message, "persist accepted webhook event") { + t.Fatalf("admission error = %q; want persistence failure", message) + } + if len(server.jobs) != 0 { + t.Fatalf("queue depth = %d; want 0 after failed persistence", len(server.jobs)) + } + if len(server.state.PendingEvents) != 0 { + t.Fatal("failed admission remained pending in memory") + } + if _, err := os.Stat(statePath); !os.IsNotExist(err) { + t.Fatalf("state file exists after failed first admission; stat error=%v", err) + } +} + +func TestWorkerCompletionWaitsForDurableSuccess(t *testing.T) { + statePath := filepath.Join(t.TempDir(), "webhook-state.json") + event := durableTestEvent("evt-durable-completion") + server := newDurableQueueServer(t, statePath, func(_ context.Context, cfg app.Config) (*app.Summary, error) { + return &app.Summary{NetworkID: cfg.NetworkID, SnapshotID: cfg.SnapshotID}, nil + }) + originalPersist := server.persistState + durableWriteStarted := make(chan struct{}) + allowDurableWrite := make(chan struct{}) + var blocked atomic.Bool + server.persistState = func(path string, state webhookState) error { + if _, completed := state.CompletedEvents[eventDedupeKey(event)]; completed && blocked.CompareAndSwap(false, true) { + close(durableWriteStarted) + <-allowDurableWrite + } + return originalPersist(path, state) + } + defer func() { + if blocked.Load() { + select { + case <-allowDurableWrite: + default: + close(allowDurableWrite) + } + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + go server.worker(ctx) + status, _ := handleDurableTestEvent(t, server, event) + if status != http.StatusAccepted { + cancel() + t.Fatalf("admission status = %d; want %d", status, http.StatusAccepted) + } + select { + case <-durableWriteStarted: + case <-time.After(2 * time.Second): + cancel() + t.Fatal("worker never reached durable success write") + } + cancel() + select { + case <-server.workerDone: + t.Fatal("worker reported completion before successful state was durable") + default: + } + close(allowDurableWrite) + server.waitForWorker() + + state := mustLoadDurableTestState(t, statePath) + if _, completed := state.CompletedEvents[eventDedupeKey(event)]; !completed { + t.Fatal("worker completed without durable dedupe state") + } + if _, pending := state.PendingEvents[eventDedupeKey(event)]; pending { + t.Fatal("worker completed with successfully processed event still pending") + } +} + +func TestRetryDelayIsExponentiallyBounded(t *testing.T) { + server := &Server{retryBaseDelay: time.Second, retryMaxDelay: 5 * time.Second} + want := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 5 * time.Second, 5 * time.Second} + for index, expected := range want { + if got := server.retryDelay(index + 1); got != expected { + t.Errorf("retryDelay(%d) = %s; want %s", index+1, got, expected) + } + } +} + +func newDurableQueueServer(t *testing.T, statePath string, run RunFunc) *Server { + t.Helper() + server, err := New(Config{ + StatePath: statePath, + Logger: log.New(io.Discard, "", 0), + Run: run, + App: app.Config{ + Host: "https://fwd.example", + Username: "user", + Password: "password", + }, + }) + if err != nil { + t.Fatalf("New() error = %v", err) + } + return server +} + +func durableTestEvent(id string) Event { + return Event{ + ID: id, + Type: "SNAPSHOT_READY", + NetworkID: "network-1", + SnapshotID: "snapshot-1", + SetupIDs: []string{"setup-1"}, + } +} + +func handleDurableTestEvent(t *testing.T, server *Server, event Event) (int, map[string]any) { + t.Helper() + payload, err := json.Marshal(event) + if err != nil { + t.Fatalf("marshal event: %v", err) + } + request := httptest.NewRequest(http.MethodPost, server.cfg.Path, bytes.NewReader(payload)) + response := httptest.NewRecorder() + server.handleEvent(response, request) + body := make(map[string]any) + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatalf("decode response %q: %v", response.Body.String(), err) + } + return response.Code, body +} + +func mustLoadDurableTestState(t *testing.T, statePath string) webhookState { + t.Helper() + state, err := loadWebhookState(statePath) + if err != nil { + t.Fatalf("load webhook state: %v", err) + } + return state +} + +func waitForDurableTestState(t *testing.T, statePath string, ready func(webhookState) bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + state := mustLoadDurableTestState(t, statePath) + if ready(state) { + return + } + if time.Now().After(deadline) { + encoded, _ := json.Marshal(state) + t.Fatalf("timed out waiting for webhook state transition: %s", encoded) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/webhook/server.go b/internal/webhook/server.go index e4d07e9..7fdb227 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -7,11 +7,14 @@ import ( "fmt" "log" "net/http" + "net/url" "sort" "strings" "sync" + "sync/atomic" "time" + "github.com/forwardnetworks/aws-sync/internal/api" "github.com/forwardnetworks/aws-sync/internal/app" ) @@ -22,6 +25,7 @@ type Config struct { Path string BasicUsername string BasicPassword string + StatePath string App app.Config Logger *log.Logger Run RunFunc @@ -36,13 +40,23 @@ type Event struct { } type Server struct { - cfg Config - logger *log.Logger - run RunFunc - jobs chan Event + cfg Config + logger *log.Logger + run RunFunc + persistState func(string, webhookState) error + jobs chan Event - seenMu sync.Mutex - seen map[string]time.Time + stateMu sync.Mutex + state webhookState + active map[string]snapshotWatermark + queued map[string]bool + scheduleGeneration map[string]uint64 + lookupSnapshotTime bool + workerRunning atomic.Bool + workerDone chan struct{} + maxAttempts int + retryBaseDelay time.Duration + retryMaxDelay time.Duration } func New(cfg Config) (*Server, error) { @@ -60,9 +74,14 @@ func New(cfg Config) (*Server, error) { if cfg.Logger == nil { cfg.Logger = log.Default() } + usingDefaultRun := cfg.Run == nil if cfg.Run == nil { cfg.Run = app.Run } + cfg.App.Unattended = true + if strings.TrimSpace(cfg.App.AuthorizationActor) == "" { + cfg.App.AuthorizationActor = "webhook" + } if strings.TrimSpace(cfg.App.Host) == "" { return nil, fmt.Errorf("Forward host is required") } @@ -72,14 +91,44 @@ func New(cfg Config) (*Server, error) { if strings.TrimSpace(cfg.App.Password) == "" { return nil, fmt.Errorf("Forward password is required") } + cfg.BasicUsername = strings.TrimSpace(cfg.BasicUsername) + cfg.BasicPassword = strings.TrimSpace(cfg.BasicPassword) + if cfg.App.Apply && (cfg.BasicUsername == "" || cfg.BasicPassword == "") { + return nil, fmt.Errorf("webhook Basic authentication username and password are required when apply is enabled") + } + if cfg.App.Apply && strings.TrimSpace(cfg.App.NetworkID) == "" { + return nil, fmt.Errorf("configured Forward network ID is required when webhook apply is enabled") + } + statePath, err := resolveStatePath(cfg.StatePath) + if err != nil { + return nil, err + } + cfg.StatePath = statePath + state, err := loadWebhookState(statePath) + if err != nil { + return nil, err + } - return &Server{ - cfg: cfg, - logger: cfg.Logger, - run: cfg.Run, - jobs: make(chan Event, 32), - seen: make(map[string]time.Time), - }, nil + server := &Server{ + cfg: cfg, + logger: cfg.Logger, + run: cfg.Run, + persistState: persistWebhookState, + jobs: make(chan Event, webhookQueueCapacity), + state: state, + active: make(map[string]snapshotWatermark), + queued: make(map[string]bool), + scheduleGeneration: make(map[string]uint64), + lookupSnapshotTime: usingDefaultRun || cfg.App.Apply || isLoopbackHost(cfg.App.Host), + workerDone: make(chan struct{}), + maxAttempts: webhookMaxAttempts, + retryBaseDelay: webhookRetryBaseDelay, + retryMaxDelay: webhookRetryMaxDelay, + } + if err := server.recoverInFlightJobs(); err != nil { + return nil, err + } + return server, nil } func (s *Server) Run(ctx context.Context) error { @@ -88,9 +137,10 @@ func (s *Server) Run(ctx context.Context) error { mux.HandleFunc(s.cfg.Path, s.handleEvent) httpServer := &http.Server{Addr: s.cfg.Listen, Handler: mux, ReadHeaderTimeout: 10 * time.Second} - go s.worker(ctx) + workerCtx, stopWorker := context.WithCancel(ctx) + go s.worker(workerCtx) go func() { - <-ctx.Done() + <-workerCtx.Done() shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = httpServer.Shutdown(shutdownCtx) @@ -98,6 +148,8 @@ func (s *Server) Run(ctx context.Context) error { s.logger.Printf("webhook server listening on %s%s", s.cfg.Listen, s.cfg.Path) err := httpServer.ListenAndServe() + stopWorker() + s.waitForWorker() if err != nil && err != http.ErrServerClosed { return err } @@ -105,7 +157,17 @@ func (s *Server) Run(ctx context.Context) error { } func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusOK, map[string]any{"ok": true, "queueDepth": len(s.jobs), "path": s.cfg.Path}) + s.stateMu.Lock() + pendingDepth := len(s.state.PendingEvents) + deadLetterDepth := len(s.state.DeadLetterEvents) + s.stateMu.Unlock() + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, + "queueDepth": len(s.jobs), + "pendingDepth": pendingDepth, + "deadLetterDepth": deadLetterDepth, + "path": s.cfg.Path, + }) } func (s *Server) handleEvent(w http.ResponseWriter, r *http.Request) { @@ -128,6 +190,9 @@ func (s *Server) handleEvent(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "only SNAPSHOT_READY events are supported") return } + if event.Type == "" { + event.Type = "SNAPSHOT_READY" + } if strings.TrimSpace(event.NetworkID) == "" { writeError(w, http.StatusBadRequest, "networkId is required") return @@ -137,15 +202,107 @@ func (s *Server) handleEvent(w http.ResponseWriter, r *http.Request) { return } event.SetupIDs = cleanSetupIDs(append(event.SetupIDs, setupIDsFromQuery(r)...)) - if s.seenBefore(event) { - writeJSON(w, http.StatusAccepted, map[string]any{"accepted": true, "duplicate": true, "networkId": event.NetworkID, "snapshotId": event.SnapshotID, "setupIds": event.SetupIDs}) + var err error + event, err = s.intersectConfiguredScope(event) + if err != nil { + writeError(w, http.StatusForbidden, err.Error()) return } + key := eventDedupeKey(event) + s.stateMu.Lock() + s.pruneCompletedLocked(time.Now().UTC()) + _, duplicate := s.state.CompletedEvents[key] + s.stateMu.Unlock() + if duplicate { + writeJSON(w, http.StatusAccepted, eventResponse(event, true)) + return + } + if err := s.rejectOlderSnapshot(r.Context(), event); err != nil { + writeError(w, http.StatusConflict, err.Error()) + return + } + + for { + done, admitted := registerProcessAdmission(s.cfg.StatePath, key) + if !admitted { + select { + case <-done: + if err := s.reloadState(); err != nil { + writeError(w, http.StatusServiceUnavailable, err.Error()) + return + } + continue + case <-r.Context().Done(): + writeError(w, http.StatusServiceUnavailable, "matching webhook job is still in progress") + return + } + } + if err := s.reloadState(); err != nil { + finishProcessAdmission(s.cfg.StatePath, key) + writeError(w, http.StatusServiceUnavailable, err.Error()) + return + } + duplicate, err := s.admitEvent(event) + if err != nil { + finishProcessAdmission(s.cfg.StatePath, key) + writeError(w, http.StatusServiceUnavailable, err.Error()) + return + } + if duplicate { + finishProcessAdmission(s.cfg.StatePath, key) + } + writeJSON(w, http.StatusAccepted, eventResponse(event, duplicate)) + return + } +} + +func (s *Server) admitEvent(event Event) (bool, error) { + key := eventDedupeKey(event) + now := time.Now().UTC() + s.stateMu.Lock() + defer s.stateMu.Unlock() + + s.pruneCompletedLocked(now) + if _, duplicate := s.state.CompletedEvents[key]; duplicate { + return true, nil + } + if job, pending := s.state.PendingEvents[key]; pending { + if job.Status == webhookJobQueued && !s.queued[key] { + select { + case s.jobs <- job.Event: + s.queued[key] = true + s.invalidateScheduleLocked(key) + default: + } + } + return true, nil + } + if len(s.state.PendingEvents) >= webhookQueueCapacity { + return false, fmt.Errorf("job queue is full") + } + + previous := cloneWebhookState(s.state) + next := cloneWebhookState(s.state) + delete(next.DeadLetterEvents, key) + next.PendingEvents[key] = pendingWebhookEvent{ + Event: event, + Status: webhookJobQueued, + AcceptedAt: now, + } + if err := s.persistState(s.cfg.StatePath, next); err != nil { + return false, fmt.Errorf("persist accepted webhook event: %w", err) + } + s.state = next select { case s.jobs <- event: - writeJSON(w, http.StatusAccepted, map[string]any{"accepted": true, "duplicate": false, "networkId": event.NetworkID, "snapshotId": event.SnapshotID, "setupIds": event.SetupIDs}) + s.queued[key] = true + return false, nil default: - writeError(w, http.StatusServiceUnavailable, "job queue is full") + if err := s.persistState(s.cfg.StatePath, previous); err != nil { + return false, fmt.Errorf("job queue is full; event remains durably pending because admission rollback failed: %w", err) + } + s.state = previous + return false, fmt.Errorf("job queue is full") } } @@ -153,7 +310,7 @@ func (s *Server) authorized(r *http.Request) bool { basicUsername := strings.TrimSpace(s.cfg.BasicUsername) basicPassword := strings.TrimSpace(s.cfg.BasicPassword) if basicUsername == "" && basicPassword == "" { - return true + return !s.cfg.App.Apply } if basicUsername == "" || basicPassword == "" { return false @@ -165,53 +322,259 @@ func (s *Server) authorized(r *http.Request) bool { } func (s *Server) worker(ctx context.Context) { + s.workerRunning.Store(true) + defer func() { + s.releasePendingAdmissions() + s.workerRunning.Store(false) + close(s.workerDone) + }() + s.startPendingJobs(ctx) for { select { case <-ctx.Done(): return case event := <-s.jobs: - cfg := s.cfg.App - cfg.NetworkID = event.NetworkID - cfg.SnapshotID = event.SnapshotID - if len(event.SetupIDs) > 0 { - cfg.SetupIDs = event.SetupIDs - } - s.logger.Printf("processing webhook event: networkId=%s snapshotId=%s setupIds=%v", event.NetworkID, event.SnapshotID, cfg.SetupIDs) - summary, err := s.run(ctx, cfg) - if err != nil { - s.logger.Printf("webhook job failed: networkId=%s snapshotId=%s setupIds=%v err=%v", event.NetworkID, event.SnapshotID, cfg.SetupIDs, err) - continue - } - encoded, err := json.Marshal(summary) - if err != nil { - s.logger.Printf("webhook job completed but summary could not be encoded: networkId=%s snapshotId=%s err=%v", event.NetworkID, event.SnapshotID, err) - continue - } - s.logger.Printf("webhook job completed: %s", encoded) + s.processJob(ctx, event) } } } -func (s *Server) seenBefore(event Event) bool { - key := event.ID - if strings.TrimSpace(key) == "" { - key = fmt.Sprintf("%s:%s:%s:%s", strings.TrimSpace(event.Type), strings.TrimSpace(event.NetworkID), strings.TrimSpace(event.SnapshotID), strings.Join(cleanSetupIDs(event.SetupIDs), ",")) +func (s *Server) waitForWorker() { + <-s.workerDone +} + +func (s *Server) processJob(ctx context.Context, event Event) { + defer finishProcessAdmission(s.cfg.StatePath, eventDedupeKey(event)) + job, exists, err := s.markJobInFlight(event) + if err != nil { + s.logger.Printf("webhook job could not be marked in-flight: networkId=%s snapshotId=%s setupIds=%v err=%v", event.NetworkID, event.SnapshotID, event.SetupIDs, err) + s.scheduleJob(ctx, event, time.Now().UTC().Add(s.retryBaseDelay)) + return + } + if !exists { + return + } + event = job.Event + cfg := s.cfg.App + cfg.NetworkID = event.NetworkID + cfg.SnapshotID = event.SnapshotID + cfg.SetupIDs = append([]string(nil), event.SetupIDs...) + + snapshotTime, snapshotTimeKnown, err := s.resolveSnapshotTime(ctx, event) + if err != nil && cfg.Apply { + s.failJob(ctx, job, fmt.Errorf("resolve snapshot ordering metadata: %w", err)) + return + } + if err != nil { + s.logger.Printf("webhook snapshot ordering unavailable for non-apply job: networkId=%s snapshotId=%s err=%v", event.NetworkID, event.SnapshotID, err) + } + if snapshotTimeKnown { + if err := s.beginSnapshot(event, snapshotTime); err != nil { + s.failJob(ctx, job, err) + return + } + defer s.endSnapshot(event) + } + + s.logger.Printf("processing webhook event: networkId=%s snapshotId=%s setupIds=%v attempt=%d/%d", event.NetworkID, event.SnapshotID, cfg.SetupIDs, job.Attempts, s.maxAttempts) + summary, err := s.run(ctx, cfg) + if err != nil { + s.failJob(ctx, job, err) + return + } + if err := s.recordSuccess(event, snapshotTime, snapshotTimeKnown); err != nil { + s.logger.Printf("webhook job completed but durable state could not be recorded; leaving it in-flight for restart recovery: networkId=%s snapshotId=%s setupIds=%v err=%v", event.NetworkID, event.SnapshotID, cfg.SetupIDs, err) + return + } + encoded, err := json.Marshal(summary) + if err != nil { + s.logger.Printf("webhook job completed but summary could not be encoded: networkId=%s snapshotId=%s err=%v", event.NetworkID, event.SnapshotID, err) + return + } + s.logger.Printf("webhook job completed: %s", encoded) +} + +func (s *Server) releasePendingAdmissions() { + s.stateMu.Lock() + keys := make([]string, 0, len(s.state.PendingEvents)) + for key := range s.state.PendingEvents { + keys = append(keys, key) + } + s.stateMu.Unlock() + for _, key := range keys { + finishProcessAdmission(s.cfg.StatePath, key) + } +} + +func (s *Server) failJob(ctx context.Context, job pendingWebhookEvent, runErr error) { + event := job.Event + if ctx.Err() != nil { + s.logger.Printf("webhook job interrupted during shutdown; leaving it in-flight for restart recovery: networkId=%s snapshotId=%s setupIds=%v err=%v", event.NetworkID, event.SnapshotID, event.SetupIDs, runErr) + return + } + nextAttempt, deadLettered, err := s.recordJobFailure(job, runErr) + if err != nil { + s.logger.Printf("webhook job failed and durable retry state could not be recorded; leaving it in-flight for restart recovery: networkId=%s snapshotId=%s setupIds=%v err=%v stateErr=%v", event.NetworkID, event.SnapshotID, event.SetupIDs, runErr, err) + return + } + if deadLettered { + s.logger.Printf("webhook job exhausted %d attempts and was dead-lettered: networkId=%s snapshotId=%s setupIds=%v err=%v", s.maxAttempts, event.NetworkID, event.SnapshotID, event.SetupIDs, runErr) + return + } + s.logger.Printf("webhook job failed; retry scheduled for %s: networkId=%s snapshotId=%s setupIds=%v err=%v", nextAttempt.Format(time.RFC3339Nano), event.NetworkID, event.SnapshotID, event.SetupIDs, runErr) + s.scheduleJob(ctx, event, *nextAttempt) +} + +func (s *Server) startPendingJobs(ctx context.Context) { + s.stateMu.Lock() + jobs := make([]pendingWebhookEvent, 0, len(s.state.PendingEvents)) + for _, job := range s.state.PendingEvents { + jobs = append(jobs, clonePendingWebhookEvent(job)) + } + s.stateMu.Unlock() + sort.Slice(jobs, func(i, j int) bool { + return jobs[i].AcceptedAt.Before(jobs[j].AcceptedAt) + }) + for _, job := range jobs { + when := time.Now().UTC() + if job.NextAttemptAt != nil { + when = *job.NextAttemptAt + } + s.scheduleJob(ctx, job.Event, when) + } +} + +func (s *Server) scheduleJob(ctx context.Context, event Event, when time.Time) { + key := eventDedupeKey(event) + s.stateMu.Lock() + s.scheduleGeneration[key]++ + generation := s.scheduleGeneration[key] + s.stateMu.Unlock() + + go func() { + delay := time.Until(when) + if delay < 0 { + delay = 0 + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return + case <-timer.C: + } + s.enqueueScheduledJob(ctx, event, generation) + }() +} + +func (s *Server) enqueueScheduledJob(ctx context.Context, event Event, generation uint64) { + key := eventDedupeKey(event) + s.stateMu.Lock() + job, exists := s.state.PendingEvents[key] + if !exists || job.Status != webhookJobQueued || s.queued[key] || s.scheduleGeneration[key] != generation { + s.stateMu.Unlock() + return + } + select { + case s.jobs <- job.Event: + s.queued[key] = true + s.stateMu.Unlock() + return + default: + s.stateMu.Unlock() + } + if ctx.Err() == nil { + s.scheduleJob(ctx, event, time.Now().UTC().Add(s.retryBaseDelay)) + } +} + +func (s *Server) invalidateScheduleLocked(key string) { + s.scheduleGeneration[key]++ +} + +func (s *Server) intersectConfiguredScope(event Event) (Event, error) { + event.NetworkID = strings.TrimSpace(event.NetworkID) + event.SnapshotID = strings.TrimSpace(event.SnapshotID) + event.ID = strings.TrimSpace(event.ID) + event.Type = strings.TrimSpace(event.Type) + + configuredNetworkID := strings.TrimSpace(s.cfg.App.NetworkID) + if configuredNetworkID != "" && event.NetworkID != configuredNetworkID { + return Event{}, fmt.Errorf("event network %s is outside configured network scope %s", event.NetworkID, configuredNetworkID) + } + if configuredNetworkID != "" { + event.NetworkID = configuredNetworkID } - now := time.Now().UTC() - cutoff := now.Add(-24 * time.Hour) - s.seenMu.Lock() - defer s.seenMu.Unlock() - for k, seenAt := range s.seen { - if seenAt.Before(cutoff) { - delete(s.seen, k) + configuredSetupIDs := cleanSetupIDs(s.cfg.App.SetupIDs) + if len(event.SetupIDs) == 0 { + event.SetupIDs = configuredSetupIDs + return event, nil + } + if len(configuredSetupIDs) == 0 { + return event, nil + } + allowed := make(map[string]struct{}, len(configuredSetupIDs)) + for _, setupID := range configuredSetupIDs { + allowed[setupID] = struct{}{} + } + for _, setupID := range event.SetupIDs { + if _, ok := allowed[setupID]; !ok { + return Event{}, fmt.Errorf("event setup %s is outside configured setup scope", setupID) } } - if _, ok := s.seen[key]; ok { - return true + return event, nil +} + +func (s *Server) resolveSnapshotTime(ctx context.Context, event Event) (time.Time, bool, error) { + if !s.lookupSnapshotTime { + return time.Time{}, false, nil + } + client, err := api.NewClient( + s.cfg.App.Host, + s.cfg.App.APIPrefix, + s.cfg.App.Username, + s.cfg.App.Password, + s.cfg.App.Insecure, + s.cfg.App.Timeout, + ) + if err != nil { + return time.Time{}, false, fmt.Errorf("create snapshot metadata client: %w", err) + } + snapshots, err := client.ListSnapshots(ctx, event.NetworkID) + if err != nil { + return time.Time{}, false, fmt.Errorf("list snapshots for ordering: %w", err) + } + for _, snapshot := range snapshots { + if strings.TrimSpace(snapshot.ID) != event.SnapshotID { + continue + } + snapshotTime, err := webhookSnapshotTimestamp(snapshot) + if err != nil { + return time.Time{}, false, err + } + return snapshotTime, true, nil + } + return time.Time{}, false, fmt.Errorf("snapshot %s was not found in network %s", event.SnapshotID, event.NetworkID) +} + +func isLoopbackHost(rawHost string) bool { + parsed, err := url.Parse(strings.TrimSpace(rawHost)) + if err != nil { + return false + } + host := strings.ToLower(parsed.Hostname()) + return host == "localhost" || host == "127.0.0.1" || host == "::1" +} + +func eventResponse(event Event, duplicate bool) map[string]any { + return map[string]any{ + "accepted": true, + "duplicate": duplicate, + "networkId": event.NetworkID, + "snapshotId": event.SnapshotID, + "setupIds": event.SetupIDs, } - s.seen[key] = now - return false } func setupIDsFromQuery(r *http.Request) []string { diff --git a/internal/webhook/server_test.go b/internal/webhook/server_test.go index 6154343..a978126 100644 --- a/internal/webhook/server_test.go +++ b/internal/webhook/server_test.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "net/http/httptest" + "path/filepath" "strings" "sync" "testing" @@ -58,6 +59,38 @@ func TestHandleEventRequiresBasicAuth(t *testing.T) { } } +func TestNewRequiresBasicAuthWhenApplyEnabled(t *testing.T) { + _, err := New(Config{ + StatePath: filepath.Join(t.TempDir(), "webhook-state.json"), + App: app.Config{ + Host: "https://fwd.example", + Username: "alice", + Password: "secret", + Apply: true, + }, + }) + if err == nil || !strings.Contains(err.Error(), "Basic authentication") { + t.Fatalf("New() error = %v; want apply-mode Basic Auth requirement", err) + } +} + +func TestNewRequiresConfiguredNetworkWhenApplyEnabled(t *testing.T) { + _, err := New(Config{ + BasicUsername: "hook", + BasicPassword: "secret", + StatePath: filepath.Join(t.TempDir(), "webhook-state.json"), + App: app.Config{ + Host: "https://fwd.example", + Username: "alice", + Password: "secret", + Apply: true, + }, + }) + if err == nil || !strings.Contains(err.Error(), "network ID is required") { + t.Fatalf("New() error = %v; want apply-mode configured-network requirement", err) + } +} + func TestHandleEventQueuesExactSnapshot(t *testing.T) { var ( mu sync.Mutex @@ -190,15 +223,46 @@ func newTestServer(t *testing.T, cfg Config) (*httptest.Server, *Server) { t.Cleanup(cancel) cfg.Listen = "127.0.0.1:0" cfg.Path = "/forward/snapshot-ready" + cfg.StatePath = filepath.Join(t.TempDir(), "webhook-state.json") cfg.Logger = log.New(io.Discard, "", 0) cfg.App = app.Config{Host: "https://fwd.example", Username: "u", Password: "p"} server, err := New(cfg) if err != nil { t.Fatalf("New() error = %v", err) } + t.Cleanup(func() { + waitForWebhookStateIdle(t, server) + }) mux := http.NewServeMux() mux.HandleFunc("/healthz", server.handleHealthz) mux.HandleFunc(server.cfg.Path, server.handleEvent) go server.worker(ctx) return httptest.NewServer(mux), server } + +func waitForWebhookStateIdle(t *testing.T, server *Server) { + t.Helper() + if !server.workerRunning.Load() { + processAdmissions.Lock() + for _, done := range processAdmissions.byStatePath[server.cfg.StatePath] { + close(done) + } + delete(processAdmissions.byStatePath, server.cfg.StatePath) + processAdmissions.Unlock() + return + } + deadline := time.Now().Add(time.Second) + for { + processAdmissions.Lock() + pending := len(processAdmissions.byStatePath[server.cfg.StatePath]) + processAdmissions.Unlock() + if pending == 0 { + return + } + if time.Now().After(deadline) { + t.Errorf("timed out waiting for webhook state writes to finish") + return + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/webhook/state.go b/internal/webhook/state.go new file mode 100644 index 0000000..703f4a1 --- /dev/null +++ b/internal/webhook/state.go @@ -0,0 +1,593 @@ +package webhook + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/forwardnetworks/aws-sync/internal/api" +) + +const ( + previousWebhookStateVersion = 1 + webhookStateVersion = 2 + dedupeRetention = 24 * time.Hour + webhookMaxAttempts = 5 + webhookRetryBaseDelay = time.Second + webhookRetryMaxDelay = 30 * time.Second + webhookQueueCapacity = 32 +) + +const ( + webhookJobQueued = "queued" + webhookJobInFlight = "in_flight" +) + +type webhookState struct { + Version int `json:"version"` + CompletedEvents map[string]time.Time `json:"completed_events"` + Watermarks map[string]snapshotWatermark `json:"snapshot_watermarks"` + PendingEvents map[string]pendingWebhookEvent `json:"pending_events"` + DeadLetterEvents map[string]deadLetterWebhookEvent `json:"dead_letter_events"` +} + +type pendingWebhookEvent struct { + Event Event `json:"event"` + Status string `json:"status"` + Attempts int `json:"attempts"` + AcceptedAt time.Time `json:"accepted_at"` + LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` + NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +type deadLetterWebhookEvent struct { + Event Event `json:"event"` + Attempts int `json:"attempts"` + AcceptedAt time.Time `json:"accepted_at"` + LastAttemptAt *time.Time `json:"last_attempt_at,omitempty"` + DeadLetteredAt time.Time `json:"dead_lettered_at"` + LastError string `json:"last_error"` +} + +type snapshotWatermark struct { + NetworkID string `json:"network_id"` + SetupID string `json:"setup_id"` + SnapshotID string `json:"snapshot_id"` + SnapshotAt time.Time `json:"snapshot_at"` + CompletedAt time.Time `json:"completed_at"` +} + +var processAdmissions = struct { + sync.Mutex + byStatePath map[string]map[string]chan struct{} +}{ + byStatePath: make(map[string]map[string]chan struct{}), +} + +func resolveStatePath(configured string) (string, error) { + if configured = strings.TrimSpace(configured); configured != "" { + return configured, nil + } + configDir, err := os.UserConfigDir() + if err != nil { + return "", fmt.Errorf("resolve webhook state directory: %w", err) + } + return filepath.Join(configDir, "awssync", "webhook-state.json"), nil +} + +func loadWebhookState(path string) (webhookState, error) { + state := newWebhookState() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return state, nil + } + if err != nil { + return webhookState{}, fmt.Errorf("read webhook state %s: %w", path, err) + } + if err := json.Unmarshal(data, &state); err != nil { + return webhookState{}, fmt.Errorf("decode webhook state %s: %w", path, err) + } + migrated := false + switch state.Version { + case previousWebhookStateVersion: + state.Version = webhookStateVersion + migrated = true + case webhookStateVersion: + default: + return webhookState{}, fmt.Errorf("webhook state %s has version %d; want %d or upgradeable version %d", path, state.Version, webhookStateVersion, previousWebhookStateVersion) + } + if state.CompletedEvents == nil { + state.CompletedEvents = make(map[string]time.Time) + } + if state.Watermarks == nil { + state.Watermarks = make(map[string]snapshotWatermark) + } + if state.PendingEvents == nil { + state.PendingEvents = make(map[string]pendingWebhookEvent) + } + if state.DeadLetterEvents == nil { + state.DeadLetterEvents = make(map[string]deadLetterWebhookEvent) + } + for key, job := range state.PendingEvents { + if job.Status != webhookJobQueued && job.Status != webhookJobInFlight { + return webhookState{}, fmt.Errorf("webhook state %s pending event %q has invalid status %q", path, key, job.Status) + } + if job.Attempts < 0 { + return webhookState{}, fmt.Errorf("webhook state %s pending event %q has negative attempts", path, key) + } + if key != eventDedupeKey(job.Event) { + return webhookState{}, fmt.Errorf("webhook state %s pending event %q does not match its scoped event key", path, key) + } + } + for key, job := range state.DeadLetterEvents { + if key != eventDedupeKey(job.Event) { + return webhookState{}, fmt.Errorf("webhook state %s dead-letter event %q does not match its scoped event key", path, key) + } + } + if migrated { + if err := persistWebhookState(path, state); err != nil { + return webhookState{}, fmt.Errorf("upgrade webhook state %s from version %d: %w", path, previousWebhookStateVersion, err) + } + } + return state, nil +} + +func newWebhookState() webhookState { + return webhookState{ + Version: webhookStateVersion, + CompletedEvents: make(map[string]time.Time), + Watermarks: make(map[string]snapshotWatermark), + PendingEvents: make(map[string]pendingWebhookEvent), + DeadLetterEvents: make(map[string]deadLetterWebhookEvent), + } +} + +func (s *Server) rejectOlderSnapshot(ctx context.Context, event Event) error { + barrier, ok := s.snapshotBarrier(event) + if !ok { + return nil + } + snapshotAt, known, err := s.resolveSnapshotTime(ctx, event) + if err != nil { + return fmt.Errorf("validate snapshot ordering: %w", err) + } + if !known { + return fmt.Errorf("snapshot ordering metadata is unavailable while watermark %s is active", barrier.SnapshotID) + } + if snapshotAt.Before(barrier.SnapshotAt) { + return olderSnapshotError(event, snapshotAt, barrier) + } + return nil +} + +func (s *Server) beginSnapshot(event Event, snapshotAt time.Time) error { + s.stateMu.Lock() + defer s.stateMu.Unlock() + if barrier, ok := s.snapshotBarrierLocked(event); ok && snapshotAt.Before(barrier.SnapshotAt) { + return olderSnapshotError(event, snapshotAt, barrier) + } + mark := snapshotWatermark{ + NetworkID: event.NetworkID, + SnapshotID: event.SnapshotID, + SnapshotAt: snapshotAt, + } + for _, key := range eventWatermarkKeys(event) { + copy := mark + copy.SetupID = watermarkSetupID(key) + s.active[key] = copy + } + return nil +} + +func (s *Server) endSnapshot(event Event) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + for _, key := range eventWatermarkKeys(event) { + delete(s.active, key) + } +} + +func (s *Server) snapshotBarrier(event Event) (snapshotWatermark, bool) { + s.stateMu.Lock() + defer s.stateMu.Unlock() + return s.snapshotBarrierLocked(event) +} + +func (s *Server) snapshotBarrierLocked(event Event) (snapshotWatermark, bool) { + var ( + barrier snapshotWatermark + found bool + ) + consider := func(mark snapshotWatermark) { + if !found || mark.SnapshotAt.After(barrier.SnapshotAt) { + barrier = mark + found = true + } + } + keys := eventWatermarkKeys(event) + for _, key := range keys { + if mark, ok := s.state.Watermarks[key]; ok { + consider(mark) + } + if mark, ok := s.active[key]; ok { + consider(mark) + } + } + networkPrefix := event.NetworkID + "\x1f" + if len(event.SetupIDs) == 0 { + for key, mark := range s.state.Watermarks { + if strings.HasPrefix(key, networkPrefix) { + consider(mark) + } + } + for key, mark := range s.active { + if strings.HasPrefix(key, networkPrefix) { + consider(mark) + } + } + } else { + wildcard := watermarkKey(event.NetworkID, "*") + if mark, ok := s.state.Watermarks[wildcard]; ok { + consider(mark) + } + if mark, ok := s.active[wildcard]; ok { + consider(mark) + } + } + return barrier, found +} + +func (s *Server) recordSuccess(event Event, snapshotAt time.Time, snapshotTimeKnown bool) error { + now := time.Now().UTC() + s.stateMu.Lock() + defer s.stateMu.Unlock() + + next := cloneWebhookState(s.state) + for key, completedAt := range next.CompletedEvents { + if completedAt.Before(now.Add(-dedupeRetention)) { + delete(next.CompletedEvents, key) + } + } + key := eventDedupeKey(event) + next.CompletedEvents[key] = now + delete(next.PendingEvents, key) + delete(next.DeadLetterEvents, key) + if snapshotTimeKnown { + for _, key := range eventWatermarkKeys(event) { + current, exists := next.Watermarks[key] + if exists && current.SnapshotAt.After(snapshotAt) { + continue + } + next.Watermarks[key] = snapshotWatermark{ + NetworkID: event.NetworkID, + SetupID: watermarkSetupID(key), + SnapshotID: event.SnapshotID, + SnapshotAt: snapshotAt, + CompletedAt: now, + } + } + } + if err := s.persistState(s.cfg.StatePath, next); err != nil { + return err + } + s.state = next + s.invalidateScheduleLocked(key) + delete(s.queued, key) + return nil +} + +func registerProcessAdmission(statePath, key string) (<-chan struct{}, bool) { + processAdmissions.Lock() + defer processAdmissions.Unlock() + admissions := processAdmissions.byStatePath[statePath] + if admissions == nil { + admissions = make(map[string]chan struct{}) + processAdmissions.byStatePath[statePath] = admissions + } + if done, exists := admissions[key]; exists { + return done, false + } + done := make(chan struct{}) + admissions[key] = done + return done, true +} + +func finishProcessAdmission(statePath, key string) { + processAdmissions.Lock() + defer processAdmissions.Unlock() + admissions := processAdmissions.byStatePath[statePath] + if admissions == nil { + return + } + if done, exists := admissions[key]; exists { + delete(admissions, key) + close(done) + } + if len(admissions) == 0 { + delete(processAdmissions.byStatePath, statePath) + } +} + +func (s *Server) reloadState() error { + s.stateMu.Lock() + defer s.stateMu.Unlock() + state, err := loadWebhookState(s.cfg.StatePath) + if err != nil { + return err + } + s.state = state + return nil +} + +func (s *Server) pruneCompletedLocked(now time.Time) { + cutoff := now.Add(-dedupeRetention) + for key, completedAt := range s.state.CompletedEvents { + if completedAt.Before(cutoff) { + delete(s.state.CompletedEvents, key) + } + } +} + +func persistWebhookState(path string, state webhookState) (err error) { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("encode webhook state: %w", err) + } + data = append(data, '\n') + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create webhook state directory: %w", err) + } + temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") + if err != nil { + return fmt.Errorf("create webhook state temp file: %w", err) + } + tempPath := temp.Name() + defer func() { + _ = temp.Close() + if err != nil { + _ = os.Remove(tempPath) + } + }() + if err = temp.Chmod(0o600); err != nil { + return err + } + if _, err = temp.Write(data); err != nil { + return err + } + if err = temp.Sync(); err != nil { + return err + } + if err = temp.Close(); err != nil { + return err + } + if err = os.Rename(tempPath, path); err != nil { + return err + } + return nil +} + +func cloneWebhookState(state webhookState) webhookState { + clone := newWebhookState() + for key, completedAt := range state.CompletedEvents { + clone.CompletedEvents[key] = completedAt + } + for key, watermark := range state.Watermarks { + clone.Watermarks[key] = watermark + } + for key, job := range state.PendingEvents { + clone.PendingEvents[key] = clonePendingWebhookEvent(job) + } + for key, job := range state.DeadLetterEvents { + clone.DeadLetterEvents[key] = cloneDeadLetterWebhookEvent(job) + } + return clone +} + +func clonePendingWebhookEvent(job pendingWebhookEvent) pendingWebhookEvent { + job.Event.SetupIDs = append([]string(nil), job.Event.SetupIDs...) + job.LastAttemptAt = cloneTimePointer(job.LastAttemptAt) + job.NextAttemptAt = cloneTimePointer(job.NextAttemptAt) + return job +} + +func cloneDeadLetterWebhookEvent(job deadLetterWebhookEvent) deadLetterWebhookEvent { + job.Event.SetupIDs = append([]string(nil), job.Event.SetupIDs...) + job.LastAttemptAt = cloneTimePointer(job.LastAttemptAt) + return job +} + +func cloneTimePointer(value *time.Time) *time.Time { + if value == nil { + return nil + } + copy := *value + return © +} + +func (s *Server) recoverInFlightJobs() error { + s.stateMu.Lock() + defer s.stateMu.Unlock() + + next := cloneWebhookState(s.state) + changed := false + for key, job := range next.PendingEvents { + if job.Status != webhookJobInFlight { + continue + } + changed = true + crashError := fmt.Sprintf("process exited while attempt %d was in-flight; completion is ambiguous", job.Attempts) + if job.Attempts >= s.maxAttempts { + delete(next.PendingEvents, key) + next.DeadLetterEvents[key] = deadLetterWebhookEvent{ + Event: job.Event, + Attempts: job.Attempts, + AcceptedAt: job.AcceptedAt, + LastAttemptAt: cloneTimePointer(job.LastAttemptAt), + DeadLetteredAt: time.Now().UTC(), + LastError: crashError, + } + continue + } + job.Status = webhookJobQueued + job.LastError = crashError + nextAttempt := time.Now().UTC() + if job.LastAttemptAt != nil { + nextAttempt = job.LastAttemptAt.Add(s.retryDelay(job.Attempts)) + } + job.NextAttemptAt = &nextAttempt + next.PendingEvents[key] = job + } + if !changed { + return nil + } + if err := s.persistState(s.cfg.StatePath, next); err != nil { + return fmt.Errorf("recover in-flight webhook jobs: %w", err) + } + s.state = next + return nil +} + +func (s *Server) markJobInFlight(event Event) (pendingWebhookEvent, bool, error) { + key := eventDedupeKey(event) + now := time.Now().UTC() + s.stateMu.Lock() + defer s.stateMu.Unlock() + + delete(s.queued, key) + s.invalidateScheduleLocked(key) + job, exists := s.state.PendingEvents[key] + if !exists { + return pendingWebhookEvent{}, false, nil + } + next := cloneWebhookState(s.state) + job.Attempts++ + job.Status = webhookJobInFlight + job.LastAttemptAt = &now + job.NextAttemptAt = nil + next.PendingEvents[key] = job + if err := s.persistState(s.cfg.StatePath, next); err != nil { + return pendingWebhookEvent{}, true, err + } + s.state = next + return job, true, nil +} + +func (s *Server) recordJobFailure(job pendingWebhookEvent, runErr error) (*time.Time, bool, error) { + key := eventDedupeKey(job.Event) + now := time.Now().UTC() + s.stateMu.Lock() + defer s.stateMu.Unlock() + + current, exists := s.state.PendingEvents[key] + if !exists { + return nil, false, nil + } + next := cloneWebhookState(s.state) + current.LastError = runErr.Error() + current.LastAttemptAt = cloneTimePointer(job.LastAttemptAt) + if current.Attempts >= s.maxAttempts { + delete(next.PendingEvents, key) + next.DeadLetterEvents[key] = deadLetterWebhookEvent{ + Event: current.Event, + Attempts: current.Attempts, + AcceptedAt: current.AcceptedAt, + LastAttemptAt: cloneTimePointer(current.LastAttemptAt), + DeadLetteredAt: now, + LastError: current.LastError, + } + if err := s.persistState(s.cfg.StatePath, next); err != nil { + return nil, false, err + } + s.state = next + s.invalidateScheduleLocked(key) + delete(s.queued, key) + return nil, true, nil + } + + nextAttempt := now.Add(s.retryDelay(current.Attempts)) + current.Status = webhookJobQueued + current.NextAttemptAt = &nextAttempt + next.PendingEvents[key] = current + if err := s.persistState(s.cfg.StatePath, next); err != nil { + return nil, false, err + } + s.state = next + return &nextAttempt, false, nil +} + +func (s *Server) retryDelay(failedAttempts int) time.Duration { + delay := s.retryBaseDelay + for attempt := 1; attempt < failedAttempts && delay < s.retryMaxDelay; attempt++ { + delay *= 2 + if delay > s.retryMaxDelay { + delay = s.retryMaxDelay + } + } + return delay +} + +func eventDedupeKey(event Event) string { + return strings.Join([]string{ + strings.TrimSpace(event.Type), + strings.TrimSpace(event.NetworkID), + strings.TrimSpace(event.SnapshotID), + strings.Join(cleanSetupIDs(event.SetupIDs), ","), + strings.TrimSpace(event.ID), + }, "\x1f") +} + +func eventWatermarkKeys(event Event) []string { + setupIDs := cleanSetupIDs(event.SetupIDs) + if len(setupIDs) == 0 { + return []string{watermarkKey(event.NetworkID, "*")} + } + keys := make([]string, 0, len(setupIDs)) + for _, setupID := range setupIDs { + keys = append(keys, watermarkKey(event.NetworkID, setupID)) + } + return keys +} + +func watermarkKey(networkID, setupID string) string { + return strings.TrimSpace(networkID) + "\x1f" + strings.TrimSpace(setupID) +} + +func watermarkSetupID(key string) string { + parts := strings.SplitN(key, "\x1f", 2) + if len(parts) != 2 { + return "" + } + return parts[1] +} + +func olderSnapshotError(event Event, snapshotAt time.Time, barrier snapshotWatermark) error { + return fmt.Errorf( + "snapshot %s at %s is older than applied/in-progress snapshot %s at %s for network/setup scope", + event.SnapshotID, + snapshotAt.Format(time.RFC3339Nano), + barrier.SnapshotID, + barrier.SnapshotAt.Format(time.RFC3339Nano), + ) +} + +func webhookSnapshotTimestamp(snapshot api.SnapshotInfo) (time.Time, error) { + for _, value := range []string{snapshot.CreatedAt, snapshot.ProcessedAt} { + value = strings.TrimSpace(value) + if value == "" { + continue + } + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, fmt.Errorf("snapshot %s has invalid timestamp %q: %w", snapshot.ID, value, err) + } + return parsed, nil + } + return time.Time{}, fmt.Errorf("snapshot %s has no processedAt or createdAt timestamp", snapshot.ID) +}