From be7ebf83006891ff4fd96f2bf2f06526162ab557 Mon Sep 17 00:00:00 2001 From: Jeroen Soeters Date: Tue, 18 Aug 2026 06:56:19 -0700 Subject: [PATCH] test(blackbox): assert cross-stack slots against the model like any other slot Cross-stack slots were excluded from both directions of the model-vs-inventory check, and a failed or canceled command's unmentioned cross-stack slots were never reverted to their snapshots, on the grounds that their persistence behavior was not derivable from the command response. That left two of the twelve slots per consumer stack entirely unasserted. The claimed nondeterminism does not reproduce on the deterministic harness: cross-stack resource updates carry per-resource terminal states like any others, and with the drift-tolerance mechanism gone and sync observation working, repeated full-chaos runs hold green with the exclusions removed. Delete all three skips and pin the restored coverage with tests for the forward check, the unexpected-inventory check, and the unmentioned-slot revert. --- tests/blackbox/executor.go | 9 ---- tests/blackbox/invariants.go | 11 ---- tests/blackbox/state_model_test.go | 83 ++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 20 deletions(-) diff --git a/tests/blackbox/executor.go b/tests/blackbox/executor.go index b6ea21711..3519bbd41 100644 --- a/tests/blackbox/executor.go +++ b/tests/blackbox/executor.go @@ -1008,15 +1008,6 @@ func correctModelFromCommandOutcome(t *testing.T, cmd *apimodel.Command, model * if model.IsAuthoritativeSlot(key.stackIdx, key.slotIdx) { continue } - // Cross-stack slots in failed/canceled commands: the agent's - // behavior for cross-stack resources is non-deterministic from - // the command response alone (creates may or may not persist, - // reconcile deletes may or may not complete before cancel). - // Skip model updates for cross-stack slots and rely on the - // model-vs-inventory check excluding them (see CheckModelVsInventory). - if pool != nil && pool.IsCrossStack(key.slotIdx) { - continue - } res := model.Resource(key.stackIdx, key.slotIdx) if res == nil || res.State == snap.State { continue diff --git a/tests/blackbox/invariants.go b/tests/blackbox/invariants.go index 67d6b7809..402bb6d56 100644 --- a/tests/blackbox/invariants.go +++ b/tests/blackbox/invariants.go @@ -503,13 +503,6 @@ func CheckModelVsInventory(model *StateModel, inventory []pkgmodel.Resource) []V for s := range model.Stacks { stack := &model.Stacks[s] for idx, res := range stack.Resources { - // Cross-stack slots in failed/canceled commands have - // non-deterministic persistence behavior — the command response - // alone can't tell us whether they were persisted. Skip them - // in model-vs-inventory assertions. - if model.Pool != nil && model.Pool.IsCrossStack(idx) { - continue - } label := model.LabelForResource(s, idx) resourceType := model.TypeForResource(idx) @@ -572,10 +565,6 @@ func CheckModelVsInventory(model *StateModel, inventory []pkgmodel.Resource) []V if expectedExistingKeys[key] { continue } - _, slotIdx, ok := model.findResourceSlot(res.Stack, res.Label) - if ok && model.Pool != nil && model.Pool.IsCrossStack(slotIdx) { - continue - } violations = append(violations, Violation{ Kind: ViolationModelInventoryMismatch, Message: fmt.Sprintf("inventory contains unexpected managed resource: stack %s resource %s type=%s", diff --git a/tests/blackbox/state_model_test.go b/tests/blackbox/state_model_test.go index c8a3e30ad..536910031 100644 --- a/tests/blackbox/state_model_test.go +++ b/tests/blackbox/state_model_test.go @@ -8,6 +8,7 @@ package blackbox import ( "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -502,6 +503,88 @@ func TestStateModel_DriftEligibilitySkipsStacksWithInFlightCommands(t *testing.T // Cross-stack slots reference a parent on the provider stack (stack 0), so an // in-flight command on the provider stack can cascade onto them. They are only // eligible for drift while the provider stack is quiescent too. +// findCrossStackSlot returns a cross-stack slot index of the model's pool. +func findCrossStackSlot(t *testing.T, model *StateModel) int { + t.Helper() + require.NotNil(t, model.Pool) + for i := range model.Pool.Slots { + if model.Pool.IsCrossStack(i) { + return i + } + } + t.Fatal("pool has no cross-stack slot") + return -1 +} + +// Cross-stack slots are asserted against inventory like any other slot: a +// slot the model expects to exist must have an inventory row. +func TestCheckModelVsInventory_CrossStackSlotIsAsserted(t *testing.T) { + model := NewStateModel(2, 10) + crossIdx := findCrossStackSlot(t, model) + model.ApplyCreated(1, []int{crossIdx}, `{"Name":"x","ParentId":"p","Value":"v1"}`) + + violations := CheckModelVsInventory(model, nil) + + found := false + label := model.LabelForResource(1, crossIdx) + for _, v := range violations { + if v.Kind == ViolationModelInventoryMismatch && strings.Contains(v.Message, label) { + found = true + } + } + assert.True(t, found, "a missing cross-stack inventory row must be reported, not skipped") +} + +// The reverse direction holds too: an inventory row for a cross-stack slot +// the model does not expect is an unexpected managed resource. +func TestCheckModelVsInventory_UnexpectedCrossStackRowIsAsserted(t *testing.T) { + model := NewStateModel(2, 10) + crossIdx := findCrossStackSlot(t, model) + label := model.LabelForResource(1, crossIdx) + + inventory := []pkgmodel.Resource{{ + Stack: "stack-1", + Label: label, + Type: model.TypeForResource(crossIdx), + NativeID: "test-9", + Managed: true, + Properties: []byte(`{"Name":"x","ParentId":"p","Value":"v1"}`), + }} + + violations := CheckModelVsInventory(model, inventory) + + found := false + for _, v := range violations { + if v.Kind == ViolationModelInventoryMismatch && strings.Contains(v.Message, "unexpected managed resource") && strings.Contains(v.Message, label) { + found = true + } + } + assert.True(t, found, "an unexpected cross-stack inventory row must be reported, not skipped") +} + +// A failed command's unmentioned cross-stack slot reverts to its snapshot +// like any other slot: the optimistic prediction must not survive a command +// that never reported an outcome for it. +func TestCorrectModelFromCommandOutcome_UnmentionedCrossStackSlotReverts(t *testing.T) { + model := NewStateModel(2, 10) + crossIdx := findCrossStackSlot(t, model) + + // Snapshot taken while the slot did not exist, then an optimistic create. + snapshots := []ResourceSnapshot{{StackIndex: 1, SlotIndex: crossIdx, State: StateNotExist}} + model.ApplyCreated(1, []int{crossIdx}, `{"Name":"x","ParentId":"p","Value":"v1"}`) + + cmd := &apimodel.Command{ + CommandID: "cmd-failed", + State: "Failed", + ResourceUpdates: nil, // the command died before reaching this slot + } + corrected := map[struct{ stackIdx, slotIdx int }]bool{} + correctModelFromCommandOutcome(t, cmd, model, model.Pool, snapshots, corrected, false) + + require.Equal(t, StateNotExist, model.Resource(1, crossIdx).State, + "an unmentioned cross-stack slot in a failed command reverts to its snapshot") +} + func TestStateModel_DriftEligibilityCrossStackNeedsQuiescentProvider(t *testing.T) { model := NewStateModel(2, 10) require.NotNil(t, model.Pool)