CI: Fail on missing preconditions and exceeded boundaries

.github/workflows/govulncheck.yaml

@@ -14,6 +14,9 @@
 
 name: govulncheck
 on:
+  # On pull requests too: a vulnerable dependency caught only by the post-merge
+  # run is already on main, and the weekly cron can delay that by a week.
+  pull_request:
   push:
     branches:
       - main
@@ -28,7 +31,7 @@
     runs-on: ubuntu-latest
     steps:
       - id: govulncheck
         uses: golang/govulncheck-action@v1
         with:
           go-version-file: go.mod
           go-package: ./...

.github/workflows/pr-workflow.yaml

@@ -26,9 +26,17 @@ on:
 # and run it in a throwaway cluster.
 permissions:
   contents: read
+# Superseded pushes to a pull request are cancelled; every commit that lands on
+# main still gets its own full run.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 jobs:
   run-tests:
     runs-on: ubuntu-latest
+    # A bound well clear of the observed runtime. Without one the platform
+    # default applies, and a wedged step burns a free-tier slot for six hours.
+    timeout-minutes: 45
     steps:
     - name: Checkout
       uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
@@ -38,8 +46,9 @@ jobs:
       uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
       with:
         go-version-file: 'go.mod'
+    # Unconditional: a released migration becoming permanent is exactly what
+    # this guards, and that happens on the push to main, not on the PR.
     - name: Verify immutable PostgreSQL migrations
-      if: github.event_name == 'pull_request'
       run: hack/verify/postgresql-migrations.sh
     - run: go test -race -v ./...
     # tools/apitool has its own module so we need to run it explicitly.
@@ -59,6 +68,9 @@ jobs:
   e2e-test-matrix:
     name: E2E (${{ matrix.dataplane }})
     runs-on: ubuntu-latest
+    # Cluster bring-up, image builds and six sequential lanes; generous, but far
+    # short of the six-hour platform default a hung lane would otherwise consume.
+    timeout-minutes: 120
     continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability
     strategy:
       fail-fast: false

cmd/ateapi/internal/store/atepg/atepg_test.go

@@ -118,6 +118,9 @@ func requirePool(t *testing.T) *pgxpool.Pool {
 		containerPool = pool
 	})
 	if containerErr != nil {
+		if dockerenv.Required() {
+			t.Fatalf("PostgreSQL testcontainer unavailable and required (CI or REQUIRE_DOCKER is set): %v", containerErr)
+		}
 		t.Skipf("PostgreSQL testcontainer unavailable (requires Docker): %v", containerErr)
 	}
 	return containerPool

cmd/ateapi/internal/store/dockerenv/dockerenv.go

@@ -60,3 +60,11 @@ func Configure(ctx context.Context) error {
 	}
 	return nil
 }
+
+// Required reports whether an unavailable container must fail the test rather
+// than skip it. Skipping is right on a workstation without Docker and wrong in
+// CI, where it turns every container-backed package into a silent pass.
+// REQUIRE_DOCKER opts a local run into the CI behavior.
+func Required() bool {
+	return os.Getenv("CI") == "true" || os.Getenv("REQUIRE_DOCKER") == "true"
+}

cmd/ateapi/internal/store/dockerenv/dockerenv_test.go

@@ -58,3 +58,30 @@ func TestConfigurePreservesEnvironment(t *testing.T) {
 		t.Errorf("DOCKER_HOST = %q, want %q", got, want)
 	}
 }
+
+func TestRequired(t *testing.T) {
+	for _, tc := range []struct {
+		name          string
+		ci            string
+		requireDocker string
+		want          bool
+	}{
+		{name: "unset", want: false},
+		{name: "CI true", ci: "true", want: true},
+		{name: "REQUIRE_DOCKER true", requireDocker: "true", want: true},
+		{name: "both true", ci: "true", requireDocker: "true", want: true},
+		// GitHub Actions sets CI=true; a bare "1" is not the contract, and
+		// treating it as one would fail closed on unrelated CI systems.
+		{name: "CI 1 is not true", ci: "1", want: false},
+		{name: "CI false", ci: "false", want: false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("CI", tc.ci)
+			t.Setenv("REQUIRE_DOCKER", tc.requireDocker)
+
+			if got := Required(); got != tc.want {
+				t.Errorf("Required() = %v, want %v (CI=%q REQUIRE_DOCKER=%q)", got, tc.want, tc.ci, tc.requireDocker)
+			}
+		})
+	}
+}

cmd/ateapi/internal/store/storetest/storetest.go

@@ -185,6 +185,9 @@ func requireAdminPool(t *testing.T) *pgxpool.Pool {
 		containerErr = fmt.Errorf("pinging PostgreSQL testcontainer after retries: %w", pingErr)
 	})
 	if containerErr != nil {
+		if dockerenv.Required() {
+			t.Fatalf("PostgreSQL testcontainer unavailable and required (CI or REQUIRE_DOCKER is set): %v", containerErr)
+		}
 		t.Skipf("PostgreSQL testcontainer unavailable (requires Docker): %v", containerErr)
 	}
 	return adminPool

hack/install-ate.sh

@@ -589,13 +589,32 @@ create_podcertificate_controller_cas() {
     --secret-namespace=podcertificate-controller-system
 }
 
+# The deadline is shared across both bundles: the signer produces them from the
+# same CA pools, so a partial result means the same fault as none at all.
 wait_for_podcertificate_trust_bundles() {
-  echo "Waiting for podcertificate ClusterTrustBundles to be ready..."
-  until run_kubectl get clustertrustbundles podidentity.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do
-    sleep 1
-  done
-  until run_kubectl get clustertrustbundles servicedns.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do
-    sleep 1
+  local timeout_secs="${ATE_INSTALL_TRUST_BUNDLE_TIMEOUT:-120}"
+  local deadline=$((SECONDS + timeout_secs))
+  local bundles=(
+    "podidentity.podcert.ate.dev:identity:primary-bundle"
+    "servicedns.podcert.ate.dev:identity:primary-bundle"
+  )
+  local bundle
+
+  echo "Waiting up to ${timeout_secs}s for podcertificate ClusterTrustBundles to be ready..."
+  for bundle in "${bundles[@]}"; do
+    while ! run_kubectl get clustertrustbundles "${bundle}" >/dev/null 2>&1; do
+      if ((SECONDS >= deadline)); then
+        echo "timed out after ${timeout_secs}s waiting for clustertrustbundle ${bundle}" >&2
+        # The signer is the usual cause; dump what it produced and why it stopped.
+        run_kubectl get clustertrustbundles -o wide >&2 || true
+        run_kubectl get pods -n podcertificate-controller-system -o wide >&2 || true
+        run_kubectl describe pods -n podcertificate-controller-system >&2 || true
+        run_kubectl logs -n podcertificate-controller-system -l app=podcertificate-controller \
+          --all-containers --tail=100 >&2 || true
+        return 1
+      fi
+      sleep 1
+    done
   done
 }
 

internal/e2e/preflight.go

@@ -19,8 +19,10 @@ import (
 	"fmt"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/agent-substrate/substrate/internal/deviceplugin"
 	"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
 )
 
@@ -61,5 +63,30 @@ func PreflightChecks() error {
 		return fmt.Errorf("ListActors RPC failed: %v", err)
 	}
 
+	// The micro-VM class needs a SandboxConfig and a node advertising /dev/kvm.
+	// Without either, worker pods stay Pending and every suite fails on its own
+	// timeout minutes later, naming neither cause.
+	if IsMicroVM() {
+		if _, err := clients.SubstrateK8s.ApiV1alpha1().SandboxConfigs().Get(ctx, SandboxClassMicroVM, metav1.GetOptions{}); err != nil {
+			return fmt.Errorf("E2E_SANDBOX_CLASS=%s but SandboxConfig/%s is missing (apply manifests/microvm/sandboxconfig-microvm.yaml.tmpl): %w",
+				SandboxClassMicroVM, SandboxClassMicroVM, err)
+		}
+
+		nodes, err := clients.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return fmt.Errorf("listing nodes for the %s preflight check: %w", deviceplugin.ResourceKVM, err)
+		}
+		var kvm int64
+		for _, node := range nodes.Items {
+			if q, ok := node.Status.Allocatable[corev1.ResourceName(deviceplugin.ResourceKVM)]; ok {
+				kvm += q.Value()
+			}
+		}
+		if kvm < 1 {
+			return fmt.Errorf("E2E_SANDBOX_CLASS=%s but no node advertises %s across %d node(s); expose /dev/kvm on the host so atelet's device plugin can advertise it",
+				SandboxClassMicroVM, deviceplugin.ResourceKVM, len(nodes.Items))
+		}
+	}
+
 	return nil
 }