diff --git a/.github/workflows/govulncheck.yaml b/.github/workflows/govulncheck.yaml index 6422d54dc1..710475f2be 100644 --- a/.github/workflows/govulncheck.yaml +++ b/.github/workflows/govulncheck.yaml @@ -14,6 +14,9 @@ name: govulncheck on: + # On pull requests too: a vulnerable dependency caught only by the post-merge + # run is already on main, and the weekly cron can delay that by a week. + pull_request: push: branches: - main diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 6a0394458b..3e20f2fe07 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -26,9 +26,17 @@ on: # and run it in a throwaway cluster. permissions: contents: read +# Superseded pushes to a pull request are cancelled; every commit that lands on +# main still gets its own full run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: run-tests: runs-on: ubuntu-latest + # A bound well clear of the observed runtime. Without one the platform + # default applies, and a wedged step burns a free-tier slot for six hours. + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -38,8 +46,9 @@ jobs: uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version-file: 'go.mod' + # Unconditional: a released migration becoming permanent is exactly what + # this guards, and that happens on the push to main, not on the PR. - name: Verify immutable PostgreSQL migrations - if: github.event_name == 'pull_request' run: hack/verify/postgresql-migrations.sh - run: go test -race -v ./... # tools/apitool has its own module so we need to run it explicitly. @@ -59,6 +68,9 @@ jobs: e2e-test-matrix: name: E2E (${{ matrix.dataplane }}) runs-on: ubuntu-latest + # Cluster bring-up, image builds and six sequential lanes; generous, but far + # short of the six-hour platform default a hung lane would otherwise consume. + timeout-minutes: 120 continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability strategy: fail-fast: false diff --git a/cmd/ateapi/internal/store/atepg/atepg_test.go b/cmd/ateapi/internal/store/atepg/atepg_test.go index d2ed623991..c0848e06d2 100644 --- a/cmd/ateapi/internal/store/atepg/atepg_test.go +++ b/cmd/ateapi/internal/store/atepg/atepg_test.go @@ -118,6 +118,9 @@ func requirePool(t *testing.T) *pgxpool.Pool { containerPool = pool }) if containerErr != nil { + if dockerenv.Required() { + t.Fatalf("PostgreSQL testcontainer unavailable and required (CI or REQUIRE_DOCKER is set): %v", containerErr) + } t.Skipf("PostgreSQL testcontainer unavailable (requires Docker): %v", containerErr) } return containerPool diff --git a/cmd/ateapi/internal/store/dockerenv/dockerenv.go b/cmd/ateapi/internal/store/dockerenv/dockerenv.go index 2264c641a1..3c601e3243 100644 --- a/cmd/ateapi/internal/store/dockerenv/dockerenv.go +++ b/cmd/ateapi/internal/store/dockerenv/dockerenv.go @@ -60,3 +60,11 @@ func Configure(ctx context.Context) error { } return nil } + +// Required reports whether an unavailable container must fail the test rather +// than skip it. Skipping is right on a workstation without Docker and wrong in +// CI, where it turns every container-backed package into a silent pass. +// REQUIRE_DOCKER opts a local run into the CI behavior. +func Required() bool { + return os.Getenv("CI") == "true" || os.Getenv("REQUIRE_DOCKER") == "true" +} diff --git a/cmd/ateapi/internal/store/dockerenv/dockerenv_test.go b/cmd/ateapi/internal/store/dockerenv/dockerenv_test.go index 17359203d1..96a5c6dc01 100644 --- a/cmd/ateapi/internal/store/dockerenv/dockerenv_test.go +++ b/cmd/ateapi/internal/store/dockerenv/dockerenv_test.go @@ -58,3 +58,30 @@ func TestConfigurePreservesEnvironment(t *testing.T) { t.Errorf("DOCKER_HOST = %q, want %q", got, want) } } + +func TestRequired(t *testing.T) { + for _, tc := range []struct { + name string + ci string + requireDocker string + want bool + }{ + {name: "unset", want: false}, + {name: "CI true", ci: "true", want: true}, + {name: "REQUIRE_DOCKER true", requireDocker: "true", want: true}, + {name: "both true", ci: "true", requireDocker: "true", want: true}, + // GitHub Actions sets CI=true; a bare "1" is not the contract, and + // treating it as one would fail closed on unrelated CI systems. + {name: "CI 1 is not true", ci: "1", want: false}, + {name: "CI false", ci: "false", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CI", tc.ci) + t.Setenv("REQUIRE_DOCKER", tc.requireDocker) + + if got := Required(); got != tc.want { + t.Errorf("Required() = %v, want %v (CI=%q REQUIRE_DOCKER=%q)", got, tc.want, tc.ci, tc.requireDocker) + } + }) + } +} diff --git a/cmd/ateapi/internal/store/storetest/storetest.go b/cmd/ateapi/internal/store/storetest/storetest.go index e60abf97d4..8378e66244 100644 --- a/cmd/ateapi/internal/store/storetest/storetest.go +++ b/cmd/ateapi/internal/store/storetest/storetest.go @@ -185,6 +185,9 @@ func requireAdminPool(t *testing.T) *pgxpool.Pool { containerErr = fmt.Errorf("pinging PostgreSQL testcontainer after retries: %w", pingErr) }) if containerErr != nil { + if dockerenv.Required() { + t.Fatalf("PostgreSQL testcontainer unavailable and required (CI or REQUIRE_DOCKER is set): %v", containerErr) + } t.Skipf("PostgreSQL testcontainer unavailable (requires Docker): %v", containerErr) } return adminPool diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 278467d1cf..2767ea645f 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -589,13 +589,32 @@ create_podcertificate_controller_cas() { --secret-namespace=podcertificate-controller-system } +# The deadline is shared across both bundles: the signer produces them from the +# same CA pools, so a partial result means the same fault as none at all. wait_for_podcertificate_trust_bundles() { - echo "Waiting for podcertificate ClusterTrustBundles to be ready..." - until run_kubectl get clustertrustbundles podidentity.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do - sleep 1 - done - until run_kubectl get clustertrustbundles servicedns.podcert.ate.dev:identity:primary-bundle >/dev/null 2>&1; do - sleep 1 + local timeout_secs="${ATE_INSTALL_TRUST_BUNDLE_TIMEOUT:-120}" + local deadline=$((SECONDS + timeout_secs)) + local bundles=( + "podidentity.podcert.ate.dev:identity:primary-bundle" + "servicedns.podcert.ate.dev:identity:primary-bundle" + ) + local bundle + + echo "Waiting up to ${timeout_secs}s for podcertificate ClusterTrustBundles to be ready..." + for bundle in "${bundles[@]}"; do + while ! run_kubectl get clustertrustbundles "${bundle}" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "timed out after ${timeout_secs}s waiting for clustertrustbundle ${bundle}" >&2 + # The signer is the usual cause; dump what it produced and why it stopped. + run_kubectl get clustertrustbundles -o wide >&2 || true + run_kubectl get pods -n podcertificate-controller-system -o wide >&2 || true + run_kubectl describe pods -n podcertificate-controller-system >&2 || true + run_kubectl logs -n podcertificate-controller-system -l app=podcertificate-controller \ + --all-containers --tail=100 >&2 || true + return 1 + fi + sleep 1 + done done } diff --git a/internal/e2e/preflight.go b/internal/e2e/preflight.go index 14cae171d3..e2cb91b4d5 100644 --- a/internal/e2e/preflight.go +++ b/internal/e2e/preflight.go @@ -19,8 +19,10 @@ import ( "fmt" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/agent-substrate/substrate/internal/deviceplugin" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -61,5 +63,30 @@ func PreflightChecks() error { return fmt.Errorf("ListActors RPC failed: %v", err) } + // The micro-VM class needs a SandboxConfig and a node advertising /dev/kvm. + // Without either, worker pods stay Pending and every suite fails on its own + // timeout minutes later, naming neither cause. + if IsMicroVM() { + if _, err := clients.SubstrateK8s.ApiV1alpha1().SandboxConfigs().Get(ctx, SandboxClassMicroVM, metav1.GetOptions{}); err != nil { + return fmt.Errorf("E2E_SANDBOX_CLASS=%s but SandboxConfig/%s is missing (apply manifests/microvm/sandboxconfig-microvm.yaml.tmpl): %w", + SandboxClassMicroVM, SandboxClassMicroVM, err) + } + + nodes, err := clients.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("listing nodes for the %s preflight check: %w", deviceplugin.ResourceKVM, err) + } + var kvm int64 + for _, node := range nodes.Items { + if q, ok := node.Status.Allocatable[corev1.ResourceName(deviceplugin.ResourceKVM)]; ok { + kvm += q.Value() + } + } + if kvm < 1 { + return fmt.Errorf("E2E_SANDBOX_CLASS=%s but no node advertises %s across %d node(s); expose /dev/kvm on the host so atelet's device plugin can advertise it", + SandboxClassMicroVM, deviceplugin.ResourceKVM, len(nodes.Items)) + } + } + return nil }