[AI] OSAC-3593: Move e2e test suite (tests/) from osac-test-infra into the osac mono-repo

.github/workflows/e2e-bmaas-full-install.yml

@@ -29,7 +29,7 @@ on:
         required: false
         default: ""
       test-infra-ref:
-        description: "Branch/ref of osac-test-infra"
+        description: "Branch/ref of osac-test-infra (orchestration only; test content is in this repo under tests/e2e/)"
         required: false
         default: "main"
 
@@ -109,7 +109,10 @@ jobs:
       fork-pr-author-association: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association || '' }}
       fork-pr-author: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.user.login || '' }}
       pr-number: ${{ github.event.pull_request.number }}
-      # Checkout osac-test-infra for tests/Containerfile (not the caller repo)
+      # osac-test-infra is still checked out for orchestration infrastructure
+      # (actions, scripts, secrets). Test content (tests/, pyproject.toml) has
+      # moved to this repo under tests/e2e/ -- the reusable workflow needs a
+      # companion update to source tests from the installer checkout instead.
       test-infra-repository: osac-project/osac-test-infra
       test-infra-ref: ${{ inputs.test-infra-ref || 'main' }}
 

.github/workflows/e2e-caas-full-install.yml

@@ -29,7 +29,7 @@ on:
         required: false
         default: ""
       test-infra-ref:
-        description: "Branch/ref of osac-test-infra"
+        description: "Branch/ref of osac-test-infra (orchestration only; test content is in this repo under tests/e2e/)"
         required: false
         default: "main"
 
@@ -120,7 +120,10 @@ jobs:
       fork-pr-author-association: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association || '' }}
       fork-pr-author: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.user.login || '' }}
       pr-number: ${{ github.event.pull_request.number }}
-      # Checkout osac-test-infra for tests/Containerfile (not the caller repo)
+      # osac-test-infra is still checked out for orchestration infrastructure
+      # (actions, scripts, secrets). Test content (tests/, pyproject.toml) has
+      # moved to this repo under tests/e2e/ -- the reusable workflow needs a
+      # companion update to source tests from the installer checkout instead.
       test-infra-repository: osac-project/osac-test-infra
       test-infra-ref: ${{ inputs.test-infra-ref || 'main' }}
       # CaaS-specific inputs (cluster-template, caas-domain, flavor-image,

.github/workflows/e2e-vmaas-full-install.yml

@@ -29,7 +29,7 @@ on:
         required: false
         default: ""
       test-infra-ref:
-        description: "Branch/ref of osac-test-infra"
+        description: "Branch/ref of osac-test-infra (orchestration only; test content is in this repo under tests/e2e/)"
         required: false
         default: "main"
 
@@ -116,7 +116,10 @@ jobs:
       fork-pr-author-association: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association || '' }}
       fork-pr-author: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.user.login || '' }}
       pr-number: ${{ github.event.pull_request.number }}
-      # Checkout osac-test-infra for tests/Containerfile (not the caller repo)
+      # osac-test-infra is still checked out for orchestration infrastructure
+      # (actions, scripts, secrets). Test content (tests/, pyproject.toml) has
+      # moved to this repo under tests/e2e/ -- the reusable workflow needs a
+      # companion update to source tests from the installer checkout instead.
       test-infra-repository: osac-project/osac-test-infra
       test-infra-ref: ${{ inputs.test-infra-ref || 'main' }}
 

AGENTS.md

@@ -13,6 +13,7 @@ OSAC (Open Sovereign AI Cloud) is a fulfillment system for provisioning Kubernet
 | `bare-metal-fulfillment-operator/` | Kubernetes operator for bare-metal host pools |
 | `osac-csi-driver/` | CSI meta-driver aggregating vendor storage drivers |
 | `osac-metering/` | Metering pipeline — collects usage events via gRPC Watch, publishes CloudEvents to Kafka, and provides Provider Adapters framework for billing integrations |
+| `tests/e2e/` | Cross-component e2e pytest suite (bmaas, vmaas, caas, catalog, storage) — runs against a deployed cluster's gRPC API |
 
 See also [`docs/`](docs/README.md) for hand-trimmed cross-component architecture and
 conventions content with no home in any single component's own `AGENTS.md`
@@ -25,7 +26,7 @@ Clone as siblings for cross-repo workflows:
 
 | Repo | Description |
 |------|-------------|
-| [osac-test-infra](https://github.com/osac-project/osac-test-infra) | E2E pytest tests against the fulfillment-service gRPC API |
+| [osac-test-infra](https://github.com/osac-project/osac-test-infra) | E2E workflow orchestration (cluster lifecycle, image builds, secrets); test content has moved to `tests/e2e/` in this repo |
 | [osac-ui](https://github.com/osac-project/osac-ui) | Web console (React, PatternFly 6) |
 | [enhancement-proposals](https://github.com/osac-project/enhancement-proposals) | PRDs and design documents (two-stage EP flow) |
 | [docs](https://github.com/osac-project/docs) | Architecture docs and guides |

docs/CONVENTIONS.md

@@ -15,6 +15,6 @@ When changing one repo, check all dependent repos in this table before submittin
 |-----------|-----------|-----|
 | `osac-operator` CRD types | `fulfillment-service` reconciler registration | New CRD types must be registered in the fulfillment-service reconciler (in-repo change, same PR) |
 | `osac-operator` CRD spec changes | `osac-aap` roles that read CRD fields | Adding a field to `ClusterOrderSpec` requires the AAP playbook to extract and use it |
-| `fulfillment-service` CLI flag changes | `osac-test-infra` test helpers | Adding `--pull-secret-file` required updating `OsacCLI.create_cluster` in the test infra (separate repo, separate PR) |
+| `fulfillment-service` CLI flag changes | `tests/e2e/` test helpers | Adding `--pull-secret-file` required updating `OsacCLI.create_cluster` in `tests/e2e/` (in-repo change, same PR) |
 
 Evidence: MGMT-24226 eval scored 3/5 because the agent fixed `fulfillment-service` and `osac-aap` but missed updating `osac-installer`'s pinned image version — this was pre-mono-repo-merge (OSAC-1739), when these were still separate repos and image tags were pinned per-commit. That specific mechanism (a separate pinned tag needing a manual re-sync via `sync-image-tags.sh`) no longer exists post-mono-repo — `osac/osac-installer/scripts/sync-image-tags.sh` was removed upstream (`OSAC-3367`) and `osac/osac-installer/values/*/values.yaml` now leaves mono-repo-resident components' image tags unpinned. See the CRD-registration and CLI-flag rows above for what still needs a cross-file check today.

osac-csi-driver/AGENTS.md

@@ -117,7 +117,7 @@ CLI flags are defined in `cmd/osac-csi-driver/main.go`. Run `go run ./cmd/osac-c
 
 - **Unit tests**: `pkg/driver/controller_test.go` — standard Go `testing` package with hand-written mocks; covers CreateVolume, DeleteVolume, Publish/Unpublish, polling, idempotency, error handling
 - **Sanity tests**: `test/sanity/` — kubernetes-csi/csi-test/v5 compliance suite against the real driver + a fake vendor backend
-- **E2E tests**: None in this component (E2E tests live in `osac-test-infra`)
+- **E2E tests**: None in this component (E2E tests live in `tests/e2e/` in this mono-repo)
 - Run all: `make test`
 
 ## Helm Charts

osac-operator/AGENTS.md

@@ -128,7 +128,7 @@ hack/sync-helm-crds.py     # Script invoked by `make helm-crds` to sync CRDs to
 - **Unit tests**: Ginkgo + Gomega with `envtest` (real etcd + kube-apiserver)
 - **Integration**: `make test-kustomize` (manifest validation) + `make test-smoke` (kind cluster)
 - **Go integration tests**: `test/integration/` (console_proxy_test.go, integration_suite_test.go, networking_test.go) — run against an already-running kind cluster via `make test-integration-kind` (`go test ./test/integration/ -v -ginkgo.v`)
-- **E2E tests**: pytest-based, live in the separate `osac-test-infra` repo; triggered by the root-level `.github/workflows/e2e-vmaas-full-install.yml`, which builds and deploys both `osac-operator` and `fulfillment-service` together
+- **E2E tests**: pytest-based, live in `tests/e2e/` in this mono-repo; triggered by the root-level `.github/workflows/e2e-vmaas-full-install.yml`, which builds and deploys both `osac-operator` and `fulfillment-service` together
 - Kind cluster defaults to `osac` (`KIND_CLUSTER_NAME` in Makefile line 81), but smoke tests create `osac-test`
 - Clean up: `kind delete cluster --name osac-test`
 - `test-kustomize` catches missing files in kustomization.yaml — always run before committing manifest changes

tests/e2e/.dockerignore

@@ -0,0 +1,5 @@
+.git
+.venv
+__pycache__
+*.pyc
+.pytest_cache

tests/e2e/Containerfile

@@ -0,0 +1,55 @@
+FROM registry.access.redhat.com/ubi9/ubi:latest
+
+ARG GRPCURL_VERSION=1.9.1
+ARG OSAC_VERSION=""
+ARG OSAC_CLI_BIN=""
+
+RUN dnf install -y python3.11 python3.11-pip make jq openssh-clients && dnf clean all
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl --retry 5 --retry-delay 2 -Lsf "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz" \
+    | tar xz --no-same-owner -C /usr/local/bin oc kubectl
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+RUN curl --retry 5 --retry-delay 2 -Lsf "https://github.com/fullstorydev/grpcurl/releases/download/v${GRPCURL_VERSION}/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz" \
+    | tar xz --no-same-owner -C /usr/local/bin grpcurl
+
+COPY ${OSAC_CLI_BIN:-.dockerignore} /tmp/osac-cli-candidate
+
+RUN set -euo pipefail; \
+    if [ -n "${OSAC_CLI_BIN}" ]; then \
+      mv /tmp/osac-cli-candidate /usr/local/bin/osac; \
+      chmod +x /usr/local/bin/osac; \
+      echo "Using pre-built OSAC CLI binary"; \
+    else \
+      rm -f /tmp/osac-cli-candidate; \
+      if [ -z "${OSAC_VERSION}" ]; then \
+        OSAC_TAG=$(curl --retry 5 --retry-delay 2 -Lsf "https://api.github.com/repos/osac-project/osac/releases?per_page=100" | jq -r '[.[] | select(.tag_name | test("^fulfillment-service/v[0-9]+\\.[0-9]+\\.[0-9]+$"))][0].tag_name // empty'); \
+        [ -n "${OSAC_TAG}" ] || { echo "ERROR: no fulfillment-service release (fulfillment-service/vX.Y.Z tag) found on osac-project/osac"; exit 1; }; \
+        OSAC_VERSION="${OSAC_TAG#fulfillment-service/v}"; \
+        echo "Resolved latest OSAC CLI version: ${OSAC_VERSION}"; \
+      else \
+        OSAC_TAG="fulfillment-service/v${OSAC_VERSION}"; \
+      fi; \
+      curl --retry 5 --retry-delay 2 -Lsfo /usr/local/bin/osac "https://github.com/osac-project/osac/releases/download/${OSAC_TAG}/osac_Linux_x86_64" \
+        || { echo "ERROR: osac binary download failed"; exit 1; }; \
+      curl --retry 5 --retry-delay 2 -Lsfo /tmp/checksums.txt "https://github.com/osac-project/osac/releases/download/${OSAC_TAG}/osac_${OSAC_VERSION}_checksums.txt" \
+        || { echo "ERROR: checksums file download failed"; exit 1; }; \
+      line="$(grep -E '[[:space:]]osac_Linux_x86_64$' /tmp/checksums.txt || true)"; \
+      [ -n "$line" ] || { echo "ERROR: osac_Linux_x86_64 entry not found in checksums file"; exit 1; }; \
+      echo "$line" | sed 's|osac_Linux_x86_64|/usr/local/bin/osac|' | sha256sum -c - \
+        || { echo "ERROR: checksum mismatch"; exit 1; }; \
+      rm -f /tmp/checksums.txt; \
+      chmod +x /usr/local/bin/osac; \
+    fi
+
+WORKDIR /tests
+
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --python python3.11
+
+COPY . .
+RUN rm -f osac-cli-bin
+
+ENV PATH="/tests/.venv/bin:$PATH"

tests/e2e/pyproject.toml

@@ -0,0 +1,63 @@
+[project]
+name = "osac-e2e-tests"
+version = "0.1.0"
+description = "OSAC end-to-end test suite"
+requires-python = ">=3.11"
+dependencies = [
+    "pytest>=8.0",
+    "pytest-xdist>=3.0",
+    "pyyaml>=6.0",
+    "websocket-client>=1.6",
+]
+
+[tool.setuptools.packages.find]
+include = ["tests*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]
+addopts = "-n 4 --dist loadfile"
+log_level = "INFO"
+# Stream log records live as tests run (works under xdist -- workers forward
+# log records to the master for display), instead of only showing captured
+# output after a test finishes or fails. Long single-test suites (e.g.
+# tests/caas/, which polls silently for up to ~90 minutes with only one test
+# to distribute across 4 xdist workers) would otherwise show nothing in a CI
+# log until the whole run completes.
+log_cli = true
+log_cli_date_format = "%Y-%m-%dT%H:%M:%S%z"
+log_cli_format = "%(asctime)s %(levelname)-8s %(message)s"
+log_cli_level = "INFO"
+# Write timestamped log records to a file that gets uploaded as an artifact.
+# Unlike log_cli, log_file is not suppressed by xdist output capture.
+# Each worker writes its own file; the gather script merges them by timestamp.
+# Path must match the container volume mount in .github/workflows/e2e-*.yml
+# Note for future maintainers:
+# Makefile writes output to ${REPORT_DIR}, e2e.log filepath is hardcoded.
+# TODO: CI test & Makefile create files in same directory with path retrieved from vars defined in files or environments.
+log_file = "/tmp/test-output/e2e.log"
+log_file_date_format = "%Y-%m-%dT%H:%M:%S%z"
+log_file_format = "%(asctime)s %(levelname)-8s %(name)s %(message)s"
+log_file_level = "INFO"
+log_file_mode = "a"
+
+[tool.basedpyright]
+reportUnusedCallResult = false
+reportAny = false
+reportExplicitAny = false
+
+[tool.ruff]
+target-version = "py311"
+line-length = 120
+
+[tool.ruff.format]
+skip-magic-trailing-comma = true
+
+[tool.ruff.lint.isort]
+split-on-trailing-comma = false
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "ANN", "B", "SIM", "RUF"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["ANN101", "ANN201", "ANN401"]

tests/e2e/tests/bmaas/conftest.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import subprocess
+import tempfile
+import uuid
+from collections.abc import Generator
+from pathlib import Path
+
+import pytest
+
+from tests.core.grpc_client import GRPCClient
+from tests.core.runner import env
+
+
+@pytest.fixture(scope="session")
+def bmi_template() -> str:
+    return env("OSAC_BMI_TEMPLATE", "bm-host-provisioning")
+
+
+@pytest.fixture(scope="session")
+def bmh_namespace() -> str:
+    return env("OSAC_BMH_NAMESPACE", "host-inventory")
+
+
+@pytest.fixture(scope="session")
+def test_run_id() -> str:
+    return str(uuid.uuid4())[:8]
+
+
+@pytest.fixture(scope="session")
+def ssh_public_key() -> Generator[str, None, None]:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        key_path = Path(tmpdir) / "bmi-test-key"
+        subprocess.run(
+            ["ssh-keygen", "-t", "ed25519", "-f", str(key_path), "-N", "", "-C", "bmi-e2e-test"],
+            capture_output=True,
+            check=True,
+        )
+        yield (key_path.with_suffix(".pub")).read_text().strip()
+
+
+@pytest.fixture(scope="session")
+def catalog_item(private_grpc: GRPCClient, bmi_template: str, test_run_id: str) -> Generator[str, None, None]:
+    name = f"e2e-bmaas-{test_run_id}"
+    print(f"\nCreating BareMetalInstanceCatalogItem: {name}")
+    item_id: str = private_grpc.create_baremetal_instance_catalog_item(
+        name=name,
+        title=f"E2E BMaaS Test ({test_run_id})",
+        description="Temporary catalog item for BMaaS E2E tests",
+        template=bmi_template,
+        field_definitions=[{"path": "ssh_public_key", "display_name": "SSH Public Key", "editable": True}],
+    )
+    print(f"CatalogItem created: {item_id}")
+
+    yield item_id
+
+    try:
+        print(f"\nDeleting BareMetalInstanceCatalogItem {item_id}...")
+        private_grpc.delete_baremetal_instance_catalog_item(item_id=item_id)
+        print(f"CatalogItem {item_id} deleted")
+    except Exception as e:
+        print(f"WARNING: Failed to delete catalog item {item_id}: {e}")