diff --git a/.github/workflows/e2e-bmaas-full-install.yml b/.github/workflows/e2e-bmaas-full-install.yml index 5dc32e0583..f41429a2fe 100644 --- a/.github/workflows/e2e-bmaas-full-install.yml +++ b/.github/workflows/e2e-bmaas-full-install.yml @@ -29,7 +29,7 @@ on: required: false default: "" test-infra-ref: - description: "Branch/ref of osac-test-infra" + description: "Branch/ref of osac-test-infra (orchestration only; test content is in this repo under tests/e2e/)" required: false default: "main" @@ -109,7 +109,10 @@ jobs: fork-pr-author-association: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association || '' }} fork-pr-author: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.user.login || '' }} pr-number: ${{ github.event.pull_request.number }} - # Checkout osac-test-infra for tests/Containerfile (not the caller repo) + # osac-test-infra is still checked out for orchestration infrastructure + # (actions, scripts, secrets). Test content (tests/, pyproject.toml) has + # moved to this repo under tests/e2e/ -- the reusable workflow needs a + # companion update to source tests from the installer checkout instead. test-infra-repository: osac-project/osac-test-infra test-infra-ref: ${{ inputs.test-infra-ref || 'main' }} diff --git a/.github/workflows/e2e-caas-full-install.yml b/.github/workflows/e2e-caas-full-install.yml index c654d6cd6c..73990c32bf 100644 --- a/.github/workflows/e2e-caas-full-install.yml +++ b/.github/workflows/e2e-caas-full-install.yml @@ -29,7 +29,7 @@ on: required: false default: "" test-infra-ref: - description: "Branch/ref of osac-test-infra" + description: "Branch/ref of osac-test-infra (orchestration only; test content is in this repo under tests/e2e/)" required: false default: "main" @@ -120,7 +120,10 @@ jobs: fork-pr-author-association: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association || '' }} fork-pr-author: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.user.login || '' }} pr-number: ${{ github.event.pull_request.number }} - # Checkout osac-test-infra for tests/Containerfile (not the caller repo) + # osac-test-infra is still checked out for orchestration infrastructure + # (actions, scripts, secrets). Test content (tests/, pyproject.toml) has + # moved to this repo under tests/e2e/ -- the reusable workflow needs a + # companion update to source tests from the installer checkout instead. test-infra-repository: osac-project/osac-test-infra test-infra-ref: ${{ inputs.test-infra-ref || 'main' }} # CaaS-specific inputs (cluster-template, caas-domain, flavor-image, diff --git a/.github/workflows/e2e-vmaas-full-install.yml b/.github/workflows/e2e-vmaas-full-install.yml index a28dfd4d0b..c249a12a5f 100644 --- a/.github/workflows/e2e-vmaas-full-install.yml +++ b/.github/workflows/e2e-vmaas-full-install.yml @@ -29,7 +29,7 @@ on: required: false default: "" test-infra-ref: - description: "Branch/ref of osac-test-infra" + description: "Branch/ref of osac-test-infra (orchestration only; test content is in this repo under tests/e2e/)" required: false default: "main" @@ -116,7 +116,10 @@ jobs: fork-pr-author-association: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.author_association || '' }} fork-pr-author: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.user.login || '' }} pr-number: ${{ github.event.pull_request.number }} - # Checkout osac-test-infra for tests/Containerfile (not the caller repo) + # osac-test-infra is still checked out for orchestration infrastructure + # (actions, scripts, secrets). Test content (tests/, pyproject.toml) has + # moved to this repo under tests/e2e/ -- the reusable workflow needs a + # companion update to source tests from the installer checkout instead. test-infra-repository: osac-project/osac-test-infra test-infra-ref: ${{ inputs.test-infra-ref || 'main' }} diff --git a/AGENTS.md b/AGENTS.md index 3f56b08a1b..a8d968cf9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ OSAC (Open Sovereign AI Cloud) is a fulfillment system for provisioning Kubernet | `bare-metal-fulfillment-operator/` | Kubernetes operator for bare-metal host pools | | `osac-csi-driver/` | CSI meta-driver aggregating vendor storage drivers | | `osac-metering/` | Metering pipeline — collects usage events via gRPC Watch, publishes CloudEvents to Kafka, and provides Provider Adapters framework for billing integrations | +| `tests/e2e/` | Cross-component e2e pytest suite (bmaas, vmaas, caas, catalog, storage) — runs against a deployed cluster's gRPC API | See also [`docs/`](docs/README.md) for hand-trimmed cross-component architecture and conventions content with no home in any single component's own `AGENTS.md` @@ -25,7 +26,7 @@ Clone as siblings for cross-repo workflows: | Repo | Description | |------|-------------| -| [osac-test-infra](https://github.com/osac-project/osac-test-infra) | E2E pytest tests against the fulfillment-service gRPC API | +| [osac-test-infra](https://github.com/osac-project/osac-test-infra) | E2E workflow orchestration (cluster lifecycle, image builds, secrets); test content has moved to `tests/e2e/` in this repo | | [osac-ui](https://github.com/osac-project/osac-ui) | Web console (React, PatternFly 6) | | [enhancement-proposals](https://github.com/osac-project/enhancement-proposals) | PRDs and design documents (two-stage EP flow) | | [docs](https://github.com/osac-project/docs) | Architecture docs and guides | diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md index cb48b3beb9..eccf0d8e3a 100644 --- a/docs/CONVENTIONS.md +++ b/docs/CONVENTIONS.md @@ -15,6 +15,6 @@ When changing one repo, check all dependent repos in this table before submittin |-----------|-----------|-----| | `osac-operator` CRD types | `fulfillment-service` reconciler registration | New CRD types must be registered in the fulfillment-service reconciler (in-repo change, same PR) | | `osac-operator` CRD spec changes | `osac-aap` roles that read CRD fields | Adding a field to `ClusterOrderSpec` requires the AAP playbook to extract and use it | -| `fulfillment-service` CLI flag changes | `osac-test-infra` test helpers | Adding `--pull-secret-file` required updating `OsacCLI.create_cluster` in the test infra (separate repo, separate PR) | +| `fulfillment-service` CLI flag changes | `tests/e2e/` test helpers | Adding `--pull-secret-file` required updating `OsacCLI.create_cluster` in `tests/e2e/` (in-repo change, same PR) | Evidence: MGMT-24226 eval scored 3/5 because the agent fixed `fulfillment-service` and `osac-aap` but missed updating `osac-installer`'s pinned image version — this was pre-mono-repo-merge (OSAC-1739), when these were still separate repos and image tags were pinned per-commit. That specific mechanism (a separate pinned tag needing a manual re-sync via `sync-image-tags.sh`) no longer exists post-mono-repo — `osac/osac-installer/scripts/sync-image-tags.sh` was removed upstream (`OSAC-3367`) and `osac/osac-installer/values/*/values.yaml` now leaves mono-repo-resident components' image tags unpinned. See the CRD-registration and CLI-flag rows above for what still needs a cross-file check today. diff --git a/osac-csi-driver/AGENTS.md b/osac-csi-driver/AGENTS.md index 4a3592ab22..72b7418f82 100644 --- a/osac-csi-driver/AGENTS.md +++ b/osac-csi-driver/AGENTS.md @@ -117,7 +117,7 @@ CLI flags are defined in `cmd/osac-csi-driver/main.go`. Run `go run ./cmd/osac-c - **Unit tests**: `pkg/driver/controller_test.go` — standard Go `testing` package with hand-written mocks; covers CreateVolume, DeleteVolume, Publish/Unpublish, polling, idempotency, error handling - **Sanity tests**: `test/sanity/` — kubernetes-csi/csi-test/v5 compliance suite against the real driver + a fake vendor backend -- **E2E tests**: None in this component (E2E tests live in `osac-test-infra`) +- **E2E tests**: None in this component (E2E tests live in `tests/e2e/` in this mono-repo) - Run all: `make test` ## Helm Charts diff --git a/osac-operator/AGENTS.md b/osac-operator/AGENTS.md index d189c67237..f8dc35f97d 100644 --- a/osac-operator/AGENTS.md +++ b/osac-operator/AGENTS.md @@ -128,7 +128,7 @@ hack/sync-helm-crds.py # Script invoked by `make helm-crds` to sync CRDs to - **Unit tests**: Ginkgo + Gomega with `envtest` (real etcd + kube-apiserver) - **Integration**: `make test-kustomize` (manifest validation) + `make test-smoke` (kind cluster) - **Go integration tests**: `test/integration/` (console_proxy_test.go, integration_suite_test.go, networking_test.go) — run against an already-running kind cluster via `make test-integration-kind` (`go test ./test/integration/ -v -ginkgo.v`) -- **E2E tests**: pytest-based, live in the separate `osac-test-infra` repo; triggered by the root-level `.github/workflows/e2e-vmaas-full-install.yml`, which builds and deploys both `osac-operator` and `fulfillment-service` together +- **E2E tests**: pytest-based, live in `tests/e2e/` in this mono-repo; triggered by the root-level `.github/workflows/e2e-vmaas-full-install.yml`, which builds and deploys both `osac-operator` and `fulfillment-service` together - Kind cluster defaults to `osac` (`KIND_CLUSTER_NAME` in Makefile line 81), but smoke tests create `osac-test` - Clean up: `kind delete cluster --name osac-test` - `test-kustomize` catches missing files in kustomization.yaml — always run before committing manifest changes diff --git a/tests/e2e/.dockerignore b/tests/e2e/.dockerignore new file mode 100644 index 0000000000..ba978d7068 --- /dev/null +++ b/tests/e2e/.dockerignore @@ -0,0 +1,5 @@ +.git +.venv +__pycache__ +*.pyc +.pytest_cache diff --git a/tests/e2e/Containerfile b/tests/e2e/Containerfile new file mode 100644 index 0000000000..996b7c143a --- /dev/null +++ b/tests/e2e/Containerfile @@ -0,0 +1,55 @@ +FROM registry.access.redhat.com/ubi9/ubi:latest + +ARG GRPCURL_VERSION=1.9.1 +ARG OSAC_VERSION="" +ARG OSAC_CLI_BIN="" + +RUN dnf install -y python3.11 python3.11-pip make jq openssh-clients && dnf clean all + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl --retry 5 --retry-delay 2 -Lsf "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz" \ + | tar xz --no-same-owner -C /usr/local/bin oc kubectl + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +RUN curl --retry 5 --retry-delay 2 -Lsf "https://github.com/fullstorydev/grpcurl/releases/download/v${GRPCURL_VERSION}/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz" \ + | tar xz --no-same-owner -C /usr/local/bin grpcurl + +COPY ${OSAC_CLI_BIN:-.dockerignore} /tmp/osac-cli-candidate + +RUN set -euo pipefail; \ + if [ -n "${OSAC_CLI_BIN}" ]; then \ + mv /tmp/osac-cli-candidate /usr/local/bin/osac; \ + chmod +x /usr/local/bin/osac; \ + echo "Using pre-built OSAC CLI binary"; \ + else \ + rm -f /tmp/osac-cli-candidate; \ + if [ -z "${OSAC_VERSION}" ]; then \ + OSAC_TAG=$(curl --retry 5 --retry-delay 2 -Lsf "https://api.github.com/repos/osac-project/osac/releases?per_page=100" | jq -r '[.[] | select(.tag_name | test("^fulfillment-service/v[0-9]+\\.[0-9]+\\.[0-9]+$"))][0].tag_name // empty'); \ + [ -n "${OSAC_TAG}" ] || { echo "ERROR: no fulfillment-service release (fulfillment-service/vX.Y.Z tag) found on osac-project/osac"; exit 1; }; \ + OSAC_VERSION="${OSAC_TAG#fulfillment-service/v}"; \ + echo "Resolved latest OSAC CLI version: ${OSAC_VERSION}"; \ + else \ + OSAC_TAG="fulfillment-service/v${OSAC_VERSION}"; \ + fi; \ + curl --retry 5 --retry-delay 2 -Lsfo /usr/local/bin/osac "https://github.com/osac-project/osac/releases/download/${OSAC_TAG}/osac_Linux_x86_64" \ + || { echo "ERROR: osac binary download failed"; exit 1; }; \ + curl --retry 5 --retry-delay 2 -Lsfo /tmp/checksums.txt "https://github.com/osac-project/osac/releases/download/${OSAC_TAG}/osac_${OSAC_VERSION}_checksums.txt" \ + || { echo "ERROR: checksums file download failed"; exit 1; }; \ + line="$(grep -E '[[:space:]]osac_Linux_x86_64$' /tmp/checksums.txt || true)"; \ + [ -n "$line" ] || { echo "ERROR: osac_Linux_x86_64 entry not found in checksums file"; exit 1; }; \ + echo "$line" | sed 's|osac_Linux_x86_64|/usr/local/bin/osac|' | sha256sum -c - \ + || { echo "ERROR: checksum mismatch"; exit 1; }; \ + rm -f /tmp/checksums.txt; \ + chmod +x /usr/local/bin/osac; \ + fi + +WORKDIR /tests + +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --python python3.11 + +COPY . . +RUN rm -f osac-cli-bin + +ENV PATH="/tests/.venv/bin:$PATH" diff --git a/tests/e2e/pyproject.toml b/tests/e2e/pyproject.toml new file mode 100644 index 0000000000..797861b185 --- /dev/null +++ b/tests/e2e/pyproject.toml @@ -0,0 +1,63 @@ +[project] +name = "osac-e2e-tests" +version = "0.1.0" +description = "OSAC end-to-end test suite" +requires-python = ">=3.11" +dependencies = [ + "pytest>=8.0", + "pytest-xdist>=3.0", + "pyyaml>=6.0", + "websocket-client>=1.6", +] + +[tool.setuptools.packages.find] +include = ["tests*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-n 4 --dist loadfile" +log_level = "INFO" +# Stream log records live as tests run (works under xdist -- workers forward +# log records to the master for display), instead of only showing captured +# output after a test finishes or fails. Long single-test suites (e.g. +# tests/caas/, which polls silently for up to ~90 minutes with only one test +# to distribute across 4 xdist workers) would otherwise show nothing in a CI +# log until the whole run completes. +log_cli = true +log_cli_date_format = "%Y-%m-%dT%H:%M:%S%z" +log_cli_format = "%(asctime)s %(levelname)-8s %(message)s" +log_cli_level = "INFO" +# Write timestamped log records to a file that gets uploaded as an artifact. +# Unlike log_cli, log_file is not suppressed by xdist output capture. +# Each worker writes its own file; the gather script merges them by timestamp. +# Path must match the container volume mount in .github/workflows/e2e-*.yml +# Note for future maintainers: +# Makefile writes output to ${REPORT_DIR}, e2e.log filepath is hardcoded. +# TODO: CI test & Makefile create files in same directory with path retrieved from vars defined in files or environments. +log_file = "/tmp/test-output/e2e.log" +log_file_date_format = "%Y-%m-%dT%H:%M:%S%z" +log_file_format = "%(asctime)s %(levelname)-8s %(name)s %(message)s" +log_file_level = "INFO" +log_file_mode = "a" + +[tool.basedpyright] +reportUnusedCallResult = false +reportAny = false +reportExplicitAny = false + +[tool.ruff] +target-version = "py311" +line-length = 120 + +[tool.ruff.format] +skip-magic-trailing-comma = true + +[tool.ruff.lint.isort] +split-on-trailing-comma = false + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "ANN", "B", "SIM", "RUF"] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["ANN101", "ANN201", "ANN401"] diff --git a/tests/e2e/tests/__init__.py b/tests/e2e/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/bmaas/__init__.py b/tests/e2e/tests/bmaas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/bmaas/conftest.py b/tests/e2e/tests/bmaas/conftest.py new file mode 100644 index 0000000000..f2dd30c676 --- /dev/null +++ b/tests/e2e/tests/bmaas/conftest.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import subprocess +import tempfile +import uuid +from collections.abc import Generator +from pathlib import Path + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.runner import env + + +@pytest.fixture(scope="session") +def bmi_template() -> str: + return env("OSAC_BMI_TEMPLATE", "bm-host-provisioning") + + +@pytest.fixture(scope="session") +def bmh_namespace() -> str: + return env("OSAC_BMH_NAMESPACE", "host-inventory") + + +@pytest.fixture(scope="session") +def test_run_id() -> str: + return str(uuid.uuid4())[:8] + + +@pytest.fixture(scope="session") +def ssh_public_key() -> Generator[str, None, None]: + with tempfile.TemporaryDirectory() as tmpdir: + key_path = Path(tmpdir) / "bmi-test-key" + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-f", str(key_path), "-N", "", "-C", "bmi-e2e-test"], + capture_output=True, + check=True, + ) + yield (key_path.with_suffix(".pub")).read_text().strip() + + +@pytest.fixture(scope="session") +def catalog_item(private_grpc: GRPCClient, bmi_template: str, test_run_id: str) -> Generator[str, None, None]: + name = f"e2e-bmaas-{test_run_id}" + print(f"\nCreating BareMetalInstanceCatalogItem: {name}") + item_id: str = private_grpc.create_baremetal_instance_catalog_item( + name=name, + title=f"E2E BMaaS Test ({test_run_id})", + description="Temporary catalog item for BMaaS E2E tests", + template=bmi_template, + field_definitions=[{"path": "ssh_public_key", "display_name": "SSH Public Key", "editable": True}], + ) + print(f"CatalogItem created: {item_id}") + + yield item_id + + try: + print(f"\nDeleting BareMetalInstanceCatalogItem {item_id}...") + private_grpc.delete_baremetal_instance_catalog_item(item_id=item_id) + print(f"CatalogItem {item_id} deleted") + except Exception as e: + print(f"WARNING: Failed to delete catalog item {item_id}: {e}") diff --git a/tests/e2e/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/e2e/tests/bmaas/test_baremetal_instance_lifecycle.py new file mode 100644 index 0000000000..06c77fe4dd --- /dev/null +++ b/tests/e2e/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import logging +from typing import Any + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_bmh_available, + wait_for_bmh_provisioned, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +logger = logging.getLogger(__name__) + +_RESTART_IN_PROGRESS: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_IN_PROGRESS" +_RESTART_FAILED: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_FAILED" + + +def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + for condition in response.get("object", {}).get("status", {}).get("conditions", []): + if condition.get("type") == condition_type: + return condition.get("status", "") + return "" + + +def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) + + +def test_baremetal_instance_lifecycle( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name = f"e2e-bmi-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + bmh_ns = "" + bmh_name = "" + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + # Verify provisioning + wait_for_bmh_provisioned(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + + image_url: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url != "", f"BMH {bmh_name} has no image URL after provisioning" + + consumer_ref: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref != "", f"BMH {bmh_name} has no consumerRef after allocation" + + online: str = k8s_hub_client.get_bmh_online(name=bmh_name, bmh_namespace=bmh_ns) + assert online == "true", f"BMH {bmh_name} should be online after provisioning, got: {online}" + + # Power off + halted = "BARE_METAL_INSTANCE_RUN_STRATEGY_HALTED" + grpc.update_baremetal_instance_run_strategy(bmi_id=bmi_id, run_strategy=halted) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "false", + retries=60, + delay=5, + description=f"{bmh_name} powered off", + ) + + # Power on + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_ALWAYS" + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on", + ) + + # Deprovision + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + + image_url_after: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url_after == "", f"BMH {bmh_name} image not cleared after deprovision: {image_url_after}" + + consumer_ref_after: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref_after == "", f"BMH {bmh_name} consumerRef not cleared: {consumer_ref_after}" + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + if bmh_name: + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + except Exception: + pass + raise + + +def test_baremetal_instance_restart( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name: str = f"e2e-bmi-restart-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + initial_trigger: int = _get_status_restart_trigger(grpc, bmi_id) + new_trigger: int = initial_trigger + 1 + logger.info("Incrementing restart_trigger from %d to %d", initial_trigger, new_trigger) + + grpc.update_baremetal_instance_restart_trigger(bmi_id=bmi_id, restart_trigger=new_trigger) + + poll_until( + fn=lambda: _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS), + until=lambda v: v == "CONDITION_STATUS_TRUE", + retries=60, + delay=2, + description=f"{bmi_id} RESTART_IN_PROGRESS condition appears", + ) + + poll_until( + fn=lambda: _get_status_restart_trigger(grpc, bmi_id), + until=lambda v: v == new_trigger, + retries=120, + delay=10, + description=f"{bmi_id} status.restart_trigger echoes {new_trigger}", + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on after restart", + ) + + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + restart_in_progress: str = _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS) + assert restart_in_progress in ("", "CONDITION_STATUS_FALSE"), ( + f"RESTART_IN_PROGRESS should have cleared after restart, got: {restart_in_progress}" + ) + + restart_failed: str = _get_condition_status(grpc, bmi_id, _RESTART_FAILED) + assert restart_failed in ("", "CONDITION_STATUS_FALSE"), ( + f"Unexpected RESTART_FAILED condition: {restart_failed}" + ) + + # Deprovision + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + except Exception: + logger.exception("Failed to delete BMI %s during cleanup", bmi_id) + raise diff --git a/tests/e2e/tests/caas/__init__.py b/tests/e2e/tests/caas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/caas/conftest.py b/tests/e2e/tests/caas/conftest.py new file mode 100644 index 0000000000..0e08aaaa3c --- /dev/null +++ b/tests/e2e/tests/caas/conftest.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import os + +import pytest + +from tests.core.runner import env + + +@pytest.fixture(scope="session") +def cluster_template() -> str: + return env("OSAC_CLUSTER_TEMPLATE", "ocp-ci-small") + + +@pytest.fixture(scope="session") +def pull_secret_path() -> str: + return env("OSAC_PULL_SECRET_PATH") + + +@pytest.fixture(scope="session") +def ssh_public_key_path() -> str: + return env("OSAC_SSH_PUBLIC_KEY_PATH", os.path.expanduser("~/.ssh/id_rsa.pub")) diff --git a/tests/e2e/tests/caas/contract b/tests/e2e/tests/caas/contract new file mode 100644 index 0000000000..8195446517 --- /dev/null +++ b/tests/e2e/tests/caas/contract @@ -0,0 +1 @@ +REQUIRED_VARS="KUBECONFIG OSAC_NAMESPACE OSAC_PULL_SECRET_PATH" diff --git a/tests/e2e/tests/caas/test_cluster_create.py b/tests/e2e/tests/caas/test_cluster_create.py new file mode 100644 index 0000000000..04347d116f --- /dev/null +++ b/tests/e2e/tests/caas/test_cluster_create.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import contextlib +import subprocess +from pathlib import Path + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_cluster_deleting, + wait_for_cluster_deletion, + wait_for_cluster_grpc_deleting_or_archived, + wait_for_cluster_grpc_removal, + wait_for_cluster_order_cr, + wait_for_cluster_progressing, + wait_for_cluster_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until, run + +# Real OCP release image used to provision the ClusterVersion(s) created by these +# tests. Fixed (not randomly generated) so repeated runs reuse the same +# ClusterVersion via GRPCClient.ensure_cluster_version instead of accumulating +# duplicates on the shared cluster. +TEST_RELEASE_IMAGE = "quay.io/openshift-release-dev/ocp-release:4.20.0-multi" + + +@pytest.mark.metering +def test_cluster_create( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + cluster_template: str, + pull_secret_path: str, + ssh_public_key_path: str, + metering: MeteringCollector, +) -> None: + name = unique_name("e2e-cluster") + uuid = cli.create_cluster( + name=name, + template=cluster_template, + template_parameter_files={"pull_secret": pull_secret_path}, + template_parameters={"ssh_public_key": Path(ssh_public_key_path).read_text().strip()}, + ) + metering.expect("osac.resource.created.v1", resource_id=uuid) + + try: + co_name = wait_for_cluster_order_cr(k8s=k8s_hub_client, uuid=uuid) + assert uuid in grpc.list_cluster_ids() + + wait_for_cluster_progressing(k8s=k8s_hub_client, name=co_name) + metering.expect("osac.resource.started.v1", resource_id=uuid) + metering.verify() + + wait_for_cluster_ready(k8s=k8s_hub_client, name=co_name) + + # Verify version resolved and propagated end-to-end: + # fulfillment-service default resolution -> ClusterOrder releaseImage -> HostedCluster image + cluster = grpc.get_cluster(cluster_id=uuid) + version_name = cluster.get("object", {}).get("spec", {}).get("version", {}).get("name", "") + assert version_name, "Cluster should have a resolved version name" + + co_release_image = k8s_hub_client.get_cluster_order_spec(name=co_name).get("releaseImage", "") + assert co_release_image, "ClusterOrder should have a resolved releaseImage" + + hosted_cluster_name = k8s_hub_client.get_cluster_order_hosted_cluster_name(name=co_name) + hosted_cluster_ns = k8s_hub_client.get_cluster_order_namespace(name=co_name) + hosted_cluster_image = run( + *k8s_hub_client._base(), + "get", + "hostedcluster", + hosted_cluster_name, + "-n", + hosted_cluster_ns, + "-o", + "jsonpath={.spec.release.image}", + ) + assert hosted_cluster_image == co_release_image, ( + f"HostedCluster image {hosted_cluster_image!r} != ClusterOrder releaseImage {co_release_image!r}" + ) + + # Derive expected N+1 count from cluster spec + node_sets = cluster.get("object", {}).get("spec", {}).get("nodeSets", {}) + expected_components = 1 + len(node_sets) + + # Verify N+1 heartbeat decomposition + metering.expect("osac.resource.heartbeat.v1", resource_id=uuid, timeout=180) + metering.verify() + + heartbeats = metering.get_all_events("osac.resource.heartbeat.v1", resource_id=uuid) + hb_components = [ev.get("data", {}).get("billing_dimensions", {}).get("component") for ev in heartbeats] + cp_count = sum(1 for c in hb_components if c == "control_plane") + worker_count = sum(1 for c in hb_components if c == "worker") + assert cp_count >= 1, f"Expected at least 1 control_plane heartbeat, got {cp_count}" + assert worker_count >= len(node_sets), ( + f"Expected at least {len(node_sets)} worker heartbeat(s), got {worker_count}" + ) + assert len(heartbeats) >= expected_components, ( + f"Expected at least {expected_components} heartbeat events (1 cp + {len(node_sets)} workers), " + f"got {len(heartbeats)}" + ) + + # Verify started.v1 carries correct resource type and cluster template + started = metering.get_event("osac.resource.started.v1", resource_id=uuid) + assert started.get("osacresourcetype") == "cluster_order" + started_bd = started.get("data", {}).get("billing_dimensions", {}) + assert started_bd.get("cluster_template") == cluster_template, ( + f"cluster_template mismatch: {started_bd.get('cluster_template')!r} != {cluster_template!r}" + ) + + # Scale a worker node set and verify updated.v1 + worker_node_set = next(iter(node_sets)) + original_size = node_sets[worker_node_set].get("size", 1) + scaled_size = original_size + 1 + cli.scale_cluster(uuid=uuid, node_set=worker_node_set, size=scaled_size) + metering.expect("osac.resource.updated.v1", resource_id=uuid, timeout=120) + metering.verify() + + updated = metering.get_event("osac.resource.updated.v1", resource_id=uuid) + updated_bd = updated.get("data", {}).get("billing_dimensions", {}) + assert updated_bd.get("node_set") == worker_node_set, ( + f"updated.v1 node_set mismatch: {updated_bd.get('node_set')!r} != {worker_node_set!r}" + ) + assert updated_bd.get("node_count") == scaled_size, ( + f"updated.v1 node_count should be {scaled_size}, got {updated_bd.get('node_count')}" + ) + + # Scale back before deletion + cli.scale_cluster(uuid=uuid, node_set=worker_node_set, size=original_size) + + cli.delete_cluster(uuid=uuid) + metering.expect("osac.resource.deleted.v1", resource_id=uuid) + + wait_for_cluster_deleting(k8s=k8s_hub_client, name=co_name) + wait_for_cluster_grpc_deleting_or_archived(grpc=grpc, uuid=uuid) + + wait_for_cluster_deletion(k8s=k8s_hub_client, name=co_name) + wait_for_cluster_grpc_removal(grpc=grpc, uuid=uuid) + metering.verify() + finally: + with contextlib.suppress(subprocess.CalledProcessError): + cli.delete_cluster(uuid=uuid) + + +def test_cluster_create_with_version( + cli: OsacCLI, + grpc: GRPCClient, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + cluster_template: str, + pull_secret_path: str, + ssh_public_key_path: str, +) -> None: + """Verify explicit --version resolution: the Cluster API resource stores the + version reference, the ClusterVersion cannot be deleted while referenced, + and the ClusterOrder CR's releaseImage is resolved from the matching + ClusterVersion. Does not wait for full provisioning — the HostedCluster + image propagation is covered by test_cluster_create.""" + version = private_grpc.ensure_cluster_version(version="4.20.0-e2e", image=TEST_RELEASE_IMAGE) + + name = unique_name("e2e-cluster-version") + uuid = cli.create_cluster( + name=name, + template=cluster_template, + version=version["name"], + template_parameter_files={"pull_secret": pull_secret_path}, + template_parameters={"ssh_public_key": Path(ssh_public_key_path).read_text().strip()}, + ) + + try: + co_name = wait_for_cluster_order_cr(k8s=k8s_hub_client, uuid=uuid) + + cluster = grpc.get_cluster(cluster_id=uuid) + assert cluster["object"]["spec"]["version"]["name"] == version["name"] + + # While the cluster references this version, deletion must be rejected + output, rc = private_grpc.call_unchecked( + service="osac.private.v1.ClusterVersions/Delete", data={"id": version["id"]} + ) + assert rc != 0, f"Expected delete to be rejected for referenced version, got: {output}" + assert "FailedPrecondition" in output or "referenced" in output.lower(), ( + f"Expected FailedPrecondition or 'referenced' in rejection, got: {output}" + ) + + release_image = poll_until( + fn=lambda: k8s_hub_client.get_cluster_order_spec(name=co_name).get("releaseImage", ""), + until=lambda v: v != "", + retries=30, + delay=5, + description=f"{co_name} ClusterOrder releaseImage resolution", + ) + assert release_image == TEST_RELEASE_IMAGE + + cli.delete_cluster(uuid=uuid) + + wait_for_cluster_deleting(k8s=k8s_hub_client, name=co_name) + wait_for_cluster_grpc_deleting_or_archived(grpc=grpc, uuid=uuid) + wait_for_cluster_deletion(k8s=k8s_hub_client, name=co_name) + wait_for_cluster_grpc_removal(grpc=grpc, uuid=uuid) + finally: + with contextlib.suppress(subprocess.CalledProcessError): + cli.delete_cluster(uuid=uuid) + + +def test_cluster_create_rejected_for_invalid_version( + grpc: GRPCClient, private_grpc: GRPCClient, cluster_template: str +) -> None: + """Verify cluster creation is rejected for disabled, obsolete, and + non-existent versions.""" + disabled = private_grpc.ensure_cluster_version(version="4.20.0-e2e-disabled", image=TEST_RELEASE_IMAGE) + private_grpc.update_cluster_version(version_id=disabled["id"], enabled=False) + + obsolete = private_grpc.ensure_cluster_version(version="4.20.0-e2e-obsolete", image=TEST_RELEASE_IMAGE) + private_grpc.update_cluster_version(version_id=obsolete["id"], state="CLUSTER_VERSION_STATE_OBSOLETE") + + def _create_with_version(version_name: str) -> tuple[str, int]: + return grpc.call_unchecked( + service="osac.public.v1.Clusters/Create", + data={"object": {"spec": {"template": {"name": cluster_template}, "version": {"name": version_name}}}}, + ) + + output, rc = _create_with_version(disabled["name"]) + assert rc != 0, f"Expected create to reject disabled version, got: {output}" + assert "disabled" in output.lower(), f"Expected 'disabled' in rejection, got: {output}" + + output, rc = _create_with_version(obsolete["name"]) + assert rc != 0, f"Expected create to reject obsolete version, got: {output}" + assert "obsolete" in output.lower(), f"Expected 'obsolete' in rejection, got: {output}" + + output, rc = _create_with_version("4-20-0-e2e-does-not-exist") + assert rc != 0, f"Expected create to reject non-existent version, got: {output}" + assert "not found" in output.lower(), f"Expected 'not found' in rejection, got: {output}" diff --git a/tests/e2e/tests/caas/test_cluster_delete_feedback_light.py b/tests/e2e/tests/caas/test_cluster_delete_feedback_light.py new file mode 100644 index 0000000000..8e14cfabb1 --- /dev/null +++ b/tests/e2e/tests/caas/test_cluster_delete_feedback_light.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import contextlib +import subprocess +from pathlib import Path + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_cluster_deleting, + wait_for_cluster_deletion, + wait_for_cluster_grpc_deleting_or_archived, + wait_for_cluster_grpc_removal, + wait_for_cluster_order_cr, + wait_for_cluster_progressing, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI + + +def test_cluster_delete_reports_deleting_state_without_provisioning( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + cluster_template: str, + pull_secret_path: str, + ssh_public_key_path: str, +) -> None: + """Verify that cluster deletion transitions through DELETING state + without waiting for full provisioning. Runs on kind without HyperShift + (OSAC-1586).""" + name = unique_name("e2e-cluster") + uuid = cli.create_cluster( + name=name, + template=cluster_template, + template_parameter_files={"pull_secret": pull_secret_path}, + template_parameters={"ssh_public_key": Path(ssh_public_key_path).read_text().strip()}, + ) + + try: + co_name = wait_for_cluster_order_cr(k8s=k8s_hub_client, uuid=uuid) + assert uuid in grpc.list_cluster_ids() + + wait_for_cluster_progressing(k8s=k8s_hub_client, name=co_name) + + cli.delete_cluster(uuid=uuid) + + wait_for_cluster_deleting(k8s=k8s_hub_client, name=co_name) + wait_for_cluster_grpc_deleting_or_archived(grpc=grpc, uuid=uuid) + wait_for_cluster_deletion(k8s=k8s_hub_client, name=co_name) + wait_for_cluster_grpc_removal(grpc=grpc, uuid=uuid) + finally: + with contextlib.suppress(subprocess.CalledProcessError): + cli.delete_cluster(uuid=uuid) diff --git a/tests/e2e/tests/catalog/__init__.py b/tests/e2e/tests/catalog/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/catalog/conftest.py b/tests/e2e/tests/catalog/conftest.py new file mode 100644 index 0000000000..a02640384e --- /dev/null +++ b/tests/e2e/tests/catalog/conftest.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import logging +import re +import subprocess +from collections.abc import Generator +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_subnet_ready, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import env + +logger = logging.getLogger(__name__) + + +def unique_name(prefix: str) -> str: + return f"{prefix}-{uuid4().hex[:8]}" + + +@pytest.fixture(scope="session") +def cluster_template() -> str: + return env("OSAC_CLUSTER_TEMPLATE", "ocp-small") + + +@pytest.fixture(scope="session") +def compute_instance_template() -> str: + return env("OSAC_VM_TEMPLATE", "ocp-virt-vm") + + +@pytest.fixture(scope="session") +def network_class(grpc: GRPCClient) -> str: + configured = env("OSAC_NETWORK_CLASS", "") + if configured: + return configured + response: dict[str, Any] = grpc.call(service="osac.public.v1.NetworkClasses/List") + items = response.get("items", []) + assert items, "No network classes found; set OSAC_NETWORK_CLASS" + return items[0]["metadata"]["name"] + + +def _delete_subnet_teardown(grpc: GRPCClient, k8s: K8sClient, *, subnet_id: str, subnet_cr_name: str) -> None: + try: + grpc.delete_subnet(subnet_id=subnet_id) + except subprocess.CalledProcessError as exc: + combined = (exc.stderr or "") + (exc.stdout or "") + if re.search(r"Code:\s*NotFound", combined): + logger.warning("Subnet %s already deleted via API", subnet_id) + else: + logger.warning("Subnet %s teardown delete failed: %s", subnet_id, combined.strip()) + return + if k8s.is_present(resource="subnet", name=subnet_cr_name): + wait_for_subnet_deletion(k8s=k8s, name=subnet_cr_name) + + +def _delete_virtual_network_teardown(grpc: GRPCClient, k8s: K8sClient, *, vn_id: str, vn_cr_name: str) -> None: + try: + grpc.delete_virtual_network(vn_id=vn_id) + except subprocess.CalledProcessError as exc: + combined = (exc.stderr or "") + (exc.stdout or "") + if re.search(r"Code:\s*NotFound", combined): + logger.warning("VirtualNetwork %s already deleted via API", vn_id) + else: + logger.warning("VirtualNetwork %s teardown delete failed: %s", vn_id, combined.strip()) + return + if k8s.is_present(resource="virtualnetwork", name=vn_cr_name): + wait_for_virtual_network_deletion(k8s=k8s, name=vn_cr_name) + + +@pytest.fixture(scope="module") +def catalog_networking( + grpc: GRPCClient, k8s_hub_client: K8sClient, network_class: str +) -> Generator[dict[str, str], None, None]: + """Create VirtualNetwork + Subnet for compute instance catalog item tests.""" + tag = uuid4().hex[:8] + vn_id: str | None = None + vn_cr_name: str | None = None + subnet_id: str | None = None + subnet_cr_name: str | None = None + + try: + vn_id = grpc.create_virtual_network( + name=f"e2e-cat-vn-{tag}", network_class=network_class, ipv4_cidr="10.200.0.0/16" + ) + vn_cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vn_cr_name) + + subnet_id = grpc.create_subnet(name=f"e2e-cat-subnet-{tag}", virtual_network=vn_id, ipv4_cidr="10.200.100.0/24") + subnet_cr_name = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_id) + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_cr_name) + + yield {"virtual_network_id": vn_id, "subnet_id": subnet_id} + except Exception: + # If setup fails, cleanup any resources that were created + logger.warning("Setup failed, cleaning up partial catalog networking resources: %s", tag) + if subnet_id: + try: + grpc.delete_subnet(subnet_id=subnet_id) + except Exception as e: + logger.warning("Failed to cleanup subnet %s: %s", subnet_id, type(e).__name__) + if vn_id: + try: + grpc.delete_virtual_network(vn_id=vn_id) + except Exception as e: + logger.warning("Failed to cleanup virtual network %s: %s", vn_id, type(e).__name__) + raise + finally: + # Normal cleanup runs regardless of setup success/failure + if subnet_id and subnet_cr_name: + _delete_subnet_teardown(grpc, k8s_hub_client, subnet_id=subnet_id, subnet_cr_name=subnet_cr_name) + if vn_id and vn_cr_name: + _delete_virtual_network_teardown(grpc, k8s_hub_client, vn_id=vn_id, vn_cr_name=vn_cr_name) + + +@pytest.fixture(scope="module") +def default_subnet_id(catalog_networking: dict[str, str]) -> str: + return catalog_networking["subnet_id"] diff --git a/tests/e2e/tests/catalog/test_catalog_item_lifecycle.py b/tests/e2e/tests/catalog/test_catalog_item_lifecycle.py new file mode 100644 index 0000000000..dadf06a3d9 --- /dev/null +++ b/tests/e2e/tests/catalog/test_catalog_item_lifecycle.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cluster_order_cr +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +# Fixed (not randomly generated) so repeated runs reuse the same ClusterVersion +# via GRPCClient.ensure_cluster_version instead of accumulating duplicates on +# the shared cluster. The version *name* ("4.20.0-e2e-catalog-item") is distinct +# from caas/test_cluster_create.py to avoid state races under xdist. +TEST_RELEASE_IMAGE = "quay.io/openshift-release-dev/ocp-release:4.20.0-multi" + + +def test_catalog_item_crud(grpc: GRPCClient, cluster_template: str) -> None: + name = unique_name("e2e-cat") + catalog_item_id = grpc.create_cluster_catalog_item(name=name, template=cluster_template, published=True) + try: + assert catalog_item_id in grpc.list_cluster_catalog_item_ids() + + item = grpc.get_cluster_catalog_item(catalog_item_id=catalog_item_id) + obj = item["object"] + assert obj["title"] == name + assert obj["template"]["name"] == cluster_template + assert obj["published"] is True + + updated_title = unique_name("e2e-cat-updated") + grpc.update_cluster_catalog_item(catalog_item_id=catalog_item_id, title=updated_title) + + item = grpc.get_cluster_catalog_item(catalog_item_id=catalog_item_id) + assert item["object"]["title"] == updated_title + + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + assert catalog_item_id not in grpc.list_cluster_catalog_item_ids() + + output, rc = grpc.call_unchecked(service="osac.public.v1.ClusterCatalogItems/Get", data={"id": catalog_item_id}) + assert rc != 0, f"Expected Get to fail after deletion, got: {output}" + + catalog_item_id = "" + finally: + if catalog_item_id: + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_unpublished_catalog_item_not_visible_in_public_api(grpc: GRPCClient, cluster_template: str) -> None: + name = unique_name("e2e-unpub") + catalog_item_id = grpc.create_cluster_catalog_item(name=name, template=cluster_template, published=False) + try: + assert catalog_item_id not in grpc.list_cluster_catalog_item_ids() + + output, rc = grpc.call_unchecked(service="osac.public.v1.ClusterCatalogItems/Get", data={"id": catalog_item_id}) + assert rc != 0, f"Expected Get to fail for unpublished item, got: {output}" + assert "not published" in output.lower() or "not found" in output.lower() + finally: + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_catalog_item_unpublish_transition(grpc: GRPCClient, cluster_template: str) -> None: + name = unique_name("e2e-trans") + catalog_item_id = grpc.create_cluster_catalog_item(name=name, template=cluster_template, published=True) + try: + assert catalog_item_id in grpc.list_cluster_catalog_item_ids() + + grpc.update_cluster_catalog_item(catalog_item_id=catalog_item_id, published=False) + + assert catalog_item_id not in grpc.list_cluster_catalog_item_ids() + + output, rc = grpc.call_unchecked(service="osac.public.v1.ClusterCatalogItems/Get", data={"id": catalog_item_id}) + assert rc != 0, f"Expected Get to fail after unpublishing, got: {output}" + finally: + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_catalog_item_field_definitions(grpc: GRPCClient, cluster_template: str) -> None: + field_defs = [ + { + "path": "spec.network.pod_cidr", + "display_name": "Pod CIDR", + "editable": True, + "default": {"stringValue": "10.128.0.0/14"}, + }, + { + "path": "spec.network.service_cidr", + "display_name": "Service CIDR", + "editable": False, + "default": {"stringValue": "172.30.0.0/16"}, + }, + ] + name = unique_name("e2e-fd") + catalog_item_id = grpc.create_cluster_catalog_item( + name=name, template=cluster_template, published=True, field_definitions=field_defs + ) + try: + item = grpc.get_cluster_catalog_item(catalog_item_id=catalog_item_id) + returned_fds = item["object"].get("fieldDefinitions", []) + assert len(returned_fds) == 2 + + pod_fd = next(fd for fd in returned_fds if fd["path"] == "spec.network.pod_cidr") + assert pod_fd["displayName"] == "Pod CIDR" + assert pod_fd["editable"] is True + + # editable=false is omitted by protobuf (default value), so we only check displayName + svc_fd = next(fd for fd in returned_fds if fd["path"] == "spec.network.service_cidr") + assert svc_fd["displayName"] == "Service CIDR" + + updated_fds = [ + { + "path": "spec.network.pod_cidr", + "display_name": "Pod Network CIDR", + "editable": True, + "default": {"stringValue": "10.128.0.0/14"}, + }, + { + "path": "spec.network.service_cidr", + "display_name": "Service CIDR", + "editable": False, + "default": {"stringValue": "172.30.0.0/16"}, + }, + ] + grpc.update_cluster_catalog_item(catalog_item_id=catalog_item_id, field_definitions=updated_fds) + + item = grpc.get_cluster_catalog_item(catalog_item_id=catalog_item_id) + returned_fds = item["object"].get("fieldDefinitions", []) + assert len(returned_fds) == 2 + pod_fd = next(fd for fd in returned_fds if fd["path"] == "spec.network.pod_cidr") + assert pod_fd["displayName"] == "Pod Network CIDR" + + reduced_fds = [ + { + "path": "spec.network.pod_cidr", + "display_name": "Pod Network CIDR", + "editable": True, + "default": {"stringValue": "10.128.0.0/14"}, + } + ] + grpc.update_cluster_catalog_item(catalog_item_id=catalog_item_id, field_definitions=reduced_fds) + + item = grpc.get_cluster_catalog_item(catalog_item_id=catalog_item_id) + returned_fds = item["object"].get("fieldDefinitions", []) + assert len(returned_fds) == 1 + assert returned_fds[0]["path"] == "spec.network.pod_cidr" + finally: + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_create_cluster_with_catalog_item(grpc: GRPCClient, cli: OsacCLI, cluster_template: str) -> None: + name = unique_name("e2e-cat") + catalog_item_id = grpc.create_cluster_catalog_item(name=name, template=cluster_template, published=True) + cluster_id = "" + try: + cluster_name = unique_name("e2e-cluster") + cluster_id = cli.create_cluster_with_catalog_item(catalog_item=catalog_item_id, name=cluster_name) + + assert cluster_id in grpc.list_cluster_ids() + + cluster = grpc.get_cluster(cluster_id=cluster_id) + assert cluster["object"]["spec"]["catalogItem"]["id"] == catalog_item_id + finally: + if cluster_id: + cli.delete_cluster(uuid=cluster_id) + poll_until( + fn=lambda: cluster_id not in grpc.list_cluster_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"Cluster {cluster_id} removal from API", + ) + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_create_cluster_with_unpublished_catalog_item_fails(grpc: GRPCClient, cluster_template: str) -> None: + name = unique_name("e2e-unpub") + catalog_item_id = grpc.create_cluster_catalog_item(name=name, template=cluster_template, published=False) + try: + output, rc = grpc.call_unchecked( + service="osac.public.v1.Clusters/Create", + data={"object": {"spec": {"catalog_item": {"id": catalog_item_id}}}}, + ) + assert rc != 0, f"Expected create to fail for unpublished catalog item, got: {output}" + assert "not published" in output.lower() or "not found" in output.lower() + finally: + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_delete_catalog_item_blocked_when_referenced(grpc: GRPCClient, cli: OsacCLI, cluster_template: str) -> None: + name = unique_name("e2e-ref") + catalog_item_id = grpc.create_cluster_catalog_item(name=name, template=cluster_template, published=True) + cluster_id = "" + try: + cluster_name = unique_name("e2e-cluster") + cluster_id = cli.create_cluster_with_catalog_item(catalog_item=catalog_item_id, name=cluster_name) + + output, rc = grpc.call_unchecked( + service="osac.private.v1.ClusterCatalogItems/Delete", data={"id": catalog_item_id} + ) + assert rc != 0, f"Expected catalog item delete to be blocked, got: {output}" + assert "referenc" in output.lower() or "in use" in output.lower() or "failed precondition" in output.lower() + finally: + if cluster_id: + cli.delete_cluster(uuid=cluster_id) + poll_until( + fn=lambda: cluster_id not in grpc.list_cluster_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"Cluster {cluster_id} removal from API", + ) + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) + + +def test_create_cluster_with_catalog_item_version( + grpc: GRPCClient, private_grpc: GRPCClient, cli: OsacCLI, k8s_hub_client: K8sClient, cluster_template: str +) -> None: + """Verify the catalog-item version resolution path: a field_definition + default for version overrides the template's own spec_defaults.version.""" + version = private_grpc.ensure_cluster_version(version="4.20.0-e2e-catalog-item", image=TEST_RELEASE_IMAGE) + + name = unique_name("e2e-cv-cat") + catalog_item_id = grpc.create_cluster_catalog_item( + name=name, + template=cluster_template, + published=True, + field_definitions=[ + { + "path": "version", + "display_name": "Version", + "editable": True, + "default": {"stringValue": version["name"]}, + } + ], + ) + cluster_id = "" + try: + cluster_name = unique_name("e2e-cv-cluster") + cluster_id = cli.create_cluster_with_catalog_item(catalog_item=catalog_item_id, name=cluster_name) + + cluster = grpc.get_cluster(cluster_id=cluster_id) + assert cluster["object"]["spec"]["version"]["name"] == version["name"] + + co_name = wait_for_cluster_order_cr(k8s=k8s_hub_client, uuid=cluster_id) + release_image = poll_until( + fn=lambda: k8s_hub_client.get_cluster_order_spec(name=co_name).get("releaseImage", ""), + until=lambda v: v != "", + retries=30, + delay=5, + description=f"{co_name} ClusterOrder releaseImage resolution", + ) + assert release_image == TEST_RELEASE_IMAGE + finally: + if cluster_id: + cli.delete_cluster(uuid=cluster_id) + poll_until( + fn=lambda: cluster_id not in grpc.list_cluster_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"Cluster {cluster_id} removal from API", + ) + grpc.delete_cluster_catalog_item(catalog_item_id=catalog_item_id) diff --git a/tests/e2e/tests/catalog/test_compute_instance_catalog_item_lifecycle.py b/tests/e2e/tests/catalog/test_compute_instance_catalog_item_lifecycle.py new file mode 100644 index 0000000000..fd865e2819 --- /dev/null +++ b/tests/e2e/tests/catalog/test_compute_instance_catalog_item_lifecycle.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.runner import poll_until + + +def test_compute_instance_catalog_item_crud(grpc: GRPCClient, compute_instance_template: str) -> None: + name = unique_name("e2e-ci-cat") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=True + ) + try: + assert catalog_item_id in grpc.list_compute_instance_catalog_item_ids() + + item = grpc.get_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + obj = item["object"] + assert obj["title"] == name + assert obj["template"]["name"] == compute_instance_template + assert obj["published"] is True + + updated_title = unique_name("e2e-ci-cat-updated") + grpc.update_compute_instance_catalog_item(catalog_item_id=catalog_item_id, title=updated_title) + + item = grpc.get_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + assert item["object"]["title"] == updated_title + + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + assert catalog_item_id not in grpc.list_compute_instance_catalog_item_ids() + + output, rc = grpc.call_unchecked( + service="osac.public.v1.ComputeInstanceCatalogItems/Get", data={"id": catalog_item_id} + ) + assert rc != 0, f"Expected Get to fail after deletion, got: {output}" + + catalog_item_id = "" + finally: + if catalog_item_id: + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + +def test_unpublished_compute_instance_catalog_item_not_visible_in_public_api( + grpc: GRPCClient, compute_instance_template: str +) -> None: + name = unique_name("e2e-ci-unpub") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=False + ) + try: + assert catalog_item_id not in grpc.list_compute_instance_catalog_item_ids() + + output, rc = grpc.call_unchecked( + service="osac.public.v1.ComputeInstanceCatalogItems/Get", data={"id": catalog_item_id} + ) + assert rc != 0, f"Expected Get to fail for unpublished item, got: {output}" + assert "not published" in output.lower() or "not found" in output.lower() + finally: + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + +def test_compute_instance_catalog_item_unpublish_transition(grpc: GRPCClient, compute_instance_template: str) -> None: + name = unique_name("e2e-ci-trans") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=True + ) + try: + assert catalog_item_id in grpc.list_compute_instance_catalog_item_ids() + + grpc.update_compute_instance_catalog_item(catalog_item_id=catalog_item_id, published=False) + + assert catalog_item_id not in grpc.list_compute_instance_catalog_item_ids() + + output, rc = grpc.call_unchecked( + service="osac.public.v1.ComputeInstanceCatalogItems/Get", data={"id": catalog_item_id} + ) + assert rc != 0, f"Expected Get to fail after unpublishing, got: {output}" + finally: + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + +def test_compute_instance_catalog_item_field_definitions(grpc: GRPCClient, compute_instance_template: str) -> None: + field_defs = [ + { + "path": "spec.instance_type", + "display_name": "Instance Type", + "editable": True, + "default": {"stringValue": "standard-2x4"}, + } + ] + name = unique_name("e2e-ci-fd") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=True, field_definitions=field_defs + ) + try: + item = grpc.get_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + returned_fds = item["object"].get("fieldDefinitions", []) + assert len(returned_fds) == 1 + + it_fd = next(fd for fd in returned_fds if fd["path"] == "spec.instance_type") + assert it_fd["displayName"] == "Instance Type" + assert it_fd["editable"] is True + + updated_fds = [ + { + "path": "spec.instance_type", + "display_name": "VM Size", + "editable": True, + "default": {"stringValue": "standard-2x4"}, + } + ] + grpc.update_compute_instance_catalog_item(catalog_item_id=catalog_item_id, field_definitions=updated_fds) + + item = grpc.get_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + returned_fds = item["object"].get("fieldDefinitions", []) + assert len(returned_fds) == 1 + it_fd = next(fd for fd in returned_fds if fd["path"] == "spec.instance_type") + assert it_fd["displayName"] == "VM Size" + + updated_fds_v2 = [ + { + "path": "spec.instance_type", + "display_name": "VM Size", + "editable": False, + "default": {"stringValue": "standard-4x8"}, + } + ] + grpc.update_compute_instance_catalog_item(catalog_item_id=catalog_item_id, field_definitions=updated_fds_v2) + + item = grpc.get_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + returned_fds = item["object"].get("fieldDefinitions", []) + assert len(returned_fds) == 1 + it_fd = next(fd for fd in returned_fds if fd["path"] == "spec.instance_type") + assert it_fd["displayName"] == "VM Size" + assert it_fd.get("editable", False) is False + finally: + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + +def test_create_compute_instance_with_catalog_item( + grpc: GRPCClient, compute_instance_template: str, default_subnet_id: str +) -> None: + name = unique_name("e2e-ci-cat") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=True + ) + ci_id = "" + try: + ci_name = unique_name("e2e-ci") + ci_id = grpc.create_compute_instance(name=ci_name, catalog_item=catalog_item_id, subnet_ids=[default_subnet_id]) + + assert ci_id in grpc.list_compute_instance_ids() + + ci = grpc.get_compute_instance(ci_id=ci_id) + assert ci["object"]["spec"]["catalogItem"]["id"] == catalog_item_id + finally: + if ci_id: + grpc.delete_compute_instance(ci_id=ci_id) + poll_until( + fn=lambda: ci_id not in grpc.list_compute_instance_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"ComputeInstance {ci_id} removal from API", + ) + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + +def test_create_compute_instance_with_unpublished_catalog_item_fails( + grpc: GRPCClient, compute_instance_template: str, default_subnet_id: str +) -> None: + name = unique_name("e2e-ci-unpub") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=False + ) + try: + output, rc = grpc.call_unchecked( + service="osac.public.v1.ComputeInstances/Create", + data={ + "object": { + "spec": { + "catalog_item": {"id": catalog_item_id}, + "network_attachments": [{"subnet": {"id": default_subnet_id}}], + } + } + }, + ) + assert rc != 0, f"Expected create to fail for unpublished catalog item, got: {output}" + assert "not published" in output.lower() or "not found" in output.lower() + finally: + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) + + +def test_delete_compute_instance_catalog_item_blocked_when_referenced( + grpc: GRPCClient, compute_instance_template: str, default_subnet_id: str +) -> None: + name = unique_name("e2e-ci-ref") + catalog_item_id = grpc.create_compute_instance_catalog_item( + name=name, template=compute_instance_template, published=True + ) + ci_id = "" + try: + ci_name = unique_name("e2e-ci") + ci_id = grpc.create_compute_instance(name=ci_name, catalog_item=catalog_item_id, subnet_ids=[default_subnet_id]) + + output, rc = grpc.call_unchecked( + service="osac.private.v1.ComputeInstanceCatalogItems/Delete", data={"id": catalog_item_id} + ) + assert rc != 0, f"Expected catalog item delete to be blocked, got: {output}" + assert "referenc" in output.lower() or "in use" in output.lower() or "failed precondition" in output.lower() + finally: + if ci_id: + grpc.delete_compute_instance(ci_id=ci_id) + poll_until( + fn=lambda: ci_id not in grpc.list_compute_instance_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"ComputeInstance {ci_id} removal from API", + ) + grpc.delete_compute_instance_catalog_item(catalog_item_id=catalog_item_id) diff --git a/tests/e2e/tests/conftest.py b/tests/e2e/tests/conftest.py new file mode 100644 index 0000000000..9a3b3f6709 --- /dev/null +++ b/tests/e2e/tests/conftest.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +import os +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.k8s_client import K8sClient +from tests.core.keycloak import get_jwt +from tests.core.keycloak_admin import ( + add_user_to_organization, + add_user_to_organization_group, + ensure_organization_group, + get_admin_token, + get_user_id, + wait_for_organization, +) +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI +from tests.core.runner import env, run + + +def pytest_configure(config: pytest.Config) -> None: + """Give each xdist worker its own log_file so records stay chronologically ordered. + + A shared log_file is interleaved across workers as they report at their own + pace; gather-osac-logs.sh merges the per-worker files back into one sorted + e2e.log artifact. + """ + config.addinivalue_line("markers", "metering: test verifies metering events via the test adapter HTTP API") + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + if worker_id is not None: + log_dir = Path(config.getini("log_file")).parent + log_dir.mkdir(parents=True, exist_ok=True) + config.option.log_file = str(log_dir / f"e2e_{worker_id}.log") + + +@pytest.fixture(scope="session") +def namespace() -> str: + return env("OSAC_NAMESPACE", "osac-devel") + + +@pytest.fixture(scope="session") +def cluster_domain() -> str: + return run("kubectl", "get", "ingress.config.openshift.io", "cluster", "-o", "jsonpath={.spec.domain}") + + +@pytest.fixture(scope="session") +def fulfillment_address(namespace: str, cluster_domain: str) -> str: + return env("OSAC_FULFILLMENT_ADDRESS", f"fulfillment-api-{namespace}.{cluster_domain}:443") + + +@pytest.fixture(scope="session") +def fulfillment_private_address(namespace: str, cluster_domain: str) -> str: + return env("OSAC_FULFILLMENT_PRIVATE_ADDRESS", f"fulfillment-internal-api-{namespace}.{cluster_domain}:443") + + +@pytest.fixture(scope="session") +def service_account() -> str: + return env("OSAC_SERVICE_ACCOUNT", "admin") + + +@pytest.fixture(scope="session") +def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPCClient: + # wait_for_cluster_ready's own budget alone can run up to 120min on cold + # EC2 hardware, plus deletion/verification steps after it -- a token this + # short can expire mid-session, failing every subsequent grpcurl call + # with UNAUTHENTICATED. Stay safely above the worst-case session length. + token: str = run( + "oc", "create", "token", service_account, "-n", namespace, "--duration", "4h", "--as", "system:admin" + ) + return GRPCClient(address=fulfillment_address, token=token) + + +@pytest.fixture(scope="session") +def private_grpc(fulfillment_private_address: str, namespace: str, service_account: str) -> GRPCClient: + token: str = run( + "oc", "create", "token", service_account, "-n", namespace, "--duration", "4h", "--as", "system:admin" + ) + return GRPCClient(address=fulfillment_private_address, token=token) + + +@pytest.fixture(scope="session", autouse=True) +def ensure_tenants(private_grpc: GRPCClient) -> None: + for name in ("tenant1", "tenant2"): + private_grpc.ensure_tenant(name=name) + + +@pytest.fixture(scope="session") +def keycloak_admin_password() -> str: + return env("OSAC_KEYCLOAK_ADMIN_PASSWORD", "admin") + + +@pytest.fixture(scope="session", autouse=True) +def setup_organization_memberships(ensure_tenants: None, keycloak_url: str, keycloak_admin_password: str) -> None: + """ + Add test users to their corresponding Keycloak organizations. + This runs after ensure_tenants creates the Tenant resources, which the + tenant controller syncs to Keycloak as organizations. + """ + # Get admin token for Keycloak admin API + admin_token = get_admin_token(keycloak_url=keycloak_url, username="admin", password=keycloak_admin_password) + + # Map of organization name -> list of usernames + org_users = {"tenant1": ["tenant1_user", "tenant1_admin"], "tenant2": ["tenant2_user", "tenant2_admin"]} + + for org_name, usernames in org_users.items(): + # Wait for the organization to be synced to Keycloak by the tenant controller + org_id = wait_for_organization(keycloak_url=keycloak_url, admin_token=admin_token, org_name=org_name) + + # Add each user to the organization + for username in usernames: + user_id = get_user_id(keycloak_url=keycloak_url, admin_token=admin_token, username=username) + add_user_to_organization( + keycloak_url=keycloak_url, + admin_token=admin_token, + org_id=org_id, + user_id=user_id, + username=username, + org_name=org_name, + ) + + # Create /members group in the organization and add all users to it + # This is required for the organization scope to include the organization in the JWT token + group_id = ensure_organization_group( + keycloak_url=keycloak_url, admin_token=admin_token, org_id=org_id, org_name=org_name + ) + + for username in usernames: + user_id = get_user_id(keycloak_url=keycloak_url, admin_token=admin_token, username=username) + add_user_to_organization_group( + keycloak_url=keycloak_url, + admin_token=admin_token, + org_id=org_id, + group_id=group_id, + user_id=user_id, + username=username, + org_name=org_name, + ) + + +@pytest.fixture(scope="session") +def k8s_hub_client(namespace: str) -> K8sClient: + return K8sClient(namespace=namespace) + + +@pytest.fixture(scope="session") +def cli(namespace: str, fulfillment_address: str, service_account: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=f"oc create token -n {namespace} {service_account} --as system:admin", + namespace=namespace, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def private_cli(namespace: str, fulfillment_private_address: str, service_account: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_private_address.rsplit(':', 1)[0]}", + token_script=f"oc create token -n {namespace} {service_account} --as system:admin", + namespace=namespace, + private=True, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def keycloak_url(cluster_domain: str) -> str: + return env("OSAC_KEYCLOAK_URL", f"https://keycloak-keycloak.{cluster_domain}") + + +@pytest.fixture(scope="session") +def jwt_password() -> str: + return env("OSAC_JWT_PASSWORD", "foobar") + + +def _make_jwt_token_script(keycloak_url: str, username: str, password: str) -> str: + return ( + f"curl -sk -X POST {keycloak_url}/realms/osac/protocol/openid-connect/token" + f" -d grant_type=password -d client_id=osac-cli" + f" -d username={username} -d password={password} -d 'scope=openid organization'" + " | python3 -c \"import sys,json;print(json.load(sys.stdin)['access_token'])\"" + ) + + +@pytest.fixture(scope="session") +def jwt_cli_user(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, "tenant1_user", jwt_password), + namespace=namespace, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, "tenant1_admin", jwt_password), + namespace=namespace, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant1_admin(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_admin", + password=jwt_password, + ), + ) + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant1(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_user", + password=jwt_password, + ), + ) + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant2(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant2_user", + password=jwt_password, + ), + ) + + +# --- Cross-cutting concern: Metering --- + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + metering_tests = [item for item in items if item.get_closest_marker("metering")] + if metering_tests and not os.environ.get("METERING_ADAPTER_URL"): + pytest.fail( + f"METERING_ADAPTER_URL is not set but {len(metering_tests)} test(s) require metering. " + "Set the env var or run with -m 'not metering' to exclude metering tests.", + pytrace=False, + ) + + +@pytest.fixture +def metering() -> Iterator[MeteringCollector]: + """Composable metering verifier. Inject into any test to verify + that lifecycle events appear via the test adapter HTTP API. + + Records a start timestamp on setup, verifies all expectations + (including CloudEvent structure validation) on teardown. + """ + collector = MeteringCollector(base_url=env("METERING_ADAPTER_URL")) + collector.start() + yield collector + try: + collector.verify() + finally: + collector.stop() diff --git a/tests/e2e/tests/core/__init__.py b/tests/e2e/tests/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/core/grpc_client.py b/tests/e2e/tests/core/grpc_client.py new file mode 100644 index 0000000000..c77ed08600 --- /dev/null +++ b/tests/e2e/tests/core/grpc_client.py @@ -0,0 +1,648 @@ +from __future__ import annotations + +import json +import re +import subprocess +import time +from collections.abc import Callable +from typing import Any + +from tests.core.runner import run, run_unchecked + +PUBLIC_API: str = "osac.public.v1" +PRIVATE_API: str = "osac.private.v1" + + +class GRPCClient: + _TOKEN_TTL: float = 60.0 + + def __init__(self, *, address: str, token: str = "", token_factory: Callable[[], str] | None = None) -> None: + self.address: str = address + self._static_token: str = token + self._token_factory: Callable[[], str] | None = token_factory + self._cached_token: str = "" + self._cached_at: float = 0.0 + + @property + def token(self) -> str: + if self._token_factory is not None: + now = time.monotonic() + if not self._cached_token or (now - self._cached_at) >= self._TOKEN_TTL: + self._cached_token = self._token_factory() + self._cached_at = now + return self._cached_token + return self._static_token + + def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> list[str]: + args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] + if data is not None: + args.extend(["-d", json.dumps(data)]) + args.extend([self.address, service]) + return args + + def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: + return json.loads(run(*self._build_args(service=service, data=data))) + + def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str], name: str | None = None) -> str: + attachments = [{"subnet": {"id": sid}} for sid in subnet_ids] + obj: dict[str, Any] = {"spec": {"catalog_item": {"id": catalog_item}, "network_attachments": attachments}} + if name is not None: + obj["metadata"] = {"name": name} + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/Create", data={"object": obj}) + return response["object"]["id"] + + def update_compute_instance_run_strategy(self, *, ci_id: str, run_strategy: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.ComputeInstances/Update", + data={ + "object": {"id": ci_id, "spec": {"run_strategy": run_strategy}}, + "updateMask": {"paths": ["spec.run_strategy"]}, + }, + ) + + def delete_compute_instance(self, *, ci_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ComputeInstances/Delete", data={"id": ci_id}) + + def list_compute_instance_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/List") + return [item["id"] for item in response.get("items", [])] + + def get_compute_instance(self, *, ci_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ComputeInstances/Get", data={"id": ci_id}) + + def get_hub(self, *, hub_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.Hubs/Get", data={"id": hub_id}) + + def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.ComputeInstances/Update", + data={ + "object": {"id": uuid, "spec": {"template": {"name": template}, "restart_requested_at": timestamp}}, + "updateMask": {"paths": ["spec.restart_requested_at"]}, + }, + ) + + # VirtualNetwork operations + + def create_virtual_network(self, *, name: str, network_class: str, ipv4_cidr: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": ipv4_cidr}, + } + }, + ) + return response["object"]["id"] + + def get_virtual_network(self, *, vn_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.VirtualNetworks/Get", data={"id": vn_id}) + + def list_virtual_network_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.VirtualNetworks/List") + return [item["id"] for item in response.get("items", [])] + + def delete_virtual_network(self, *, vn_id: str) -> None: + self.call(service=f"{PUBLIC_API}.VirtualNetworks/Delete", data={"id": vn_id}) + + # Subnet operations + + def create_subnet(self, *, name: str, virtual_network: str, ipv4_cidr: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.Subnets/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"virtual_network": {"id": virtual_network}, "ipv4_cidr": ipv4_cidr}, + } + }, + ) + return response["object"]["id"] + + def get_subnet(self, *, subnet_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.Subnets/Get", data={"id": subnet_id}) + + def list_subnet_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Subnets/List") + return [item["id"] for item in response.get("items", [])] + + def delete_subnet(self, *, subnet_id: str) -> None: + self.call(service=f"{PUBLIC_API}.Subnets/Delete", data={"id": subnet_id}) + + def call_unchecked(self, *, service: str, data: dict[str, Any] | None = None) -> tuple[str, int]: + return run_unchecked(*self._build_args(service=service, data=data)) + + # Cluster operations + + def list_cluster_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Clusters/List") + return [item["id"] for item in response.get("items", [])] + + def get_cluster(self, *, cluster_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.Clusters/Get", data={"id": cluster_id}) + + # SecurityGroup operations + + def create_security_group(self, *, name: str, virtual_network: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", + data={"object": {"metadata": {"name": name}, "spec": {"virtual_network": {"id": virtual_network}}}}, + ) + return response["object"]["id"] + + def get_security_group(self, *, sg_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.SecurityGroups/Get", data={"id": sg_id}) + + def list_security_group_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.SecurityGroups/List") + return [item["id"] for item in response.get("items", [])] + + def delete_security_group(self, *, sg_id: str) -> None: + self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) + + # Console operations + + def create_console_session( + self, *, resource_type: str, resource_id: str, console_type: str, client_id: str = "" + ) -> dict[str, Any]: + data: dict[str, Any] = { + "object": {"resourceType": resource_type, "resourceId": resource_id, "type": console_type} + } + if client_id: + data["object"]["clientId"] = client_id + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ConsoleSessions/Create", data=data) + return response["object"] + + # Tenant operations + + def ensure_tenant(self, *, name: str, retries: int = 10, delay: int = 5) -> None: + # OSAC-3553: retry the transient `code = Unavailable` / connection-refused + # transport failure the first call can hit after `helm --wait` returns, + # before the route converges. AlreadyExists is idempotent success; any + # other error still fails fast. Note the two error shapes are matched + # differently: a completed RPC prints grpcurl's `Code: ` block + # (AlreadyExists), while a transport failure prints the Go status string + # `code = Unavailable desc = ...` -- see tests/vmaas/external_ip/conftest.py. + for attempt in range(retries): + try: + self.call(service=f"{PRIVATE_API}.Tenants/Create", data={"object": {"metadata": {"name": name}}}) + return + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if re.search(r"Code:\s*AlreadyExists", output): + return + if attempt < retries - 1 and ("Unavailable" in output or "connection refused" in output.lower()): + time.sleep(delay) + continue + raise RuntimeError(f"Failed to create tenant '{name}': {output}") from e + + # ExternalIPPool operations (private API only) + + def create_external_ip_pool( + self, + *, + name: str, + cidrs: list[str], + ip_family: str = "IP_FAMILY_IPV4", + implementation_strategy: str = "metallb-l2", + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ExternalIPPools/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "cidrs": cidrs, + "ip_family": ip_family, + "implementation_strategy": implementation_strategy, + }, + } + }, + ) + return response["object"]["id"] + + def get_external_ip_pool(self, *, pool_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ExternalIPPools/Get", data={"id": pool_id}) + + def list_external_ip_pool_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ExternalIPPools/List") + return [item["id"] for item in response.get("items", [])] + + def delete_external_ip_pool(self, *, pool_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ExternalIPPools/Delete", data={"id": pool_id}) + + # ExternalIP operations (public API) + + def create_external_ip(self, *, name: str, pool: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ExternalIPs/Create", + data={"object": {"metadata": {"name": name}, "spec": {"pool": {"id": pool}}}}, + ) + return response["object"]["id"] + + def get_external_ip(self, *, external_ip_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ExternalIPs/Get", data={"id": external_ip_id}) + + def list_external_ip_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ExternalIPs/List") + return [item["id"] for item in response.get("items", [])] + + def delete_external_ip(self, *, external_ip_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ExternalIPs/Delete", data={"id": external_ip_id}) + + # ExternalIPAttachment operations (public API) + + def create_external_ip_attachment(self, *, name: str, external_ip: str, compute_instance: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ExternalIPAttachments/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"external_ip": {"id": external_ip}, "compute_instance": {"id": compute_instance}}, + } + }, + ) + return response["object"]["id"] + + def get_external_ip_attachment(self, *, attachment_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Get", data={"id": attachment_id}) + + def list_external_ip_attachment_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/List") + return [item["id"] for item in response.get("items", [])] + + def delete_external_ip_attachment(self, *, attachment_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Delete", data={"id": attachment_id}) + + # ClusterCatalogItem operations + + def create_cluster_catalog_item( + self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None + ) -> str: + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": name, + "template": {"name": template}, + "published": published, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Create", data={"object": obj}) + return response["object"]["id"] + + def get_cluster_catalog_item(self, *, catalog_item_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/Get", data={"id": catalog_item_id}) + + def list_cluster_catalog_item_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/List") + return [item["id"] for item in response.get("items", [])] + + def update_cluster_catalog_item(self, *, catalog_item_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_cluster_catalog_item requires at least one field to update") + obj: dict[str, Any] = {"id": catalog_item_id, **fields} + data: dict[str, Any] = {"object": obj, "update_mask": {"paths": list(fields.keys())}} + return self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Update", data=data) + + def delete_cluster_catalog_item(self, *, catalog_item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Delete", data={"id": catalog_item_id}) + + # ComputeInstanceCatalogItem operations + + def create_compute_instance_catalog_item( + self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None + ) -> str: + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": name, + "template": {"name": template}, + "published": published, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Create", data={"object": obj} + ) + return response["object"]["id"] + + def get_compute_instance_catalog_item(self, *, catalog_item_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/Get", data={"id": catalog_item_id}) + + def list_compute_instance_catalog_item_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/List") + return [item["id"] for item in response.get("items", [])] + + def update_compute_instance_catalog_item(self, *, catalog_item_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_compute_instance_catalog_item requires at least one field to update") + obj: dict[str, Any] = {"id": catalog_item_id, **fields} + data: dict[str, Any] = {"object": obj, "update_mask": {"paths": list(fields.keys())}} + return self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Update", data=data) + + def delete_compute_instance_catalog_item(self, *, catalog_item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Delete", data={"id": catalog_item_id}) + + # InstanceType operations (private API only) + + def create_instance_type( + self, *, name: str, cores: int, memory_gib: int, description: str = "", gpu: dict[str, Any] | None = None + ) -> str: + spec: dict[str, Any] = {"cores": cores, "memory_gib": memory_gib, "description": description} + if gpu is not None: + spec["gpu"] = gpu + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.InstanceTypes/Create", data={"object": {"metadata": {"name": name}, "spec": spec}} + ) + return response["object"]["id"] + + def get_instance_type(self, *, name: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.InstanceTypes/Get", data={"id": name}) + + def list_instance_type_names(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.InstanceTypes/List") + return [item["metadata"]["name"] for item in response.get("items", [])] + + def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]: + return self.call( + service=f"{PRIVATE_API}.InstanceTypes/Update", + data={"object": {"id": name, "spec": {"state": state}}, "updateMask": {"paths": ["spec.state"]}}, + ) + + def delete_instance_type(self, *, name: str) -> None: + self.call(service=f"{PRIVATE_API}.InstanceTypes/Delete", data={"id": name}) + + # ClusterVersion operations (private API only) + + def create_cluster_version( + self, + *, + version: str, + image: str, + enabled: bool = True, + is_default: bool = False, + state: str = "CLUSTER_VERSION_STATE_ACTIVE", + ) -> dict[str, str]: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ClusterVersions/Create", + data={ + "object": { + "spec": { + "version": version, + "image": image, + "enabled": enabled, + "is_default": is_default, + "state": state, + } + } + }, + ) + cluster_version: dict[str, Any] = response["object"] + return {"id": cluster_version["id"], "name": cluster_version["metadata"]["name"]} + + def get_cluster_version(self, *, version_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ClusterVersions/Get", data={"id": version_id}) + + def list_cluster_version_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterVersions/List") + return [item["id"] for item in response.get("items", [])] + + def update_cluster_version(self, *, version_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_cluster_version requires at least one field to update") + return self.call( + service=f"{PRIVATE_API}.ClusterVersions/Update", + data={ + "object": {"id": version_id, "spec": dict(fields)}, + "updateMask": {"paths": [f"spec.{k}" for k in fields]}, + }, + ) + + def delete_cluster_version(self, *, version_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ClusterVersions/Delete", data={"id": version_id}) + + def ensure_cluster_version(self, *, version: str, image: str) -> dict[str, str]: + """Create a ClusterVersion, tolerating AlreadyExists left behind by a prior failed run. + + Returns {"id": ..., "name": ...} for the resolved ClusterVersion.""" + try: + return self.create_cluster_version(version=version, image=image) + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if not re.search(r"Code:\s*AlreadyExists", output): + raise RuntimeError(f"Failed to create cluster version '{version}': {output}") from e + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterVersions/List") + for item in response.get("items", []): + if item.get("spec", {}).get("version") == version: + return {"id": item["id"], "name": item["metadata"]["name"]} + raise RuntimeError(f"Cluster version '{version}' reported AlreadyExists but not found in list") + + # BareMetalInstance operations (public API) + + def list_baremetal_instance_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.BareMetalInstances/List") + return [item["id"] for item in response.get("items", [])] + + def get_baremetal_instance(self, *, bmi_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.BareMetalInstances/Get", data={"id": bmi_id}) + + def get_baremetal_instance_state(self, *, bmi_id: str) -> str: + response: dict[str, Any] = self.get_baremetal_instance(bmi_id=bmi_id) + return response.get("object", {}).get("status", {}).get("state", "") + + def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"run_strategy": run_strategy}}, + "updateMask": {"paths": ["spec.run_strategy"]}, + }, + ) + + def update_baremetal_instance_restart_trigger(self, *, bmi_id: str, restart_trigger: int) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"restart_trigger": restart_trigger}}, + "updateMask": {"paths": ["spec.restart_trigger"]}, + }, + ) + + def delete_baremetal_instance(self, *, bmi_id: str) -> None: + self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id}) + + # BareMetalInstanceCatalogItem operations (private API for admin setup) + + def create_baremetal_instance_catalog_item( + self, + *, + name: str, + title: str, + description: str, + template: str, + field_definitions: list[dict[str, Any]] | None = None, + ) -> str: + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": title, + "description": description, + "template": {"name": template}, + "published": True, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Create", data={"object": obj} + ) + return response["object"]["id"] + + def delete_baremetal_instance_catalog_item(self, *, item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Delete", data={"id": item_id}) + + # DiskImage operations (public API) + + def create_disk_image( + self, + *, + source_ref: str, + architecture: list[str] | None = None, + source_type: str = "SOURCE_TYPE_REGISTRY", + guest_os_family: str = "GUEST_OS_FAMILY_LINUX", + name: str | None = None, + ) -> str: + spec: dict[str, Any] = { + "source_type": source_type, + "source_ref": source_ref, + "guest_os_family": guest_os_family, + "architecture": architecture or ["ARCHITECTURE_AMD64"], + } + obj: dict[str, Any] = {"spec": spec} + if name is not None: + obj["metadata"] = {"name": name} + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.DiskImages/Create", data={"object": obj}) + return response["object"]["id"] + + def get_disk_image(self, *, disk_image_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.DiskImages/Get", data={"id": disk_image_id}) + + def list_disk_image_ids(self, *, filter_expr: str | None = None) -> list[str]: + data: dict[str, Any] | None = {"filter": filter_expr} if filter_expr else None + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.DiskImages/List", data=data) + return [item["id"] for item in response.get("items", [])] + + def update_disk_image_lifecycle(self, *, disk_image_id: str, lifecycle: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.DiskImages/Update", + data={ + "object": {"id": disk_image_id, "spec": {"lifecycle": lifecycle}}, + "updateMask": {"paths": ["spec.lifecycle"]}, + }, + ) + + def delete_disk_image(self, *, disk_image_id: str) -> None: + self.call(service=f"{PUBLIC_API}.DiskImages/Delete", data={"id": disk_image_id}) + + # ComputeInstance creation with explicit DiskImage (public API) + + def create_compute_instance_with_disk_image( + self, + *, + template: str, + disk_image_name: str, + subnet_ids: list[str], + instance_type: str | None = None, + name: str | None = None, + ) -> dict[str, Any]: + attachments = [{"subnet": {"id": sid}} for sid in subnet_ids] + spec: dict[str, Any] = { + "template": {"name": template}, + "disk_image": {"name": disk_image_name}, + "network_attachments": attachments, + } + if instance_type is not None: + spec["instance_type"] = {"name": instance_type} + obj: dict[str, Any] = {"spec": spec} + if name is not None: + obj["metadata"] = {"name": name} + return self.call(service=f"{PUBLIC_API}.ComputeInstances/Create", data={"object": obj}) + + # ComputeInstanceTemplate operations (private API) + + def create_compute_instance_template( + self, *, name: str, title: str, description: str, spec_defaults: dict[str, Any] | None = None + ) -> str: + obj: dict[str, Any] = {"metadata": {"name": name}, "title": title, "description": description} + if spec_defaults is not None: + obj["spec_defaults"] = spec_defaults + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ComputeInstanceTemplates/Create", data={"object": obj} + ) + return response["object"]["id"] + + def delete_compute_instance_template(self, *, template_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ComputeInstanceTemplates/Delete", data={"id": template_id}) + + # Generic filtered list + + def list_with_filter(self, *, service: str, filter_expr: str) -> list[dict[str, Any]]: + response: dict[str, Any] = self.call(service=service, data={"filter": filter_expr}) + return response.get("items", []) + + # NATGateway operations (public API) + + def create_nat_gateway(self, *, name: str, virtual_network_name: str, external_ip_name: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.NATGateways/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "virtual_network": {"name": virtual_network_name}, + "external_ip": {"name": external_ip_name}, + }, + } + }, + ) + return response["object"]["id"] + + def delete_nat_gateway(self, *, nat_gateway_id: str) -> None: + self.call(service=f"{PUBLIC_API}.NATGateways/Delete", data={"id": nat_gateway_id}) + + # RoleBinding operations (public API) + + def create_role_binding(self, *, name: str, role_name: str, user_names: list[str]) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.RoleBindings/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"role": {"name": role_name}, "users": [{"name": u} for u in user_names]}, + } + }, + ) + return response["object"]["id"] + + def get_role_binding(self, *, role_binding_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.RoleBindings/Get", data={"id": role_binding_id}) + + def delete_role_binding(self, *, role_binding_id: str) -> None: + self.call(service=f"{PUBLIC_API}.RoleBindings/Delete", data={"id": role_binding_id}) + + # ProjectMembership operations (public API) + + def create_project_membership( + self, *, name: str, user_names: list[str], role: str = "PROJECT_MEMBERSHIP_ROLE_VIEWER" + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ProjectMemberships/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"role": role, "users": [{"name": u} for u in user_names]}, + } + }, + ) + return response["object"]["id"] + + def delete_project_membership(self, *, membership_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ProjectMemberships/Delete", data={"id": membership_id}) diff --git a/tests/e2e/tests/core/helpers.py b/tests/e2e/tests/core/helpers.py new file mode 100644 index 0000000000..13cc11695c --- /dev/null +++ b/tests/e2e/tests/core/helpers.py @@ -0,0 +1,704 @@ +from __future__ import annotations + +import re +import subprocess +from typing import Any + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until, run_unchecked + +_POOL_READY_STATE = "EXTERNAL_IP_POOL_STATE_READY" + + +def assert_grpc_rejected(exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], code: str) -> None: + exc = exc_info.value + combined: str = (exc.stderr or "") + (exc.stdout or "") + assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" + + +def assert_grpc_field_violation( + exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], *, field_path: str +) -> None: + assert_grpc_rejected(exc_info, "InvalidArgument") + exc = exc_info.value + combined: str = (exc.stderr or "") + (exc.stdout or "") + assert field_path in combined, ( + f"Expected FieldViolation containing '{field_path}' in error output, got: {combined.strip()}" + ) + + +def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_compute_instance_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"CR for {uuid}", + ) + + +def wait_for_provision(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_condition_status(name=name, condition_type="Provisioned", checked=False), + until=lambda v: v == "True", + retries=120, + delay=5, + description=f"{name} Provisioned condition", + ) + + +def wait_for_running(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_phase(name=name, checked=False), + until=lambda v: v == "Running", + retries=90, + delay=10, + description=f"{name} Running", + ) + + +def wait_for_restart(*, k8s: K8sClient, name: str, initial: str, restart_ts: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_last_restarted_at(name=name), + until=lambda v: v != "" and v != initial and v >= restart_ts, + retries=30, + delay=10, + description=f"{name} lastRestartedAt update", + ) + + +def wait_for_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="computeinstance", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} deletion", + ) + + +def wait_for_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_compute_instance_ids(), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"{uuid} removed from gRPC list", + ) + + +def wait_for_virtual_network_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_virtual_network_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"VirtualNetwork CR for {uuid}", + ) + + +def wait_for_virtual_network_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_virtual_network_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} VirtualNetwork Ready", + ) + + +def wait_for_virtual_network_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="virtualnetwork", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} VirtualNetwork deletion", + ) + + +def wait_for_subnet_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_subnet_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"Subnet CR for {uuid}", + ) + + +def wait_for_subnet_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_subnet_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} Subnet Ready", + ) + + +def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="subnet", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} Subnet deletion", + ) + + +def wait_for_external_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_external_ip_pool_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"ExternalIPPool CR for {uuid}", + ) + + +def wait_for_external_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_external_ip_pool_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} ExternalIPPool Ready", + ) + + +def wait_for_external_ip_pool_grpc_ready(*, private_grpc: GRPCClient, pool_id: str) -> None: + """Poll the private gRPC API until the pool state is READY. + + The K8s CR status may report Ready before the fulfillment-service database + has been updated by the controller feedback loop. Polling via gRPC closes + this race so that subsequent ExternalIP creation does not hit + FailedPrecondition. + """ + + def _state() -> str: + try: + pool = private_grpc.get_external_ip_pool(pool_id=pool_id) + except subprocess.CalledProcessError: + return "" + return pool.get("object", {}).get("status", {}).get("state", "") + + poll_until( + fn=_state, + until=lambda v: v == _POOL_READY_STATE, + retries=30, + delay=2, + description=f"ExternalIPPool {pool_id} gRPC READY", + ) + + +def wait_for_external_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="externalippool", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} ExternalIPPool deletion", + ) + + +def wait_for_external_ip_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_external_ip_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"ExternalIP CR for {uuid}", + ) + + +def wait_for_external_ip_allocated(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_external_ip_state(name=name, checked=False), + until=lambda v: v == "Allocated", + retries=60, + delay=5, + description=f"{name} ExternalIP Allocated", + ) + + +def wait_for_external_ip_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="externalip", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} ExternalIP deletion", + ) + + +def wait_for_external_ip_attachment_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_external_ip_attachment_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"ExternalIPAttachment CR for {uuid}", + ) + + +def wait_for_external_ip_attachment_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_external_ip_attachment_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} ExternalIPAttachment Ready", + ) + + +def wait_for_external_ip_attachment_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="externalipattachment", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} ExternalIPAttachment deletion", + ) + + +def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_cluster_order_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"ClusterOrder CR for {uuid}", + ) + + +def wait_for_cluster_progressing(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Progressing", + retries=30, + delay=2, + description=f"{name} ClusterOrder Progressing phase", + ) + + +def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: + # Must stay safely above osac-aap's own wait_for_clusteroperators_retries + # budget (60 min) plus earlier steps in the same AAP job (create hosted + # cluster, retrieve kubeconfig, etc.), or this times out first with a + # less useful error while the ClusterOrder is still legitimately Progressing. + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=480, + delay=15, + description=f"{name} ClusterOrder Ready", + ) + + +def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: + # HACK: HyperShift has multiple teardown bugs where controllers leave orphaned state + # that deadlocks HostedCluster deletion. We force-clean on every poll iteration: + # + # 1. AgentCluster deprovision finalizer: capi-provider-agent is killed during teardown + # before removing its finalizer, blocking namespace termination. + # https://github.com/openshift/hypershift/blob/main/hypershift-operator/controllers/hostedcluster/karpenter.go#L88 + # + # 2. Agent labels: the CAPI provider sometimes fails to clear + # clusterdeployment-namespace from agents after HostedCluster deletion. The delete + # playbook's detach_and_unlabel skips agents that still have this label set, leaving + # the clusterorder label stuck and blocking agent reuse for subsequent tests. + # + # 3. Machine pre-terminate hooks: the CAPI provider sets a pre-terminate hook + # annotation on Machines, but is killed before removing it. The CAPI Machine + # controller waits forever for the annotation to be removed, blocking the entire + # deletion cascade (Machine → MachineSet → CAPI Cluster → HostedCluster). + def _check_deleted() -> bool: + _force_cleanup_agentcluster_finalizers(k8s=k8s, name=name) + _force_cleanup_agent_labels(k8s=k8s, name=name) + _force_cleanup_machine_preterminate_hooks(k8s=k8s, name=name) + return not k8s.is_present(resource="clusterorder", name=name) + + poll_until( + fn=_check_deleted, until=lambda v: v is True, retries=120, delay=10, description=f"{name} ClusterOrder deletion" + ) + + +def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None: + # HCP namespace: {osac-ns}-{co-name}-{hc-name}, where hc-name == co-name + hc_ns = f"{k8s.namespace}-{name}" + cp_ns = f"{hc_ns}-{name}" + finalizer = "agentclustercapi-provider.agent-install.openshift.io/deprovision" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, + "get", + "agentclusters.capi-provider.agent-install.openshift.io", + "-n", + cp_ns, + "-o", + f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", + ) + if rc != 0 or not output.strip(): + return + for ac_name in output.strip().split(): + finalizers_json, rc = run_unchecked( + *base_args, + "get", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "-o", + "jsonpath={.metadata.finalizers}", + ) + if rc != 0 or finalizer not in finalizers_json: + continue + import json + + idx = json.loads(finalizers_json).index(finalizer) + run_unchecked( + *base_args, + "patch", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "--type=json", + f'-p=[{{"op": "remove", "path": "/metadata/finalizers/{idx}"}}]', + ) + + +def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: + agent_ns = "hardware-inventory" + clusterorder_label = "osac.openshift.io/clusterorder" + clusterdeployment_ns_label = "agent-install.openshift.io/clusterdeployment-namespace" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, + "get", + "agents.agent-install.openshift.io", + "-n", + agent_ns, + "-l", + f"{clusterorder_label}={name}", + "-o", + "jsonpath={.items[*].metadata.name}", + ) + if rc != 0 or not output.strip(): + return + for agent_name in output.strip().split(): + run_unchecked( + *base_args, + "label", + f"agents.agent-install.openshift.io/{agent_name}", + "-n", + agent_ns, + f"{clusterorder_label}-", + f"{clusterdeployment_ns_label}-", + ) + + +def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> None: + cp_ns = f"{k8s.namespace}-{name}-{name}" + hook = "pre-terminate.delete.hook.machine.cluster.x-k8s.io/agentmachine" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, "get", "machines.cluster.x-k8s.io", "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}" + ) + if rc != 0 or not output.strip(): + return + for machine_name in output.strip().split(): + run_unchecked(*base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", "-n", cp_ns, f"{hook}-") + + +def wait_for_cluster_deleting(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Deleting", + retries=30, + delay=5, + description=f"{name} ClusterOrder Deleting phase", + ) + + +def wait_for_cluster_grpc_deleting_or_archived(*, grpc: GRPCClient, uuid: str) -> None: + """Succeed if we catch CLUSTER_STATE_DELETING or if the cluster is already archived. + + The DELETING window in the fulfillment-service is extremely short (one + Update + Signal round-trip). Polling for the exact state is racey; accepting + either DELETING or 'already gone' makes the assertion reliable. + """ + + def _done() -> bool: + try: + cluster = grpc.get_cluster(cluster_id=uuid) + state = cluster.get("object", {}).get("status", {}).get("state", "") + return state == "CLUSTER_STATE_DELETING" + except subprocess.CalledProcessError: + return True + + poll_until( + fn=_done, + until=lambda v: v is True, + retries=30, + delay=2, + description=f"{uuid} gRPC DELETING or already archived", + ) + + +def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + # retry_on_error=True: a flaky grpcurl call hitting a momentarily-busy + # route right after heavy cluster-deletion activity shouldn't fail the + # whole test on the first hiccup. + poll_until( + fn=lambda: uuid not in grpc.list_cluster_ids(), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{uuid} removed from gRPC cluster list", + retry_on_error=True, + ) + + +def wait_for_security_group_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_security_group_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"SecurityGroup CR for {uuid}", + ) + + +def wait_for_security_group_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_security_group_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} SecurityGroup Ready", + ) + + +def wait_for_security_group_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="securitygroup", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} SecurityGroup deletion", + ) + + +# Tenant helpers + + +def wait_for_tenant_cr(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.is_present(resource="tenant", name=name), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"Tenant CR {name}", + ) + + +def wait_for_tenant_condition(*, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True") -> None: + def _check() -> str: + if not k8s.is_present(resource="tenant", name=name): + raise AssertionError(f"Tenant {name} disappeared before {condition_type}={expected_status}") + phase: str = k8s.get_tenant_phase(name=name, checked=False) + if phase == "Failed": + cond_status = k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) + if cond_status != expected_status: + raise AssertionError(f"Tenant {name} entered Failed phase before {condition_type}={expected_status}") + return k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) + + poll_until( + fn=_check, + until=lambda v: v == expected_status, + retries=120, + delay=5, + description=f"Tenant {name} {condition_type}={expected_status}", + ) + + +def wait_for_tenant_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="tenant", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"Tenant {name} deletion", + ) + + +# CaaS cluster storage helpers + + +def wait_for_cluster_order_condition( + *, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True" +) -> None: + def _check() -> str: + if not k8s.is_present(resource="clusterorder", name=name): + raise AssertionError(f"ClusterOrder {name} disappeared before {condition_type}={expected_status}") + phase: str = k8s.get_cluster_order_phase(name=name, checked=False) + cond_status = k8s.get_cluster_order_condition_status(name=name, condition_type=condition_type, checked=False) + if phase == "Failed" and cond_status != expected_status: + raise AssertionError(f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}") + return cond_status + + poll_until( + fn=_check, + until=lambda v: v == expected_status, + retries=120, + delay=10, + description=f"ClusterOrder {name} {condition_type}={expected_status}", + ) + + +def wait_for_tenant_cluster_storage_entry(*, k8s: K8sClient, tenant_name: str, cluster_name: str) -> dict[str, Any]: + def _check() -> dict[str, Any] | None: + entries = k8s.get_tenant_cluster_storage(name=tenant_name, checked=False) + for entry in entries: + if entry.get("clusterName") == cluster_name and entry.get("ready") is True: + return entry + return None + + return poll_until( + fn=_check, + until=lambda v: v is not None, + retries=60, + delay=10, + description=f"Tenant {tenant_name} clusterStorage entry for {cluster_name}", + ) + + +def wait_for_tenant_cluster_storage_entry_removed(*, k8s: K8sClient, tenant_name: str, cluster_name: str) -> None: + poll_until( + fn=lambda: all( + entry.get("clusterName") != cluster_name + for entry in k8s.get_tenant_cluster_storage(name=tenant_name, checked=False) + ), + until=lambda v: v is True, + retries=60, + delay=10, + description=f"Tenant {tenant_name} clusterStorage entry for {cluster_name} removed", + ) + + +# Storage resource helpers + + +def wait_for_storage_classes_by_tenant(*, k8s: K8sClient, tenant_name: str, min_count: int = 1) -> list[str]: + return poll_until( + fn=lambda: k8s.list_storage_class_names_by_tenant(tenant_name=tenant_name), + until=lambda v: len(v) >= min_count, + retries=120, + delay=5, + description=f"StorageClasses for tenant {tenant_name} (>= {min_count})", + ) + + +def wait_for_storage_classes_removed(*, k8s: K8sClient, tenant_name: str) -> None: + poll_until( + fn=lambda: k8s.count_storage_classes_by_tenant(tenant_name=tenant_name), + until=lambda v: v == 0, + retries=120, + delay=5, + description=f"StorageClasses for tenant {tenant_name} removed", + ) + + +def wait_for_secrets_removed(*, k8s: K8sClient, tenant_name: str, namespace: str) -> None: + poll_until( + fn=lambda: k8s.count_secrets_by_tenant(tenant_name=tenant_name, namespace=namespace), + until=lambda v: v == 0, + retries=120, + delay=5, + description=f"Secrets for tenant {tenant_name} in {namespace} removed", + ) + + +# BareMetalInstance helpers + + +def wait_for_bmi_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_baremetal_instance_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"BareMetalInstance CR for {uuid}", + ) + + +def wait_for_bmi_running(*, grpc: GRPCClient, bmi_id: str) -> None: + def _check_state() -> str: + state: str = grpc.get_baremetal_instance_state(bmi_id=bmi_id) + assert "FAILED" not in state, f"BareMetalInstance {bmi_id} entered {state}" + return state + + poll_until( + fn=_check_state, + until=lambda v: v == "BARE_METAL_INSTANCE_STATE_RUNNING", + retries=120, + delay=10, + description=f"{bmi_id} RUNNING", + ) + + +def wait_for_bmi_deletion(*, k8s: K8sClient, name: str) -> None: + # 2700s (45min), not the old 1200s (20min): the deprovision AAP job this + # blocks on retries with exponential backoff up to a 30-minute ceiling + # (osac-operator's shared pkg/provisioning, BackoffMaxDelay) when it fails + # -- e.g. under the AAP job-pod attach flakiness tracked in OSAC-3499. A + # 20-minute window can time out here while the operator is still correctly + # retrying and would have succeeded; 45 minutes gives it room for a worst-case + # backoff wait plus job execution time, without silently swallowing an + # actually-stuck deletion (still fails, just later). + poll_until( + fn=lambda: not k8s.is_present(resource="baremetalinstance", name=name), + until=lambda v: v is True, + retries=270, + delay=10, + description=f"{name} BareMetalInstance deletion", + ) + + +def wait_for_bmi_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_baremetal_instance_ids(), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{uuid} removed from gRPC BareMetalInstance list", + ) + + +def wait_for_bmh_provisioned(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None: + def _check() -> str: + state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace) + assert state != "error", f"BMH {name} entered error state" + return state + + poll_until( + fn=_check, until=lambda v: v == "provisioned", retries=120, delay=10, description=f"{name} BMH provisioned" + ) + + +def wait_for_bmh_available(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None: + def _check() -> str: + state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace) + assert state != "error", f"BMH {name} entered error state" + return state + + poll_until( + fn=_check, + until=lambda v: v in ("available", "ready"), + retries=120, + delay=10, + description=f"{name} BMH available", + ) diff --git a/tests/e2e/tests/core/k8s_client.py b/tests/e2e/tests/core/k8s_client.py new file mode 100644 index 0000000000..e0e2662602 --- /dev/null +++ b/tests/e2e/tests/core/k8s_client.py @@ -0,0 +1,499 @@ +from __future__ import annotations + +import json +import subprocess +from typing import Any + +from tests.core.runner import run, run_unchecked + + +class K8sClient: + def __init__(self, *, namespace: str, kubeconfig: str | None = None) -> None: + self.namespace: str = namespace + self.kubeconfig: str | None = kubeconfig + + def _base(self) -> list[str]: + args: list[str] = ["kubectl"] + if self.kubeconfig is not None: + args.extend(["--kubeconfig", self.kubeconfig]) + args.extend(["--as", "system:admin"]) + return args + + def _get(self, *args: str, checked: bool = True) -> tuple[str, int]: + if checked: + return run(*self._base(), *args), 0 + return run_unchecked(*self._base(), *args) + + # Generic kubectl operations + + def get_json(self, *, resource: str, name: str) -> dict[str, Any]: + return json.loads(run(*self._base(), "get", resource, name, "-n", self.namespace, "-o", "json")) + + def get_jsonpath(self, *, resource: str, name: str, jsonpath: str) -> str: + return run(*self._base(), "get", resource, name, "-n", self.namespace, "-o", f"jsonpath={jsonpath}") + + def get_by_label(self, *, resource: str, label: str, jsonpath: str) -> str: + return run(*self._base(), "get", resource, "-n", self.namespace, "-l", label, "-o", f"jsonpath={jsonpath}") + + def patch(self, *, resource: str, name: str, patch: str) -> tuple[str, int]: + return run_unchecked(*self._base(), "patch", resource, name, "-n", self.namespace, "--type=merge", "-p", patch) + + def apply(self, *, manifest: str) -> None: + args: list[str] = [*self._base(), "apply", "-f", "-"] + subprocess.run(args, input=manifest, capture_output=True, text=True, check=True) + + def delete(self, *, resource: str, name: str, wait: bool = True) -> None: + args = [*self._base(), "delete", resource, name, "-n", self.namespace] + if not wait: + args.append("--wait=false") + run(*args, timeout=600) + + def is_present(self, *, resource: str, name: str) -> bool: + _, rc = run_unchecked(*self._base(), "get", resource, name, "-n", self.namespace) + return rc == 0 + + def count_by_label_all_namespaces(self, *, resource: str, label: str) -> int: + output: str = run(*self._base(), "get", resource, "-A", "-l", label, "--no-headers") + if not output: + return 0 + return len(output.strip().splitlines()) + + # ComputeInstance queries + + def get_compute_instance_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "computeinstance", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/computeinstance-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_compute_instance_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "computeinstance", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_compute_instance_last_restarted_at(self, *, name: str) -> str: + return self.get_jsonpath(resource="computeinstance", name=name, jsonpath="{.status.lastRestartedAt}") + + def get_compute_instance_latest_job_id(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0]["jobID"] + + def get_compute_instance_latest_job_state(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + + def get_compute_instance_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_compute_instance_vm_namespace(self, *, name: str) -> str: + return self.get_jsonpath( + resource="computeinstance", name=name, jsonpath="{.status.virtualMachineReference.namespace}" + ) + + # VirtualNetwork queries + + def get_virtual_network_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "virtualnetwork", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/virtualnetwork-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_virtual_network_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "virtualnetwork", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # Subnet queries + + def get_subnet_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "subnet", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/subnet-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_subnet_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "subnet", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # VirtualMachine/VMI queries (explicit namespace — may be on a different cluster) + + def get_vmi_creation_timestamp(self, *, vmi_namespace: str, compute_instance_name: str) -> str: + return run( + *self._base(), + "get", + "virtualmachineinstance", + "-n", + vmi_namespace, + "-l", + f"osac.openshift.io/computeinstance={compute_instance_name}", + "-o", + "jsonpath={.items[0].metadata.creationTimestamp}", + ) + + def get_vm_printable_status(self, *, name: str, vm_namespace: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "virtualmachine", + name, + "-n", + vm_namespace, + "-o", + "jsonpath={.status.printableStatus}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_vm_run_strategy(self, *, name: str, vm_namespace: str) -> str: + return run( + *self._base(), "get", "virtualmachine", name, "-n", vm_namespace, "-o", "jsonpath={.spec.runStrategy}" + ) + + # ExternalIPPool queries + + def get_external_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "externalippool", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/externalippool-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_external_ip_pool_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "externalippool", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # ExternalIP queries + + def get_external_ip_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "externalip", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/externalip-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_external_ip_state(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "externalip", name, "-n", self.namespace, "-o", "jsonpath={.status.state}", checked=checked + ) + return output if rc == 0 else "" + + # ExternalIPAttachment queries + + def get_external_ip_attachment_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "externalipattachment", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/externalipattachment-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_external_ip_attachment_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "externalipattachment", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # ClusterOrder queries + + def get_cluster_order_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "clusterorder", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/clusterorder-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_cluster_order_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "clusterorder", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_cluster_order_latest_job_id(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0]["jobID"] + + def get_cluster_order_latest_job_state(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + + def get_cluster_order_hosted_cluster_name(self, *, name: str) -> str: + return self.get_jsonpath( + resource="clusterorder", name=name, jsonpath="{.status.clusterReference.hostedClusterName}" + ) + + def get_cluster_order_namespace(self, *, name: str) -> str: + return self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.status.clusterReference.namespace}") + + def get_cluster_order_spec(self, *, name: str) -> dict[str, Any]: + output = self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.spec}") + return json.loads(output) if output else {} + + def get_cluster_order_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_cluster_order_finalizers(self, *, name: str, checked: bool = True) -> list[str]: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("metadata", {}).get("finalizers", []) + + # Tenant queries + + def get_tenant_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "tenant", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_tenant_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_tenant_storage_classes(self, *, name: str, checked: bool = True) -> list[dict[str, str]]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("status", {}).get("storageClasses", []) + + def get_tenant_finalizers(self, *, name: str, checked: bool = True) -> list[str]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("metadata", {}).get("finalizers", []) + + def get_tenant_cluster_storage(self, *, name: str, checked: bool = True) -> list[dict[str, Any]]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("status", {}).get("clusterStorage", []) + + # Cluster-scoped storage resource queries (no -n flag) + + def count_storage_classes_by_tenant(self, *, tenant_name: str) -> int: + output, rc = self._get( + "get", "storageclass", "-l", f"osac.openshift.io/tenant={tenant_name}", "--no-headers", checked=False + ) + if rc != 0 or not output.strip(): + return 0 + return len(output.strip().splitlines()) + + def list_storage_class_names_by_tenant(self, *, tenant_name: str) -> list[str]: + output, rc = self._get( + "get", + "storageclass", + "-l", + f"osac.openshift.io/tenant={tenant_name}", + "-o", + "jsonpath={.items[*].metadata.name}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return output.strip().split() + + def get_storage_class_labels(self, *, name: str) -> dict[str, str]: + output, rc = self._get("get", "storageclass", name, "-o", "json", checked=False) + if rc != 0: + return {} + return json.loads(output).get("metadata", {}).get("labels", {}) + + # Namespaced secret queries with explicit namespace (for cross-namespace lookups) + + def count_secrets_by_tenant(self, *, tenant_name: str, namespace: str) -> int: + output, rc = self._get( + "get", + "secret", + "-n", + namespace, + "-l", + f"osac.openshift.io/tenant={tenant_name}", + "--no-headers", + checked=False, + ) + if rc != 0 or not output.strip(): + return 0 + return len(output.strip().splitlines()) + + # SecurityGroup queries + + def get_security_group_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "securitygroup", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/securitygroup-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_security_group_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "securitygroup", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # BareMetalInstance queries + + def get_baremetal_instance_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "baremetalinstance", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/baremetalinstance-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_baremetal_instance_external_host_id(self, *, name: str) -> str: + return self.get_jsonpath(resource="baremetalinstance", name=name, jsonpath="{.spec.externalHostID}") + + # BareMetalHost queries (explicit namespace — BMHs live in a different namespace) + + def get_bmh_provisioning_state(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", + "baremetalhost", + name, + "-n", + bmh_namespace, + "-o", + "jsonpath={.status.provisioning.state}", + checked=False, + ) + return output if rc == 0 else "" + + def get_bmh_image_url(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.image.url}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_consumer_ref(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.consumerRef.name}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_online(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.online}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_powered_on(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.status.poweredOn}", checked=False + ) + return output if rc == 0 else "" diff --git a/tests/e2e/tests/core/keycloak.py b/tests/e2e/tests/core/keycloak.py new file mode 100644 index 0000000000..55a3859706 --- /dev/null +++ b/tests/e2e/tests/core/keycloak.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json + +from tests.core.runner import poll_until, run_unchecked + + +def get_jwt(*, keycloak_url: str, realm: str, client_id: str, username: str, password: str) -> str: + """Get JWT token from Keycloak with retry logic for startup flakiness.""" + token_url = f"{keycloak_url}/realms/{realm}/protocol/openid-connect/token" + + def try_get_token() -> str | None: + stdout, returncode = run_unchecked( + "curl", + "-sk", + "--fail-with-body", + "--max-time", + "10", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + f"client_id={client_id}", + "-d", + f"username={username}", + "-d", + f"password={password}", + "-d", + "scope=openid organization", + ) + + if returncode != 0: + return None + + try: + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get JWT from Keycloak: {error}") + return token + except json.JSONDecodeError: + # Malformed response - Keycloak might still be starting + return None + + return poll_until( + fn=try_get_token, until=lambda token: token is not None, retries=24, delay=5, description="Keycloak JWT token" + ) diff --git a/tests/e2e/tests/core/keycloak_admin.py b/tests/e2e/tests/core/keycloak_admin.py new file mode 100644 index 0000000000..8ed048243b --- /dev/null +++ b/tests/e2e/tests/core/keycloak_admin.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import json +import time +from typing import Any +from urllib.parse import urlencode + +from tests.core.runner import run + + +def get_admin_token(*, keycloak_url: str, username: str, password: str) -> str: + """Get an admin access token for Keycloak admin API calls.""" + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + stdout: str = run( + "curl", + "-sk", + "--fail-with-body", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + "client_id=admin-cli", + "-d", + f"username={username}", + "-d", + f"password={password}", + ) + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get admin token from Keycloak: {error}") + return token + + +def keycloak_admin_request( + *, keycloak_url: str, admin_token: str, method: str, path: str, data: Any = None +) -> tuple[int, bytes]: + """ + Make an authenticated request to the Keycloak admin API for the 'osac' realm. + The path is relative to /admin/realms/osac (e.g., "/organizations", "/users/{id}"). + Returns (status_code, response_body). + """ + url = f"{keycloak_url}/admin/realms/osac{path}" + args = [ + "curl", + "-sk", + "-w", + "\n%{http_code}", + "-X", + method, + "-H", + f"Authorization: Bearer {admin_token}", + "-H", + "Content-Type: application/json", + ] + if data is not None: + if isinstance(data, str): + args.extend(["-d", data]) + else: + args.extend(["-d", json.dumps(data)]) + + args.append(url) + + output: str = run(*args) + lines = output.strip().split("\n") + status_code = int(lines[-1]) + body = "\n".join(lines[:-1]).encode("utf-8") + + return status_code, body + + +def wait_for_organization(*, keycloak_url: str, admin_token: str, org_name: str, timeout_seconds: int = 60) -> str: + """ + Wait for an organization to be synced to Keycloak and return its ID. + Polls with exponential backoff until the organization exists or timeout is reached. + """ + start_time = time.time() + interval = 1.0 + max_interval = 10.0 + + while time.time() - start_time < timeout_seconds: + query = urlencode({"exact": "true", "search": org_name}) + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/organizations?{query}" + ) + + if status != 200: + raise RuntimeError(f"Failed to query organizations: status={status} body={body.decode()}") + + orgs: list[dict[str, Any]] = json.loads(body) + if len(orgs) > 0: + org_id: str = orgs[0]["id"] + return org_id + + time.sleep(interval) + interval = min(interval * 2, max_interval) + + raise RuntimeError(f"Organization '{org_name}' not found in Keycloak after {timeout_seconds}s") + + +def get_user_id(*, keycloak_url: str, admin_token: str, username: str) -> str: + """Get a user's ID by username.""" + query = urlencode({"username": username, "exact": "true"}) + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/users?{query}" + ) + + if status != 200: + raise RuntimeError(f"Failed to get user '{username}': status={status} body={body.decode()}") + + users: list[dict[str, Any]] = json.loads(body) + if len(users) == 0: + raise RuntimeError(f"User '{username}' not found in Keycloak") + + return users[0]["id"] + + +def add_user_to_organization( + *, keycloak_url: str, admin_token: str, org_id: str, user_id: str, username: str, org_name: str +) -> None: + """Add a user to a Keycloak organization.""" + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="POST", + path=f"/organizations/{org_id}/members", + data=user_id, + ) + + # 201 Created, 204 No Content, or 409 Conflict (already a member) are all acceptable + # 400 with "Duplicate resource error" is also acceptable (Keycloak returns this when user is already a member) + if status == 400: + error_msg = body.decode().lower() + if "duplicate" not in error_msg: + raise RuntimeError( + f"Failed to add user '{username}' to organization '{org_name}': status={status} body={body.decode()}" + ) + elif status not in (201, 204, 409): + raise RuntimeError( + f"Failed to add user '{username}' to organization '{org_name}': status={status} body={body.decode()}" + ) + + +def ensure_organization_group(*, keycloak_url: str, admin_token: str, org_id: str, org_name: str) -> str: + """ + Ensure a /members group exists in the organization and return its ID. + Creates the group if it doesn't exist. + """ + group_name = "/members" + group_payload = {"name": group_name} + + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="POST", + path=f"/organizations/{org_id}/groups", + data=group_payload, + ) + + # 201 Created or 409 Conflict (already exists) are acceptable + if status == 201: + group_resp: dict[str, Any] = json.loads(body) + return group_resp["id"] + elif status == 409: + # Group already exists, need to fetch it + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/organizations/{org_id}/groups" + ) + + if status != 200: + raise RuntimeError( + f"Failed to get groups for organization '{org_name}': status={status} body={body.decode()}" + ) + + groups: list[dict[str, Any]] = json.loads(body) + for g in groups: + if g.get("name") == group_name: + return g["id"] + + raise RuntimeError(f"Failed to find group '{group_name}' in organization '{org_name}'") + else: + raise RuntimeError( + f"Failed to create group '{group_name}' in organization '{org_name}': status={status} body={body.decode()}" + ) + + +def add_user_to_organization_group( + *, keycloak_url: str, admin_token: str, org_id: str, group_id: str, user_id: str, username: str, org_name: str +) -> None: + """Add a user to a group within a Keycloak organization.""" + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="PUT", + path=f"/organizations/{org_id}/groups/{group_id}/members/{user_id}", + ) + + # 200 OK, 201 Created, 204 No Content, or 409 Conflict are all acceptable + # 400 with "Duplicate resource error" is also acceptable (Keycloak returns this when user is already a member) + if status == 400: + error_msg = body.decode().lower() + if "duplicate" not in error_msg: + raise RuntimeError( + f"Failed to add user '{username}' to group in organization '{org_name}': " + f"status={status} body={body.decode()}" + ) + elif status not in (200, 201, 204, 409): + raise RuntimeError( + f"Failed to add user '{username}' to group in organization '{org_name}': " + f"status={status} body={body.decode()}" + ) diff --git a/tests/e2e/tests/core/metering.py b/tests/e2e/tests/core/metering.py new file mode 100644 index 0000000000..942020fb7b --- /dev/null +++ b/tests/e2e/tests/core/metering.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import json +import logging +import time +import urllib.error +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from tests.core.runner import poll_until + +logger = logging.getLogger(__name__) + + +def _parse_time(s: str) -> datetime: + if s.endswith("Z"): + s = s[:-1] + "+00:00" + return datetime.fromisoformat(s) + + +@dataclass +class ExpectedEvent: + event_type: str + resource_id: str + timeout: int = 60 + + +class MeteringCollector: + """Collects CloudEvents from the metering test adapter HTTP API and verifies expectations. + + The test adapter runs inside the Kubernetes cluster and exposes + ``GET /events?type={event_type}&resource_id={resource_id}&since={RFC3339}`` + which returns a JSON array of CloudEvents. + + Usage in tests:: + + metering.expect("osac.resource.created.v1", resource_id=uuid) + # ... test actions ... + # metering.verify() runs automatically on fixture teardown + """ + + def __init__(self, *, base_url: str) -> None: + self._base_url = base_url.rstrip("/") + self._expectations: list[ExpectedEvent] = [] + self._matched: set[str] = set() + self._verified: dict[tuple[str, str], dict[str, Any]] = {} + self._start_time: str = "" + + def start(self) -> None: + """Record the start time so event queries only return events after this point.""" + self._start_time = datetime.now(UTC).isoformat() + + def stop(self) -> None: + """No-op -- no background resources to clean up.""" + + def expect(self, event_type: str, resource_id: str, *, timeout: int = 60) -> None: + self._expectations.append(ExpectedEvent(event_type=event_type, resource_id=resource_id, timeout=timeout)) + + def verify(self) -> None: + if not self._expectations: + if not self._verified: + logger.warning("verify() called with no expectations — did you forget to call expect()?") + return + for exp in self._expectations: + event = self._poll_for_event(exp) + self._validate_structure(event, exp) + self._verified[(exp.event_type, exp.resource_id)] = event + self._expectations.clear() + + def get_event(self, event_type: str, resource_id: str, *, timeout: int = 60) -> dict[str, Any]: + """Return a verified event, or poll for one if not yet verified.""" + key = (event_type, resource_id) + if key in self._verified: + return self._verified[key] + exp = ExpectedEvent(event_type=event_type, resource_id=resource_id, timeout=timeout) + return self._poll_for_event(exp) + + def assert_no_events(self, event_type: str, resource_id: str, *, since: str, within: int = 60) -> None: + """Assert that no events of the given type appear for the resource within a time window.""" + time.sleep(within) + events = self._fetch_events(event_type, resource_id) + since_dt = datetime.fromisoformat(since) + matching = [ + ev + for ev in events + if ev.get("type") == event_type + and ( + ev.get("resource_id") == resource_id + or ev.get("osacresourceid") == resource_id + or ev.get("data", {}).get("resource_id") == resource_id + ) + and _parse_time(ev.get("time", "")) >= since_dt + ] + assert not matching, ( + f"Expected no {event_type} events for {resource_id} after {since}, " + f"but found {len(matching)}: {[{'id': e.get('id'), 'time': e.get('time')} for e in matching]}" + ) + + def get_all_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: + """Return all events matching type and resource_id (for N+1 count assertions).""" + return self._fetch_events(event_type, resource_id) + + def _fetch_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: + params = urlencode({"type": event_type, "resource_id": resource_id, "since": self._start_time}) + url = f"{self._base_url}/events?{params}" + last_exc: OSError | None = None + for attempt in range(3): + try: + req = Request(url) + with urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, OSError) as exc: + last_exc = exc + logger.warning("Transient HTTP error fetching events (attempt %d/3): %s", attempt + 1, exc) + time.sleep(2) + raise last_exc # type: ignore[misc] + + def _poll_for_event(self, expected: ExpectedEvent) -> dict[str, Any]: + result: list[dict[str, Any]] = [] + + def find() -> bool: + try: + events = self._fetch_events(expected.event_type, expected.resource_id) + except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc: + logger.warning("Transient error fetching metering events, will retry: %s", exc) + return False + for ev in events: + ev_id = ev.get("id", "") + if ev_id and ev_id in self._matched: + continue + if ev.get("type") == expected.event_type and ( + ev.get("resource_id") == expected.resource_id + or ev.get("osacresourceid") == expected.resource_id + or ev.get("data", {}).get("resource_id") == expected.resource_id + ): + if ev_id: + self._matched.add(ev_id) + result.append(ev) + return True + return False + + poll_until( + fn=find, + until=lambda found: found is True, + retries=expected.timeout // 2, + delay=2, + description=f"metering event type={expected.event_type} resource_id={expected.resource_id}", + ) + return result[0] + + @staticmethod + def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: + assert event.get("specversion") == "1.0", f"Wrong specversion: {event.get('specversion')}" + assert event.get("source") in ("osac-metering", "osac-metering/reconciler"), ( + f"Wrong source: {event.get('source')}" + ) + assert event.get("id"), "Missing event id" + assert event.get("time"), "Missing event time" + assert event.get("osacresourceid") == expected.resource_id, ( + f"Wrong osacresourceid: {event.get('osacresourceid')}" + ) + assert event.get("osacresourcetype") in ("compute_instance", "cluster_order"), ( + f"Wrong osacresourcetype: {event.get('osacresourcetype')}" + ) + assert event.get("osactenant"), "Missing osactenant" + + data = event.get("data", {}) + assert data.get("resource_id") == expected.resource_id, f"Wrong resource_id in data: {data.get('resource_id')}" + assert data.get("resource_type"), "Missing resource_type in data" + assert data.get("tenant_id"), "Missing tenant_id in data" + assert data.get("schema_version") == "v1", f"Wrong schema_version: {data.get('schema_version')}" + assert "project_id" in data, "Missing project_id in data" + assert "billing_dimensions" in data, "Missing billing_dimensions in data" + + if expected.event_type != "osac.resource.heartbeat.v1": + assert data.get("transition_time"), "Missing transition_time in data" + try: + datetime.fromisoformat(data["transition_time"]) + except (ValueError, TypeError) as exc: + raise AssertionError(f"Invalid RFC3339 transition_time: {data.get('transition_time')}") from exc + + transition_types = {"osac.resource.started.v1", "osac.resource.suspended.v1", "osac.resource.resumed.v1"} + if expected.event_type in transition_types: + assert "previous_state" in data, f"Missing previous_state in {expected.event_type}" + assert "duration_seconds" in data, f"Missing duration_seconds in {expected.event_type}" + + if expected.event_type == "osac.resource.suspended.v1": + valid = ("RUNNING", "STOPPING", "STARTING", "PROGRESSING", "READY") + assert data.get("previous_state") in valid, ( + f"suspended.v1 previous_state should be one of {valid}, got {data.get('previous_state')!r}" + ) + + if expected.event_type == "osac.resource.resumed.v1": + valid = ("STOPPED", "PAUSED", "FAILED", "DELETE_FAILED") + assert data.get("previous_state") in valid, ( + f"resumed.v1 previous_state should be one of {valid}, got {data.get('previous_state')!r}" + ) + + resource_type = event.get("osacresourcetype") + if resource_type == "compute_instance": + _validate_vmaas_billing(event) + elif resource_type == "cluster_order": + _validate_caas_billing(event, expected.event_type) + + +def _validate_vmaas_billing(event: dict[str, Any]) -> None: + bd = event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("instance_type"), "Missing or empty instance_type in billing_dimensions" + assert bd.get("image_ref"), "Missing or empty image_ref in billing_dimensions" + assert "boot_disk_size_gib" in bd, "Missing boot_disk_size_gib in billing_dimensions" + assert isinstance(bd["boot_disk_size_gib"], (int, float)), ( + f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" + ) + + +def _validate_caas_billing(event: dict[str, Any], event_type: str) -> None: + bd = event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("cluster_template"), "Missing cluster_template in billing_dimensions" + # created.v1 and deleted.v1 use topLevelDims (cluster_template + + # release_image only, no per-component decomposition) + if event_type in ("osac.resource.created.v1", "osac.resource.deleted.v1"): + assert bd.get("release_image"), "Missing release_image in billing_dimensions" + return + assert bd.get("component") in ("control_plane", "worker"), ( + f"component should be control_plane or worker, got {bd.get('component')!r}" + ) + assert bd.get("node_set"), "Missing node_set in billing_dimensions" + assert bd.get("host_type"), "Missing host_type in billing_dimensions" + assert isinstance(bd.get("node_count"), (int, float)) and bd["node_count"] > 0, ( + f"node_count should be positive number, got {bd.get('node_count')!r}" + ) diff --git a/tests/e2e/tests/core/osac_cli.py b/tests/e2e/tests/core/osac_cli.py new file mode 100644 index 0000000000..eb472913d2 --- /dev/null +++ b/tests/e2e/tests/core/osac_cli.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import re +import shutil +import tempfile +from typing import Any + +from tests.core.runner import run, run_unchecked + + +class OsacCLI: + def __init__( + self, + *, + binary: str, + address: str, + token_script: str, + namespace: str, + default_instance_type: str | None = None, + default_disk_image: str | None = None, + private: bool = False, + ) -> None: + self.binary: str = binary + self.namespace: str = namespace + self._address: str = address + self._token_script: str = token_script + self._private: bool = private + self.default_instance_type: str | None = default_instance_type + self.default_disk_image: str | None = default_disk_image + # Each OsacCLI instance gets its own config directory so that parallel + # xdist workers (or multiple CLI fixtures) don't overwrite each other's + # login credentials via the shared ~/.config/osac/config.json. + self._config_dir: str = tempfile.mkdtemp(prefix="osac-config-") + login_args = ["login", "--address", address, "--insecure", "--token-script", token_script] + if self._private: + login_args.append("--private") + self._run(*login_args) + + def close(self) -> None: + shutil.rmtree(self._config_dir, ignore_errors=True) + + @property + def config_dir(self) -> str: + return self._config_dir + + def _run(self, *args: str, timeout: int = 300) -> str: + return run(self.binary, "--config", self._config_dir, *args, timeout=timeout) + + def _run_unchecked(self, *args: str, timeout: int = 300) -> tuple[str, int]: + return run_unchecked(self.binary, "--config", self._config_dir, *args, timeout=timeout) + + def relogin(self) -> None: + login_args = ["login", "--address", self._address, "--insecure", "--token-script", self._token_script] + if self._private: + login_args.append("--private") + self._run(*login_args) + + @staticmethod + def _parse_uuid(stdout: str) -> str: + match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) + assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" + return match.group(1) + + def create_hub(self, *, hub_id: str, kubeconfig: str) -> None: + self._run("create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) + + def create_compute_instance( + self, + *, + template: str, + name: str | None = None, + network_attachments: list[dict[str, Any]] | None = None, + boot_disk_size: int = 20, + disk_image: str | None = None, + run_strategy: str = "Always", + user_data_secret_ref: str | None = None, + instance_type: str | None = None, + ) -> str: + args: list[str] = [ + "create", + "computeinstance", + "--template", + template, + "--boot-disk-size", + str(boot_disk_size), + "--run-strategy", + run_strategy, + ] + if name is not None: + args.extend(["--name", name]) + + effective_instance_type = instance_type if instance_type is not None else self.default_instance_type + if effective_instance_type is not None: + args.extend(["--instance-type", effective_instance_type]) + else: + raise ValueError("instance_type or default_instance_type must be set") + + effective_disk_image = disk_image if disk_image is not None else self.default_disk_image + if effective_disk_image is not None: + args.extend(["--disk-image", effective_disk_image]) + else: + raise ValueError("disk_image or default_disk_image must be set") + + # Add network attachments + if network_attachments is not None: + for idx, attachment in enumerate(network_attachments): + subnet = attachment.get("subnet") + if not subnet or not isinstance(subnet, str): + raise ValueError(f"network_attachments[{idx}]: 'subnet' must be a non-empty string, got {subnet!r}") + + security_groups = attachment.get("security_groups", []) + if not isinstance(security_groups, list): + raise ValueError( + f"network_attachments[{idx}]: 'security_groups' must be a list," + f" got {type(security_groups).__name__}" + ) + + if security_groups and not all(isinstance(sg, str) and sg for sg in security_groups): + raise ValueError(f"network_attachments[{idx}]: all security_groups must be non-empty strings") + + # Build network-attachment flag value + # Format: subnet=,security-groups=, + parts = [f"subnet={subnet}"] + if security_groups: + sg_list = ",".join(security_groups) + parts.append(f"security-groups={sg_list}") + + args.extend(["--network-attachment", ",".join(parts)]) + + if user_data_secret_ref is not None: + args.extend(["--user-data", user_data_secret_ref]) + + return self._parse_uuid(self._run(*args)) + + def delete_compute_instance(self, *, uuid: str) -> None: + self._run("delete", "computeinstance", uuid) + + def create_instance_type( + self, + *, + name: str, + cores: int, + memory_gib: int, + description: str = "", + gpu_pci_device_selector: str = "", + gpu_resource_name: str = "", + gpu_count: int = 0, + ) -> str: + args: list[str] = [ + "create", + "instancetype", + "--name", + name, + "--cores", + str(cores), + "--memory-gib", + str(memory_gib), + ] + if description: + args.extend(["--description", description]) + if gpu_pci_device_selector: + args.extend(["--gpu-pci-device-selector", gpu_pci_device_selector]) + if gpu_resource_name: + args.extend(["--gpu-resource-name", gpu_resource_name]) + if gpu_count: + args.extend(["--gpu-count", str(gpu_count)]) + return self._parse_uuid(self._run(*args)) + + def describe_instance_type(self, *, name: str) -> str: + return self._run("describe", "instancetype", name) + + def delete_instance_type(self, *, name: str) -> None: + self._run("delete", "instancetype", name) + + def create_cluster( + self, + *, + template: str, + name: str | None = None, + pull_secret_file: str | None = None, + ssh_public_key_file: str | None = None, + version: str | None = None, + template_parameters: dict[str, str] | None = None, + template_parameter_files: dict[str, str] | None = None, + ) -> str: + args: list[str] = ["create", "cluster", "--template", template] + if name is not None: + args.extend(["--name", name]) + if pull_secret_file is not None: + args.extend(["--pull-secret-file", pull_secret_file]) + if ssh_public_key_file is not None: + args.extend(["--ssh-public-key-file", ssh_public_key_file]) + if version is not None: + args.extend(["--version", version]) + if template_parameters is not None: + for key, value in template_parameters.items(): + args.extend(["-p", f"{key}={value}"]) + if template_parameter_files is not None: + for key, path in template_parameter_files.items(): + args.extend(["-f", f"{key}={path}"]) + + return self._parse_uuid(self._run(*args)) + + def get(self, resource: str, *, output: str | None = None) -> str: + args: list[str] = ["get", resource] + if output is not None: + args.extend(["-o", output]) + return self._run(*args) + + def get_cluster_credential(self, credential: str, *, uuid: str) -> str: + return self._run("get", credential, uuid) + + def get_unchecked(self, resource: str) -> tuple[str, int]: + return self._run_unchecked("get", resource) + + def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str, version: str | None = None) -> str: + args = ["create", "cluster", "--catalog-item", catalog_item, "--name", name] + if version is not None: + args.extend(["--version", version]) + return self._parse_uuid(self._run(*args)) + + def create_compute_instance_with_catalog_item( + self, *, catalog_item: str, name: str | None = None, subnet: str | None = None + ) -> str: + args: list[str] = ["create", "computeinstance", "--catalog-item", catalog_item] + if name is not None: + args.extend(["--name", name]) + if subnet is not None: + args.extend(["--network-attachment", f"subnet={subnet}"]) + return self._parse_uuid(self._run(*args)) + + def scale_cluster(self, *, uuid: str, node_set: str, size: int) -> None: + self._run("scale", "cluster", uuid, "--node-set", node_set, "--size", str(size)) + + def delete_cluster(self, *, uuid: str) -> None: + self._run("delete", "cluster", uuid) + + def create_baremetal_instance( + self, *, name: str, catalog_item: str, ssh_key: str | None = None, user_data: str | None = None + ) -> str: + args: list[str] = ["create", "baremetalinstance", "--name", name, "--catalog-item", catalog_item] + if ssh_key is not None: + args.extend(["--ssh-key", ssh_key]) + if user_data is not None: + args.extend(["--user-data", user_data]) + return self._parse_uuid(self._run(*args)) + + def delete_baremetal_instance(self, *, uuid: str) -> None: + self._run("delete", "baremetalinstance", uuid) diff --git a/tests/e2e/tests/core/runner.py b/tests/e2e/tests/core/runner.py new file mode 100644 index 0000000000..1d4f16aa7a --- /dev/null +++ b/tests/e2e/tests/core/runner.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import logging +import os +import subprocess +import time +from collections.abc import Callable +from typing import TypeVar + +T = TypeVar("T") + +logger = logging.getLogger(__name__) + +# How often to log a "still waiting" progress line during a poll_until call, +# regardless of its own retry/delay settings -- long polls (e.g. waiting up +# to ~60 minutes for a bare-metal cluster to become Ready) would otherwise +# print nothing until they finish or time out. +_PROGRESS_LOG_INTERVAL_S = 30 + + +def run(*args: str, timeout: int = 300) -> str: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=True) + return result.stdout.strip() + + +def run_unchecked(*args: str, timeout: int = 300) -> tuple[str, int]: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False) + combined = (result.stdout.strip() + "\n" + result.stderr.strip()).strip() + return combined, result.returncode + + +def poll_until( + *, + fn: Callable[[], T], + until: Callable[[T], bool], + retries: int = 60, + delay: int = 5, + description: str, + retry_on_error: bool = False, +) -> T: + value: T | None = None + last_error: subprocess.CalledProcessError | None = None + start = time.monotonic() + last_logged = start + logger.info("Waiting for %s...", description) + for attempt in range(retries): + # retry_on_error is opt-in: most callers' fn() raising CalledProcessError + # indicates a real bug (bad namespace, typo'd resource, auth + # misconfig) and should fail fast, not be swallowed for the whole + # retry budget. Only pollers built on a checked (raising) subprocess + # call where a transient hiccup is expected -- e.g. a flaky grpcurl + # call hitting a momentarily-busy route -- should set this. + if retry_on_error: + try: + value = fn() + except subprocess.CalledProcessError as exc: + last_error = exc + value = None + else: + last_error = None + if until(value): + logger.info("%s — done after %.0fs", description, time.monotonic() - start) + return value + else: + value = fn() + if until(value): + logger.info("%s — done after %.0fs", description, time.monotonic() - start) + return value + now = time.monotonic() + if now - last_logged >= _PROGRESS_LOG_INTERVAL_S: + logger.info( + "Still waiting for %s (attempt %d/%d, %.0fs elapsed, last value: %r%s)", + description, + attempt + 1, + retries, + now - start, + value, + f", last error: {last_error}" if last_error else "", + ) + last_logged = now + time.sleep(delay) + if last_error is not None: + raise TimeoutError( + f"{description} — timeout after {retries * delay}s, last call failed: {last_error}" + ) from last_error + raise TimeoutError(f"{description} — timeout after {retries * delay}s, last value: {value!r}") + + +def env(name: str, default: str | None = None) -> str: + value = os.environ.get(name, default) + if value is None: + msg = f"Required environment variable {name} is not set" + raise RuntimeError(msg) + return value diff --git a/tests/e2e/tests/references/__init__.py b/tests/e2e/tests/references/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/references/conftest.py b/tests/e2e/tests/references/conftest.py new file mode 100644 index 0000000000..f871299e51 --- /dev/null +++ b/tests/e2e/tests/references/conftest.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import logging +import re +import subprocess +from collections.abc import Generator +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import PUBLIC_API, GRPCClient +from tests.core.helpers import ( + wait_for_security_group_cr, + wait_for_security_group_deletion, + wait_for_security_group_ready, + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_subnet_ready, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="session") +def ref_test_run_id() -> str: + return uuid4().hex[:8] + + +@pytest.fixture(scope="session") +def ref_network_class(grpc: GRPCClient) -> str: + response: dict[str, Any] = grpc.call(service=f"{PUBLIC_API}.NetworkClasses/List") + items = response.get("items", []) + assert items, "No network classes found; set OSAC_NETWORK_CLASS or deploy a NetworkClass" + return items[0]["metadata"]["name"] + + +@pytest.fixture(scope="session") +def ref_virtual_network( + grpc: GRPCClient, k8s_hub_client: K8sClient, ref_network_class: str, ref_test_run_id: str +) -> Generator[dict[str, str], None, None]: + vn_name = f"ref-vn-{ref_test_run_id}" + vn_id: str | None = None + vn_cr_name: str | None = None + + try: + vn_id = grpc.create_virtual_network(name=vn_name, network_class=ref_network_class, ipv4_cidr="10.210.0.0/16") + vn_cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vn_cr_name) + yield {"id": vn_id, "name": vn_name, "cr_name": vn_cr_name} + except Exception: + if vn_id: + try: + grpc.delete_virtual_network(vn_id=vn_id) + except Exception as e: + logger.warning("Failed to cleanup VN %s: %s", vn_id, type(e).__name__) + raise + finally: + if vn_id and vn_cr_name: + _safe_delete_vn(grpc, k8s_hub_client, vn_id=vn_id, vn_cr_name=vn_cr_name) + + +@pytest.fixture(scope="session") +def ref_subnet( + grpc: GRPCClient, k8s_hub_client: K8sClient, ref_virtual_network: dict[str, str], ref_test_run_id: str +) -> Generator[dict[str, str], None, None]: + subnet_name = f"ref-subnet-{ref_test_run_id}" + subnet_id: str | None = None + subnet_cr_name: str | None = None + + try: + response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.Subnets/Create", + data={ + "object": { + "metadata": {"name": subnet_name}, + "spec": {"virtual_network": {"name": ref_virtual_network["name"]}, "ipv4_cidr": "10.210.100.0/24"}, + } + }, + ) + subnet_id = response["object"]["id"] + subnet_cr_name = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_id) + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_cr_name) + yield {"id": subnet_id, "name": subnet_name, "cr_name": subnet_cr_name} + except Exception: + if subnet_id: + try: + grpc.delete_subnet(subnet_id=subnet_id) + except Exception as e: + logger.warning("Failed to cleanup subnet %s: %s", subnet_id, type(e).__name__) + raise + finally: + if subnet_id and subnet_cr_name: + _safe_delete_subnet(grpc, k8s_hub_client, subnet_id=subnet_id, subnet_cr_name=subnet_cr_name) + + +@pytest.fixture(scope="session") +def ref_security_group( + grpc: GRPCClient, k8s_hub_client: K8sClient, ref_virtual_network: dict[str, str], ref_test_run_id: str +) -> Generator[dict[str, str], None, None]: + sg_name = f"ref-sg-{ref_test_run_id}" + sg_id: str | None = None + sg_cr_name: str | None = None + + try: + response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", + data={ + "object": { + "metadata": {"name": sg_name}, + "spec": {"virtual_network": {"name": ref_virtual_network["name"]}}, + } + }, + ) + sg_id = response["object"]["id"] + sg_cr_name = wait_for_security_group_cr(k8s=k8s_hub_client, uuid=sg_id) + wait_for_security_group_ready(k8s=k8s_hub_client, name=sg_cr_name) + yield {"id": sg_id, "name": sg_name, "cr_name": sg_cr_name} + except Exception: + if sg_id: + try: + grpc.delete_security_group(sg_id=sg_id) + except Exception as e: + logger.warning("Failed to cleanup SG %s: %s", sg_id, type(e).__name__) + raise + finally: + if sg_id and sg_cr_name: + _safe_delete_sg(grpc, k8s_hub_client, sg_id=sg_id, sg_cr_name=sg_cr_name) + + +def _safe_delete_vn(grpc: GRPCClient, k8s: K8sClient, *, vn_id: str, vn_cr_name: str) -> None: + try: + grpc.delete_virtual_network(vn_id=vn_id) + except subprocess.CalledProcessError as exc: + combined = (exc.stderr or "") + (exc.stdout or "") + if not re.search(r"Code:\s*NotFound", combined): + logger.warning("VN %s teardown failed: %s", vn_id, combined.strip()) + return + if k8s.is_present(resource="virtualnetwork", name=vn_cr_name): + wait_for_virtual_network_deletion(k8s=k8s, name=vn_cr_name) + + +def _safe_delete_subnet(grpc: GRPCClient, k8s: K8sClient, *, subnet_id: str, subnet_cr_name: str) -> None: + try: + grpc.delete_subnet(subnet_id=subnet_id) + except subprocess.CalledProcessError as exc: + combined = (exc.stderr or "") + (exc.stdout or "") + if not re.search(r"Code:\s*NotFound", combined): + logger.warning("Subnet %s teardown failed: %s", subnet_id, combined.strip()) + return + if k8s.is_present(resource="subnet", name=subnet_cr_name): + wait_for_subnet_deletion(k8s=k8s, name=subnet_cr_name) + + +def _safe_delete_sg(grpc: GRPCClient, k8s: K8sClient, *, sg_id: str, sg_cr_name: str) -> None: + try: + grpc.delete_security_group(sg_id=sg_id) + except subprocess.CalledProcessError as exc: + combined = (exc.stderr or "") + (exc.stdout or "") + if not re.search(r"Code:\s*NotFound", combined): + logger.warning("SG %s teardown failed: %s", sg_id, combined.strip()) + return + if k8s.is_present(resource="securitygroup", name=sg_cr_name): + wait_for_security_group_deletion(k8s=k8s, name=sg_cr_name) diff --git a/tests/e2e/tests/references/test_cluster_baremetal_references.py b/tests/e2e/tests/references/test_cluster_baremetal_references.py new file mode 100644 index 0000000000..07cd8a5531 --- /dev/null +++ b/tests/e2e/tests/references/test_cluster_baremetal_references.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import logging +import re +import subprocess +from collections.abc import Generator +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import PRIVATE_API, PUBLIC_API, GRPCClient +from tests.core.helpers import assert_grpc_field_violation +from tests.core.osac_cli import OsacCLI +from tests.core.runner import env + +logger = logging.getLogger(__name__) + +_ENV_SKIP_PATTERNS = [re.compile(r"no host type"), re.compile(r"no instance type")] + + +def _create_cluster_or_skip(cli: OsacCLI, *, catalog_item: str, name: str, version: str) -> str: + try: + return cli.create_cluster_with_catalog_item(catalog_item=catalog_item, name=name, version=version) + except subprocess.CalledProcessError as exc: + output = (exc.stdout or "") + (exc.stderr or "") + for pat in _ENV_SKIP_PATTERNS: + if pat.search(output): + pytest.skip(f"Cluster creation not viable in this environment: {output.strip()}") + raise + + +@pytest.fixture(scope="module") +def cluster_template(private_grpc: GRPCClient) -> str: + configured = env("OSAC_CLUSTER_TEMPLATE", "") + if configured: + return configured + response: dict[str, Any] = private_grpc.call(service=f"{PRIVATE_API}.ClusterTemplates/List") + items = response.get("items", []) + assert items, "No ClusterTemplates found; set OSAC_CLUSTER_TEMPLATE or deploy a template" + return items[0]["metadata"]["name"] + + +@pytest.fixture(scope="module") +def cluster_version(grpc: GRPCClient, private_grpc: GRPCClient) -> Generator[str, None, None]: + configured = env("OSAC_CLUSTER_VERSION", "") + if configured: + yield configured + return + response: dict[str, Any] = grpc.call(service=f"{PUBLIC_API}.ClusterVersions/List") + items = response.get("items", []) + if items: + yield items[0]["metadata"]["name"] + return + tag = uuid4().hex[:8] + name = f"ref-cv-{tag}" + cv_response: dict[str, Any] = private_grpc.call( + service=f"{PRIVATE_API}.ClusterVersions/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "version": f"4.17.{int(tag, 16) % 10000}", + "image": "quay.io/openshift-release-dev/ocp-release:4.17.0-multi", + }, + } + }, + ) + cv_id = cv_response["object"]["id"] + try: + yield name + finally: + try: + private_grpc.call(service=f"{PRIVATE_API}.ClusterVersions/Delete", data={"id": cv_id}) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup ClusterVersion %s", cv_id) + + +@pytest.fixture(scope="module") +def bmi_template(private_grpc: GRPCClient) -> str: + configured = env("OSAC_BMI_TEMPLATE", "") + if configured: + return configured + response: dict[str, Any] = private_grpc.call(service=f"{PRIVATE_API}.BareMetalInstanceTemplates/List") + items = response.get("items", []) + assert items, "No BareMetalInstanceTemplates found; set OSAC_BMI_TEMPLATE or deploy a template" + return items[0]["metadata"]["name"] + + +class TestClusterBareMetalReferences: + """OSAC-3110: Cluster and bare metal resource reference tests.""" + + def test_cluster_provisioning_chain_by_name( + self, private_grpc: GRPCClient, grpc: GRPCClient, cli: OsacCLI, cluster_template: str, cluster_version: str + ): + tag = uuid4().hex[:8] + cat_name = f"ref-cl-cat-{tag}" + + cat_id = private_grpc.create_cluster_catalog_item(name=cat_name, template=cluster_template) + cluster_id: str | None = None + try: + cat_response = grpc.get_cluster_catalog_item(catalog_item_id=cat_id) + tmpl_ref = cat_response["object"]["template"] + assert tmpl_ref.get("name") == cluster_template + assert tmpl_ref.get("id"), "template.id should be auto-populated in catalog item" + + cluster_id = _create_cluster_or_skip( + cli, catalog_item=cat_name, name=f"ref-cl-{tag}", version=cluster_version + ) + cluster = grpc.get_cluster(cluster_id=cluster_id) + spec = cluster["object"]["spec"] + cat_ref = spec.get("catalog_item", spec.get("catalogItem", {})) + assert cat_ref.get("name") == cat_name + assert cat_ref.get("id") == cat_id + finally: + if cluster_id: + try: + cli.delete_cluster(uuid=cluster_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup cluster %s", cluster_id) + try: + private_grpc.delete_cluster_catalog_item(catalog_item_id=cat_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup cluster catalog item %s", cat_id) + + def test_baremetal_instance_chain_by_name(self, private_grpc: GRPCClient, grpc: GRPCClient, bmi_template: str): + tag = uuid4().hex[:8] + cat_name = f"ref-bmi-cat-{tag}" + + cat_id = private_grpc.create_baremetal_instance_catalog_item( + name=cat_name, title=cat_name, description="Reference test", template=bmi_template + ) + bmi_id: str | None = None + try: + cat_response: dict[str, Any] = private_grpc.call( + service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Get", data={"id": cat_id} + ) + tmpl_ref = cat_response["object"]["template"] + assert tmpl_ref.get("name") == bmi_template + assert tmpl_ref.get("id"), "template.id should be auto-populated in BMI catalog item" + + bmi_response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.BareMetalInstances/Create", + data={"object": {"metadata": {"name": f"ref-bmi-{tag}"}, "spec": {"catalog_item": {"name": cat_name}}}}, + ) + bmi_id = bmi_response["object"]["id"] + spec = bmi_response["object"]["spec"] + cat_ref = spec.get("catalog_item", spec.get("catalogItem", {})) + assert cat_ref.get("name") == cat_name + assert cat_ref.get("id") == cat_id + finally: + if bmi_id: + try: + grpc.delete_baremetal_instance(bmi_id=bmi_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup BMI %s", bmi_id) + try: + private_grpc.delete_baremetal_instance_catalog_item(item_id=cat_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup BMI catalog item %s", cat_id) + + def test_cross_tenant_cluster_template_reference( + self, + private_grpc: GRPCClient, + jwt_grpc_tenant1: GRPCClient, + jwt_cli_user: OsacCLI, + cluster_template: str, + cluster_version: str, + ): + tag = uuid4().hex[:8] + cat_name = f"ref-xt-cl-cat-{tag}" + + cat_id = private_grpc.create_cluster_catalog_item(name=cat_name, template=cluster_template) + cluster_id: str | None = None + try: + cluster_id = _create_cluster_or_skip( + jwt_cli_user, catalog_item=cat_name, name=f"ref-xt-cl-{tag}", version=cluster_version + ) + cluster = jwt_grpc_tenant1.get_cluster(cluster_id=cluster_id) + spec = cluster["object"]["spec"] + cat_ref = spec.get("catalog_item", spec.get("catalogItem", {})) + assert cat_ref.get("name") == cat_name + assert cat_ref.get("id") == cat_id + finally: + if cluster_id: + try: + jwt_cli_user.delete_cluster(uuid=cluster_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup cross-tenant cluster %s", cluster_id) + try: + private_grpc.delete_cluster_catalog_item(catalog_item_id=cat_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup cross-tenant catalog item %s", cat_id) + + def test_invalid_cluster_template_name_returns_error(self, private_grpc: GRPCClient): + tag = uuid4().hex[:8] + with pytest.raises(subprocess.CalledProcessError) as exc_info: + private_grpc.create_cluster_catalog_item(name=f"ref-bad-cat-{tag}", template="nonexistent-template") + assert_grpc_field_violation(exc_info, field_path="template") diff --git a/tests/e2e/tests/references/test_compute_references.py b/tests/e2e/tests/references/test_compute_references.py new file mode 100644 index 0000000000..a95b406ba2 --- /dev/null +++ b/tests/e2e/tests/references/test_compute_references.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import logging +import subprocess +from collections.abc import Generator +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import PRIVATE_API, PUBLIC_API, GRPCClient +from tests.core.helpers import ( + assert_grpc_field_violation, + wait_for_cr, + wait_for_deletion, + wait_for_provision, + wait_for_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import env + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="module") +def compute_template() -> str: + return env("OSAC_VM_TEMPLATE", "ocp-virt-vm") + + +@pytest.fixture(scope="module") +def ref_instance_type(private_grpc: GRPCClient) -> Generator[str, None, None]: + tag = uuid4().hex[:8] + name = f"ref-it-{tag}" + private_grpc.create_instance_type(name=name, cores=2, memory_gib=4) + yield name + try: + private_grpc.delete_instance_type(name=name) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup instance type %s", name) + + +@pytest.fixture(scope="module") +def ref_ci_catalog_item(private_grpc: GRPCClient, compute_template: str) -> Generator[str, None, None]: + tag = uuid4().hex[:8] + name = f"ref-ci-cat-{tag}" + cat_id = private_grpc.create_compute_instance_catalog_item(name=name, template=compute_template) + yield cat_id + try: + private_grpc.delete_compute_instance_catalog_item(catalog_item_id=cat_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup catalog item %s", cat_id) + + +def _ci_create_data( + name: str, cat_item_name: str, subnet_name: str, sg_name: str, instance_type: str +) -> dict[str, Any]: + return { + "object": { + "metadata": {"name": name}, + "spec": { + "catalog_item": {"name": cat_item_name}, + "instance_type": {"name": instance_type}, + "network_attachments": [{"subnet": {"name": subnet_name}, "security_groups": [{"name": sg_name}]}], + }, + } + } + + +class TestComputeReferences: + """OSAC-3100: Compute resource reference tests.""" + + def test_compute_instance_full_chain_by_name( + self, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + ref_subnet: dict[str, str], + ref_security_group: dict[str, str], + ref_ci_catalog_item: str, + ref_instance_type: str, + ): + tag = uuid4().hex[:8] + ci_name = f"ref-ci-chain-{tag}" + cat_item = grpc.get_compute_instance_catalog_item(catalog_item_id=ref_ci_catalog_item) + cat_item_name = cat_item["object"]["metadata"]["name"] + + response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.ComputeInstances/Create", + data=_ci_create_data( + ci_name, cat_item_name, ref_subnet["name"], ref_security_group["name"], ref_instance_type + ), + ) + ci_id = response["object"]["id"] + try: + spec = response["object"]["spec"] + cat_ref = spec.get("catalog_item", spec.get("catalogItem", {})) + assert cat_ref.get("name") == cat_item_name + assert cat_ref.get("id") == ref_ci_catalog_item + + att = spec.get("network_attachments", spec.get("networkAttachments", [{}]))[0] + subnet_ref = att.get("subnet", {}) + assert subnet_ref.get("name") == ref_subnet["name"] + assert subnet_ref.get("id") == ref_subnet["id"] + + sg_ref = att.get("security_groups", att.get("securityGroups", [{}]))[0] + assert sg_ref.get("name") == ref_security_group["name"] + assert sg_ref.get("id") == ref_security_group["id"] + finally: + grpc.delete_compute_instance(ci_id=ci_id) + try: + cr_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_id) + wait_for_deletion(k8s=k8s_hub_client, name=cr_name) + except (subprocess.CalledProcessError, AssertionError, TimeoutError): + logger.warning("Cleanup wait failed for compute instance %s", ci_id) + + def test_compute_instance_reaches_running_with_name_refs( + self, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + ref_subnet: dict[str, str], + ref_security_group: dict[str, str], + ref_ci_catalog_item: str, + ref_instance_type: str, + ): + tag = uuid4().hex[:8] + ci_name = f"ref-ci-run-{tag}" + cat_item = grpc.get_compute_instance_catalog_item(catalog_item_id=ref_ci_catalog_item) + cat_item_name = cat_item["object"]["metadata"]["name"] + + response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.ComputeInstances/Create", + data=_ci_create_data( + ci_name, cat_item_name, ref_subnet["name"], ref_security_group["name"], ref_instance_type + ), + ) + ci_id = response["object"]["id"] + cr_name = None + try: + cr_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_id) + wait_for_provision(k8s=k8s_hub_client, name=cr_name) + wait_for_running(k8s=k8s_hub_client, name=cr_name) + + phase = k8s_hub_client.get_compute_instance_phase(name=cr_name) + assert phase == "Running" + finally: + grpc.delete_compute_instance(ci_id=ci_id) + if cr_name: + try: + wait_for_deletion(k8s=k8s_hub_client, name=cr_name) + except (subprocess.CalledProcessError, AssertionError, TimeoutError): + logger.warning("Cleanup wait failed for compute instance %s", ci_id) + + def test_invalid_subnet_name_returns_array_indexed_field_path( + self, grpc: GRPCClient, ref_subnet: dict[str, str], ref_ci_catalog_item: str, ref_instance_type: str + ): + tag = uuid4().hex[:8] + cat_item = grpc.get_compute_instance_catalog_item(catalog_item_id=ref_ci_catalog_item) + cat_item_name = cat_item["object"]["metadata"]["name"] + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.call( + service=f"{PUBLIC_API}.ComputeInstances/Create", + data={ + "object": { + "metadata": {"name": f"ref-ci-bad-sg-{tag}"}, + "spec": { + "catalog_item": {"name": cat_item_name}, + "instance_type": {"name": ref_instance_type}, + "network_attachments": [ + { + "subnet": {"name": ref_subnet["name"]}, + "security_groups": [{"name": "nonexistent-sg"}], + } + ], + }, + } + }, + ) + assert_grpc_field_violation(exc_info, field_path="security_groups") + + def test_cross_tenant_template_reference( + self, + jwt_grpc_tenant1: GRPCClient, + private_grpc: GRPCClient, + compute_template: str, + ref_subnet: dict[str, str], + ref_security_group: dict[str, str], + ref_instance_type: str, + ): + tag = uuid4().hex[:8] + cat_name = f"ref-xt-cat-{tag}" + cat_id = private_grpc.create_compute_instance_catalog_item(name=cat_name, template=compute_template) + ci_id = None + try: + response: dict[str, Any] = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.ComputeInstances/Create", + data=_ci_create_data( + f"ref-ci-xt-{tag}", cat_name, ref_subnet["name"], ref_security_group["name"], ref_instance_type + ), + ) + ci_id = response["object"]["id"] + spec = response["object"]["spec"] + cat_ref = spec.get("catalog_item", spec.get("catalogItem", {})) + assert cat_ref.get("name") == cat_name + assert cat_ref.get("id") == cat_id + finally: + if ci_id: + try: + jwt_grpc_tenant1.delete_compute_instance(ci_id=ci_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup compute instance %s", ci_id) + try: + private_grpc.delete_compute_instance_catalog_item(catalog_item_id=cat_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup cross-tenant catalog item %s", cat_id) + + def test_instance_type_deprecation_replacement_resolves(self, private_grpc: GRPCClient): + tag = uuid4().hex[:8] + active_name = f"ref-it-active-{tag}" + deprecated_name = f"ref-it-depr-{tag}" + + active_id = private_grpc.create_instance_type(name=active_name, cores=2, memory_gib=4) + private_grpc.create_instance_type(name=deprecated_name, cores=2, memory_gib=4) + try: + private_grpc.call( + service=f"{PRIVATE_API}.InstanceTypes/Update", + data={ + "object": { + "id": deprecated_name, + "spec": { + "state": "INSTANCE_TYPE_STATE_DEPRECATED", + "deprecation": {"replacement": {"name": active_name}}, + }, + }, + "updateMask": {"paths": ["spec.state", "spec.deprecation"]}, + }, + ) + + result = private_grpc.get_instance_type(name=deprecated_name) + deprecation = result["object"]["spec"]["deprecation"] + replacement = deprecation["replacement"] + assert replacement.get("name") == active_name + assert replacement.get("id") == active_id + finally: + try: + private_grpc.delete_instance_type(name=deprecated_name) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup deprecated instance type %s", deprecated_name) + try: + private_grpc.delete_instance_type(name=active_name) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup active instance type %s", active_name) diff --git a/tests/e2e/tests/references/test_iam_references.py b/tests/e2e/tests/references/test_iam_references.py new file mode 100644 index 0000000000..fce2d4fb36 --- /dev/null +++ b/tests/e2e/tests/references/test_iam_references.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import contextlib +import logging +import subprocess +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import PUBLIC_API, GRPCClient +from tests.core.helpers import assert_grpc_field_violation +from tests.core.osac_cli import OsacCLI + +logger = logging.getLogger(__name__) + +TENANT_ADMIN_USER = "tenant1_admin" +TENANT_USER = "tenant1_user" +TENANT_ADMIN_ROLE = "tenant-admin" + + +@pytest.fixture(scope="module", autouse=True) +def _register_test_users(jwt_grpc_tenant1: GRPCClient, jwt_cli_admin: OsacCLI) -> None: + """Trigger user auto-registration by making authenticated API calls.""" + with contextlib.suppress(subprocess.CalledProcessError): + jwt_grpc_tenant1.call(service=f"{PUBLIC_API}.RoleBindings/List") + jwt_cli_admin.get_unchecked("rolebinding") + + +def _skip_if_users_not_found(exc: subprocess.CalledProcessError) -> None: + stderr = exc.stderr or "" + if "not found" in stderr.lower(): + pytest.skip("Test users not registered in OSAC; Keycloak user sync may be pending") + + +class TestIAMReferences: + """OSAC-3114: IAM resource reference tests.""" + + def test_role_binding_with_role_and_users_by_name(self, grpc: GRPCClient): + tag = uuid4().hex[:8] + rb_name = f"ref-rb-{tag}" + + try: + rb_id = grpc.create_role_binding(name=rb_name, role_name=TENANT_ADMIN_ROLE, user_names=[TENANT_ADMIN_USER]) + except subprocess.CalledProcessError as exc: + _skip_if_users_not_found(exc) + raise + try: + response = grpc.get_role_binding(role_binding_id=rb_id) + spec = response["object"]["spec"] + + role_ref = spec["role"] + assert role_ref.get("name") == TENANT_ADMIN_ROLE + assert role_ref.get("id"), "role.id should be auto-populated" + + users = spec["users"] + assert len(users) >= 1 + user_ref = users[0] + assert user_ref.get("name") == TENANT_ADMIN_USER + assert user_ref.get("id"), "user.id should be auto-populated" + finally: + try: + grpc.delete_role_binding(role_binding_id=rb_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup role binding %s", rb_id) + + def test_project_membership_by_name(self, grpc: GRPCClient): + tag = uuid4().hex[:8] + pm_name = f"ref-pm-{tag}" + + try: + pm_id = grpc.create_project_membership(name=pm_name, user_names=[TENANT_USER]) + except subprocess.CalledProcessError as exc: + _skip_if_users_not_found(exc) + raise + try: + response: dict[str, Any] = grpc.call(service=f"{PUBLIC_API}.ProjectMemberships/Get", data={"id": pm_id}) + spec = response["object"]["spec"] + + users = spec["users"] + assert len(users) >= 1 + user_ref = users[0] + assert user_ref.get("name") == TENANT_USER + assert user_ref.get("id"), "user.id should be auto-populated" + finally: + try: + grpc.delete_project_membership(membership_id=pm_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup project membership %s", pm_id) + + def test_invalid_role_name_returns_error(self, grpc: GRPCClient): + tag = uuid4().hex[:8] + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_role_binding( + name=f"ref-bad-rb-{tag}", role_name="nonexistent-role", user_names=[TENANT_ADMIN_USER] + ) + assert_grpc_field_violation(exc_info, field_path="role") + + def test_role_binding_with_multiple_users_by_name(self, grpc: GRPCClient): + tag = uuid4().hex[:8] + rb_name = f"ref-rb-multi-{tag}" + + try: + rb_id = grpc.create_role_binding( + name=rb_name, role_name=TENANT_ADMIN_ROLE, user_names=[TENANT_ADMIN_USER, TENANT_USER] + ) + except subprocess.CalledProcessError as exc: + _skip_if_users_not_found(exc) + raise + try: + response = grpc.get_role_binding(role_binding_id=rb_id) + users = response["object"]["spec"]["users"] + resolved_names = {u.get("name") for u in users} + assert TENANT_ADMIN_USER in resolved_names + assert TENANT_USER in resolved_names + for u in users: + assert u.get("id"), f"user.id should be auto-populated for {u.get('name')}" + finally: + try: + grpc.delete_role_binding(role_binding_id=rb_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup role binding %s", rb_id) diff --git a/tests/e2e/tests/references/test_ip_management_references.py b/tests/e2e/tests/references/test_ip_management_references.py new file mode 100644 index 0000000000..34d22463bc --- /dev/null +++ b/tests/e2e/tests/references/test_ip_management_references.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import logging +import subprocess +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import PRIVATE_API, PUBLIC_API, GRPCClient +from tests.core.helpers import assert_grpc_field_violation + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="module") +def ref_eip_pool(private_grpc: GRPCClient) -> str: + response: dict[str, Any] = private_grpc.call(service=f"{PRIVATE_API}.ExternalIPPools/List") + items = response.get("items", []) + if not items: + pytest.skip("No ExternalIPPools found; deploy at least one pool") + return items[0]["metadata"]["name"] + + +@pytest.fixture(scope="module") +def ref_eip_pool_id(private_grpc: GRPCClient, ref_eip_pool: str) -> str: + items = private_grpc.list_with_filter( + service=f"{PRIVATE_API}.ExternalIPPools/List", filter_expr=f'this.metadata.name == "{ref_eip_pool}"' + ) + assert items, f"ExternalIPPool '{ref_eip_pool}' not found" + return items[0]["id"] + + +class TestIPManagementReferences: + """OSAC-3105: IP management resource reference tests.""" + + def test_external_ip_from_pool_by_name(self, grpc: GRPCClient, ref_eip_pool: str, ref_eip_pool_id: str): + tag = uuid4().hex[:8] + eip_name = f"ref-eip-{tag}" + + response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.ExternalIPs/Create", + data={"object": {"metadata": {"name": eip_name}, "spec": {"pool": {"name": ref_eip_pool}}}}, + ) + eip_id = response["object"]["id"] + try: + pool_ref = response["object"]["spec"]["pool"] + assert pool_ref.get("name") == ref_eip_pool + assert pool_ref.get("id") == ref_eip_pool_id + finally: + grpc.delete_external_ip(external_ip_id=eip_id) + + def test_nat_gateway_by_name( + self, grpc: GRPCClient, ref_virtual_network: dict[str, str], ref_eip_pool: str, ref_test_run_id: str + ): + tag = uuid4().hex[:8] + eip_name = f"ref-nat-eip-{tag}" + + eip_response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.ExternalIPs/Create", + data={"object": {"metadata": {"name": eip_name}, "spec": {"pool": {"name": ref_eip_pool}}}}, + ) + eip_id = eip_response["object"]["id"] + nat_id: str | None = None + try: + nat_name = f"ref-nat-{tag}" + nat_id = grpc.create_nat_gateway( + name=nat_name, virtual_network_name=ref_virtual_network["name"], external_ip_name=eip_name + ) + + nat_response = grpc.call(service=f"{PUBLIC_API}.NATGateways/Get", data={"id": nat_id}) + spec = nat_response["object"]["spec"] + + vn_ref = spec["virtual_network"] + assert vn_ref.get("name") == ref_virtual_network["name"] + assert vn_ref.get("id") == ref_virtual_network["id"] + + eip_ref = spec["external_ip"] + assert eip_ref.get("name") == eip_name + assert eip_ref.get("id") == eip_id + finally: + if nat_id: + try: + grpc.delete_nat_gateway(nat_gateway_id=nat_id) + except subprocess.CalledProcessError: + logger.warning("Failed to cleanup NATGateway %s", nat_id) + grpc.delete_external_ip(external_ip_id=eip_id) + + def test_invalid_attachment_target_returns_field_path(self, grpc: GRPCClient, ref_eip_pool: str): + tag = uuid4().hex[:8] + eip_name = f"ref-att-eip-{tag}" + + eip_response: dict[str, Any] = grpc.call( + service=f"{PUBLIC_API}.ExternalIPs/Create", + data={"object": {"metadata": {"name": eip_name}, "spec": {"pool": {"name": ref_eip_pool}}}}, + ) + eip_id = eip_response["object"]["id"] + try: + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.call( + service=f"{PUBLIC_API}.ExternalIPAttachments/Create", + data={ + "object": { + "metadata": {"name": f"ref-att-bad-{tag}"}, + "spec": {"external_ip": {"name": eip_name}, "compute_instance": {"name": "nonexistent-ci"}}, + } + }, + ) + assert_grpc_field_violation(exc_info, field_path="compute_instance") + finally: + grpc.delete_external_ip(external_ip_id=eip_id) + + def test_cross_tenant_pool_reference(self, jwt_grpc_tenant1: GRPCClient, ref_eip_pool: str, ref_eip_pool_id: str): + tag = uuid4().hex[:8] + eip_name = f"ref-xt-eip-{tag}" + + response: dict[str, Any] = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.ExternalIPs/Create", + data={"object": {"metadata": {"name": eip_name}, "spec": {"pool": {"name": ref_eip_pool}}}}, + ) + eip_id = response["object"]["id"] + try: + pool_ref = response["object"]["spec"]["pool"] + assert pool_ref.get("name") == ref_eip_pool + assert pool_ref.get("id") == ref_eip_pool_id + finally: + jwt_grpc_tenant1.delete_external_ip(external_ip_id=eip_id) diff --git a/tests/e2e/tests/references/test_networking_references.py b/tests/e2e/tests/references/test_networking_references.py new file mode 100644 index 0000000000..2b9f1720ce --- /dev/null +++ b/tests/e2e/tests/references/test_networking_references.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import logging +import subprocess + +import pytest + +from tests.core.grpc_client import PUBLIC_API, GRPCClient +from tests.core.helpers import ( + assert_grpc_field_violation, + wait_for_security_group_cr, + wait_for_security_group_deletion, + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, +) +from tests.core.k8s_client import K8sClient + +logger = logging.getLogger(__name__) + + +class TestNetworkingReferences: + """OSAC-3095: Networking resource reference tests.""" + + def test_create_virtual_network_with_network_class_by_name( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, ref_network_class: str, ref_test_run_id: str + ): + vn_name = f"ref-vn-nc-{ref_test_run_id}" + response = grpc.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": { + "metadata": {"name": vn_name}, + "spec": {"network_class": {"name": ref_network_class}, "ipv4_cidr": "10.211.0.0/16"}, + } + }, + ) + vn_id = response["object"]["id"] + try: + vn = grpc.get_virtual_network(vn_id=vn_id) + nc_ref = vn["object"]["spec"].get("network_class", vn["object"]["spec"].get("networkClass", {})) + assert nc_ref.get("name") == ref_network_class or nc_ref.get("id"), ( + "network_class reference should contain name or resolved id" + ) + finally: + grpc.delete_virtual_network(vn_id=vn_id) + try: + vn_cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=vn_cr_name) + except (subprocess.CalledProcessError, AssertionError, TimeoutError): + logger.warning("Cleanup wait failed for virtual network %s", vn_id) + + def test_create_subnet_with_virtual_network_by_name( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, ref_virtual_network: dict[str, str], ref_test_run_id: str + ): + subnet_name = f"ref-sub-vn-{ref_test_run_id}" + response = grpc.call( + service=f"{PUBLIC_API}.Subnets/Create", + data={ + "object": { + "metadata": {"name": subnet_name}, + "spec": {"virtual_network": {"name": ref_virtual_network["name"]}, "ipv4_cidr": "10.210.200.0/24"}, + } + }, + ) + subnet_id = response["object"]["id"] + try: + subnet = grpc.call(service=f"{PUBLIC_API}.Subnets/Get", data={"id": subnet_id}) + vn_ref = subnet["object"]["spec"].get("virtual_network", subnet["object"]["spec"].get("virtualNetwork", {})) + assert vn_ref.get("name") == ref_virtual_network["name"] or vn_ref.get("id") == ref_virtual_network["id"] + finally: + grpc.delete_subnet(subnet_id=subnet_id) + try: + subnet_cr_name = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_id) + wait_for_subnet_deletion(k8s=k8s_hub_client, name=subnet_cr_name) + except (subprocess.CalledProcessError, AssertionError, TimeoutError): + logger.warning("Cleanup wait failed for subnet %s", subnet_id) + + def test_create_security_group_with_virtual_network_by_name( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, ref_virtual_network: dict[str, str], ref_test_run_id: str + ): + sg_name = f"ref-sg-vn-{ref_test_run_id}" + response = grpc.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", + data={ + "object": { + "metadata": {"name": sg_name}, + "spec": {"virtual_network": {"name": ref_virtual_network["name"]}}, + } + }, + ) + sg_id = response["object"]["id"] + try: + sg = grpc.call(service=f"{PUBLIC_API}.SecurityGroups/Get", data={"id": sg_id}) + vn_ref = sg["object"]["spec"].get("virtual_network", sg["object"]["spec"].get("virtualNetwork", {})) + assert vn_ref.get("name") == ref_virtual_network["name"] or vn_ref.get("id") == ref_virtual_network["id"] + finally: + grpc.delete_security_group(sg_id=sg_id) + try: + sg_cr_name = wait_for_security_group_cr(k8s=k8s_hub_client, uuid=sg_id) + wait_for_security_group_deletion(k8s=k8s_hub_client, name=sg_cr_name) + except (subprocess.CalledProcessError, AssertionError, TimeoutError): + logger.warning("Cleanup wait failed for security group %s", sg_id) + + def test_invalid_virtual_network_reference_returns_field_path(self, grpc: GRPCClient, ref_test_run_id: str): + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.call( + service=f"{PUBLIC_API}.Subnets/Create", + data={ + "object": { + "metadata": {"name": f"ref-bad-vn-{ref_test_run_id}"}, + "spec": {"virtual_network": {"name": "nonexistent-vn"}, "ipv4_cidr": "10.212.0.0/24"}, + } + }, + ) + assert_grpc_field_violation(exc_info, field_path="spec.virtual_network") + + def test_cel_filter_by_virtual_network_name( + self, grpc: GRPCClient, ref_subnet: dict[str, str], ref_virtual_network: dict[str, str] + ): + vn_name = ref_virtual_network["name"] + items = grpc.list_with_filter( + service=f"{PUBLIC_API}.Subnets/List", filter_expr=f'this.spec.virtual_network.name == "{vn_name}"' + ) + found_ids = [item["id"] for item in items] + assert ref_subnet["id"] in found_ids, ( + f"Expected subnet {ref_subnet['id']} in filter results for VN name '{vn_name}', got {found_ids}" + ) + + def test_cross_tenant_network_class_reference( + self, jwt_grpc_tenant1: GRPCClient, k8s_hub_client: K8sClient, ref_network_class: str, ref_test_run_id: str + ): + vn_name = f"ref-vn-xt-{ref_test_run_id}" + response = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": { + "metadata": {"name": vn_name}, + "spec": {"network_class": {"name": ref_network_class}, "ipv4_cidr": "10.213.0.0/16"}, + } + }, + ) + vn_id = response["object"]["id"] + try: + vn = jwt_grpc_tenant1.get_virtual_network(vn_id=vn_id) + nc_ref = vn["object"]["spec"].get("network_class", vn["object"]["spec"].get("networkClass", {})) + assert nc_ref.get("name") == ref_network_class or nc_ref.get("id"), ( + "network_class reference should contain name or resolved id for cross-tenant reference" + ) + finally: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) + try: + vn_cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=vn_cr_name) + except (subprocess.CalledProcessError, AssertionError, TimeoutError): + logger.warning("Cleanup wait failed for virtual network %s", vn_id) diff --git a/tests/e2e/tests/storage/__init__.py b/tests/e2e/tests/storage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/storage/conftest.py b/tests/e2e/tests/storage/conftest.py new file mode 100644 index 0000000000..17bc01120a --- /dev/null +++ b/tests/e2e/tests/storage/conftest.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import os + +import pytest + +from tests.core.runner import env, run_unchecked + + +def _storage_controller_configured(namespace: str) -> bool: + """Auto-detect whether the storage controller is enabled. + + Checks two sources: direct env vars on the operator deployment + and envFrom secrets (e.g. osac-config) that may inject the var at runtime. + """ + output, rc = run_unchecked( + "kubectl", + "--as", + "system:admin", + "get", + "deployment", + "-n", + namespace, + "-l", + "control-plane=controller-manager", + "-o", + "jsonpath={.items[*].spec.template.spec.containers[*].env[*].name}", + ) + if rc == 0 and ( + "OSAC_STORAGE_BACKEND_AAP_PROVISION_TEMPLATE" in output or "OSAC_ENABLE_STORAGE_CONTROLLER" in output + ): + return True + + # Check envFrom secrets for the storage env var + secret_names, rc = run_unchecked( + "kubectl", + "--as", + "system:admin", + "get", + "deployment", + "-n", + namespace, + "-l", + "control-plane=controller-manager", + "-o", + "jsonpath={.items[*].spec.template.spec.containers[*].envFrom[*].secretRef.name}", + ) + if rc != 0: + return False + for secret_name in secret_names.strip().split(): + data_keys, rc = run_unchecked( + "kubectl", "--as", "system:admin", "get", "secret", secret_name, "-n", namespace, "-o", "jsonpath={.data}" + ) + if rc == 0 and "OSAC_STORAGE_BACKEND_AAP_PROVISION_TEMPLATE" in data_keys: + return True + if rc == 0 and "OSAC_ENABLE_STORAGE_CONTROLLER" in data_keys: + return True + return False + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + namespace: str = env("OSAC_NAMESPACE", "osac-devel") + if not _storage_controller_configured(namespace): + skip_storage = pytest.mark.skip( + reason="Storage controller not configured" + " (OSAC_ENABLE_STORAGE_CONTROLLER not found in operator deployment or envFrom secrets)" + ) + for item in items: + if "storage" in str(item.fspath): + item.add_marker(skip_storage) + return + + if not os.environ.get("OSAC_PULL_SECRET_PATH"): + skip_caas = pytest.mark.skip(reason="CaaS infrastructure not configured (OSAC_PULL_SECRET_PATH not set)") + for item in items: + if "test_caas_cluster_storage" in str(item.fspath): + item.add_marker(skip_caas) + + +@pytest.fixture(scope="session") +def storage_config_namespace() -> str: + return env("OSAC_STORAGE_CONFIG_NAMESPACE", "osac-system") + + +@pytest.fixture(scope="session") +def cluster_template() -> str: + return env("OSAC_CLUSTER_TEMPLATE", "ocp-ci-small") + + +@pytest.fixture(scope="session") +def pull_secret_path() -> str: + return env("OSAC_PULL_SECRET_PATH") + + +@pytest.fixture(scope="session") +def ssh_public_key_path() -> str: + return env("OSAC_SSH_PUBLIC_KEY_PATH", os.path.expanduser("~/.ssh/id_rsa.pub")) diff --git a/tests/e2e/tests/storage/test_caas_cluster_storage.py b/tests/e2e/tests/storage/test_caas_cluster_storage.py new file mode 100644 index 0000000000..56cf729da8 --- /dev/null +++ b/tests/e2e/tests/storage/test_caas_cluster_storage.py @@ -0,0 +1,140 @@ +"""E2E test for CaaS cluster storage provisioning lifecycle (OSAC-1123). + +Validates the CaaS storage path (Stage 3) of the Storage Controller: +1. ClusterOrder reaches Ready with tenant annotation +2. Storage controller adds cluster-storage finalizer to ClusterOrder +3. AAP provisioning job installs CSI + StorageClass on CaaS cluster +4. ClusterStorageReady condition set on ClusterOrder +5. Tenant.status.clusterStorage[] updated with cluster entry +6. Teardown removes finalizer and clusterStorage entry on ClusterOrder deletion + +Requires: storage controller enabled, AAP templates configured, CaaS +infrastructure configured (cluster template, pull secret). +""" + +from __future__ import annotations + +import contextlib +import json +import textwrap +from pathlib import Path +from uuid import uuid4 + +from tests.catalog.conftest import unique_name +from tests.core.helpers import ( + wait_for_cluster_deletion, + wait_for_cluster_order_condition, + wait_for_cluster_order_cr, + wait_for_cluster_ready, + wait_for_tenant_cluster_storage_entry, + wait_for_tenant_cluster_storage_entry_removed, + wait_for_tenant_condition, + wait_for_tenant_cr, + wait_for_tenant_deletion, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +_NAMESPACE_MANIFEST = textwrap.dedent("""\ + apiVersion: v1 + kind: Namespace + metadata: + name: {name} +""") + +_TENANT_MANIFEST = textwrap.dedent("""\ + apiVersion: osac.openshift.io/v1alpha1 + kind: Tenant + metadata: + name: {name} + namespace: {namespace} + spec: {{}} +""") + + +def test_caas_cluster_storage_lifecycle( + k8s_hub_client: K8sClient, cli: OsacCLI, cluster_template: str, pull_secret_path: str, ssh_public_key_path: str +) -> None: + tenant_name: str = f"test-caas-storage-{uuid4().hex[:8]}" + namespace: str = k8s_hub_client.namespace + co_name: str | None = None + cluster_uuid: str | None = None + + try: + # --- Setup: create Tenant and its namespace --- + k8s_hub_client.apply(manifest=_NAMESPACE_MANIFEST.format(name=tenant_name)) + k8s_hub_client.apply(manifest=_TENANT_MANIFEST.format(name=tenant_name, namespace=namespace)) + wait_for_tenant_cr(k8s=k8s_hub_client, name=tenant_name) + + # --- VMaaS storage stages must complete before CaaS path runs --- + wait_for_tenant_condition(k8s=k8s_hub_client, name=tenant_name, condition_type="StorageBackendReady") + + # --- Create ClusterOrder and associate with our Tenant --- + name = unique_name("e2e-cluster") + cluster_uuid = cli.create_cluster( + name=name, + template=cluster_template, + template_parameter_files={"pull_secret": pull_secret_path}, + template_parameters={"ssh_public_key": Path(ssh_public_key_path).read_text().strip()}, + ) + co_name = wait_for_cluster_order_cr(k8s=k8s_hub_client, uuid=cluster_uuid) + + _, rc = k8s_hub_client.patch( + resource="clusterorder", + name=co_name, + patch=json.dumps({"metadata": {"annotations": {"osac.openshift.io/tenant": tenant_name}}}), + ) + assert rc == 0, f"Failed to patch ClusterOrder {co_name} with tenant annotation" + + # --- Wait for cluster provisioning --- + wait_for_cluster_ready(k8s=k8s_hub_client, name=co_name) + + # --- Verify CaaS storage provisioning --- + _verify_provisioning(k8s=k8s_hub_client, tenant_name=tenant_name, co_name=co_name) + + finally: + # --- Teardown: delete ClusterOrder and verify storage cleanup --- + if cluster_uuid is not None: + with contextlib.suppress(Exception): + cli.delete_cluster(uuid=cluster_uuid) + if co_name is not None and k8s_hub_client.is_present(resource="clusterorder", name=co_name): + _verify_teardown(k8s=k8s_hub_client, tenant_name=tenant_name, co_name=co_name) + + if k8s_hub_client.is_present(resource="tenant", name=tenant_name): + k8s_hub_client.delete(resource="tenant", name=tenant_name, wait=False) + wait_for_tenant_deletion(k8s=k8s_hub_client, name=tenant_name) + + if k8s_hub_client.is_present(resource="namespace", name=tenant_name): + k8s_hub_client.delete(resource="namespace", name=tenant_name, wait=False) + + +def _verify_provisioning(*, k8s: K8sClient, tenant_name: str, co_name: str) -> None: + # --- cluster-storage finalizer added to ClusterOrder --- + poll_until( + fn=lambda: "osac.openshift.io/cluster-storage" in k8s.get_cluster_order_finalizers(name=co_name, checked=False), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"cluster-storage finalizer on {co_name}", + ) + + # --- ClusterStorageReady=True on ClusterOrder --- + wait_for_cluster_order_condition(k8s=k8s, name=co_name, condition_type="ClusterStorageReady") + + # --- Tenant.status.clusterStorage has entry for this cluster --- + entry = wait_for_tenant_cluster_storage_entry(k8s=k8s, tenant_name=tenant_name, cluster_name=co_name) + assert entry.get("ready") is True, f"clusterStorage entry for {co_name} is not ready: {entry}" + + +def _verify_teardown(*, k8s: K8sClient, tenant_name: str, co_name: str) -> None: + # --- ClusterOrder fully deleted (storage finalizer released) --- + wait_for_cluster_deletion(k8s=k8s, name=co_name) + + # --- Tenant.status.clusterStorage entry removed --- + if k8s.is_present(resource="tenant", name=tenant_name): + wait_for_tenant_cluster_storage_entry_removed(k8s=k8s, tenant_name=tenant_name, cluster_name=co_name) + + # --- Tenant still healthy --- + phase: str = k8s.get_tenant_phase(name=tenant_name, checked=False) + assert phase != "Failed", f"Tenant {tenant_name} entered Failed phase after ClusterOrder deletion" diff --git a/tests/e2e/tests/storage/test_tenant_storage_lifecycle.py b/tests/e2e/tests/storage/test_tenant_storage_lifecycle.py new file mode 100644 index 0000000000..41da0ca5b3 --- /dev/null +++ b/tests/e2e/tests/storage/test_tenant_storage_lifecycle.py @@ -0,0 +1,109 @@ +"""E2E test for tenant storage onboarding lifecycle (OSAC-77). + +Validates the full storage provisioning lifecycle managed by the Storage Controller: +Stage 1 (StorageBackendReady) -> Stage 2 (ClusterStorageReady) -> ordered teardown. + +Requires: storage controller enabled, AAP templates configured, storage backend +reachable (real VAST or mock VMS server). +""" + +from __future__ import annotations + +import textwrap +from uuid import uuid4 + +from tests.core.helpers import wait_for_tenant_condition, wait_for_tenant_cr, wait_for_tenant_deletion +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + +_NAMESPACE_MANIFEST = textwrap.dedent("""\ + apiVersion: v1 + kind: Namespace + metadata: + name: {name} +""") + +_TENANT_MANIFEST = textwrap.dedent("""\ + apiVersion: osac.openshift.io/v1alpha1 + kind: Tenant + metadata: + name: {name} + namespace: {namespace} + spec: {{}} +""") + + +def test_tenant_storage_lifecycle(k8s_hub_client: K8sClient, storage_config_namespace: str) -> None: + tenant_name: str = f"test-storage-{uuid4().hex[:8]}" + namespace: str = k8s_hub_client.namespace + + k8s_hub_client.apply(manifest=_NAMESPACE_MANIFEST.format(name=tenant_name)) + k8s_hub_client.apply(manifest=_TENANT_MANIFEST.format(name=tenant_name, namespace=namespace)) + + try: + _verify_provisioning( + k8s=k8s_hub_client, tenant_name=tenant_name, storage_config_namespace=storage_config_namespace + ) + finally: + _trigger_teardown(k8s=k8s_hub_client, tenant_name=tenant_name) + _verify_teardown(k8s=k8s_hub_client, tenant_name=tenant_name) + + +def _verify_provisioning(*, k8s: K8sClient, tenant_name: str, storage_config_namespace: str) -> None: + # --- Tenant CR exists (we created it directly) --- + wait_for_tenant_cr(k8s=k8s, name=tenant_name) + + # --- Storage finalizer set (requires Phase=Ready first, then storage controller reconcile) --- + poll_until( + fn=lambda: "osac.openshift.io/storage" in k8s.get_tenant_finalizers(name=tenant_name), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"Storage finalizer on {tenant_name}", + ) + + # --- Stage 1: StorageBackendReady --- + wait_for_tenant_condition(k8s=k8s, name=tenant_name, condition_type="StorageBackendReady") + + hub_secret_count: int = k8s.count_secrets_by_tenant(tenant_name=tenant_name, namespace=storage_config_namespace) + assert hub_secret_count >= 1, ( + f"Expected hub Secret for tenant {tenant_name} in {storage_config_namespace}, found {hub_secret_count}" + ) + + # --- Stage 2: ClusterStorageReady --- + wait_for_tenant_condition(k8s=k8s, name=tenant_name, condition_type="ClusterStorageReady") + + # --- status.storageClasses populated with name + tier --- + resolved_scs: list[dict[str, str]] = k8s.get_tenant_storage_classes(name=tenant_name) + assert len(resolved_scs) >= 1, f"Expected status.storageClasses populated, got {resolved_scs}" + for sc in resolved_scs: + assert sc.get("name"), f"Resolved SC missing name: {sc}" + assert sc.get("tier"), f"Resolved SC missing tier: {sc}" + + # --- Resolved SCs reference real StorageClasses --- + for sc in resolved_scs: + assert k8s.is_present(resource="storageclass", name=sc["name"]), ( + f"status.storageClasses references non-existent SC: {sc['name']}" + ) + + # --- If per-tenant SCs exist, verify labels --- + tenant_sc_names: list[str] = k8s.list_storage_class_names_by_tenant(tenant_name=tenant_name) + for sc_name in tenant_sc_names: + labels: dict[str, str] = k8s.get_storage_class_labels(name=sc_name) + assert labels.get("osac.openshift.io/storage-tier"), ( + f"StorageClass {sc_name} missing osac.openshift.io/storage-tier label" + ) + + +def _trigger_teardown(*, k8s: K8sClient, tenant_name: str) -> None: + k8s.delete(resource="tenant", name=tenant_name, wait=False) + + +def _verify_teardown(*, k8s: K8sClient, tenant_name: str) -> None: + # --- Tenant CR fully deleted (storage finalizer released = all cleanup done) --- + wait_for_tenant_deletion(k8s=k8s, name=tenant_name) + + # Finalizer removal confirms deprovision jobs succeeded (SCs + Secrets cleaned up by AAP). + + # --- Clean up tenant namespace --- + k8s.delete(resource="namespace", name=tenant_name, wait=False) diff --git a/tests/e2e/tests/vmaas/__init__.py b/tests/e2e/tests/vmaas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/vmaas/conftest.py b/tests/e2e/tests/vmaas/conftest.py new file mode 100644 index 0000000000..9f11c38bbd --- /dev/null +++ b/tests/e2e/tests/vmaas/conftest.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import os +import subprocess +import time +import uuid +from collections.abc import Iterator + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_subnet_ready, + wait_for_tenant_condition, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import env + +DEFAULT_IT_CORES: int = 2 +DEFAULT_IT_MEMORY_GIB: int = 4 + +# Shared DiskImage source for CLI-based ComputeInstance scaffolding. Pinned (not +# :latest) to match the source_ref the gRPC AC tests provision and for CI stability. +DEFAULT_DISK_IMAGE_SOURCE_REF: str = "quay.io/containerdisks/fedora:41" + + +@pytest.fixture(scope="session") +def k8s_virt_client(namespace: str) -> K8sClient: + vm_kubeconfig: str = os.environ["OSAC_VM_KUBECONFIG"] + return K8sClient(namespace=namespace, kubeconfig=vm_kubeconfig) + + +@pytest.fixture(scope="session") +def vm_template() -> str: + return env("OSAC_VM_TEMPLATE", "ocp-virt-vm") + + +@pytest.fixture(scope="session") +def network_class() -> str: + return env("OSAC_NETWORK_CLASS", "cudn-net") + + +@pytest.fixture(scope="session") +def test_run_id() -> str: + """Unique ID for this test run to avoid resource name conflicts.""" + return str(uuid.uuid4())[:8] + + +@pytest.fixture(scope="session") +def default_networking( + grpc: GRPCClient, k8s_hub_client: K8sClient, network_class: str, test_run_id: str +) -> dict[str, str]: + """ + Create default networking resources (VirtualNetwork + Subnet) for VM tests. + + Uses unique names per test run to avoid conflicts with leftover resources + from interrupted previous runs. + + Returns: + dict with keys: 'virtual_network_id', 'subnet_id' + """ + # Track created resources for cleanup on setup failure + vn_id: str | None = None + vn_cr_name: str | None = None + subnet_id: str | None = None + subnet_cr_name: str | None = None + + try: + # Create virtual network with unique name + vn_name = f"test-vn-{test_run_id}" + print(f"\nCreating VirtualNetwork: {vn_name}") + vn_id = grpc.create_virtual_network(name=vn_name, network_class=network_class, ipv4_cidr="10.200.0.0/16") + vn_cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + print(f"Waiting for VirtualNetwork {vn_cr_name} to become Ready...") + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vn_cr_name) + print(f"VirtualNetwork {vn_cr_name} is Ready") + + # Create subnet with unique name + subnet_name = f"test-subnet-{test_run_id}" + print(f"Creating Subnet: {subnet_name}") + subnet_id = grpc.create_subnet(name=subnet_name, virtual_network=vn_id, ipv4_cidr="10.200.100.0/24") + subnet_cr_name = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_id) + print(f"Waiting for Subnet {subnet_cr_name} to become Ready...") + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_cr_name) + print(f"Subnet {subnet_cr_name} is Ready") + + yield { + "virtual_network_id": vn_id, + "virtual_network_cr_name": vn_cr_name, + "subnet_id": subnet_id, + "subnet_cr_name": subnet_cr_name, + } + except Exception: + # If setup fails, cleanup any resources that were created + print(f"\nSetup failed, cleaning up partial resources: {test_run_id}") + if subnet_id and subnet_cr_name: + try: + grpc.delete_subnet(subnet_id=subnet_id) + except Exception as e: + print(f"WARNING: Failed to cleanup subnet {subnet_id}: {e}") + if vn_id and vn_cr_name: + try: + grpc.delete_virtual_network(vn_id=vn_id) + except Exception as e: + print(f"WARNING: Failed to cleanup virtual network {vn_id}: {e}") + raise # Re-raise original exception + finally: + # Normal cleanup runs regardless of setup success/failure + # Only attempt cleanup if resources were successfully created + if vn_id and vn_cr_name and subnet_id and subnet_cr_name: + print(f"\nCleaning up test networking resources: {test_run_id}") + + # Delete subnet first + try: + print(f"Deleting Subnet {subnet_id}...") + grpc.delete_subnet(subnet_id=subnet_id) + wait_for_subnet_deletion(k8s=k8s_hub_client, name=subnet_cr_name) + print(f"Subnet {subnet_id} deleted") + except Exception as e: + print(f"WARNING: Failed to delete subnet {subnet_id}: {e}") + + # Delete virtual network + try: + print(f"Deleting VirtualNetwork {vn_id}...") + grpc.delete_virtual_network(vn_id=vn_id) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=vn_cr_name) + print(f"VirtualNetwork {vn_id} deleted") + except Exception as e: + print(f"WARNING: Failed to delete virtual network {vn_id}: {e}") + + +@pytest.fixture(scope="session") +def default_subnet(default_networking: dict[str, str]) -> str: + """Convenience fixture that returns just the subnet ID (for CLI/gRPC API usage).""" + return default_networking["subnet_id"] + + +@pytest.fixture(scope="session") +def default_subnet_ref(default_networking: dict[str, str]) -> str: + """Convenience fixture that returns the subnet CR name (for K8s API usage).""" + return default_networking["subnet_cr_name"] + + +@pytest.fixture(scope="session") +def default_instance_type(private_grpc: GRPCClient, test_run_id: str) -> Iterator[str]: + """Create a default ACTIVE instance type for VM tests; clean up after.""" + it_name = f"e2e-default-it-{test_run_id}" + private_grpc.create_instance_type( + name=it_name, cores=DEFAULT_IT_CORES, memory_gib=DEFAULT_IT_MEMORY_GIB, description="Default E2E instance type" + ) + yield it_name + try: + private_grpc.delete_instance_type(name=it_name) + except subprocess.CalledProcessError as e: + output = ((e.stdout or "") + (e.stderr or "")).lower() + if "not found" not in output: + raise + + +@pytest.fixture(scope="session") +def default_disk_image(grpc: GRPCClient, test_run_id: str) -> Iterator[str]: + """Create a default AVAILABLE Linux DiskImage for CLI-based VM tests; clean up after. + + Mirrors ``default_instance_type`` but diverges in two API-driven ways: + DiskImages/Create is a PUBLIC API (uses ``grpc``, not ``private_grpc``), and a + DiskImage carries its own UUID distinct from its name — so we capture the id + returned at create and delete by id, while yielding the name the CLI's + ``--disk-image`` flag needs. + """ + di_name = f"e2e-default-di-{test_run_id}" + di_id = grpc.create_disk_image( + name=di_name, + source_ref=DEFAULT_DISK_IMAGE_SOURCE_REF, + guest_os_family="GUEST_OS_FAMILY_LINUX", + architecture=["ARCHITECTURE_AMD64"], + ) + yield di_name + try: + grpc.delete_disk_image(disk_image_id=di_id) + except subprocess.CalledProcessError as e: + output = ((e.stdout or "") + (e.stderr or "")).lower() + # Tolerate only an already-deleted DiskImage; surface in-use/failedprecondition (leaked CI). + if "not found" not in output: + raise + + +@pytest.fixture(scope="session", autouse=True) +def _wait_for_tenant_storage_ready(k8s_hub_client: K8sClient, namespace: str) -> None: + """Wait for ClusterStorageReady=True before any VMaaS tests if storage is configured. + + When storageFulfillment is enabled, the storage controller triggers AAP jobs to + create per-tenant StorageClasses. Those jobs consume AAP executor slots. Compute + instance provisioning jobs that run while storage jobs are still queued time out + after 600s waiting for an executor. This fixture waits for storage setup to + complete before tests begin, eliminating the queue contention. + + Polls for up to 30s to detect whether the ClusterStorageReady condition exists; + if it never appears the environment has no storage configured and we skip the wait. + """ + condition: str = "" + for attempt in range(6): # 6 x 5s sleeps, 7 reads total + condition = k8s_hub_client.get_tenant_condition_status( + name=namespace, condition_type="ClusterStorageReady", checked=False + ) + if condition: + break + if attempt < 5: + time.sleep(5) + else: + # Final read after the last sleep — avoids missing the condition + # if it appears during the delay after the 6th check. + time.sleep(5) + condition = k8s_hub_client.get_tenant_condition_status( + name=namespace, condition_type="ClusterStorageReady", checked=False + ) + + if not condition or condition == "True": + return + + wait_for_tenant_condition(k8s=k8s_hub_client, name=namespace, condition_type="ClusterStorageReady") + + +@pytest.fixture(scope="session", autouse=True) +def _set_cli_default_instance_type(cli: OsacCLI, default_instance_type: str) -> None: + """Wire the session-scoped default instance type into the shared CLI fixture.""" + cli.default_instance_type = default_instance_type + + +@pytest.fixture(scope="session", autouse=True) +def _set_cli_default_disk_image(cli: OsacCLI, default_disk_image: str) -> None: + """Wire the session-scoped default disk image into the shared CLI fixture.""" + cli.default_disk_image = default_disk_image diff --git a/tests/e2e/tests/vmaas/external_ip/__init__.py b/tests/e2e/tests/vmaas/external_ip/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/tests/vmaas/external_ip/conftest.py b/tests/e2e/tests/vmaas/external_ip/conftest.py new file mode 100644 index 0000000000..20a88aa8d2 --- /dev/null +++ b/tests/e2e/tests/vmaas/external_ip/conftest.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import logging +import subprocess +from collections.abc import Callable, Generator +from uuid import uuid4 + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_cr, + wait_for_deletion, + wait_for_external_ip_attachment_deletion, + wait_for_external_ip_deletion, + wait_for_external_ip_pool_cr, + wait_for_external_ip_pool_deletion, + wait_for_external_ip_pool_grpc_ready, + wait_for_external_ip_pool_ready, + wait_for_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.vmaas.external_ip.helpers import allocate_worker_subnet, create_ip + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="class") +def make_pool(grpc: GRPCClient, private_grpc: GRPCClient, k8s_hub_client: K8sClient) -> Generator[..., None, None]: + created: list[tuple[str, str]] = [] + + def _make(*, prefix: int = 24, name_prefix: str = "test-pool") -> tuple[str, str]: + pool_name = f"{name_prefix}-{uuid4().hex[:8]}" + subnet = allocate_worker_subnet(prefix=prefix) + pool_id = private_grpc.create_external_ip_pool(name=pool_name, cidrs=[str(subnet)]) + pool_cr_name = wait_for_external_ip_pool_cr(k8s=k8s_hub_client, uuid=pool_id) + wait_for_external_ip_pool_ready(k8s=k8s_hub_client, name=pool_cr_name) + wait_for_external_ip_pool_grpc_ready(private_grpc=private_grpc, pool_id=pool_id) + created.append((pool_id, pool_cr_name)) + return pool_id, pool_cr_name + + yield _make + + for pool_id, pool_cr_name in reversed(created): + if not k8s_hub_client.is_present(resource="externalippool", name=pool_cr_name): + continue + try: + _cleanup_pool_children(grpc, k8s_hub_client, pool_id) + private_grpc.delete_external_ip_pool(pool_id=pool_id) + except subprocess.CalledProcessError as exc: + stderr = exc.stderr or "" + if "Unavailable" in stderr or "no route to host" in stderr or "connection" in stderr.lower(): + logger.warning("ExternalIPPool %s teardown skipped — gRPC unreachable: %s", pool_id, stderr.strip()) + continue + if "NotFound" not in stderr: + logger.warning("ExternalIPPool %s teardown delete failed: %s", pool_id, stderr.strip()) + continue + logger.warning("ExternalIPPool %s already deleted", pool_id) + wait_for_external_ip_pool_deletion(k8s=k8s_hub_client, name=pool_cr_name) + + +def _cleanup_pool_children(grpc: GRPCClient, k8s: K8sClient, pool_id: str) -> None: + """Delete all attachments and IPs belonging to a pool before deleting it.""" + pool_ip_ids: set[str] = set() + for ip_id in grpc.list_external_ip_ids(): + try: + ip_obj = grpc.get_external_ip(external_ip_id=ip_id) + if ip_obj["object"]["spec"].get("pool", {}).get("id") == pool_id: + pool_ip_ids.add(ip_id) + except subprocess.CalledProcessError: + continue + + for att_id in grpc.list_external_ip_attachment_ids(): + try: + att = grpc.get_external_ip_attachment(attachment_id=att_id) + if att["object"]["spec"].get("external_ip", {}).get("id") in pool_ip_ids: + grpc.delete_external_ip_attachment(attachment_id=att_id) + att_name = k8s.get_external_ip_attachment_name(uuid=att_id, checked=False) + if att_name: + wait_for_external_ip_attachment_deletion(k8s=k8s, name=att_name) + except subprocess.CalledProcessError: + continue + + for ip_id in pool_ip_ids: + try: + grpc.delete_external_ip(external_ip_id=ip_id) + ip_name = k8s.get_external_ip_name(uuid=ip_id, checked=False) + if ip_name: + wait_for_external_ip_deletion(k8s=k8s, name=ip_name) + except subprocess.CalledProcessError: + continue + + +@pytest.fixture +def external_ip_pool(make_pool: Callable[..., tuple[str, str]]) -> tuple[str, str]: + return make_pool() + + +@pytest.fixture(scope="class") +def small_pool(make_pool: Callable[..., tuple[str, str]]) -> tuple[str, str]: + """A /30 pool with 2 usable IPs.""" + return make_pool(prefix=30, name_prefix="test-small-capacity-pool") + + +@pytest.fixture(scope="class") +def created_ips(grpc: GRPCClient) -> Generator[list[tuple[str, str]], None, None]: + """Track IPs across chained tests; clean up any survivors on teardown.""" + ips: list[tuple[str, str]] = [] + yield ips + for ip_id, _ in reversed(ips): + try: + grpc.delete_external_ip(external_ip_id=ip_id) + except subprocess.CalledProcessError: + logger.warning("ExternalIP %s teardown failed, may need manual cleanup", ip_id) + + +@pytest.fixture +def external_ip( + grpc: GRPCClient, k8s_hub_client: K8sClient, external_ip_pool: tuple[str, str] +) -> Generator[tuple[str, str], None, None]: + pool_id, _ = external_ip_pool + ip_id, ip_cr_name = create_ip(grpc, k8s_hub_client, pool_id) + yield ip_id, ip_cr_name + if k8s_hub_client.is_present(resource="externalip", name=ip_cr_name): + try: + grpc.delete_external_ip(external_ip_id=ip_id) + except subprocess.CalledProcessError as exc: + logger.warning("ExternalIP %s gRPC delete failed in teardown: %s", ip_id, (exc.stderr or "").strip()) + wait_for_external_ip_deletion(k8s=k8s_hub_client, name=ip_cr_name) + + +@pytest.fixture(scope="class") +def make_compute_instances( + cli: OsacCLI, k8s_hub_client: K8sClient, vm_template: str, default_subnet: str +) -> Generator[Callable[..., tuple[tuple[str, str], ...]], None, None]: + created: list[tuple[str, str]] = [] + + def _make(count: int = 2) -> tuple[tuple[str, str], ...]: + instances: list[tuple[str, str]] = [] + for _ in range(count): + name = unique_name("e2e-ci") + uuid = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + name = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + created.append((uuid, name)) + instances.append((uuid, name)) + for _, name in instances: + wait_for_running(k8s=k8s_hub_client, name=name) + return tuple(instances) + + yield _make + + for ci_uuid, ci_name in reversed(created): + if not k8s_hub_client.is_present(resource="computeinstance", name=ci_name): + continue + try: + cli.delete_compute_instance(uuid=ci_uuid) + except subprocess.CalledProcessError as exc: + logger.warning("ComputeInstance %s teardown failed: %s", ci_uuid, (exc.stderr or "").strip()) + continue + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) diff --git a/tests/e2e/tests/vmaas/external_ip/helpers.py b/tests/e2e/tests/vmaas/external_ip/helpers.py new file mode 100644 index 0000000000..09f92be81f --- /dev/null +++ b/tests/e2e/tests/vmaas/external_ip/helpers.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import ipaddress +import logging +import os +from typing import Any +from uuid import uuid4 + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_external_ip_allocated, wait_for_external_ip_cr +from tests.core.k8s_client import K8sClient + +logger = logging.getLogger(__name__) + +# This network is a portion of the RFC 1918 private range +IPV4_NETWORK: str = "172.27.0.0/16" + + +def allocate_worker_subnet(prefix: int = 24) -> ipaddress.IPv4Network: + """ + Allocate subnet with worker-based namespacing to prevent conflicts in parallel execution. + + Each pytest-xdist worker gets its own /16 subdivision of 172.27.0.0/16: + - Worker 0 (gw0): 172.27.0.x + - Worker 1 (gw1): 172.27.1.x + - Worker 2 (gw2): 172.27.2.x + - Worker 3 (gw3): 172.27.3.x + + Within each worker, a sequential counter ensures unique, deterministic CIDRs. + """ + # Get pytest-xdist worker ID (e.g., "gw0", "gw1", etc.) + worker_id = os.environ.get("PYTEST_XDIST_WORKER", "gw0") + worker_num = int(worker_id.replace("gw", "")) if worker_id.startswith("gw") else 0 + + # Use a sequential counter within this worker's address space + if not hasattr(allocate_worker_subnet, "_counter"): + allocate_worker_subnet._counter = 0 + + counter = allocate_worker_subnet._counter + allocate_worker_subnet._counter += 1 + + if prefix == 24: + # /24 blocks use the lower half of 172.27.0.0/16 (172.27.0.0 - 172.27.127.255) + # Each worker gets 32 /24 blocks + # Worker 0: 172.27.0.0/24, 172.27.1.0/24, ..., 172.27.31.0/24 + # Worker 1: 172.27.32.0/24, 172.27.33.0/24, ..., 172.27.63.0/24 + # Worker 2: 172.27.64.0/24, 172.27.65.0/24, ..., 172.27.95.0/24 + # Worker 3: 172.27.96.0/24, 172.27.97.0/24, ..., 172.27.127.0/24 + third_octet = worker_num * 32 + counter + if third_octet > 127: + raise RuntimeError(f"Worker {worker_id} exhausted /24 address space (counter={counter})") + cidr = f"172.27.{third_octet}.0/24" + elif prefix == 30: + # /30 blocks use the upper half of 172.27.0.0/16 (172.27.128.0 - 172.27.255.255) + # to avoid overlap with /24 blocks + # Each worker gets 32 third octets, each with 64 /30 blocks + # Worker 0: 172.27.128.x - 172.27.159.x + # Worker 1: 172.27.160.x - 172.27.191.x + # Worker 2: 172.27.192.x - 172.27.223.x + # Worker 3: 172.27.224.x - 172.27.255.x + third_octet = 128 + worker_num * 32 + (counter // 64) + fourth_octet = (counter % 64) * 4 + if third_octet > 255: + raise RuntimeError(f"Worker {worker_id} exhausted /30 address space (counter={counter})") + cidr = f"172.27.{third_octet}.{fourth_octet}/30" + else: + raise NotImplementedError(f"Prefix /{prefix} not supported") + + return ipaddress.IPv4Network(cidr) + + +def pool_status(private_grpc: GRPCClient, pool_id: str) -> dict[str, Any]: + pool = private_grpc.get_external_ip_pool(pool_id=pool_id) + raw = pool["object"]["status"] + return { + "total": int(raw.get("total", 0)), + "allocated": int(raw.get("allocated", 0)), + "available": int(raw.get("available", 0)), + } + + +def create_ip(grpc: GRPCClient, k8s: K8sClient, pool_id: str) -> tuple[str, str]: + ip_name: str = f"test-ip-{uuid4().hex[:8]}" + ip_id: str = grpc.create_external_ip(name=ip_name, pool=pool_id) + ip_cr_name: str = wait_for_external_ip_cr(k8s=k8s, uuid=ip_id) + wait_for_external_ip_allocated(k8s=k8s, name=ip_cr_name) + return ip_id, ip_cr_name + + +def delete_ip(grpc: GRPCClient, ip_id: str) -> None: + grpc.delete_external_ip(external_ip_id=ip_id) diff --git a/tests/e2e/tests/vmaas/external_ip/test_external_ip_pool_capacity.py b/tests/e2e/tests/vmaas/external_ip/test_external_ip_pool_capacity.py new file mode 100644 index 0000000000..4544783bee --- /dev/null +++ b/tests/e2e/tests/vmaas/external_ip/test_external_ip_pool_capacity.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import subprocess +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import assert_grpc_rejected, wait_for_external_ip_pool_deletion +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until +from tests.vmaas.external_ip.helpers import create_ip, delete_ip, pool_status + + +class TestPoolCapacity: + """ + Tests for ExternalIPPool capacity management. + + Note: Tests are order-dependent - each builds on prevous state. This is a + purposeful tradeoff to greatly reduce test execution time. + """ + + def test_capacity_initialized_from_cidr(self, small_pool: tuple[str, str], private_grpc: GRPCClient) -> None: + pool_id, _ = small_pool + status = pool_status(private_grpc, pool_id) + assert status["total"] == 2, f"Expected small pool to have 2 usable IPs, got {status['total']}" + assert status["available"] == 2 + assert status["allocated"] == 0 + + def test_allocation_decrements_available( + self, + small_pool: tuple[str, str], + grpc: GRPCClient, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + created_ips: list[tuple[str, str]], + ) -> None: + pool_id, _ = small_pool + created_ips.append(create_ip(grpc, k8s_hub_client, pool_id)) + status = pool_status(private_grpc, pool_id) + assert status["total"] == 2 + assert status["allocated"] == 1 + assert status["available"] == 1 + + def test_exhaustion_rejects_creation( + self, + small_pool: tuple[str, str], + grpc: GRPCClient, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + created_ips: list[tuple[str, str]], + ) -> None: + pool_id, _ = small_pool + created_ips.append(create_ip(grpc, k8s_hub_client, pool_id)) + + status = pool_status(private_grpc, pool_id) + assert status["available"] == 0, f"Pool should be full, available={status['available']}" + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_external_ip(name=f"test-ip-{uuid4().hex[:8]}", pool=pool_id) + assert_grpc_rejected(exc_info, "FailedPrecondition") + + def test_release_restores_capacity( + self, + small_pool: tuple[str, str], + grpc: GRPCClient, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + created_ips: list[tuple[str, str]], + ) -> None: + pool_id, _ = small_pool + ip_id, _ = created_ips.pop(0) + delete_ip(grpc, ip_id) + + poll_until( + fn=lambda: pool_status(private_grpc, pool_id)["available"], + until=lambda v: v == 1, + retries=30, + delay=2, + description="Pool available restored to 1 after IP release", + ) + + created_ips.append(create_ip(grpc, k8s_hub_client, pool_id)) + status = pool_status(private_grpc, pool_id) + assert status["allocated"] == 2 + assert status["available"] == 0 + + def test_pool_deletion_blocked_while_ips_allocated( + self, small_pool: tuple[str, str], private_grpc: GRPCClient + ) -> None: + pool_id, _ = small_pool + with pytest.raises(subprocess.CalledProcessError) as exc_info: + private_grpc.delete_external_ip_pool(pool_id=pool_id) + assert_grpc_rejected(exc_info, "FailedPrecondition") + + def test_pool_deletion_succeeds_after_all_ips_released( + self, + small_pool: tuple[str, str], + grpc: GRPCClient, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + created_ips: list[tuple[str, str]], + ) -> None: + pool_id, pool_cr_name = small_pool + for ip_id, _ in created_ips: + delete_ip(grpc, ip_id) + created_ips.clear() + + poll_until( + fn=lambda: pool_status(private_grpc, pool_id)["allocated"], + until=lambda v: v == 0, + retries=30, + delay=2, + description="Pool allocated drops to 0", + ) + + private_grpc.delete_external_ip_pool(pool_id=pool_id) + wait_for_external_ip_pool_deletion(k8s=k8s_hub_client, name=pool_cr_name) diff --git a/tests/e2e/tests/vmaas/external_ip/test_external_ip_pool_lifecycle.py b/tests/e2e/tests/vmaas/external_ip/test_external_ip_pool_lifecycle.py new file mode 100644 index 0000000000..e9c0dc2f0c --- /dev/null +++ b/tests/e2e/tests/vmaas/external_ip/test_external_ip_pool_lifecycle.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import subprocess +from collections.abc import Callable +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + assert_grpc_rejected, + wait_for_external_ip_allocated, + wait_for_external_ip_attachment_cr, + wait_for_external_ip_attachment_deletion, + wait_for_external_ip_attachment_ready, + wait_for_external_ip_deletion, + wait_for_external_ip_pool_deletion, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + + +class TestExternalIPPoolLifecycle: + def test_attach_detach_reattach( + self, + external_ip_pool: tuple[str, str], + external_ip: tuple[str, str], + make_compute_instances: Callable[..., tuple[tuple[str, str], ...]], + grpc: GRPCClient, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + ) -> None: + pool_id, pool_cr_name = external_ip_pool + ip_id, ip_cr_name = external_ip + (ci1_uuid, _ci1_name), (ci2_uuid, _ci2_name) = make_compute_instances(2) + + assert pool_id in private_grpc.list_external_ip_pool_ids() + assert ip_id in grpc.list_external_ip_ids() + wait_for_external_ip_allocated(k8s=k8s_hub_client, name=ip_cr_name) + + # --- Attach ExternalIP to ComputeInstance 1 --- + att_id: str = grpc.create_external_ip_attachment( + name=f"test-att-{uuid4().hex[:8]}", external_ip=ip_id, compute_instance=ci1_uuid + ) + att_cr_name: str = wait_for_external_ip_attachment_cr(k8s=k8s_hub_client, uuid=att_id) + wait_for_external_ip_attachment_ready(k8s=k8s_hub_client, name=att_cr_name) + + ip_obj = grpc.get_external_ip(external_ip_id=ip_id) + assert ip_obj["object"]["status"].get("attached") is True + attached_ip_address: str = ip_obj["object"]["status"]["address"] + assert attached_ip_address, "ExternalIP should have an allocated address" + + # --- Detach (delete attachment) --- + grpc.delete_external_ip_attachment(attachment_id=att_id) + wait_for_external_ip_attachment_deletion(k8s=k8s_hub_client, name=att_cr_name) + + poll_until( + fn=lambda: grpc.get_external_ip(external_ip_id=ip_id)["object"]["status"].get("attached"), + until=lambda v: v is not True, + retries=30, + delay=5, + description=f"ExternalIP {ip_id} detached", + ) + + # --- Re-attach same IP to ComputeInstance 2 --- + att2_id: str = grpc.create_external_ip_attachment( + name=f"test-att-{uuid4().hex[:8]}", external_ip=ip_id, compute_instance=ci2_uuid + ) + att2_cr_name: str = wait_for_external_ip_attachment_cr(k8s=k8s_hub_client, uuid=att2_id) + wait_for_external_ip_attachment_ready(k8s=k8s_hub_client, name=att2_cr_name) + + ip_obj = grpc.get_external_ip(external_ip_id=ip_id) + assert ip_obj["object"]["status"]["address"] == attached_ip_address, ( + "Re-attached ExternalIP should keep the same address" + ) + + # --- Cleanup: delete attachment, ExternalIP, then pool --- + grpc.delete_external_ip_attachment(attachment_id=att2_id) + wait_for_external_ip_attachment_deletion(k8s=k8s_hub_client, name=att2_cr_name) + + grpc.delete_external_ip(external_ip_id=ip_id) + wait_for_external_ip_deletion(k8s=k8s_hub_client, name=ip_cr_name) + poll_until( + fn=lambda: ip_id not in grpc.list_external_ip_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"ExternalIP {ip_id} removal from API", + ) + + private_grpc.delete_external_ip_pool(pool_id=pool_id) + wait_for_external_ip_pool_deletion(k8s=k8s_hub_client, name=pool_cr_name) + poll_until( + fn=lambda: pool_id not in private_grpc.list_external_ip_pool_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"ExternalIPPool {pool_id} removal from API", + ) + + def test_validation_rejections( + self, + external_ip_pool: tuple[str, str], + external_ip: tuple[str, str], + make_compute_instances: Callable[..., tuple[tuple[str, str], ...]], + grpc: GRPCClient, + k8s_hub_client: K8sClient, + ) -> None: + _pool_id, _pool_cr_name = external_ip_pool + ip_id, ip_cr_name = external_ip + ci1_uuid, _ = make_compute_instances(1)[0] + + wait_for_external_ip_allocated(k8s=k8s_hub_client, name=ip_cr_name) + + # Nonexistent ComputeInstance + fake_ci_uuid: str = str(uuid4()) + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_external_ip_attachment( + name=f"test-att-{uuid4().hex[:8]}", external_ip=ip_id, compute_instance=fake_ci_uuid + ) + assert_grpc_rejected(exc_info, "InvalidArgument") + + # Nonexistent ExternalIP + fake_ip_uuid: str = str(uuid4()) + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_external_ip_attachment( + name=f"test-att-{uuid4().hex[:8]}", external_ip=fake_ip_uuid, compute_instance=ci1_uuid + ) + assert_grpc_rejected(exc_info, "InvalidArgument") + + # Duplicate attachment — attach same ExternalIP twice + att_id: str = grpc.create_external_ip_attachment( + name=f"test-att-{uuid4().hex[:8]}", external_ip=ip_id, compute_instance=ci1_uuid + ) + att_cr_name: str = wait_for_external_ip_attachment_cr(k8s=k8s_hub_client, uuid=att_id) + wait_for_external_ip_attachment_ready(k8s=k8s_hub_client, name=att_cr_name) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_external_ip_attachment( + name=f"test-att-{uuid4().hex[:8]}", external_ip=ip_id, compute_instance=ci1_uuid + ) + assert_grpc_rejected(exc_info, "FailedPrecondition") + + grpc.delete_external_ip_attachment(attachment_id=att_id) + wait_for_external_ip_attachment_deletion(k8s=k8s_hub_client, name=att_cr_name) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_api_fields.py b/tests/e2e/tests/vmaas/test_compute_instance_api_fields.py new file mode 100644 index 0000000000..4392436bef --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_api_fields.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion, wait_for_provision, wait_for_running +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +TEST_BOOT_DISK_SIZE: int = 20 +TEST_RUN_STRATEGY: str = "Always" + + +def test_compute_instance_api_fields( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + default_subnet: str, + vm_template: str, +) -> None: + ci_uuid: str = cli.create_compute_instance( + template=vm_template, + name=unique_name("e2e-api"), + network_attachments=[{"subnet": default_subnet}], + boot_disk_size=TEST_BOOT_DISK_SIZE, + run_strategy=TEST_RUN_STRATEGY, + ) + + instance_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=ci_uuid) + + wait_for_provision(k8s=k8s_hub_client, name=instance_name) + wait_for_running(k8s=k8s_hub_client, name=instance_name) + + vm_ns: str = k8s_hub_client.get_compute_instance_vm_namespace(name=instance_name) + + # runStrategy mutability: Always -> Halted + _, rc = k8s_hub_client.patch( + resource="computeinstance", name=instance_name, patch='{"spec":{"runStrategy":"Halted"}}' + ) + assert rc == 0, "runStrategy patch to Halted should succeed" + + poll_until( + fn=lambda: k8s_virt_client.get_vm_printable_status(name=instance_name, vm_namespace=vm_ns, checked=False), + until=lambda v: v == "Stopped", + retries=30, + delay=10, + description=f"{instance_name} VM stopped", + ) + + vm_strategy: str = k8s_virt_client.get_vm_run_strategy(name=instance_name, vm_namespace=vm_ns) + vm_status: str = k8s_virt_client.get_vm_printable_status(name=instance_name, vm_namespace=vm_ns, checked=False) + assert vm_strategy == "Halted", f"VM runStrategy should be Halted, got {vm_strategy}" + assert vm_status == "Stopped", f"VM should be Stopped, got {vm_status}" + + # runStrategy mutability: Halted -> Always + _, rc = k8s_hub_client.patch( + resource="computeinstance", name=instance_name, patch='{"spec":{"runStrategy":"Always"}}' + ) + assert rc == 0, "runStrategy patch to Always should succeed" + + poll_until( + fn=lambda: k8s_virt_client.get_vm_printable_status(name=instance_name, vm_namespace=vm_ns, checked=False), + until=lambda v: v == "Running", + retries=30, + delay=10, + description=f"{instance_name} VM running", + ) + + vm_strategy = k8s_virt_client.get_vm_run_strategy(name=instance_name, vm_namespace=vm_ns) + vm_status = k8s_virt_client.get_vm_printable_status(name=instance_name, vm_namespace=vm_ns, checked=False) + assert vm_strategy == "Always", f"VM runStrategy should be Always, got {vm_strategy}" + assert vm_status == "Running", f"VM should be Running, got {vm_status}" + + # Immutability: cores + output, rc = k8s_hub_client.patch(resource="computeinstance", name=instance_name, patch='{"spec":{"cores":8}}') + assert rc != 0, "cores field should be immutable" + assert "cores is immutable" in output, f"Expected immutability error, got: {output}" + + # Immutability: memoryGiB + output, rc = k8s_hub_client.patch(resource="computeinstance", name=instance_name, patch='{"spec":{"memoryGiB":16}}') + assert rc != 0, "memoryGiB field should be immutable" + assert "memoryGiB is immutable" in output, f"Expected immutability error, got: {output}" + + # Immutability: image + output, rc = k8s_hub_client.patch( + resource="computeinstance", + name=instance_name, + patch='{"spec":{"image":{"sourceRef":"quay.io/fedora/fedora:latest"}}}', + ) + assert rc != 0, "image field should be immutable" + assert "image is immutable" in output, f"Expected immutability error, got: {output}" + + cli.delete_compute_instance(uuid=ci_uuid) + wait_for_deletion(k8s=k8s_hub_client, name=instance_name) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_cli_fields.py b/tests/e2e/tests/vmaas/test_compute_instance_cli_fields.py new file mode 100644 index 0000000000..8699b7fd10 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_cli_fields.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import base64 +from typing import Any + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion, wait_for_provision, wait_for_running +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.vmaas.conftest import DEFAULT_IT_CORES, DEFAULT_IT_MEMORY_GIB + +TEST_BOOT_DISK_SIZE: int = 20 +TEST_RUN_STRATEGY: str = "Always" +TEST_USER_DATA: str = "#cloud-config\npackages:\n - vim\n" + + +def test_compute_instance_cli_explicit_fields( + cli: OsacCLI, grpc: GRPCClient, k8s_hub_client: K8sClient, default_subnet: str, vm_template: str +) -> None: + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + template=vm_template, + name=name, + network_attachments=[{"subnet": default_subnet}], + boot_disk_size=TEST_BOOT_DISK_SIZE, + run_strategy=TEST_RUN_STRATEGY, + user_data_secret_ref=TEST_USER_DATA, + ) + + assert uuid in grpc.list_compute_instance_ids() + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + + ci_spec: dict[str, Any] = k8s_hub_client.get_json(resource="computeinstance", name=ci_name) + spec: dict[str, Any] = ci_spec["spec"] + assert spec["cores"] == DEFAULT_IT_CORES, f"cores mismatch: {spec['cores']} != {DEFAULT_IT_CORES}" + assert spec["memoryGiB"] == DEFAULT_IT_MEMORY_GIB, ( + f"memoryGiB mismatch: {spec['memoryGiB']} != {DEFAULT_IT_MEMORY_GIB}" + ) + + # Verify osac.openshift.io/instance-type-name label is set by reconciler + labels: dict[str, str] = ci_spec["metadata"].get("labels", {}) + it_label: str | None = labels.get("osac.openshift.io/instance-type-name") + assert it_label is not None, f"osac.openshift.io/instance-type-name label should be set, got: {it_label!r}" + assert len(it_label) > 0, f"osac.openshift.io/instance-type-name label should be non-empty, got: {it_label!r}" + + assert spec["bootDisk"]["sizeGiB"] == TEST_BOOT_DISK_SIZE, ( + f"bootDisk.sizeGiB mismatch: {spec['bootDisk']['sizeGiB']} != {TEST_BOOT_DISK_SIZE}" + ) + assert spec["runStrategy"] == TEST_RUN_STRATEGY, f"runStrategy mismatch: {spec['runStrategy']}" + + expected_secret_name: str = f"{uuid}-user-data" + assert spec["userDataSecretRef"]["name"] == expected_secret_name, ( + f"userDataSecretRef.name mismatch: {spec['userDataSecretRef']['name']} != {expected_secret_name}" + ) + + secret: dict[str, Any] = k8s_hub_client.get_json(resource="secret", name=expected_secret_name) + expected_b64: str = base64.b64encode(TEST_USER_DATA.encode()).decode() + assert secret["data"]["userdata"] == expected_b64, "User data Secret content does not match" + + owner_refs: list[dict[str, Any]] = secret["metadata"]["ownerReferences"] + assert len(owner_refs) > 0, "Secret missing ownerReferences" + assert owner_refs[0]["kind"] == "ComputeInstance" + assert owner_refs[0]["name"] == ci_name + + wait_for_provision(k8s=k8s_hub_client, name=ci_name) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + cli.delete_compute_instance(uuid=uuid) + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_creation.py b/tests/e2e/tests/vmaas/test_compute_instance_creation.py new file mode 100644 index 0000000000..dcec6457b4 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_creation.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_cr, + wait_for_deletion, + wait_for_grpc_removal, + wait_for_provision, + wait_for_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI + + +@pytest.mark.metering +def test_compute_instance_lifecycle( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + vm_template: str, + default_subnet: str, + metering: MeteringCollector, +) -> None: + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + metering.expect("osac.resource.created.v1", resource_id=uuid) + + assert uuid in grpc.list_compute_instance_ids() + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + wait_for_provision(k8s=k8s_hub_client, name=ci_name) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + metering.expect("osac.resource.started.v1", resource_id=uuid) + metering.verify() + + # Verify billing dimensions values match the created resource + created_event = metering.get_event("osac.resource.created.v1", resource_id=uuid) + bd = created_event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("instance_type") == cli.default_instance_type, ( + f"billing_dimensions.instance_type mismatch: {bd.get('instance_type')!r} != {cli.default_instance_type!r}" + ) + + # Verify VM exists on virt cluster + vmi_ns: str = k8s_hub_client.get_compute_instance_vm_namespace(name=ci_name) + vmi_ts: str = k8s_virt_client.get_vmi_creation_timestamp(vmi_namespace=vmi_ns, compute_instance_name=ci_name) + assert vmi_ts != "", f"No VMI found on virt cluster for {ci_name}" + + cli.delete_compute_instance(uuid=uuid) + metering.expect("osac.resource.deleted.v1", resource_id=uuid) + + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=uuid) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_delete_during_provision.py b/tests/e2e/tests/vmaas/test_compute_instance_delete_during_provision.py new file mode 100644 index 0000000000..2239076abf --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_delete_during_provision.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import time + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion, wait_for_grpc_removal +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +TERMINAL_JOB_STATES: tuple[str, ...] = ("Canceled", "Failed", "Succeeded") + + +def _get_deprovision_status(k8s: K8sClient, *, name: str) -> str: + if not k8s.is_present(resource="computeinstance", name=name): + return "cr_deleted" + return k8s.get_compute_instance_latest_job_state(name=name, job_type="deprovision", checked=False) + + +def _wait_for_provision_termination(k8s: K8sClient, *, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_latest_job_state(name=name, job_type="provision", checked=False), + until=lambda v: v in (*TERMINAL_JOB_STATES, ""), + retries=60, + delay=1, + description=f"provision termination for {name}", + ) + + +def _wait_for_deprovision_completion(k8s: K8sClient, *, name: str) -> str: + result: str = poll_until( + fn=lambda: _get_deprovision_status(k8s, name=name), + until=lambda v: v in (*TERMINAL_JOB_STATES, "cr_deleted"), + retries=120, + delay=1, + description=f"deprovision completion for {name}", + ) + assert result != "Failed", f"Deprovision job failed for {name}" + return result + + +def _verify_no_duplicate_deprovision(k8s: K8sClient, *, name: str) -> None: + if not k8s.is_present(resource="computeinstance", name=name): + return + first_id: str = k8s.get_compute_instance_latest_job_id(name=name, job_type="deprovision", checked=False) + time.sleep(10) + if not k8s.is_present(resource="computeinstance", name=name): + return + second_id: str = k8s.get_compute_instance_latest_job_id(name=name, job_type="deprovision", checked=False) + assert first_id == second_id, f"Deprovision job ID changed: {first_id} -> {second_id}" + + +@pytest.mark.metering +def test_compute_instance_delete_during_provision( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + vm_template: str, + default_subnet: str, + metering: MeteringCollector, +) -> None: + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + metering.expect("osac.resource.created.v1", resource_id=uuid) + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + + poll_until( + fn=lambda: k8s_hub_client.get_compute_instance_phase(name=ci_name, checked=False), + until=lambda v: v == "Starting", + retries=30, + delay=2, + description=f"{ci_name} Starting", + ) + deprov_job_id: str = k8s_hub_client.get_compute_instance_latest_job_id( + name=ci_name, job_type="deprovision", checked=False + ) + assert deprov_job_id == "", "No deprovision job should exist before deletion" + + cli.delete_compute_instance(uuid=uuid) + metering.expect("osac.resource.deleted.v1", resource_id=uuid) + + _wait_for_provision_termination(k8s_hub_client, name=ci_name) + _wait_for_deprovision_completion(k8s_hub_client, name=ci_name) + _verify_no_duplicate_deprovision(k8s_hub_client, name=ci_name) + + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=uuid) + + orphan_count: int = k8s_virt_client.count_by_label_all_namespaces( + resource="virtualmachine", label=f"osac.openshift.io/computeinstance={ci_name}" + ) + assert orphan_count == 0, f"Found {orphan_count} orphaned VMs for {ci_name}" diff --git a/tests/e2e/tests/vmaas/test_compute_instance_disk_image.py b/tests/e2e/tests/vmaas/test_compute_instance_disk_image.py new file mode 100644 index 0000000000..b0e2b14eb1 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_disk_image.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import subprocess +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import PUBLIC_API, GRPCClient +from tests.core.helpers import ( + assert_grpc_rejected, + wait_for_cr, + wait_for_deletion, + wait_for_grpc_removal, + wait_for_provision, + wait_for_running, +) +from tests.core.k8s_client import K8sClient + +SOURCE_REF = "quay.io/containerdisks/fedora:41" + + +def _unique_name(prefix: str = "e2e-cidi") -> str: + return f"{prefix}-{uuid4().hex[:8]}" + + +def test_compute_instance_with_disk_image( + grpc: GRPCClient, vm_template: str, default_subnet: str, default_instance_type: str, k8s_hub_client: K8sClient +) -> None: + """AC-1 / TC-FR7-01: Create CI with DiskImage reference, verify VM runs with correct image.""" + di_name = _unique_name("e2e-di") + di_id: str | None = None + ci_id: str | None = None + ci_name: str | None = None + + try: + di_id = grpc.create_disk_image( + name=di_name, + source_ref=SOURCE_REF, + guest_os_family="GUEST_OS_FAMILY_LINUX", + architecture=["ARCHITECTURE_AMD64"], + ) + assert di_id, "create_disk_image should return a non-empty ID" + + response = grpc.create_compute_instance_with_disk_image( + template=vm_template, + disk_image_name=di_name, + subnet_ids=[default_subnet], + instance_type=default_instance_type, + name=_unique_name("e2e-ci"), + ) + ci_id = response["object"]["id"] + assert ci_id, "create_compute_instance should return a non-empty ID" + + ci_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_id) + wait_for_provision(k8s=k8s_hub_client, name=ci_name) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + ci_obj = grpc.get_compute_instance(ci_id=ci_id) + assert ci_obj["object"]["spec"]["diskImage"]["id"] == di_id + + source_ref = k8s_hub_client.get_jsonpath( + resource="computeinstance", name=ci_name, jsonpath="{.spec.image.sourceRef}" + ) + assert source_ref == SOURCE_REF, f"CRD sourceRef should be {SOURCE_REF}, got {source_ref}" + + guest_os = k8s_hub_client.get_jsonpath( + resource="computeinstance", name=ci_name, jsonpath="{.spec.guestOSFamily}" + ) + assert guest_os == "linux", f"CRD guestOSFamily should be 'linux', got {guest_os}" + finally: + if ci_id is not None: + grpc.delete_compute_instance(ci_id=ci_id) + if ci_name is not None: + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=ci_id) + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_obsolete_disk_image_blocks_creation( + grpc: GRPCClient, vm_template: str, default_subnet: str, default_instance_type: str +) -> None: + """AC-2 / TC-FR7-04: OBSOLETE DiskImage blocks ComputeInstance creation.""" + di_name = _unique_name("e2e-di") + di_id: str | None = None + + try: + di_id = grpc.create_disk_image(name=di_name, source_ref=SOURCE_REF) + + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_DEPRECATED") + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_OBSOLETE") + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_compute_instance_with_disk_image( + template=vm_template, + disk_image_name=di_name, + subnet_ids=[default_subnet], + instance_type=default_instance_type, + ) + assert_grpc_rejected(exc_info, "FailedPrecondition") + finally: + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_deprecated_disk_image_allows_creation_with_warning( + grpc: GRPCClient, vm_template: str, default_subnet: str, default_instance_type: str +) -> None: + """AC-3 / TC-FR7-05: DEPRECATED DiskImage allows creation with warning.""" + di_name = _unique_name("e2e-di") + di_id: str | None = None + ci_id: str | None = None + + try: + di_id = grpc.create_disk_image(name=di_name, source_ref=SOURCE_REF) + + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_DEPRECATED") + + response = grpc.create_compute_instance_with_disk_image( + template=vm_template, + disk_image_name=di_name, + subnet_ids=[default_subnet], + instance_type=default_instance_type, + name=_unique_name("e2e-ci"), + ) + ci_id = response["object"]["id"] + assert ci_id, "CI creation with DEPRECATED DiskImage should succeed" + + warnings: list[str] = response.get("warnings", []) + assert any("deprecated" in w.lower() for w in warnings), ( + f"Response should contain deprecation warning, got: {warnings}" + ) + finally: + if ci_id is not None: + grpc.delete_compute_instance(ci_id=ci_id) + wait_for_grpc_removal(grpc=grpc, uuid=ci_id) + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_template_disk_image_default( + grpc: GRPCClient, private_grpc: GRPCClient, default_subnet: str, default_instance_type: str +) -> None: + """AC-4 / TC-FR8-01: Template disk_image default applied when user omits disk_image.""" + di_name = _unique_name("e2e-di") + di_id: str | None = None + template_id: str | None = None + ci_id: str | None = None + + try: + di_id = grpc.create_disk_image(name=di_name, source_ref=SOURCE_REF) + + template_name = _unique_name("e2e-tmpl") + template_id = private_grpc.create_compute_instance_template( + name=template_name, + title="E2E DiskImage default test", + description="Template with disk_image in spec_defaults", + spec_defaults={"disk_image": {"name": di_name}}, + ) + + attachments = [{"subnet": {"id": default_subnet}}] + # disk_image is deliberately omitted here — it must be inherited from the + # template's spec_defaults. boot_disk/run_strategy are supplied directly + # because the custom template intentionally defaults only disk_image. + spec: dict[str, Any] = { + "template": {"name": template_name}, + "instance_type": {"name": default_instance_type}, + "network_attachments": attachments, + "boot_disk": {"size_gib": 20}, + "run_strategy": "Always", + } + response = grpc.call( + service=f"{PUBLIC_API}.ComputeInstances/Create", + data={"object": {"metadata": {"name": _unique_name("e2e-ci")}, "spec": spec}}, + ) + ci_id = response["object"]["id"] + assert ci_id, "CI creation from template should succeed" + + ci_obj = grpc.get_compute_instance(ci_id=ci_id) + disk_image_ref = ci_obj["object"]["spec"].get("diskImage", {}) + assert disk_image_ref.get("id") == di_id, ( + f"CI should inherit disk_image from template default, got: {disk_image_ref}" + ) + finally: + if ci_id is not None: + grpc.delete_compute_instance(ci_id=ci_id) + wait_for_grpc_removal(grpc=grpc, uuid=ci_id) + if template_id is not None: + private_grpc.delete_compute_instance_template(template_id=template_id) + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_deletion_protection( + grpc: GRPCClient, vm_template: str, default_subnet: str, default_instance_type: str, k8s_hub_client: K8sClient +) -> None: + """AC-5 / TC-FR12-01 + TC-FR12-04: Deletion protection lifecycle.""" + di_name = _unique_name("e2e-di") + di_id: str | None = None + ci_id: str | None = None + ci_name: str | None = None + + try: + di_id = grpc.create_disk_image(name=di_name, source_ref=SOURCE_REF) + + response = grpc.create_compute_instance_with_disk_image( + template=vm_template, + disk_image_name=di_name, + subnet_ids=[default_subnet], + instance_type=default_instance_type, + name=_unique_name("e2e-ci"), + ) + ci_id = response["object"]["id"] + ci_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_id) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.delete_disk_image(disk_image_id=di_id) + assert_grpc_rejected(exc_info, "FailedPrecondition") + + grpc.delete_compute_instance(ci_id=ci_id) + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=ci_id) + ci_id = None + + deleted_di_id = di_id + grpc.delete_disk_image(disk_image_id=di_id) + di_id = None + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.get_disk_image(disk_image_id=deleted_di_id) + assert_grpc_rejected(exc_info, "NotFound") + finally: + if ci_id is not None: + grpc.delete_compute_instance(ci_id=ci_id) + if ci_name is not None: + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=ci_id) + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_heartbeat.py b/tests/e2e/tests/vmaas/test_compute_instance_heartbeat.py new file mode 100644 index 0000000000..7e8faa3758 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_heartbeat.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_cr, + wait_for_deletion, + wait_for_grpc_removal, + wait_for_provision, + wait_for_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI + + +@pytest.mark.metering +def test_compute_instance_heartbeat( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + vm_template: str, + default_subnet: str, + metering: MeteringCollector, +) -> None: + """Verify heartbeat events appear for a billable RUNNING VM. + + The metering system emits periodic heartbeat events (default interval 60s) + for every VM in the RUNNING state. This test creates a VM, waits for it to + reach RUNNING, then verifies that at least one heartbeat event is emitted + within 180s. + """ + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + wait_for_provision(k8s=k8s_hub_client, name=ci_name) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + metering.expect("osac.resource.heartbeat.v1", resource_id=uuid, timeout=180) + metering.verify() # verify heartbeat arrived BEFORE deleting + + cli.delete_compute_instance(uuid=uuid) + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=uuid) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_instance_type.py b/tests/e2e/tests/vmaas/test_compute_instance_instance_type.py new file mode 100644 index 0000000000..387bcbe97f --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_instance_type.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import re +import subprocess +from collections.abc import Iterator +from typing import Any +from uuid import uuid4 + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import run_unchecked + +IT_CORES: int = 2 +IT_MEMORY_GIB: int = 4 + + +def _build_create_ci_args( + cli: OsacCLI, vm_template: str, default_subnet: str, it_name: str, disk_image: str, ci_name: str | None = None +) -> list[str]: + args = [cli.binary, "--config", cli.config_dir, "create", "computeinstance"] + if ci_name is not None: + args += ["--name", ci_name] + args += [ + "--template", + vm_template, + "--network-attachment", + f"subnet={default_subnet}", + "--instance-type", + it_name, + "--boot-disk-size", + "20", + "--disk-image", + disk_image, + "--run-strategy", + "Always", + ] + return args + + +@pytest.fixture +def active_instance_type(private_grpc: GRPCClient) -> Iterator[str]: + """Create an ACTIVE instance type for testing; clean up after.""" + it_name = f"e2e-ci-it-{uuid4().hex[:8]}" + private_grpc.create_instance_type( + name=it_name, cores=IT_CORES, memory_gib=IT_MEMORY_GIB, description="E2E compute instance test type" + ) + yield it_name + try: + private_grpc.delete_instance_type(name=it_name) + except subprocess.CalledProcessError as e: + output = ((e.stdout or "") + (e.stderr or "")).lower() + if "not found" not in output: + raise + + +def test_compute_instance_happy_path( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + default_subnet: str, + vm_template: str, + active_instance_type: str, +) -> None: + ci_uuid: str | None = None + ci_name: str | None = None + + try: + name = unique_name("e2e-ci") + ci_uuid = cli.create_compute_instance( + name=name, + template=vm_template, + network_attachments=[{"subnet": default_subnet}], + instance_type=active_instance_type, + ) + assert ci_uuid in grpc.list_compute_instance_ids(), f"ComputeInstance {ci_uuid} not found in list after create" + + # Wait for CR and verify reconciler expansion + ci_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_uuid) + ci_obj: dict[str, Any] = k8s_hub_client.get_json(resource="computeinstance", name=ci_name) + spec: dict[str, Any] = ci_obj["spec"] + assert spec["cores"] == IT_CORES, ( + f"E2E-02: reconciler should expand cores from instance type: {spec['cores']} != {IT_CORES}" + ) + assert spec["memoryGiB"] == IT_MEMORY_GIB, ( + f"E2E-02: reconciler should expand memory from instance type: {spec['memoryGiB']} != {IT_MEMORY_GIB}" + ) + + # Verify osac.openshift.io/instance-type-name label (E2E-03) + labels: dict[str, str] = ci_obj["metadata"].get("labels", {}) + assert labels.get("osac.openshift.io/instance-type-name") == active_instance_type, ( + f"E2E-03: osac.openshift.io/instance-type-name label mismatch: " + f"{labels.get('osac.openshift.io/instance-type-name')!r} != {active_instance_type!r}" + ) + finally: + if ci_uuid is not None: + cli.delete_compute_instance(uuid=ci_uuid) + if ci_name is not None: + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + + +def test_compute_instance_deletion_protection( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + default_subnet: str, + vm_template: str, + active_instance_type: str, +) -> None: + ci_uuid: str | None = None + ci_name: str | None = None + + try: + name = unique_name("e2e-ci") + ci_uuid = cli.create_compute_instance( + name=name, + template=vm_template, + network_attachments=[{"subnet": default_subnet}], + instance_type=active_instance_type, + ) + assert ci_uuid in grpc.list_compute_instance_ids(), f"ComputeInstance {ci_uuid} not found in list after create" + ci_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_uuid) + + output, rc = run_unchecked( + cli.binary, "--config", cli.config_dir, "delete", "instancetype", active_instance_type + ) + assert rc != 0, "delete should be rejected when ComputeInstance references instance type" + error_lower = output.lower() + assert any(term in error_lower for term in ["409", "conflict", "referenced", "failedprecondition", "in use"]), ( + f"Expected conflict/reference error, got: {output}" + ) + finally: + if ci_uuid is not None: + cli.delete_compute_instance(uuid=ci_uuid) + if ci_name is not None: + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + + +def test_compute_instance_deprecated_warning( + cli: OsacCLI, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + default_subnet: str, + vm_template: str, + active_instance_type: str, + default_disk_image: str, +) -> None: + deprecated_ci_uuid: str | None = None + deprecated_ci_name: str | None = None + + try: + private_grpc.update_instance_type(name=active_instance_type, state="INSTANCE_TYPE_STATE_DEPRECATED") + + dep_output, dep_rc = run_unchecked( + *_build_create_ci_args( + cli, + vm_template, + default_subnet, + active_instance_type, + default_disk_image, + ci_name=unique_name("e2e-ci"), + ) + ) + assert dep_rc == 0, f"create with DEPRECATED type should succeed, got: {dep_output}" + assert "deprecat" in dep_output.lower() or "warning" in dep_output.lower(), ( + f"Expected deprecation warning in output, got: {dep_output}" + ) + + uuid_match: re.Match[str] | None = re.search(r"'([^']+)'", dep_output) + assert uuid_match is not None, f"Failed to parse UUID from CLI output: {dep_output}" + deprecated_ci_uuid = uuid_match.group(1) + deprecated_ci_name = wait_for_cr(k8s=k8s_hub_client, uuid=deprecated_ci_uuid) + finally: + if deprecated_ci_uuid is not None: + cli.delete_compute_instance(uuid=deprecated_ci_uuid) + if deprecated_ci_name is not None: + wait_for_deletion(k8s=k8s_hub_client, name=deprecated_ci_name) + + +def test_compute_instance_nonexistent_instance_type( + cli: OsacCLI, k8s_hub_client: K8sClient, default_subnet: str, vm_template: str, default_disk_image: str +) -> None: + missing_it_name = f"nonexistent-it-{uuid4().hex[:8]}" + ci_name = f"e2e-neg-{uuid4().hex[:8]}" + output, rc = run_unchecked( + *_build_create_ci_args(cli, vm_template, default_subnet, missing_it_name, default_disk_image, ci_name=ci_name) + ) + if rc == 0: + uuid_match = re.search(r"'([^']+)'", output) + if uuid_match: + ci_uuid = uuid_match.group(1) + cr_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_uuid) + cli.delete_compute_instance(uuid=ci_uuid) + wait_for_deletion(k8s=k8s_hub_client, name=cr_name) + elif k8s_hub_client.is_present(resource="computeinstance", name=ci_name): + pytest.fail(f"ComputeInstance CR {ci_name} leaked but UUID could not be parsed from output: {output}") + assert rc != 0, f"create with nonexistent instance type should fail, got: {output}" + error_lower = output.lower() + assert any(term in error_lower for term in ["not found", "404", "notfound"]), ( + f"Expected not-found error, got: {output}" + ) + + +def test_compute_instance_obsolete_instance_type( + cli: OsacCLI, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + default_subnet: str, + vm_template: str, + active_instance_type: str, + default_disk_image: str, +) -> None: + private_grpc.update_instance_type(name=active_instance_type, state="INSTANCE_TYPE_STATE_DEPRECATED") + private_grpc.update_instance_type(name=active_instance_type, state="INSTANCE_TYPE_STATE_OBSOLETE") + + ci_name = f"e2e-obs-{uuid4().hex[:8]}" + output, rc = run_unchecked( + *_build_create_ci_args( + cli, vm_template, default_subnet, active_instance_type, default_disk_image, ci_name=ci_name + ) + ) + if rc == 0: + uuid_match = re.search(r"'([^']+)'", output) + if uuid_match: + ci_uuid = uuid_match.group(1) + cr_name = wait_for_cr(k8s=k8s_hub_client, uuid=ci_uuid) + cli.delete_compute_instance(uuid=ci_uuid) + wait_for_deletion(k8s=k8s_hub_client, name=cr_name) + elif k8s_hub_client.is_present(resource="computeinstance", name=ci_name): + pytest.fail(f"ComputeInstance CR {ci_name} leaked but UUID could not be parsed from output: {output}") + assert rc != 0, f"create with OBSOLETE instance type should be rejected, got: {output}" + error_lower = output.lower() + assert any(term in error_lower for term in ["obsolete", "rejected", "failedprecondition", "409", "conflict"]), ( + f"Expected rejection error for obsolete instance type, got: {output}" + ) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_restart.py b/tests/e2e/tests/vmaas/test_compute_instance_restart.py new file mode 100644 index 0000000000..185b6d2831 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_restart.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion, wait_for_restart, wait_for_running +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + + +def _wait_for_new_vmi(k8s_virt: K8sClient, *, vmi_namespace: str, ci_name: str, original_ts: str) -> str: + poll_until( + fn=lambda: k8s_virt.get_vmi_creation_timestamp(vmi_namespace=vmi_namespace, compute_instance_name=ci_name), + until=lambda v: v != original_ts, + retries=30, + delay=10, + description=f"new VMI for {ci_name}", + ) + return k8s_virt.get_vmi_creation_timestamp(vmi_namespace=vmi_namespace, compute_instance_name=ci_name) + + +@pytest.mark.metering +def test_compute_instance_restart( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + vm_template: str, + default_subnet: str, + metering: MeteringCollector, +) -> None: + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + metering.expect("osac.resource.created.v1", resource_id=uuid) + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + metering.expect("osac.resource.started.v1", resource_id=uuid) + + vmi_ns: str = k8s_hub_client.get_compute_instance_vm_namespace(name=ci_name) + original_vmi_ts: str = k8s_virt_client.get_vmi_creation_timestamp( + vmi_namespace=vmi_ns, compute_instance_name=ci_name + ) + initial_last_restarted: str = k8s_hub_client.get_compute_instance_last_restarted_at(name=ci_name) + + restart_ts: str = datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + grpc.update_restart(uuid=uuid, template=vm_template, timestamp=restart_ts) + + metering.expect("osac.resource.suspended.v1", resource_id=uuid) + metering.expect("osac.resource.resumed.v1", resource_id=uuid) + + wait_for_restart(k8s=k8s_hub_client, name=ci_name, initial=initial_last_restarted, restart_ts=restart_ts) + + final_last_restarted: str = k8s_hub_client.get_compute_instance_last_restarted_at(name=ci_name) + assert final_last_restarted != "" + assert final_last_restarted != initial_last_restarted + assert final_last_restarted >= restart_ts + + new_vmi_ts: str = _wait_for_new_vmi( + k8s_virt_client, vmi_namespace=vmi_ns, ci_name=ci_name, original_ts=original_vmi_ts + ) + assert new_vmi_ts > original_vmi_ts + + restart_failed: str = k8s_hub_client.get_jsonpath( + resource="computeinstance", name=ci_name, jsonpath='{.status.conditions[?(@.type=="RestartFailed")].status}' + ) + assert restart_failed in ("", "False") + + cli.delete_compute_instance(uuid=uuid) + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_restart_negative.py b/tests/e2e/tests/vmaas/test_compute_instance_restart_negative.py new file mode 100644 index 0000000000..3ad07108b2 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_restart_negative.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import time +from datetime import UTC, datetime + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion, wait_for_restart, wait_for_running +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI + + +def test_compute_instance_restart_past_timestamp_ignored( + cli: OsacCLI, grpc: GRPCClient, k8s_hub_client: K8sClient, vm_template: str, default_subnet: str +) -> None: + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + template=vm_template, name=name, network_attachments=[{"subnet": default_subnet}] + ) + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + # Do a real restart first + initial_last_restarted: str = k8s_hub_client.get_compute_instance_last_restarted_at(name=ci_name) + restart_ts: str = datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + grpc.update_restart(uuid=uuid, template=vm_template, timestamp=restart_ts) + wait_for_restart(k8s=k8s_hub_client, name=ci_name, initial=initial_last_restarted, restart_ts=restart_ts) + + # Past timestamp should be ignored + saved_last_restarted: str = k8s_hub_client.get_compute_instance_last_restarted_at(name=ci_name) + grpc.update_restart(uuid=uuid, template=vm_template, timestamp="2020-01-01T00:00:00Z") + time.sleep(15) + + current_last_restarted: str = k8s_hub_client.get_compute_instance_last_restarted_at(name=ci_name) + assert current_last_restarted == saved_last_restarted, ( + f"Past timestamp should be ignored: was {saved_last_restarted!r}, now {current_last_restarted!r}" + ) + + cli.delete_compute_instance(uuid=uuid) + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_short_lived_metering.py b/tests/e2e/tests/vmaas/test_compute_instance_short_lived_metering.py new file mode 100644 index 0000000000..1a36cdb025 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_short_lived_metering.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_cr, wait_for_deletion, wait_for_grpc_removal +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI + + +@pytest.mark.metering +def test_short_lived_vm_metering( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + vm_template: str, + default_subnet: str, + metering: MeteringCollector, +) -> None: + """Verify metering captures events for a VM created and deleted within 30s (CAP-4). + + A resource existing for 30 seconds must appear in usage data. This validates + sub-minute billing granularity by creating a VM and immediately deleting it + without waiting for it to reach Running. + """ + uuid: str = cli.create_compute_instance( + name=unique_name("e2e-ci"), template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + metering.expect("osac.resource.created.v1", resource_id=uuid) + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + + cli.delete_compute_instance(uuid=uuid) + metering.expect("osac.resource.deleted.v1", resource_id=uuid) + + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=uuid) diff --git a/tests/e2e/tests/vmaas/test_compute_instance_stop_metering.py b/tests/e2e/tests/vmaas/test_compute_instance_stop_metering.py new file mode 100644 index 0000000000..53fe358c4a --- /dev/null +++ b/tests/e2e/tests/vmaas/test_compute_instance_stop_metering.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import time +from datetime import UTC, datetime + +import pytest + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_cr, + wait_for_deletion, + wait_for_grpc_removal, + wait_for_provision, + wait_for_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + + +@pytest.mark.metering +def test_compute_instance_stop_metering( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + vm_template: str, + default_subnet: str, + metering: MeteringCollector, +) -> None: + """Verify metering captures suspended.v1 when a running VM is stopped. + + Also verifies that heartbeats stop after the VM becomes non-billable. + """ + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + + ci_name: str = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + wait_for_provision(k8s=k8s_hub_client, name=ci_name) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + + metering.expect("osac.resource.created.v1", resource_id=uuid) + metering.expect("osac.resource.started.v1", resource_id=uuid) + metering.verify() + + grpc.update_compute_instance_run_strategy(ci_id=uuid, run_strategy="Halted") + + vm_ns: str = k8s_hub_client.get_compute_instance_vm_namespace(name=ci_name) + poll_until( + fn=lambda: k8s_virt_client.get_vm_printable_status(name=ci_name, vm_namespace=vm_ns, checked=False), + until=lambda v: v == "Stopped", + retries=30, + delay=10, + description=f"{ci_name} VM stopped", + ) + + metering.expect("osac.resource.suspended.v1", resource_id=uuid, timeout=120) + metering.verify() + + # The VM was explicitly stopped from RUNNING — verify the suspended event + # closed the billing interval with a positive duration. + suspended_event = metering.get_event("osac.resource.suspended.v1", resource_id=uuid) + suspended_data = suspended_event.get("data", {}) + assert suspended_data.get("previous_state") == "RUNNING", ( + f"Stop from RUNNING should have previous_state=RUNNING, got {suspended_data.get('previous_state')!r}" + ) + ds = suspended_data.get("duration_seconds") + assert isinstance(ds, (int, float)) and ds > 0, f"Stop from RUNNING should have positive duration_seconds, got {ds}" + + # Verify heartbeats stop after VM becomes non-billable. + # The heartbeat generator queries the projection DB, which is updated + # after the Kafka publish (publishAndUpsert). One more heartbeat sweep + # may fire against the stale projection. Wait past one full heartbeat + # interval so in-flight heartbeats drain, then assert silence. + time.sleep(90) + no_more_after = datetime.now(UTC).isoformat() + metering.assert_no_events("osac.resource.heartbeat.v1", resource_id=uuid, since=no_more_after, within=90) + + cli.delete_compute_instance(uuid=uuid) + metering.expect("osac.resource.deleted.v1", resource_id=uuid) + + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + wait_for_grpc_removal(grpc=grpc, uuid=uuid) diff --git a/tests/e2e/tests/vmaas/test_console.py b/tests/e2e/tests/vmaas/test_console.py new file mode 100644 index 0000000000..674a795e3b --- /dev/null +++ b/tests/e2e/tests/vmaas/test_console.py @@ -0,0 +1,444 @@ +from __future__ import annotations + +import base64 +import contextlib +import json +import logging +import queue +import ssl +import subprocess +import threading +import time +from collections.abc import Callable, Iterator +from datetime import UTC, datetime +from typing import Any + +import pytest +import websocket + +from tests.catalog.conftest import unique_name +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + assert_grpc_rejected, + wait_for_cr, + wait_for_deletion, + wait_for_provision, + wait_for_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI + +logger = logging.getLogger(__name__) + +CONSOLE_WS_PATH = "/api/fulfillment/v1/console_sessions/connect" +CONSOLE_GRPC_SERVICE = "osac.public.v1.ConsoleProxy/Connect" + + +@pytest.fixture(scope="module") +def console_vm( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + k8s_virt_client: K8sClient, + vm_template: str, + default_subnet: str, +) -> Iterator[dict[str, str]]: + """Create a single compute instance for all console tests in this module.""" + print("\nCreating console test VM...") + name = unique_name("e2e-ci") + uuid: str = cli.create_compute_instance( + name=name, template=vm_template, network_attachments=[{"subnet": default_subnet}] + ) + ci_name: str | None = None + try: + ci_name = wait_for_cr(k8s=k8s_hub_client, uuid=uuid) + print(f"Waiting for {ci_name} to provision and reach Running...") + wait_for_provision(k8s=k8s_hub_client, name=ci_name) + wait_for_running(k8s=k8s_hub_client, name=ci_name) + print(f"Console test VM {ci_name} is Running") + + yield {"uuid": uuid, "name": ci_name} + finally: + print(f"\nCleaning up console test VM {uuid}...") + try: + cli.delete_compute_instance(uuid=uuid) + if ci_name is not None: + wait_for_deletion(k8s=k8s_hub_client, name=ci_name) + except Exception as e: + print(f"WARNING: Failed to cleanup console VM {uuid}: {e}") + + +# --------------------------------------------------------------------------- +# Transport helpers +# --------------------------------------------------------------------------- + + +def _ws_url(fulfillment_address: str) -> str: + """Build the WebSocket console proxy URL from the fulfillment address.""" + host: str = fulfillment_address.rsplit(":", 1)[0] + return f"wss://{host}{CONSOLE_WS_PATH}" + + +def _ws_connect(url: str, ticket: str, timeout: int = 30) -> websocket.WebSocket: + """Open a WebSocket connection to the console proxy with the given ticket.""" + logger.info("WS connecting to %s (timeout=%ds)", url, timeout) + ws = websocket.create_connection( + url, + header={"Authorization": f"Bearer {ticket}"}, + sslopt={"cert_reqs": ssl.CERT_NONE}, + subprotocols=["binary"], + timeout=timeout, + ) + logger.info("WS connected") + return ws + + +def _ws_recv(ws: websocket.WebSocket, timeout: float) -> str | None: + """Read one WebSocket message, returning decoded text or None on timeout.""" + ws.settimeout(timeout) + try: + data = ws.recv() + if isinstance(data, bytes): + return data.decode(errors="replace") + return data if data else None + except websocket.WebSocketTimeoutException: + return None + + +def _ws_try_connect(url: str, ticket: str) -> bool: + """Try to open a WebSocket connection. Returns True if the session is + established, False if rejected. + + Handles both handshake-level rejection (old behavior: server returns + non-101) and post-upgrade rejection (new behavior: server upgrades to + 101, then sends a Close frame with an error code). + """ + # TODO(console-ws-errors): Remove the handshake-rejection path once + # the server always uses post-upgrade close frames for setup errors. + try: + ws = _ws_connect(url, ticket, timeout=10) + except Exception as exc: + logger.info("WS try-connect rejected at handshake: %s: %s", type(exc).__name__, exc) + return False + try: + ws.settimeout(3) + data = ws.recv() + if data == "": + logger.info("WS try-connect rejected via close frame after upgrade") + return False + return True + except websocket.WebSocketTimeoutException: + return True + except Exception as exc: + logger.info("WS try-connect failed after upgrade: %s: %s", type(exc).__name__, exc) + return False + finally: + with contextlib.suppress(Exception): + ws.close() + ws.shutdown() + + +def _grpc_popen(address: str, ticket: str) -> subprocess.Popen: + """Start a gRPC console stream subprocess. Caller must manage stdin/lifecycle.""" + return subprocess.Popen( + ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {ticket}", "-d", "@", address, CONSOLE_GRPC_SERVICE], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +class _GrpcSession: + """Interactive gRPC console session via grpcurl subprocess.""" + + def __init__(self, address: str, ticket: str) -> None: + self._proc = _grpc_popen(address, ticket) + self._output_q: queue.Queue[str | None] = queue.Queue() + self._connected = threading.Event() + self._reader = threading.Thread(target=self._read_stdout, daemon=True) + self._reader.start() + + def _read_stdout(self) -> None: + assert self._proc.stdout is not None + json_buf = "" + for raw_line in self._proc.stdout: + line = raw_line.decode() + json_buf += line + if line.strip() == "}": + try: + obj = json.loads(json_buf) + json_buf = "" + if "output" in obj and "data" in obj["output"]: + self._output_q.put(base64.b64decode(obj["output"]["data"]).decode(errors="replace")) + elif "status" in obj: + msg = obj["status"].get("message", "") + logger.info("gRPC status: %s", msg) + if msg == "Connected": + self._connected.set() + except json.JSONDecodeError: + pass # incomplete object, keep accumulating lines + self._output_q.put(None) + + def send(self, data: bytes) -> None: + assert self._proc.stdin is not None + encoded = base64.b64encode(data).decode() + msg = json.dumps({"input": {"data": encoded}}) + "\n" + self._proc.stdin.write(msg.encode()) + self._proc.stdin.flush() + + def recv(self, timeout: float) -> str | None: + """Read next decoded console output chunk. Returns None on timeout/EOF.""" + try: + return self._output_q.get(timeout=timeout) + except queue.Empty: + return None + + def wait_connected(self, timeout: float = 10.0) -> bool: + """Block until the gRPC session reports Connected. Returns False on timeout.""" + return self._connected.wait(timeout=timeout) + + @property + def alive(self) -> bool: + return self._proc.poll() is None + + def close(self) -> None: + with contextlib.suppress(OSError): + if self._proc.stdin and not self._proc.stdin.closed: + self._proc.stdin.close() + self._proc.stdin = None # prevent communicate() from flushing closed stdin + with contextlib.suppress(OSError): + self._proc.kill() + self._proc.communicate() + + +def _grpc_stream(address: str, ticket: str) -> tuple[str, str, int]: + """Open a gRPC console stream via grpcurl, close stdin, and collect output. + + Returns (stdout, stderr, returncode). + """ + logger.info("gRPC stream opening to %s", address) + proc = _grpc_popen(address, ticket) + try: + proc.stdin.close() # type: ignore[union-attr] + proc.stdin = None # prevent communicate() from flushing closed stdin + stdout, stderr = proc.communicate(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + raise + except BaseException: + proc.kill() + proc.communicate() + raise + stdout_str, stderr_str = stdout.decode(), stderr.decode() + logger.info( + "gRPC stream result: rc=%d, stdout=%d bytes, stderr=%d bytes", proc.returncode, len(stdout_str), len(stderr_str) + ) + if stdout_str: + logger.info("gRPC stdout: %s%s", stdout_str[:1024], "..." if len(stdout_str) > 1024 else "") + if stderr_str: + logger.info("gRPC stderr: %s%s", stderr_str[:1024], "..." if len(stderr_str) > 1024 else "") + return stdout_str, stderr_str, proc.returncode + + +def _create_ticket(grpc: GRPCClient, vm_uuid: str, *, client_id: str = "") -> dict[str, Any]: + """Create a serial console session and return the session object.""" + logger.info("Creating console ticket for VM %s", vm_uuid) + result = grpc.create_console_session( + resource_type="CONSOLE_RESOURCE_TYPE_COMPUTE_INSTANCE", + resource_id=vm_uuid, + console_type="CONSOLE_TYPE_SERIAL", + client_id=client_id, + ) + logger.info("Ticket created, expires at %s", result.get("expiresAt", "unknown")) + return result + + +# --------------------------------------------------------------------------- +# Shared interactive console helper +# --------------------------------------------------------------------------- + + +def _wait_for_login_prompt( + *, + send: Callable[[bytes], None], + recv: Callable[[float], str | None], + timeout: float = 300.0, + enter_interval: float = 15.0, +) -> str: + """Send enter periodically and wait for 'login' prompt in console output. + + Returns accumulated console output. Raises AssertionError on timeout. + """ + accumulated = "" + deadline = time.monotonic() + timeout + next_enter = time.monotonic() + + while time.monotonic() < deadline: + if time.monotonic() >= next_enter: + send(b"\n") + logger.info("Sent enter to console") + next_enter = time.monotonic() + enter_interval + + remaining = deadline - time.monotonic() + chunk = recv(min(remaining, 5.0)) + + if chunk: + accumulated += chunk + truncated = chunk[:1024] + logger.info("Received %d bytes: %s%s", len(chunk), truncated, "..." if len(chunk) > 1024 else "") + if "login" in accumulated.lower(): + logger.info("Found login prompt (%d bytes total)", len(accumulated)) + return accumulated + + raise AssertionError( + f"Console did not show login prompt within {timeout:.0f}s. Received {len(accumulated)} bytes total." + ) + + +# --------------------------------------------------------------------------- +# Positive tests, WebSocket transport +# --------------------------------------------------------------------------- + + +def test_console_serial_websocket(console_vm: dict[str, str], grpc: GRPCClient, fulfillment_address: str) -> None: + """Connect to the serial console via WebSocket and verify interactive login prompt.""" + session: dict[str, Any] = _create_ticket(grpc, console_vm["uuid"]) + url: str = _ws_url(fulfillment_address) + ws = _ws_connect(url, session["ticket"]) + try: + _wait_for_login_prompt(send=ws.send_binary, recv=lambda t, _ws=ws: _ws_recv(_ws, t)) + finally: + with contextlib.suppress(Exception): + ws.close() + + +# --------------------------------------------------------------------------- +# Positive tests, gRPC stream transport +# --------------------------------------------------------------------------- + + +def test_console_serial_grpc_stream(console_vm: dict[str, str], grpc: GRPCClient, fulfillment_address: str) -> None: + """Connect to the serial console via gRPC bidi stream and verify interactive login prompt.""" + session: dict[str, Any] = _create_ticket(grpc, console_vm["uuid"]) + gs = _GrpcSession(fulfillment_address, session["ticket"]) + try: + _wait_for_login_prompt(send=gs.send, recv=gs.recv) + finally: + gs.close() + + +# --------------------------------------------------------------------------- +# Ticket reuse, single-use JTI enforcement +# --------------------------------------------------------------------------- + + +def test_console_ticket_reuse_rejected(console_vm: dict[str, str], grpc: GRPCClient, fulfillment_address: str) -> None: + """A ticket should work exactly once. 4 subsequent attempts must all fail.""" + session: dict[str, Any] = _create_ticket(grpc, console_vm["uuid"]) + ticket: str = session["ticket"] + url: str = _ws_url(fulfillment_address) + + results: list[bool] = [] + for i in range(5): + success: bool = _ws_try_connect(url, ticket) + results.append(success) + logger.info("Ticket reuse attempt %d: %s", i + 1, "success" if success else "rejected") + + assert results[0] is True, "First use of ticket must succeed" + assert all(r is False for r in results[1:]), f"Subsequent uses must fail (JTI single-use), got: {results}" + + +# --------------------------------------------------------------------------- +# Concurrent session, only one session per resource +# --------------------------------------------------------------------------- + + +def test_console_concurrent_session_rejected( + console_vm: dict[str, str], grpc: GRPCClient, fulfillment_address: str +) -> None: + """A second connection to the same resource must be rejected while one is active.""" + session1: dict[str, Any] = _create_ticket(grpc, console_vm["uuid"]) + session2: dict[str, Any] = _create_ticket(grpc, console_vm["uuid"]) + + gs1 = _GrpcSession(fulfillment_address, session1["ticket"]) + try: + assert gs1.wait_connected(), "First session failed to connect" + assert gs1.alive, "First session should still be running" + + stdout, stderr, rc = _grpc_stream(fulfillment_address, session2["ticket"]) + combined: str = stdout + stderr + logger.info("Concurrent session2 rc=%d", rc) + assert rc != 0, f"Second connection should fail, got: {combined!r}" + assert "FailedPrecondition" in combined or "session already active" in combined, ( + f"Expected FailedPrecondition or 'session already active', got: {combined!r}" + ) + + assert gs1.alive, "First session died while second was rejected" + finally: + gs1.close() + + +# --------------------------------------------------------------------------- +# Expired ticket +# --------------------------------------------------------------------------- + + +def test_console_expired_ticket_rejected( + console_vm: dict[str, str], grpc: GRPCClient, fulfillment_address: str +) -> None: + """A ticket used after its expiresAt must be rejected.""" + session: dict[str, Any] = _create_ticket(grpc, console_vm["uuid"]) + ticket: str = session["ticket"] + expires_at_str: str = session["expiresAt"] + + expires_at = datetime.fromisoformat(expires_at_str.replace("Z", "+00:00")) + seconds_until_expiry: float = (expires_at - datetime.now(tz=UTC)).total_seconds() + wait_seconds: float = min(60.0, max(0.0, seconds_until_expiry) + 15.0) + + logger.info("Ticket expires at %s, waiting %.0fs past expiry", expires_at_str, wait_seconds) + time.sleep(wait_seconds) + + url: str = _ws_url(fulfillment_address) + success: bool = _ws_try_connect(url, ticket) + logger.info("Expired ticket connect result: %s", success) + assert success is False, "Expired ticket must be rejected" + + +# --------------------------------------------------------------------------- +# Negative tests, no running VM required +# --------------------------------------------------------------------------- + + +def test_console_session_nonexistent_vm(grpc: GRPCClient) -> None: + """Creating a console session for a non-existent VM should fail.""" + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.create_console_session( + resource_type="CONSOLE_RESOURCE_TYPE_COMPUTE_INSTANCE", + resource_id="00000000-0000-0000-0000-000000000000", + console_type="CONSOLE_TYPE_SERIAL", + ) + assert_grpc_rejected(exc_info, "NotFound") + + +def test_console_invalid_ticket_websocket(fulfillment_address: str) -> None: + """Connecting with a garbage ticket must be rejected. + + Old behavior: server rejects at handshake (non-101 HTTP status). + New behavior: server upgrades to 101, then sends Close frame (3000). + """ + url: str = _ws_url(fulfillment_address) + # TODO(console-ws-errors): Remove the handshake-rejection path once + # the server always uses post-upgrade close frames for setup errors. + try: + ws = _ws_connect(url, "not-a-valid-ticket", timeout=10) + except websocket.WebSocketException: + return + try: + ws.settimeout(5) + data = ws.recv() + assert data == "", f"Expected close frame after upgrade, got: {data!r}" + finally: + with contextlib.suppress(Exception): + ws.close() + ws.shutdown() diff --git a/tests/e2e/tests/vmaas/test_disk_image_lifecycle.py b/tests/e2e/tests/vmaas/test_disk_image_lifecycle.py new file mode 100644 index 0000000000..07bfd02ecc --- /dev/null +++ b/tests/e2e/tests/vmaas/test_disk_image_lifecycle.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import subprocess +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import assert_grpc_rejected + +SOURCE_REF = "quay.io/containerdisks/fedora:41" + + +def _unique_name(prefix: str = "e2e-di") -> str: + return f"{prefix}-{uuid4().hex[:8]}" + + +def test_disk_image_crud_provider_admin(grpc: GRPCClient) -> None: + """AC-1: Provider Admin registers a global DiskImage, lists and gets it. + AC-7: Delete DiskImage succeeds when unreferenced.""" + di_id: str | None = None + + try: + di_id = grpc.create_disk_image( + name=_unique_name(), + source_ref=SOURCE_REF, + guest_os_family="GUEST_OS_FAMILY_LINUX", + architecture=["ARCHITECTURE_AMD64"], + ) + assert di_id, "create_disk_image should return a non-empty ID" + + assert di_id in grpc.list_disk_image_ids(), "DiskImage should appear in list after create" + + obj = grpc.get_disk_image(disk_image_id=di_id) + spec = obj["object"]["spec"] + assert spec["sourceRef"] == SOURCE_REF + assert spec["guestOsFamily"] == "GUEST_OS_FAMILY_LINUX" + assert "ARCHITECTURE_AMD64" in spec["architecture"] + assert spec.get("lifecycle", "DISK_IMAGE_LIFECYCLE_AVAILABLE") == "DISK_IMAGE_LIFECYCLE_AVAILABLE" + + deleted_id = di_id + grpc.delete_disk_image(disk_image_id=di_id) + di_id = None + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.get_disk_image(disk_image_id=deleted_id) + assert_grpc_rejected(exc_info, "NotFound") + finally: + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_tenant_admin(jwt_grpc_tenant1_admin: GRPCClient) -> None: + """AC-2: Tenant Admin registers a tenant-scoped DiskImage.""" + di_id: str | None = None + + try: + di_id = jwt_grpc_tenant1_admin.create_disk_image( + name=_unique_name(), source_ref=SOURCE_REF, architecture=["ARCHITECTURE_AMD64"] + ) + assert di_id, "Tenant Admin should be able to create a tenant-scoped DiskImage" + + obj = jwt_grpc_tenant1_admin.get_disk_image(disk_image_id=di_id) + tenant = obj["object"].get("metadata", {}).get("tenant", "") + assert tenant, "Tenant-scoped DiskImage should have metadata.tenant set" + + assert di_id in jwt_grpc_tenant1_admin.list_disk_image_ids(), ( + "Tenant Admin should see their own DiskImage in list" + ) + finally: + if di_id is not None: + jwt_grpc_tenant1_admin.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_tenant_user(jwt_grpc_tenant1: GRPCClient) -> None: + """AC-3: Tenant User registers a tenant-scoped DiskImage.""" + di_id: str | None = None + + try: + di_id = jwt_grpc_tenant1.create_disk_image( + name=_unique_name(), source_ref=SOURCE_REF, architecture=["ARCHITECTURE_AMD64"] + ) + assert di_id, "Tenant User should be able to create a tenant-scoped DiskImage" + + obj = jwt_grpc_tenant1.get_disk_image(disk_image_id=di_id) + tenant = obj["object"].get("metadata", {}).get("tenant", "") + assert tenant, "Tenant-scoped DiskImage should have metadata.tenant set" + finally: + if di_id is not None: + jwt_grpc_tenant1.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_deprecation_lifecycle(grpc: GRPCClient) -> None: + """AC-4: Deprecation workflow — deprecate, verify listing, obsolete, verify hidden.""" + di_id: str | None = None + + try: + di_id = grpc.create_disk_image(name=_unique_name(), source_ref=SOURCE_REF) + + # Deprecate + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_DEPRECATED") + obj = grpc.get_disk_image(disk_image_id=di_id) + spec = obj["object"]["spec"] + assert spec["lifecycle"] == "DISK_IMAGE_LIFECYCLE_DEPRECATED" + assert spec.get("deprecation", {}).get("deprecationTimestamp"), ( + "deprecation_timestamp should be auto-set on deprecation" + ) + + # Deprecated images still visible in default list + assert di_id in grpc.list_disk_image_ids(), "DEPRECATED DiskImage should still appear in default list" + + # Obsolete + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_OBSOLETE") + obj = grpc.get_disk_image(disk_image_id=di_id) + spec = obj["object"]["spec"] + assert spec["lifecycle"] == "DISK_IMAGE_LIFECYCLE_OBSOLETE" + assert spec.get("deprecation", {}).get("obsolescenceTimestamp"), ( + "obsolescence_timestamp should be auto-set on obsolescence" + ) + + # Obsolete images hidden from default list + assert di_id not in grpc.list_disk_image_ids(), "OBSOLETE DiskImage should be excluded from default list" + finally: + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_reactivation(grpc: GRPCClient) -> None: + """AC-5: Reactivation — obsolete image reactivated, visible in default list again.""" + di_id: str | None = None + + try: + di_id = grpc.create_disk_image(name=_unique_name(), source_ref=SOURCE_REF) + + # Drive to OBSOLETE + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_DEPRECATED") + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_OBSOLETE") + assert di_id not in grpc.list_disk_image_ids(), "OBSOLETE should be hidden" + + # Reactivate + grpc.update_disk_image_lifecycle(disk_image_id=di_id, lifecycle="DISK_IMAGE_LIFECYCLE_AVAILABLE") + obj = grpc.get_disk_image(disk_image_id=di_id) + spec = obj["object"]["spec"] + assert spec["lifecycle"] == "DISK_IMAGE_LIFECYCLE_AVAILABLE" + dep = spec.get("deprecation", {}) + assert not dep.get("deprecationTimestamp") and not dep.get("obsolescenceTimestamp"), ( + "deprecation timestamps should be cleared on reactivation" + ) + + assert di_id in grpc.list_disk_image_ids(), "Reactivated DiskImage should be visible in default list" + finally: + if di_id is not None: + grpc.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_tenant_isolation(jwt_grpc_tenant1: GRPCClient, jwt_grpc_tenant2: GRPCClient) -> None: + """AC-6: Tenant isolation — Tenant B cannot see/access Tenant A's images.""" + di_id: str | None = None + + try: + di_id = jwt_grpc_tenant1.create_disk_image(name=_unique_name(), source_ref=SOURCE_REF) + + # Tenant 1 can see it + assert di_id in jwt_grpc_tenant1.list_disk_image_ids(), "Tenant 1 should see its own DiskImage" + + # Tenant 2 cannot see it in list + assert di_id not in jwt_grpc_tenant2.list_disk_image_ids(), ( + "Tenant 2 should NOT see Tenant 1's DiskImage in list" + ) + + # Tenant 2 cannot get it by ID + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant2.get_disk_image(disk_image_id=di_id) + assert_grpc_rejected(exc_info, "NotFound") + finally: + if di_id is not None: + jwt_grpc_tenant1.delete_disk_image(disk_image_id=di_id) + + +def test_disk_image_obsolete_filtered_list(grpc: GRPCClient) -> None: + """AC-4: OBSOLETE images excluded from default list but visible with explicit filter.""" + di_available_id: str | None = None + di_obsolete_id: str | None = None + + try: + di_available_id = grpc.create_disk_image(name=_unique_name("e2e-di-avail"), source_ref=SOURCE_REF) + di_obsolete_id = grpc.create_disk_image(name=_unique_name("e2e-di-obs"), source_ref=SOURCE_REF) + + grpc.update_disk_image_lifecycle(disk_image_id=di_obsolete_id, lifecycle="DISK_IMAGE_LIFECYCLE_DEPRECATED") + grpc.update_disk_image_lifecycle(disk_image_id=di_obsolete_id, lifecycle="DISK_IMAGE_LIFECYCLE_OBSOLETE") + + # Default list: available visible, obsolete hidden + default_ids = grpc.list_disk_image_ids() + assert di_available_id in default_ids, "AVAILABLE DiskImage should be in default list" + assert di_obsolete_id not in default_ids, "OBSOLETE DiskImage should be excluded from default list" + + # Explicit filter: obsolete visible + # 3 = DISK_IMAGE_LIFECYCLE_OBSOLETE (CEL uses proto enum numeric values) + obsolete_ids = grpc.list_disk_image_ids(filter_expr="this.spec.lifecycle == 3") + assert di_obsolete_id in obsolete_ids, "OBSOLETE DiskImage should be visible with explicit lifecycle filter" + finally: + if di_obsolete_id is not None: + grpc.delete_disk_image(disk_image_id=di_obsolete_id) + if di_available_id is not None: + grpc.delete_disk_image(disk_image_id=di_available_id) diff --git a/tests/e2e/tests/vmaas/test_instance_type_lifecycle.py b/tests/e2e/tests/vmaas/test_instance_type_lifecycle.py new file mode 100644 index 0000000000..5974252bbc --- /dev/null +++ b/tests/e2e/tests/vmaas/test_instance_type_lifecycle.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import subprocess +from typing import Any +from uuid import uuid4 + +from tests.core.grpc_client import PRIVATE_API, GRPCClient +from tests.core.osac_cli import OsacCLI + +TEST_CORES: int = 4 +TEST_MEMORY_GIB: int = 8 + +TEST_GPU: dict[str, Any] = {"pci_device_selector": "10DE:20B0", "resource_name": "nvidia.com/A100", "count": 1} + + +def _assert_state_transition(private_grpc: GRPCClient, it_name: str, target_state: str) -> None: + """Update an InstanceType to *target_state* and verify the transition.""" + private_grpc.update_instance_type(name=it_name, state=target_state) + response = private_grpc.get_instance_type(name=it_name) + actual = response["object"]["spec"]["state"] + assert actual == target_state, f"state transition to {target_state}: {actual} != {target_state}" + + +def test_instance_type_lifecycle(cli: OsacCLI, private_grpc: GRPCClient) -> None: + it_name: str = f"e2e-lifecycle-{uuid4().hex[:8]}" + + try: + # 1. CREATE via private gRPC (admin operation) + private_grpc.create_instance_type( + name=it_name, cores=TEST_CORES, memory_gib=TEST_MEMORY_GIB, description="Lifecycle test type" + ) + names: list[str] = private_grpc.list_instance_type_names() + assert it_name in names, f"InstanceType {it_name} not found in list after create: {names}" + + # 2. GET via gRPC (verify API fields) + response: dict = private_grpc.get_instance_type(name=it_name) + obj: dict = response["object"] + assert obj["spec"]["cores"] == TEST_CORES, f"spec.cores mismatch: {obj['spec']['cores']} != {TEST_CORES}" + assert obj["spec"]["memoryGib"] == TEST_MEMORY_GIB, ( + f"spec.memoryGib mismatch: {obj['spec']['memoryGib']} != {TEST_MEMORY_GIB}" + ) + assert obj["spec"]["state"] == "INSTANCE_TYPE_STATE_ACTIVE", ( + f"spec.state mismatch: {obj['spec']['state']} != INSTANCE_TYPE_STATE_ACTIVE" + ) + assert obj["metadata"]["name"] == it_name, f"metadata.name mismatch: {obj['metadata']['name']} != {it_name}" + + # Verify CLI describe works + cli_output = cli.describe_instance_type(name=it_name) + assert it_name in cli_output, f"CLI describe should show {it_name}: {cli_output}" + + # 3. STATE TRANSITION: ACTIVE -> DEPRECATED + _assert_state_transition(private_grpc, it_name, "INSTANCE_TYPE_STATE_DEPRECATED") + + # 4. STATE TRANSITION: DEPRECATED -> OBSOLETE + _assert_state_transition(private_grpc, it_name, "INSTANCE_TYPE_STATE_OBSOLETE") + + # 5. STATE TRANSITION: OBSOLETE -> ACTIVE (reactivation) + _assert_state_transition(private_grpc, it_name, "INSTANCE_TYPE_STATE_ACTIVE") + + # 6. DELETE via private gRPC (admin operation) + private_grpc.delete_instance_type(name=it_name) + names = private_grpc.list_instance_type_names() + assert it_name not in names, f"InstanceType {it_name} still in list after delete: {names}" + + # 7. NEGATIVE: get after delete should fail + output, rc = private_grpc.call_unchecked(service=f"{PRIVATE_API}.InstanceTypes/Get", data={"id": it_name}) + assert rc != 0, f"get after delete should fail, but rc={rc}, output: {output}" + error_lower = output.lower() + assert any(term in error_lower for term in ["not found", "404", "notfound"]), ( + f"Expected not-found error after delete, got: {output}" + ) + + finally: + try: + private_grpc.delete_instance_type(name=it_name) + except subprocess.CalledProcessError as e: + output = ((e.stdout or "") + (e.stderr or "")).lower() + if "not found" not in output: + raise + + +def test_create_instance_type_via_cli(private_cli: OsacCLI, private_grpc: GRPCClient) -> None: + it_name: str = f"e2e-cli-it-{uuid4().hex[:8]}" + + try: + private_cli.create_instance_type( + name=it_name, + cores=TEST_CORES, + memory_gib=TEST_MEMORY_GIB, + description="CLI create test type", + gpu_pci_device_selector=TEST_GPU["pci_device_selector"], + gpu_resource_name=TEST_GPU["resource_name"], + gpu_count=TEST_GPU["count"], + ) + + response: dict = private_grpc.get_instance_type(name=it_name) + spec: dict = response["object"]["spec"] + assert spec["cores"] == TEST_CORES, f"spec.cores mismatch: {spec['cores']} != {TEST_CORES}" + assert spec["memoryGib"] == TEST_MEMORY_GIB, ( + f"spec.memoryGib mismatch: {spec['memoryGib']} != {TEST_MEMORY_GIB}" + ) + assert "gpu" in spec, f"spec.gpu missing from response: {spec}" + gpu: dict = spec["gpu"] + assert gpu["pciDeviceSelector"] == TEST_GPU["pci_device_selector"], ( + f"gpu.pciDeviceSelector mismatch: {gpu['pciDeviceSelector']} != {TEST_GPU['pci_device_selector']}" + ) + assert gpu["resourceName"] == TEST_GPU["resource_name"], ( + f"gpu.resourceName mismatch: {gpu['resourceName']} != {TEST_GPU['resource_name']}" + ) + assert gpu["count"] == TEST_GPU["count"], f"gpu.count mismatch: {gpu['count']} != {TEST_GPU['count']}" + + finally: + try: + private_grpc.delete_instance_type(name=it_name) + except subprocess.CalledProcessError as e: + output = ((e.stdout or "") + (e.stderr or "")).lower() + if "not found" not in output: + raise + + +def test_gpu_instance_type(private_grpc: GRPCClient) -> None: + gpu_name: str = f"e2e-gpu-lifecycle-{uuid4().hex[:8]}" + nogpu_name: str = f"e2e-nogpu-lifecycle-{uuid4().hex[:8]}" + + try: + # 1. CREATE GPU-enabled InstanceType and verify GPU fields + private_grpc.create_instance_type( + name=gpu_name, + cores=TEST_CORES, + memory_gib=TEST_MEMORY_GIB, + description="GPU lifecycle test type", + gpu=TEST_GPU, + ) + + response: dict = private_grpc.get_instance_type(name=gpu_name) + spec: dict = response["object"]["spec"] + assert "gpu" in spec, f"spec.gpu missing from response: {spec}" + gpu: dict = spec["gpu"] + assert gpu["pciDeviceSelector"] == TEST_GPU["pci_device_selector"], ( + f"gpu.pciDeviceSelector mismatch: {gpu['pciDeviceSelector']} != {TEST_GPU['pci_device_selector']}" + ) + assert gpu["resourceName"] == TEST_GPU["resource_name"], ( + f"gpu.resourceName mismatch: {gpu['resourceName']} != {TEST_GPU['resource_name']}" + ) + assert gpu["count"] == TEST_GPU["count"], f"gpu.count mismatch: {gpu['count']} != {TEST_GPU['count']}" + + # 2. LIST: GPU types are distinguishable from non-GPU types + private_grpc.create_instance_type( + name=nogpu_name, cores=TEST_CORES, memory_gib=TEST_MEMORY_GIB, description="Non-GPU lifecycle test type" + ) + + list_response: dict[str, Any] = private_grpc.call(service=f"{PRIVATE_API}.InstanceTypes/List") + items: list[dict[str, Any]] = list_response.get("items", []) + gpu_items = [i for i in items if i["metadata"]["name"] == gpu_name] + nogpu_items = [i for i in items if i["metadata"]["name"] == nogpu_name] + + assert len(gpu_items) == 1, f"GPU InstanceType {gpu_name} not found in list" + assert len(nogpu_items) == 1, f"Non-GPU InstanceType {nogpu_name} not found in list" + assert "gpu" in gpu_items[0]["spec"], f"GPU type should have gpu field: {gpu_items[0]['spec']}" + assert "gpu" not in nogpu_items[0]["spec"], f"Non-GPU type should not have gpu field: {nogpu_items[0]['spec']}" + + # 3. IMMUTABILITY: update attempt with different GPU is rejected + output, rc = private_grpc.call_unchecked( + service=f"{PRIVATE_API}.InstanceTypes/Update", + data={ + "object": { + "id": gpu_name, + "spec": { + "gpu": {"pci_device_selector": "10DE:2204", "resource_name": "nvidia.com/A10", "count": 2} + }, + } + }, + ) + assert rc != 0, f"update should reject GPU field change, but rc={rc}, output: {output}" + assert "immutable" in output.lower(), f"Expected immutability error, got: {output}" + + response = private_grpc.get_instance_type(name=gpu_name) + assert response["object"]["spec"]["gpu"] == gpu, "GPU fields should be unchanged after rejected update" + + finally: + for name in (gpu_name, nogpu_name): + try: + private_grpc.delete_instance_type(name=name) + except subprocess.CalledProcessError as e: + output = ((e.stdout or "") + (e.stderr or "")).lower() + if "not found" not in output: + raise diff --git a/tests/e2e/tests/vmaas/test_jwt_auth_smoke.py b/tests/e2e/tests/vmaas/test_jwt_auth_smoke.py new file mode 100644 index 0000000000..a1efa3676a --- /dev/null +++ b/tests/e2e/tests/vmaas/test_jwt_auth_smoke.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import time +from uuid import uuid4 + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import run_unchecked + +CLIENT_LISTABLE_RESOURCES = [ + "clustertemplates", + "clusters", + "computeinstancetemplates", + "computeinstances", + "hosttypes", + "networkclasses", + "externalips", + "rolebindings", + "roles", + "securitygroups", + "subnets", + "virtualnetworks", +] + + +# JWT List access for all resource types + + +@pytest.mark.parametrize("resource", CLIENT_LISTABLE_RESOURCES) +def test_jwt_user_can_list(jwt_cli_user: OsacCLI, resource: str) -> None: + jwt_cli_user.get(resource) + + +@pytest.mark.parametrize("resource", CLIENT_LISTABLE_RESOURCES) +def test_jwt_admin_can_list(jwt_cli_admin: OsacCLI, resource: str) -> None: + jwt_cli_admin.get(resource) + + +# Authorization boundary: Users API + + +def test_jwt_user_denied_users(jwt_cli_user: OsacCLI) -> None: + jwt_cli_user.relogin() + output, rc = jwt_cli_user.get_unchecked("users") + assert rc != 0, f"Regular user should be denied access to users, got: {output}" + assert "denied" in output.lower() or "permission" in output.lower() + + +def test_jwt_admin_can_list_users(jwt_cli_admin: OsacCLI) -> None: + jwt_cli_admin.relogin() + jwt_cli_admin.get("users") + + +# Invalid token rejection + + +def test_invalid_token_rejected(fulfillment_address: str) -> None: + output, rc = run_unchecked( + "grpcurl", + "-insecure", + "-H", + "Authorization: Bearer invalid", + fulfillment_address, + "osac.public.v1.ComputeInstances/List", + ) + assert rc != 0, f"Invalid token should be rejected, got: {output}" + + +# JWT CRUD lifecycle + + +def test_jwt_virtual_network_lifecycle(jwt_grpc_tenant1: GRPCClient) -> None: + vn_name: str = f"jwt-smoke-{uuid4().hex[:8]}" + vn_id: str = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class="cudn-net", ipv4_cidr="10.200.0.0/16" + ) + assert vn_id in jwt_grpc_tenant1.list_virtual_network_ids() + + vn: dict = jwt_grpc_tenant1.get_virtual_network(vn_id=vn_id) + assert vn["object"]["metadata"]["name"] == vn_name + + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) + + +def test_jwt_security_group_lifecycle(jwt_grpc_tenant1: GRPCClient) -> None: + vn_name: str = f"jwt-sg-vnet-{uuid4().hex[:8]}" + vn_id: str = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class="cudn-net", ipv4_cidr="10.202.0.0/16" + ) + + for _ in range(90): + vn = jwt_grpc_tenant1.get_virtual_network(vn_id=vn_id) + if vn["object"].get("status", {}).get("state") == "VIRTUAL_NETWORK_STATE_READY": + break + time.sleep(2) + else: + pytest.fail(f"VirtualNetwork {vn_id} did not reach READY state within 180s") + + sg_name: str = f"jwt-sg-{uuid4().hex[:8]}" + sg_id: str = jwt_grpc_tenant1.create_security_group(name=sg_name, virtual_network=vn_id) + assert sg_id in jwt_grpc_tenant1.list_security_group_ids() + + jwt_grpc_tenant1.delete_security_group(sg_id=sg_id) + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) + + +# Multi-tenant isolation + + +def test_jwt_tenant_isolation(jwt_grpc_tenant1: GRPCClient, jwt_grpc_tenant2: GRPCClient) -> None: + vn_name: str = f"tenant-iso-{uuid4().hex[:8]}" + vn_id: str = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class="cudn-net", ipv4_cidr="10.201.0.0/16" + ) + + assert vn_id in jwt_grpc_tenant1.list_virtual_network_ids() + assert vn_id not in jwt_grpc_tenant2.list_virtual_network_ids() + + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) diff --git a/tests/e2e/tests/vmaas/test_metadata_name_validation.py b/tests/e2e/tests/vmaas/test_metadata_name_validation.py new file mode 100644 index 0000000000..f0a1218e45 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_metadata_name_validation.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import re +import subprocess +import uuid + +import pytest + +from tests.core.grpc_client import PUBLIC_API, GRPCClient +from tests.core.helpers import ( + assert_grpc_rejected, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + + +def _grpc_error_message(exc: subprocess.CalledProcessError) -> str: + combined = (exc.stderr or "") + (exc.stdout or "") + match = re.search(r"Message:\s*(.+)", combined) + return match.group(1).strip() if match else "" + + +class TestVirtualNetworkNameValidation: + """Validates Metadata.name enforcement on VirtualNetwork (tenant-scoped).""" + + def test_create_without_name(self, grpc: GRPCClient, network_class: str) -> None: + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={"object": {"spec": {"network_class": {"name": network_class}, "ipv4_cidr": "10.100.0.0/16"}}}, + ) + assert_grpc_rejected(exc_info, "InvalidArgument") + grpc_msg = _grpc_error_message(exc_info.value) + assert "metadata is required" in grpc_msg.lower(), "gRPC rejection should reference metadata is required" + + @pytest.mark.parametrize( + "invalid_name", + ["Test-VNet", "test_vnet!", "-starts-with-hyphen", "ends-with-hyphen-", "has spaces", "ALLCAPS"], + ids=["uppercase-mixed", "special-chars", "leading-hyphen", "trailing-hyphen", "spaces", "all-uppercase"], + ) + def test_create_with_invalid_name(self, grpc: GRPCClient, network_class: str, invalid_name: str) -> None: + with pytest.raises(subprocess.CalledProcessError) as exc_info: + grpc.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": { + "metadata": {"name": invalid_name}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": "10.100.0.0/16"}, + } + }, + ) + assert_grpc_rejected(exc_info, "InvalidArgument") + + def test_create_with_valid_dns_name(self, grpc: GRPCClient, k8s_hub_client: K8sClient, network_class: str) -> None: + vn_name = f"test-valid-dns-{uuid.uuid4().hex[:8]}" + vn_id: str = grpc.create_virtual_network(name=vn_name, network_class=network_class, ipv4_cidr="10.110.0.0/16") + cr_name: str | None = None + try: + cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=cr_name) + + vn: dict = grpc.get_virtual_network(vn_id=vn_id) + assert vn["object"]["metadata"]["name"] == vn_name + finally: + grpc.delete_virtual_network(vn_id=vn_id) + if cr_name: + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=cr_name) + poll_until( + fn=lambda: vn_id not in grpc.list_virtual_network_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"VirtualNetwork {vn_id} removal from API", + ) diff --git a/tests/e2e/tests/vmaas/test_name_immutability.py b/tests/e2e/tests/vmaas/test_name_immutability.py new file mode 100644 index 0000000000..3ef95d7781 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_name_immutability.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import subprocess +import uuid + +import pytest + +from tests.core.grpc_client import PUBLIC_API, GRPCClient +from tests.core.helpers import assert_grpc_rejected + + +def _grpc_error_output(exc: subprocess.CalledProcessError) -> str: + return (exc.stderr or "") + (exc.stdout or "") + + +class TestVirtualNetworkImmutability: + """Immutable fields on VirtualNetwork cannot be changed after creation.""" + + def test_update_name_rejected(self, jwt_grpc_tenant1: GRPCClient, network_class: str) -> None: + vn_name = f"imm-name-{uuid.uuid4().hex[:8]}" + vn_id: str | None = None + try: + vn_id = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.130.0.0/16" + ) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.VirtualNetworks/Update", + data={ + "object": { + "id": vn_id, + "metadata": {"name": f"renamed-{vn_name}"}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": "10.130.0.0/16"}, + }, + "update_mask": {"paths": ["metadata.name"]}, + }, + ) + assert_grpc_rejected(exc_info, "InvalidArgument") + output = _grpc_error_output(exc_info.value) + assert "metadata.name" in output.lower() or "immutable" in output.lower(), ( + f"Error should mention 'metadata.name' or 'immutable', got: {output}" + ) + finally: + if vn_id: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) + + @pytest.mark.skip( + reason="Cannot test tenant/project immutability" + " - referencing non-existent tenant fails with PermissionDenied before immutability check" + ) + @pytest.mark.parametrize( + ("field_path", "field_key", "field_value"), + [("metadata.tenant", "tenant", "different-tenant"), ("metadata.project", "project", "different-project")], + ids=["tenant", "project"], + ) + def test_update_tenant_or_project_rejected( + self, jwt_grpc_tenant1: GRPCClient, network_class: str, field_path: str, field_key: str, field_value: str + ) -> None: + vn_name = f"imm-tp-{uuid.uuid4().hex[:8]}" + vn_id: str | None = None + try: + vn_id = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.131.0.0/16" + ) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.VirtualNetworks/Update", + data={ + "object": { + "id": vn_id, + "metadata": {"name": vn_name, field_key: field_value}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": "10.131.0.0/16"}, + }, + "update_mask": {"paths": [field_path]}, + }, + ) + assert_grpc_rejected(exc_info, "InvalidArgument") + output = _grpc_error_output(exc_info.value) + assert "immutable" in output.lower() or field_path in output.lower(), ( + f"Error should mention 'immutable' or '{field_path}', got: {output}" + ) + finally: + if vn_id: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) diff --git a/tests/e2e/tests/vmaas/test_name_uniqueness.py b/tests/e2e/tests/vmaas/test_name_uniqueness.py new file mode 100644 index 0000000000..161f3b232e --- /dev/null +++ b/tests/e2e/tests/vmaas/test_name_uniqueness.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import re +import subprocess +import uuid + +import pytest + +from tests.core.grpc_client import PRIVATE_API, PUBLIC_API, GRPCClient +from tests.core.helpers import ( + assert_grpc_rejected, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient + + +def _grpc_error_message(exc: subprocess.CalledProcessError) -> str: + combined = (exc.stderr or "") + (exc.stdout or "") + match = re.search(r"Message:\s*(.+)", combined) + return match.group(1).strip() if match else "" + + +class TestVirtualNetworkProjectScopedUniqueness: + """VirtualNetwork names must be unique within a (tenant, project) scope.""" + + def test_duplicate_name_rejected(self, jwt_grpc_tenant1: GRPCClient, network_class: str) -> None: + vn_name = f"dup-vn-{uuid.uuid4().hex[:8]}" + vn_id: str | None = None + try: + vn_id = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.120.0.0/16" + ) + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.121.0.0/16" + ) + assert_grpc_rejected(exc_info, "AlreadyExists") + msg = _grpc_error_message(exc_info.value) + assert "virtual network" in msg.lower(), f"Error should mention 'virtual network', got: {msg}" + finally: + if vn_id: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) + + def test_duplicate_name_during_deletion_rejected( + self, jwt_grpc_tenant1: GRPCClient, k8s_hub_client: K8sClient, network_class: str + ) -> None: + vn_name = f"del-dup-{uuid.uuid4().hex[:8]}" + vn_id = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.122.0.0/16" + ) + cr_name = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=cr_name) + + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.123.0.0/16" + ) + assert_grpc_rejected(exc_info, "AlreadyExists") + + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=cr_name) + + def test_same_name_different_tenants_succeeds( + self, jwt_grpc_tenant1: GRPCClient, jwt_grpc_tenant2: GRPCClient, network_class: str + ) -> None: + vn_name = f"cross-t-{uuid.uuid4().hex[:8]}" + vn_id_t1: str | None = None + vn_id_t2: str | None = None + try: + vn_id_t1 = jwt_grpc_tenant1.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.124.0.0/16" + ) + vn_id_t2 = jwt_grpc_tenant2.create_virtual_network( + name=vn_name, network_class=network_class, ipv4_cidr="10.125.0.0/16" + ) + assert vn_id_t1 != vn_id_t2 + finally: + if vn_id_t1: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id_t1) + if vn_id_t2: + jwt_grpc_tenant2.delete_virtual_network(vn_id=vn_id_t2) + + @pytest.mark.skip( + reason="Projects feature not yet implemented - projects must exist before resources can reference them" + ) + def test_same_name_different_projects_succeeds(self, jwt_grpc_tenant1: GRPCClient, network_class: str) -> None: + vn_name = f"cross-p-{uuid.uuid4().hex[:8]}" + vn_id_p1: str | None = None + vn_id_p2: str | None = None + try: + vn_id_p1 = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": { + "metadata": {"name": vn_name, "project": "project-alpha"}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": "10.126.0.0/16"}, + } + }, + )["object"]["id"] + vn_id_p2 = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": { + "metadata": {"name": vn_name, "project": "project-beta"}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": "10.127.0.0/16"}, + } + }, + )["object"]["id"] + assert vn_id_p1 != vn_id_p2 + finally: + if vn_id_p1: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id_p1) + if vn_id_p2: + jwt_grpc_tenant1.delete_virtual_network(vn_id=vn_id_p2) + + +class TestNetworkClassPlatformScopedUniqueness: + """NetworkClass names are platform-scoped: globally unique across all tenants.""" + + @pytest.mark.skip(reason="NetworkClassSpec fields unknown - implementation_strategy not a valid field") + def test_duplicate_network_class_name_rejected(self, private_grpc: GRPCClient, network_class: str) -> None: + with pytest.raises(subprocess.CalledProcessError) as exc_info: + private_grpc.call( + service=f"{PRIVATE_API}.NetworkClasses/Create", + data={"object": {"metadata": {"name": network_class}, "spec": {"implementation_strategy": "ovn"}}}, + ) + assert_grpc_rejected(exc_info, "AlreadyExists") + + +class TestRoleGlobalUniqueness: + """Roles are globally unique: same name in different tenants is rejected.""" + + @pytest.mark.skip(reason="JWT tokens lack permission to create Roles - requires admin credentials") + def test_duplicate_role_name_different_tenants_rejected( + self, jwt_grpc_tenant1: GRPCClient, jwt_grpc_tenant2: GRPCClient + ) -> None: + role_name = f"e2e-role-{uuid.uuid4().hex[:8]}" + role_id: str | None = None + try: + response = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.Roles/Create", data={"object": {"metadata": {"name": role_name}}} + ) + role_id = response["object"]["id"] + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant2.call( + service=f"{PUBLIC_API}.Roles/Create", data={"object": {"metadata": {"name": role_name}}} + ) + assert_grpc_rejected(exc_info, "AlreadyExists") + finally: + if role_id: + jwt_grpc_tenant1.call(service=f"{PUBLIC_API}.Roles/Delete", data={"id": role_id}) + + +class TestUserTenantScopedUniqueness: + """User names are tenant-scoped: unique within a tenant, allowed across tenants.""" + + @pytest.mark.skip(reason="JWT tokens lack permission to create Users - requires admin credentials") + def test_duplicate_user_name_same_tenant_rejected(self, jwt_grpc_tenant1: GRPCClient) -> None: + user_name = f"e2e-user-{uuid.uuid4().hex[:8]}" + user_id: str | None = None + try: + response = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.Users/Create", data={"object": {"metadata": {"name": user_name}}} + ) + user_id = response["object"]["id"] + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.Users/Create", data={"object": {"metadata": {"name": user_name}}} + ) + assert_grpc_rejected(exc_info, "AlreadyExists") + finally: + if user_id: + jwt_grpc_tenant1.call(service=f"{PUBLIC_API}.Users/Delete", data={"id": user_id}) + + @pytest.mark.skip(reason="JWT tokens lack permission to create Users - requires admin credentials") + def test_same_user_name_different_tenants_succeeds( + self, jwt_grpc_tenant1: GRPCClient, jwt_grpc_tenant2: GRPCClient + ) -> None: + user_name = f"e2e-user-{uuid.uuid4().hex[:8]}" + user_id_t1: str | None = None + user_id_t2: str | None = None + try: + response_t1 = jwt_grpc_tenant1.call( + service=f"{PUBLIC_API}.Users/Create", data={"object": {"metadata": {"name": user_name}}} + ) + user_id_t1 = response_t1["object"]["id"] + + response_t2 = jwt_grpc_tenant2.call( + service=f"{PUBLIC_API}.Users/Create", data={"object": {"metadata": {"name": user_name}}} + ) + user_id_t2 = response_t2["object"]["id"] + + assert user_id_t1 != user_id_t2 + finally: + if user_id_t1: + jwt_grpc_tenant1.call(service=f"{PUBLIC_API}.Users/Delete", data={"id": user_id_t1}) + if user_id_t2: + jwt_grpc_tenant2.call(service=f"{PUBLIC_API}.Users/Delete", data={"id": user_id_t2}) diff --git a/tests/e2e/tests/vmaas/test_security_group_lifecycle.py b/tests/e2e/tests/vmaas/test_security_group_lifecycle.py new file mode 100644 index 0000000000..a546bc03c3 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_security_group_lifecycle.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from uuid import uuid4 + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_security_group_cr, + wait_for_security_group_deletion, + wait_for_security_group_ready, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + + +def test_security_group_lifecycle(grpc: GRPCClient, k8s_hub_client: K8sClient, network_class: str) -> None: + vn_name: str = f"sg-test-vnet-{uuid4().hex[:8]}" + vn_id: str = grpc.create_virtual_network(name=vn_name, network_class=network_class, ipv4_cidr="10.210.0.0/16") + vn_cr_name: str = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vn_cr_name) + + sg_name: str = f"sg-test-{uuid4().hex[:8]}" + sg_id: str = grpc.create_security_group(name=sg_name, virtual_network=vn_id) + + sg_cr_name: str = wait_for_security_group_cr(k8s=k8s_hub_client, uuid=sg_id) + assert sg_id in grpc.list_security_group_ids() + + sg: dict = grpc.get_security_group(sg_id=sg_id) + assert sg["object"]["metadata"]["name"] == sg_name + + wait_for_security_group_ready(k8s=k8s_hub_client, name=sg_cr_name) + + grpc.delete_security_group(sg_id=sg_id) + wait_for_security_group_deletion(k8s=k8s_hub_client, name=sg_cr_name) + poll_until( + fn=lambda: sg_id not in grpc.list_security_group_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"SecurityGroup {sg_id} removal from API", + ) + + grpc.delete_virtual_network(vn_id=vn_id) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=vn_cr_name) diff --git a/tests/e2e/tests/vmaas/test_subnet_lifecycle.py b/tests/e2e/tests/vmaas/test_subnet_lifecycle.py new file mode 100644 index 0000000000..04296a7fd7 --- /dev/null +++ b/tests/e2e/tests/vmaas/test_subnet_lifecycle.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from uuid import uuid4 + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_subnet_ready, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + + +def test_subnet_lifecycle(grpc: GRPCClient, k8s_hub_client: K8sClient, network_class: str) -> None: + vn_name: str = f"test-vnet-{uuid4().hex[:8]}" + vn_id: str = grpc.create_virtual_network(name=vn_name, network_class=network_class, ipv4_cidr="10.200.0.0/16") + vn_cr_name: str = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vn_cr_name) + + subnet_name: str = f"test-subnet-{uuid4().hex[:8]}" + subnet_id: str = grpc.create_subnet(name=subnet_name, virtual_network=vn_id, ipv4_cidr="10.200.1.0/24") + subnet_cr_name: str = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_id) + + assert subnet_id in grpc.list_subnet_ids() + + subnet: dict = grpc.get_subnet(subnet_id=subnet_id) + assert subnet["object"]["metadata"]["name"] == subnet_name + + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_cr_name) + + grpc.delete_subnet(subnet_id=subnet_id) + wait_for_subnet_deletion(k8s=k8s_hub_client, name=subnet_cr_name) + poll_until( + fn=lambda: subnet_id not in grpc.list_subnet_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"Subnet {subnet_id} removal from API", + ) + + grpc.delete_virtual_network(vn_id=vn_id) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=vn_cr_name) + poll_until( + fn=lambda: vn_id not in grpc.list_virtual_network_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"VirtualNetwork {vn_id} removal from API", + ) diff --git a/tests/e2e/tests/vmaas/test_virtual_network_lifecycle.py b/tests/e2e/tests/vmaas/test_virtual_network_lifecycle.py new file mode 100644 index 0000000000..7b38ed594e --- /dev/null +++ b/tests/e2e/tests/vmaas/test_virtual_network_lifecycle.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from uuid import uuid4 + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + + +def test_virtual_network_lifecycle(grpc: GRPCClient, k8s_hub_client: K8sClient, network_class: str) -> None: + vn_name: str = f"test-vnet-{uuid4().hex[:8]}" + vn_id: str = grpc.create_virtual_network(name=vn_name, network_class=network_class, ipv4_cidr="10.100.0.0/16") + cr_name: str = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vn_id) + + assert vn_id in grpc.list_virtual_network_ids() + + vn: dict = grpc.get_virtual_network(vn_id=vn_id) + assert vn["object"]["metadata"]["name"] == vn_name + + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=cr_name) + + grpc.delete_virtual_network(vn_id=vn_id) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=cr_name) + poll_until( + fn=lambda: vn_id not in grpc.list_virtual_network_ids(), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"VirtualNetwork {vn_id} removal from API", + ) diff --git a/tests/e2e/uv.lock b/tests/e2e/uv.lock new file mode 100644 index 0000000000..c4ac6c2cf7 --- /dev/null +++ b/tests/e2e/uv.lock @@ -0,0 +1,169 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "osac-e2e-tests" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "pytest" }, + { name = "pytest-xdist" }, + { name = "pyyaml" }, + { name = "websocket-client" }, +] + +[package.metadata] +requires-dist = [ + { name = "pytest", specifier = ">=8.0" }, + { name = "pytest-xdist", specifier = ">=3.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "websocket-client", specifier = ">=1.6" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +]