From d5d61974e9a890d7a4c412faca502f7d29c55f0a Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Sun, 22 Mar 2026 04:38:44 +0200 Subject: [PATCH 001/112] MGMT-22783: add pytest e2e test suite for vmaas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port all ansible e2e tests to pytest with 1:1 parity. The hub creation test was removed — hub creation is implicitly verified by every compute instance test since they all go through the fulfillment api on the hub. Tests support a two-cluster topology where the hub cluster runs the osac operator and the remote cluster runs the kubevirt VMs. The k8s client takes a kubeconfig parameter, and tests that need to inspect VMs on the remote cluster use a separate OSAC_VM_KUBECONFIG env var. See MGMT-23623 for the remote cluster setup script. Tests: - compute instance lifecycle (create, wait for running, delete) - delete during provision - restart (trigger via grpc, verify new vmi creation timestamp) - restart negative (past timestamp ignored) - api fields (explicit cpu/memory/disk via grpc) - cli fields (explicit cpu/memory/disk via fulfillment-cli) Infrastructure: - k8s client with two-kubeconfig support (hub + remote) - grpc client wrapping grpcurl - fulfillment-cli wrapper with typed methods - poll_until helper matching ansible retry semantics - Makefile with MAKEFILE_TARGET dispatch for ci - pyproject.toml with ruff and basedpyright config --- tests/__init__.py | 0 tests/conftest.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..78480d5519 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os + +import pytest + +from tests.fulfillment_cli import FulfillmentCLI +from tests.grpc_client import GRPCClient +from tests.k8s_client import K8sClient +from tests.runner import env, run + + +@pytest.fixture(scope="session") +def namespace() -> str: + return env("OSAC_NAMESPACE", "osac-devel") + + +@pytest.fixture(scope="session") +def cluster_domain() -> str: + return run("kubectl", "get", "ingress.config.openshift.io", "cluster", "-o", "jsonpath={.spec.domain}") + + +@pytest.fixture(scope="session") +def fulfillment_address(namespace: str, cluster_domain: str) -> str: + return env("OSAC_FULFILLMENT_ADDRESS", f"fulfillment-api-{namespace}.{cluster_domain}:443") + + +@pytest.fixture(scope="session") +def service_account() -> str: + return env("OSAC_SERVICE_ACCOUNT", "admin") + + +@pytest.fixture(scope="session") +def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPCClient: + token: str = run( + "oc", "create", "token", service_account, "-n", namespace, "--duration", "1h", "--as", "system:admin" + ) + return GRPCClient(address=fulfillment_address, token=token) + + +@pytest.fixture(scope="session") +def k8s_hub_client(namespace: str) -> K8sClient: + return K8sClient(namespace=namespace) + + +@pytest.fixture(scope="session") +def k8s_virt_client(namespace: str) -> K8sClient: + vm_kubeconfig: str = os.environ["OSAC_VM_KUBECONFIG"] + return K8sClient(namespace=namespace, kubeconfig=vm_kubeconfig) + + +@pytest.fixture(scope="session") +def cli(namespace: str, fulfillment_address: str, service_account: str) -> FulfillmentCLI: + return FulfillmentCLI( + binary=env("FULFILLMENT_CLI_PATH", "fulfillment-cli"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=f"oc create token -n {namespace} {service_account} --as system:admin", + namespace=namespace, + ) + + +@pytest.fixture(scope="session") +def vm_template() -> str: + return env("OSAC_VM_TEMPLATE", "osac.templates.ocp_virt_vm") From 52e0f5ab89ff6eb133ea04b67996f05b963d679c Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 14 Apr 2026 16:11:11 +0300 Subject: [PATCH 002/112] MGMT-22635: add VMaaS networking tests and test infrastructure improvements --- tests/conftest.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 78480d5519..b75c30c35c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,5 @@ from __future__ import annotations -import os - import pytest from tests.fulfillment_cli import FulfillmentCLI @@ -43,12 +41,6 @@ def k8s_hub_client(namespace: str) -> K8sClient: return K8sClient(namespace=namespace) -@pytest.fixture(scope="session") -def k8s_virt_client(namespace: str) -> K8sClient: - vm_kubeconfig: str = os.environ["OSAC_VM_KUBECONFIG"] - return K8sClient(namespace=namespace, kubeconfig=vm_kubeconfig) - - @pytest.fixture(scope="session") def cli(namespace: str, fulfillment_address: str, service_account: str) -> FulfillmentCLI: return FulfillmentCLI( @@ -57,8 +49,3 @@ def cli(namespace: str, fulfillment_address: str, service_account: str) -> Fulfi token_script=f"oc create token -n {namespace} {service_account} --as system:admin", namespace=namespace, ) - - -@pytest.fixture(scope="session") -def vm_template() -> str: - return env("OSAC_VM_TEMPLATE", "osac.templates.ocp_virt_vm") From 09d584359c13c7c4ca1a5562e9395a27f43401b1 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 21 Apr 2026 20:44:52 +0300 Subject: [PATCH 003/112] MGMT-22635: fix test defaults for CI environment - Change default CLI binary from fulfillment-cli to osac - Fix OSAC_NETWORK_CLASS default from osac.templates.cudn_net to cudn_net to match actual NetworkClass implementation_strategy --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index b75c30c35c..b29b28c107 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,7 +44,7 @@ def k8s_hub_client(namespace: str) -> K8sClient: @pytest.fixture(scope="session") def cli(namespace: str, fulfillment_address: str, service_account: str) -> FulfillmentCLI: return FulfillmentCLI( - binary=env("FULFILLMENT_CLI_PATH", "fulfillment-cli"), + binary=env("FULFILLMENT_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", token_script=f"oc create token -n {namespace} {service_account} --as system:admin", namespace=namespace, From 486bb39e20dd3e6fbfc9ede3d6a83a46e8471c63 Mon Sep 17 00:00:00 2001 From: Will Gordon Date: Tue, 28 Apr 2026 14:34:15 -0400 Subject: [PATCH 004/112] MGMT-23845: renames fulfillment-cli to osac in osac-test-infra Generated-By: Claude Code (Anthropic) Signed-off-by: Will Gordon --- tests/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b29b28c107..35d568dabf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import pytest -from tests.fulfillment_cli import FulfillmentCLI +from tests.osac_cli import OsacCLI from tests.grpc_client import GRPCClient from tests.k8s_client import K8sClient from tests.runner import env, run @@ -42,9 +42,9 @@ def k8s_hub_client(namespace: str) -> K8sClient: @pytest.fixture(scope="session") -def cli(namespace: str, fulfillment_address: str, service_account: str) -> FulfillmentCLI: - return FulfillmentCLI( - binary=env("FULFILLMENT_CLI_PATH", "osac"), +def cli(namespace: str, fulfillment_address: str, service_account: str) -> OsacCLI: + return OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", token_script=f"oc create token -n {namespace} {service_account} --as system:admin", namespace=namespace, From b80fcff057350d9f11457a5e7315a907e6d0f605 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 5 May 2026 07:58:39 +0300 Subject: [PATCH 005/112] NO_ISSUE: remove unused code and refactor tests structure to have better control on run_if_changed tests triggers on release --- tests/conftest.py | 8 +- tests/core/__init__.py | 0 tests/core/grpc_client.py | 83 +++++++++++++ tests/core/helpers.py | 145 +++++++++++++++++++++++ tests/core/k8s_client.py | 237 ++++++++++++++++++++++++++++++++++++++ tests/core/osac_cli.py | 83 +++++++++++++ tests/core/runner.py | 40 +++++++ 7 files changed, 592 insertions(+), 4 deletions(-) create mode 100644 tests/core/__init__.py create mode 100644 tests/core/grpc_client.py create mode 100644 tests/core/helpers.py create mode 100644 tests/core/k8s_client.py create mode 100644 tests/core/osac_cli.py create mode 100644 tests/core/runner.py diff --git a/tests/conftest.py b/tests/conftest.py index 35d568dabf..5b10e6603c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,10 +2,10 @@ import pytest -from tests.osac_cli import OsacCLI -from tests.grpc_client import GRPCClient -from tests.k8s_client import K8sClient -from tests.runner import env, run +from tests.core.grpc_client import GRPCClient +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import env, run @pytest.fixture(scope="session") diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py new file mode 100644 index 0000000000..c6cfe75aa2 --- /dev/null +++ b/tests/core/grpc_client.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import json +from typing import Any + +from tests.core.runner import run + +PUBLIC_API: str = "osac.public.v1" +PRIVATE_API: str = "osac.private.v1" + + +class GRPCClient: + def __init__(self, *, address: str, token: str) -> None: + self.address: str = address + self.token: str = token + + def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: + args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] + if data is not None: + args.extend(["-d", json.dumps(data)]) + args.extend([self.address, service]) + return json.loads(run(*args)) + + def list_compute_instance_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/List") + return [item["id"] for item in response.get("items", [])] + + def get_hub(self, *, hub_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.Hubs/Get", data={"id": hub_id}) + + def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.ComputeInstances/Update", + data={ + "object": {"id": uuid, "spec": {"template": template, "restart_requested_at": timestamp}}, + "updateMask": {"paths": ["spec.restart_requested_at"]}, + }, + ) + + # VirtualNetwork operations + + def create_virtual_network(self, *, name: str, network_class: str, ipv4_cidr: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={ + "object": {"metadata": {"name": name}, "spec": {"network_class": network_class, "ipv4_cidr": ipv4_cidr}} + }, + ) + return response["object"]["id"] + + def list_virtual_network_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.VirtualNetworks/List") + return [item["id"] for item in response.get("items", [])] + + def delete_virtual_network(self, *, vn_id: str) -> None: + self.call(service=f"{PUBLIC_API}.VirtualNetworks/Delete", data={"id": vn_id}) + + # Subnet operations + + def create_subnet(self, *, name: str, virtual_network: str, ipv4_cidr: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.Subnets/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"virtual_network": virtual_network, "ipv4_cidr": ipv4_cidr}, + } + }, + ) + return response["object"]["id"] + + def list_subnet_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Subnets/List") + return [item["id"] for item in response.get("items", [])] + + def delete_subnet(self, *, subnet_id: str) -> None: + self.call(service=f"{PUBLIC_API}.Subnets/Delete", data={"id": subnet_id}) + + # Cluster operations + + def list_cluster_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Clusters/List") + return [item["id"] for item in response.get("items", [])] diff --git a/tests/core/helpers.py b/tests/core/helpers.py new file mode 100644 index 0000000000..534e8017f8 --- /dev/null +++ b/tests/core/helpers.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until + + +def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_compute_instance_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"CR for {uuid}", + ) + + +def wait_for_provision(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_latest_job_state(name=name, job_type="provision", checked=False), + until=lambda v: v == "Succeeded", + retries=120, + delay=5, + description=f"provision Succeeded for {name}", + ) + + +def wait_for_running(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_phase(name=name, checked=False), + until=lambda v: v == "Running", + retries=90, + delay=10, + description=f"{name} Running", + ) + + +def wait_for_restart(*, k8s: K8sClient, name: str, initial: str, restart_ts: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_last_restarted_at(name=name), + until=lambda v: v != "" and v != initial and v >= restart_ts, + retries=30, + delay=10, + description=f"{name} lastRestartedAt update", + ) + + +def wait_for_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="computeinstance", name=name), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{name} deletion", + ) + + +def wait_for_virtual_network_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_virtual_network_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"VirtualNetwork CR for {uuid}", + ) + + +def wait_for_virtual_network_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_virtual_network_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} VirtualNetwork Ready", + ) + + +def wait_for_virtual_network_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="virtualnetwork", name=name), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{name} VirtualNetwork deletion", + ) + + +def wait_for_subnet_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_subnet_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"Subnet CR for {uuid}", + ) + + +def wait_for_subnet_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_subnet_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} Subnet Ready", + ) + + +def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="subnet", name=name), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{name} Subnet deletion", + ) + + +def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_cluster_order_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"ClusterOrder CR for {uuid}", + ) + + +def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: + def _check_phase() -> str: + phase: str = k8s.get_cluster_order_phase(name=name, checked=False) + assert phase != "Failed", f"{name} ClusterOrder entered Failed phase" + return phase + + poll_until( + fn=_check_phase, until=lambda v: v == "Ready", retries=120, delay=15, description=f"{name} ClusterOrder Ready" + ) + + +def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="clusterorder", name=name), + until=lambda v: v is True, + retries=120, + delay=10, + description=f"{name} ClusterOrder deletion", + ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py new file mode 100644 index 0000000000..29c1a6d01f --- /dev/null +++ b/tests/core/k8s_client.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import json +import subprocess +from typing import Any + +from tests.core.runner import run, run_unchecked + + +class K8sClient: + def __init__(self, *, namespace: str, kubeconfig: str | None = None) -> None: + self.namespace: str = namespace + self.kubeconfig: str | None = kubeconfig + + def _base(self) -> list[str]: + args: list[str] = ["kubectl"] + if self.kubeconfig is not None: + args.extend(["--kubeconfig", self.kubeconfig]) + args.extend(["--as", "system:admin"]) + return args + + def _get(self, *args: str, checked: bool = True) -> tuple[str, int]: + if checked: + return run(*self._base(), *args), 0 + return run_unchecked(*self._base(), *args) + + # Generic kubectl operations + + def get_json(self, *, resource: str, name: str) -> dict[str, Any]: + return json.loads(run(*self._base(), "get", resource, name, "-n", self.namespace, "-o", "json")) + + def get_jsonpath(self, *, resource: str, name: str, jsonpath: str) -> str: + return run(*self._base(), "get", resource, name, "-n", self.namespace, "-o", f"jsonpath={jsonpath}") + + def get_by_label(self, *, resource: str, label: str, jsonpath: str) -> str: + return run(*self._base(), "get", resource, "-n", self.namespace, "-l", label, "-o", f"jsonpath={jsonpath}") + + def patch(self, *, resource: str, name: str, patch: str) -> tuple[str, int]: + return run_unchecked(*self._base(), "patch", resource, name, "-n", self.namespace, "--type=merge", "-p", patch) + + def apply(self, *, manifest: str) -> None: + args: list[str] = [*self._base(), "apply", "-f", "-"] + subprocess.run(args, input=manifest, capture_output=True, text=True, check=True) + + def delete(self, *, resource: str, name: str) -> None: + run(*self._base(), "delete", resource, name, "-n", self.namespace) + + def is_present(self, *, resource: str, name: str) -> bool: + _, rc = run_unchecked(*self._base(), "get", resource, name, "-n", self.namespace) + return rc == 0 + + def count_by_label_all_namespaces(self, *, resource: str, label: str) -> int: + output: str = run(*self._base(), "get", resource, "-A", "-l", label, "--no-headers") + if not output: + return 0 + return len(output.strip().splitlines()) + + # ComputeInstance queries + + def get_compute_instance_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "computeinstance", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/computeinstance-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_compute_instance_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "computeinstance", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_compute_instance_last_restarted_at(self, *, name: str) -> str: + return self.get_jsonpath(resource="computeinstance", name=name, jsonpath="{.status.lastRestartedAt}") + + def get_compute_instance_latest_job_id(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0]["jobID"] + + def get_compute_instance_latest_job_state(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + + def get_compute_instance_vm_namespace(self, *, name: str) -> str: + return self.get_jsonpath( + resource="computeinstance", name=name, jsonpath="{.status.virtualMachineReference.namespace}" + ) + + # VirtualNetwork queries + + def get_virtual_network_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "virtualnetwork", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/virtualnetwork-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_virtual_network_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "virtualnetwork", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # Subnet queries + + def get_subnet_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "subnet", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/subnet-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_subnet_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "subnet", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # VirtualMachine/VMI queries (explicit namespace — may be on a different cluster) + + def get_vmi_creation_timestamp(self, *, vmi_namespace: str, compute_instance_name: str) -> str: + return run( + *self._base(), + "get", + "virtualmachineinstance", + "-n", + vmi_namespace, + "-l", + f"osac.openshift.io/computeinstance={compute_instance_name}", + "-o", + "jsonpath={.items[0].metadata.creationTimestamp}", + ) + + def get_vm_printable_status(self, *, name: str, vm_namespace: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "virtualmachine", + name, + "-n", + vm_namespace, + "-o", + "jsonpath={.status.printableStatus}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_vm_run_strategy(self, *, name: str, vm_namespace: str) -> str: + return run( + *self._base(), "get", "virtualmachine", name, "-n", vm_namespace, "-o", "jsonpath={.spec.runStrategy}" + ) + + # ClusterOrder queries + + def get_cluster_order_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "clusterorder", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/clusterorder-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_cluster_order_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "clusterorder", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_cluster_order_latest_job_id(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0]["jobID"] + + def get_cluster_order_latest_job_state(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + + def get_cluster_order_hosted_cluster_name(self, *, name: str) -> str: + return self.get_jsonpath( + resource="clusterorder", name=name, jsonpath="{.status.clusterReference.hostedClusterName}" + ) + + def get_cluster_order_namespace(self, *, name: str) -> str: + return self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.status.clusterReference.namespace}") diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py new file mode 100644 index 0000000000..ce379f401e --- /dev/null +++ b/tests/core/osac_cli.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import re + +from tests.core.runner import run + + +class OsacCLI: + def __init__(self, *, binary: str, address: str, token_script: str, namespace: str) -> None: + self.binary: str = binary + self.namespace: str = namespace + run(binary, "login", "--address", address, "--insecure", "--token-script", token_script) + + def create_hub(self, *, hub_id: str, kubeconfig: str) -> None: + run(self.binary, "create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) + + def create_compute_instance( + self, + *, + template: str, + cores: int = 2, + memory_gib: int = 4, + boot_disk_size: int = 20, + image: str = "quay.io/containerdisks/fedora:latest", + image_source_type: str = "registry", + run_strategy: str = "Always", + user_data_secret_ref: str | None = None, + ) -> str: + args: list[str] = [ + self.binary, + "create", + "computeinstance", + "--template", + template, + "--cores", + str(cores), + "--memory-gib", + str(memory_gib), + "--boot-disk-size", + str(boot_disk_size), + "--image", + image, + "--image-source-type", + image_source_type, + "--run-strategy", + run_strategy, + ] + if user_data_secret_ref is not None: + args.extend(["--user-data", user_data_secret_ref]) + + stdout: str = run(*args) + match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) + assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" + return match.group(1) + + def delete_compute_instance(self, *, uuid: str) -> None: + run(self.binary, "delete", "computeinstance", uuid) + + def create_cluster( + self, + *, + template: str, + name: str | None = None, + template_parameters: dict[str, str] | None = None, + template_parameter_files: dict[str, str] | None = None, + ) -> str: + args: list[str] = [self.binary, "create", "cluster", "--template", template] + if name is not None: + args.extend(["--name", name]) + if template_parameters is not None: + for key, value in template_parameters.items(): + args.extend(["-p", f"{key}={value}"]) + if template_parameter_files is not None: + for key, path in template_parameter_files.items(): + args.extend(["-f", f"{key}={path}"]) + + stdout: str = run(*args) + match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) + assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" + return match.group(1) + + def delete_cluster(self, *, uuid: str) -> None: + run(self.binary, "delete", "cluster", uuid) diff --git a/tests/core/runner.py b/tests/core/runner.py new file mode 100644 index 0000000000..51786d091b --- /dev/null +++ b/tests/core/runner.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import os +import subprocess +import time +from collections.abc import Callable +from typing import TypeVar + +T = TypeVar("T") + + +def run(*args: str, timeout: int = 300) -> str: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=True) + return result.stdout.strip() + + +def run_unchecked(*args: str, timeout: int = 300) -> tuple[str, int]: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False) + combined = (result.stdout.strip() + "\n" + result.stderr.strip()).strip() + return combined, result.returncode + + +def poll_until( + *, fn: Callable[[], T], until: Callable[[T], bool], retries: int = 60, delay: int = 5, description: str +) -> T: + value: T | None = None + for _ in range(retries): + value = fn() + if until(value): + return value + time.sleep(delay) + raise TimeoutError(f"{description} — timeout after {retries * delay}s, last value: {value!r}") + + +def env(name: str, default: str | None = None) -> str: + value = os.environ.get(name, default) + if value is None: + msg = f"Required environment variable {name} is not set" + raise RuntimeError(msg) + return value From e8d7cb9d80f7aadaa721ec29cd4e2648c4d04a38 Mon Sep 17 00:00:00 2001 From: Elad Tabak Date: Sun, 10 May 2026 12:38:26 +0300 Subject: [PATCH 006/112] OSAC-150: add E2E test for explicit cluster configuration fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add --pull-secret-file and --ssh-public-key-file params to OsacCLI.create_cluster - Add get_cluster method to GRPCClient - Add get_cluster_order_spec method to K8sClient - Add test_cluster_explicit_fields: verifies explicit fields flow through CLI → API (with redaction) → ClusterOrder CR (with typed fields) Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Elad Tabak --- tests/core/grpc_client.py | 3 +++ tests/core/k8s_client.py | 4 ++++ tests/core/osac_cli.py | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index c6cfe75aa2..890c667f93 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -81,3 +81,6 @@ def delete_subnet(self, *, subnet_id: str) -> None: def list_cluster_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Clusters/List") return [item["id"] for item in response.get("items", [])] + + def get_cluster(self, *, cluster_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.Clusters/Get", data={"id": cluster_id}) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 29c1a6d01f..f1fb00f4bd 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -235,3 +235,7 @@ def get_cluster_order_hosted_cluster_name(self, *, name: str) -> str: def get_cluster_order_namespace(self, *, name: str) -> str: return self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.status.clusterReference.namespace}") + + def get_cluster_order_spec(self, *, name: str) -> dict[str, Any]: + output = self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.spec}") + return json.loads(output) if output else {} diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index ce379f401e..0e9f1bfc60 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -61,12 +61,18 @@ def create_cluster( *, template: str, name: str | None = None, + pull_secret_file: str | None = None, + ssh_public_key_file: str | None = None, template_parameters: dict[str, str] | None = None, template_parameter_files: dict[str, str] | None = None, ) -> str: args: list[str] = [self.binary, "create", "cluster", "--template", template] if name is not None: args.extend(["--name", name]) + if pull_secret_file is not None: + args.extend(["--pull-secret-file", pull_secret_file]) + if ssh_public_key_file is not None: + args.extend(["--ssh-public-key-file", ssh_public_key_file]) if template_parameters is not None: for key, value in template_parameters.items(): args.extend(["-p", f"{key}={value}"]) From c53c191fbe285fdc326da473b2ac7b0da7626a0d Mon Sep 17 00:00:00 2001 From: Omer Vishlitzky Date: Wed, 13 May 2026 00:25:58 +0300 Subject: [PATCH 007/112] fix gRPC deletion race after CR removal The feedback controller removes its finalizer before sending the Signal RPC to fulfillment-service. This means the CR disappears from kubectl before the fulfillment-service archives the DB record. Tests that assert immediately after wait_for_deletion hit a race where the UUID is still in the gRPC list. Replace bare assertions with poll_until via wait_for_grpc_removal (up to 60s). In the normal case the UUID is gone on the first poll. --- tests/core/helpers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 534e8017f8..d05c07ee96 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +from tests.core.grpc_client import GRPCClient from tests.core.k8s_client import K8sClient from tests.core.runner import poll_until @@ -54,6 +55,16 @@ def wait_for_deletion(*, k8s: K8sClient, name: str) -> None: ) +def wait_for_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_compute_instance_ids(), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"{uuid} removed from gRPC list", + ) + + def wait_for_virtual_network_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_virtual_network_name(uuid=uuid, checked=False), From ccc3a3fb1332de8a48219c16eff49b69eb2a2521 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 13 May 2026 12:29:41 +0300 Subject: [PATCH 008/112] NO-ISSUE: add JWT auth, multi-tenant, and SecurityGroup E2E tests The E2E tests authenticate exclusively via Kubernetes service account tokens, never via Keycloak JWT, and have no multi-tenant isolation or SecurityGroup lifecycle coverage. New test infrastructure: - OsacCLI.get() and get_unchecked() for osac get - Keycloak JWT token helper (tests/core/keycloak.py) - JWT fixtures: jwt_cli_user, jwt_cli_admin, jwt_grpc_tenant1/2 - GRPCClient: SecurityGroup CRUD, Get for VNet/Subnet/ComputeInstance - K8sClient: SecurityGroup CR queries - Helpers: SecurityGroup wait functions New tests: - JWT List access for all 12 public API resource types x 2 users (24) - Authorization boundary: regular user denied Users, admin allowed (2) - Invalid token rejection (1) - JWT VirtualNetwork lifecycle: create/get/list/delete via JWT (1) - JWT SecurityGroup lifecycle: create/list/delete via JWT (1) - Multi-tenant isolation: tenant1 resource invisible to tenant2 (1) New test file: SecurityGroup lifecycle (SA token): - Create VNet, create SecurityGroup, wait for CR and Ready - Verify Get returns correct name - Delete SecurityGroup and VNet, verify cleanup Extended existing tests with Get assertions: - test_virtual_network_lifecycle: Get after create - test_subnet_lifecycle: Get after create - test_compute_instance_creation: Get after create Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/conftest.py | 56 +++++++++++++++++++++++++++++++++++++++ tests/core/grpc_client.py | 28 ++++++++++++++++++++ tests/core/helpers.py | 30 +++++++++++++++++++++ tests/core/k8s_client.py | 22 +++++++++++++++ tests/core/keycloak.py | 33 +++++++++++++++++++++++ tests/core/osac_cli.py | 16 ++++++++++- 6 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 tests/core/keycloak.py diff --git a/tests/conftest.py b/tests/conftest.py index 5b10e6603c..1aca9ff641 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ from tests.core.grpc_client import GRPCClient from tests.core.k8s_client import K8sClient +from tests.core.keycloak import get_jwt from tests.core.osac_cli import OsacCLI from tests.core.runner import env, run @@ -49,3 +50,58 @@ def cli(namespace: str, fulfillment_address: str, service_account: str) -> OsacC token_script=f"oc create token -n {namespace} {service_account} --as system:admin", namespace=namespace, ) + + +@pytest.fixture(scope="session") +def keycloak_url(cluster_domain: str) -> str: + return env("OSAC_KEYCLOAK_URL", f"https://keycloak-keycloak.{cluster_domain}") + + +@pytest.fixture(scope="session") +def jwt_password() -> str: + return env("OSAC_JWT_PASSWORD", "foobar") + + +def _make_jwt_token_script(keycloak_url: str, username: str, password: str) -> str: + return ( + f"curl -sk -X POST {keycloak_url}/realms/osac/protocol/openid-connect/token" + f" -d grant_type=password -d client_id=osac-cli" + f" -d username={username} -d password={password} -d scope=openid" + " | python3 -c \"import sys,json;print(json.load(sys.stdin)['access_token'])\"" + ) + + +@pytest.fixture(scope="session") +def jwt_cli_user(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> OsacCLI: + return OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, "my_user", jwt_password), + namespace=namespace, + ) + + +@pytest.fixture(scope="session") +def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> OsacCLI: + return OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, "tenant1_admin", jwt_password), + namespace=namespace, + ) + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant1(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + token: str = get_jwt( + keycloak_url=keycloak_url, realm="osac", client_id="osac-cli", username="tenant1_user", password=jwt_password + ) + return GRPCClient(address=fulfillment_address, token=token) + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant2(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + token: str = get_jwt( + keycloak_url=keycloak_url, realm="osac", client_id="osac-cli", username="tenant2_user", password=jwt_password + ) + return GRPCClient(address=fulfillment_address, token=token) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 890c667f93..dc79a4e15b 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -25,6 +25,9 @@ def list_compute_instance_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/List") return [item["id"] for item in response.get("items", [])] + def get_compute_instance(self, *, ci_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ComputeInstances/Get", data={"id": ci_id}) + def get_hub(self, *, hub_id: str) -> dict[str, Any]: return self.call(service=f"{PRIVATE_API}.Hubs/Get", data={"id": hub_id}) @@ -48,6 +51,9 @@ def create_virtual_network(self, *, name: str, network_class: str, ipv4_cidr: st ) return response["object"]["id"] + def get_virtual_network(self, *, vn_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.VirtualNetworks/Get", data={"id": vn_id}) + def list_virtual_network_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.VirtualNetworks/List") return [item["id"] for item in response.get("items", [])] @@ -69,6 +75,9 @@ def create_subnet(self, *, name: str, virtual_network: str, ipv4_cidr: str) -> s ) return response["object"]["id"] + def get_subnet(self, *, subnet_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.Subnets/Get", data={"id": subnet_id}) + def list_subnet_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Subnets/List") return [item["id"] for item in response.get("items", [])] @@ -84,3 +93,22 @@ def list_cluster_ids(self) -> list[str]: def get_cluster(self, *, cluster_id: str) -> dict[str, Any]: return self.call(service=f"{PUBLIC_API}.Clusters/Get", data={"id": cluster_id}) + + # SecurityGroup operations + + def create_security_group(self, *, name: str, virtual_network: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", + data={"object": {"metadata": {"name": name}, "spec": {"virtual_network": virtual_network}}}, + ) + return response["object"]["id"] + + def get_security_group(self, *, sg_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.SecurityGroups/Get", data={"id": sg_id}) + + def list_security_group_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.SecurityGroups/List") + return [item["id"] for item in response.get("items", [])] + + def delete_security_group(self, *, sg_id: str) -> None: + self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index d05c07ee96..40f8dacb3c 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -154,3 +154,33 @@ def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: delay=10, description=f"{name} ClusterOrder deletion", ) + + +def wait_for_security_group_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_security_group_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"SecurityGroup CR for {uuid}", + ) + + +def wait_for_security_group_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_security_group_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} SecurityGroup Ready", + ) + + +def wait_for_security_group_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="securitygroup", name=name), + until=lambda v: v is True, + retries=30, + delay=5, + description=f"{name} SecurityGroup deletion", + ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index f1fb00f4bd..3b1628e98b 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -239,3 +239,25 @@ def get_cluster_order_namespace(self, *, name: str) -> str: def get_cluster_order_spec(self, *, name: str) -> dict[str, Any]: output = self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.spec}") return json.loads(output) if output else {} + + # SecurityGroup queries + + def get_security_group_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "securitygroup", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/securitygroup-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_security_group_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "securitygroup", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" diff --git a/tests/core/keycloak.py b/tests/core/keycloak.py new file mode 100644 index 0000000000..428c0021f9 --- /dev/null +++ b/tests/core/keycloak.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json + +from tests.core.runner import run + + +def get_jwt(*, keycloak_url: str, realm: str, client_id: str, username: str, password: str) -> str: + token_url = f"{keycloak_url}/realms/{realm}/protocol/openid-connect/token" + stdout: str = run( + "curl", + "-sk", + "--fail-with-body", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + f"client_id={client_id}", + "-d", + f"username={username}", + "-d", + f"password={password}", + "-d", + "scope=openid", + ) + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get JWT from Keycloak for user '{username}': {error}") + return token diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 0e9f1bfc60..3dc2b4b283 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -2,15 +2,20 @@ import re -from tests.core.runner import run +from tests.core.runner import run, run_unchecked class OsacCLI: def __init__(self, *, binary: str, address: str, token_script: str, namespace: str) -> None: self.binary: str = binary self.namespace: str = namespace + self._address: str = address + self._token_script: str = token_script run(binary, "login", "--address", address, "--insecure", "--token-script", token_script) + def relogin(self) -> None: + run(self.binary, "login", "--address", self._address, "--insecure", "--token-script", self._token_script) + def create_hub(self, *, hub_id: str, kubeconfig: str) -> None: run(self.binary, "create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) @@ -85,5 +90,14 @@ def create_cluster( assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" return match.group(1) + def get(self, resource: str, *, output: str | None = None) -> str: + args: list[str] = [self.binary, "get", resource] + if output is not None: + args.extend(["-o", output]) + return run(*args) + + def get_unchecked(self, resource: str) -> tuple[str, int]: + return run_unchecked(self.binary, "get", resource) + def delete_cluster(self, *, uuid: str) -> None: run(self.binary, "delete", "cluster", uuid) From b5d080e198748be1073330cadb9ed46e94cee249 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 20 May 2026 12:34:39 +0300 Subject: [PATCH 009/112] MGMT-22635: add CaaS test coverage and test-caas Makefile target Add test-caas Makefile target, kubeconfig/password retrieval tests, template immutability test, and fix cluster grpc removal to use polling instead of one-shot assertion. Update default template to ocp_ci_small for CI environments. --- tests/core/helpers.py | 10 ++++++++++ tests/core/osac_cli.py | 3 +++ 2 files changed, 13 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 40f8dacb3c..194e3ae7b5 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -156,6 +156,16 @@ def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: ) +def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_cluster_ids(), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{uuid} removed from gRPC cluster list", + ) + + def wait_for_security_group_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_security_group_name(uuid=uuid, checked=False), diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 3dc2b4b283..b88f27196f 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -96,6 +96,9 @@ def get(self, resource: str, *, output: str | None = None) -> str: args.extend(["-o", output]) return run(*args) + def get_cluster_credential(self, credential: str, *, uuid: str) -> str: + return run(self.binary, "get", credential, uuid) + def get_unchecked(self, resource: str) -> tuple[str, int]: return run_unchecked(self.binary, "get", resource) From 51089e581b74cc5faf8b48bdf7aa06aec466d717 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 26 May 2026 17:30:18 +0300 Subject: [PATCH 010/112] bump deletion timeouts from 300s to 600s When an AAP deprovision job fails (intermittent receptor worker stream drop, ~2-5% rate), the operator retries with exponential backoff. The retry succeeds within ~7 minutes but the previous 300s (5 min) timeout on kubectl delete and deletion wait polls expires before the retry completes, failing the test. Bump all deletion waits to 120 retries x 5s = 600s (10 min) and kubectl delete timeout to 600s to accommodate the operator retry. --- tests/core/helpers.py | 8 ++++---- tests/core/k8s_client.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 194e3ae7b5..93bf9f2d58 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -49,7 +49,7 @@ def wait_for_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="computeinstance", name=name), until=lambda v: v is True, - retries=60, + retries=120, delay=5, description=f"{name} deletion", ) @@ -89,7 +89,7 @@ def wait_for_virtual_network_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="virtualnetwork", name=name), until=lambda v: v is True, - retries=60, + retries=120, delay=5, description=f"{name} VirtualNetwork deletion", ) @@ -119,7 +119,7 @@ def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="subnet", name=name), until=lambda v: v is True, - retries=60, + retries=120, delay=5, description=f"{name} Subnet deletion", ) @@ -190,7 +190,7 @@ def wait_for_security_group_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="securitygroup", name=name), until=lambda v: v is True, - retries=30, + retries=120, delay=5, description=f"{name} SecurityGroup deletion", ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 3b1628e98b..7990e78a5c 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -43,7 +43,7 @@ def apply(self, *, manifest: str) -> None: subprocess.run(args, input=manifest, capture_output=True, text=True, check=True) def delete(self, *, resource: str, name: str) -> None: - run(*self._base(), "delete", resource, name, "-n", self.namespace) + run(*self._base(), "delete", resource, name, "-n", self.namespace, timeout=600) def is_present(self, *, resource: str, name: str) -> bool: _, rc = run_unchecked(*self._base(), "get", resource, name, "-n", self.namespace) From 617b8ce66f47c984654ce45777a6226b30e8ee19 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 26 May 2026 19:44:48 +0300 Subject: [PATCH 011/112] helpers: fail fast when provision job or phase enters Failed wait_for_provision and wait_for_running poll until success but do not check for terminal failure. When provisioning fails immediately (e.g. DataVolumeError), each test wastes 10-15 minutes of the 60-minute CI timeout before raising TimeoutError. Assert state/phase != Failed so the test fails immediately with a clear error instead of burning the timeout budget. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 194e3ae7b5..b24fefdb3f 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -16,8 +16,13 @@ def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_provision(*, k8s: K8sClient, name: str) -> None: + def _check_state() -> str: + state: str = k8s.get_compute_instance_latest_job_state(name=name, job_type="provision", checked=False) + assert state != "Failed", f"{name} provision job entered Failed state" + return state + poll_until( - fn=lambda: k8s.get_compute_instance_latest_job_state(name=name, job_type="provision", checked=False), + fn=_check_state, until=lambda v: v == "Succeeded", retries=120, delay=5, @@ -26,8 +31,13 @@ def wait_for_provision(*, k8s: K8sClient, name: str) -> None: def wait_for_running(*, k8s: K8sClient, name: str) -> None: + def _check_phase() -> str: + phase: str = k8s.get_compute_instance_phase(name=name, checked=False) + assert phase != "Failed", f"{name} entered Failed phase" + return phase + poll_until( - fn=lambda: k8s.get_compute_instance_phase(name=name, checked=False), + fn=_check_phase, until=lambda v: v == "Running", retries=90, delay=10, From 7b6ae5a5cfed6af45d6535c40193bf649c8cf752 Mon Sep 17 00:00:00 2001 From: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Date: Wed, 27 May 2026 11:52:02 -0400 Subject: [PATCH 012/112] fix: use Provisioned condition instead of job state in wait_for_provision Previously wait_for_provision checked the latest AAP job state and failed immediately if it saw "Failed". The operator retries failed provision jobs with exponential backoff, so a transient AAP failure followed by a successful retry is normal behavior. Asserting on individual job state made the test fragile and caused false failures in CI. The fix polls the Provisioned status condition instead, which reflects the end result (infrastructure provisioned) regardless of how many AAP job attempts it took. Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Assisted-by: Cursor/Claude Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> --- tests/core/helpers.py | 13 +++++-------- tests/core/k8s_client.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index f6db6dbcc9..7581b73a28 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -16,17 +16,14 @@ def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_provision(*, k8s: K8sClient, name: str) -> None: - def _check_state() -> str: - state: str = k8s.get_compute_instance_latest_job_state(name=name, job_type="provision", checked=False) - assert state != "Failed", f"{name} provision job entered Failed state" - return state - poll_until( - fn=_check_state, - until=lambda v: v == "Succeeded", + fn=lambda: k8s.get_compute_instance_condition_status( + name=name, condition_type="Provisioned", checked=False + ), + until=lambda v: v == "True", retries=120, delay=5, - description=f"provision Succeeded for {name}", + description=f"{name} Provisioned condition", ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 7990e78a5c..8b68fae71c 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -102,6 +102,16 @@ def get_compute_instance_latest_job_state(self, *, name: str, job_type: str, che return "" return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + def get_compute_instance_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + def get_compute_instance_vm_namespace(self, *, name: str) -> str: return self.get_jsonpath( resource="computeinstance", name=name, jsonpath="{.status.virtualMachineReference.namespace}" From 219dd4c1585460cb0463c795ce790617f5cad1a3 Mon Sep 17 00:00:00 2001 From: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Date: Wed, 27 May 2026 12:05:30 -0400 Subject: [PATCH 013/112] fix: fast-fail on terminal Failed phase during provision wait Add a phase check so wait_for_provision fails immediately if the ComputeInstance reaches Failed phase, rather than waiting the full timeout. Transient AAP job failures (which keep the phase at Starting) are still tolerated until the Provisioned condition becomes True. Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Assisted-by: Cursor/Claude Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> --- tests/core/helpers.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 7581b73a28..c058f8b6ad 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -16,10 +16,15 @@ def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_provision(*, k8s: K8sClient, name: str) -> None: - poll_until( - fn=lambda: k8s.get_compute_instance_condition_status( + def _check_provisioned() -> str: + phase: str = k8s.get_compute_instance_phase(name=name, checked=False) + assert phase != "Failed", f"{name} entered Failed phase before Provisioned=True" + return k8s.get_compute_instance_condition_status( name=name, condition_type="Provisioned", checked=False - ), + ) + + poll_until( + fn=_check_provisioned, until=lambda v: v == "True", retries=120, delay=5, From cf2608de8a26f8324390a3733537656afbc03197 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Thu, 28 May 2026 00:40:09 +0300 Subject: [PATCH 014/112] fix: harden CaaS tests against fragile assertions and hypershift deletion bug - test_cluster_explicit_fields: use template parameters (-p/-f) instead of deprecated --pull-secret-file/--ssh-public-key-file CLI flags, update assertions to check templateParameters instead of top-level spec fields - test_cluster_order_delete_during_provision: check CR phase (Progressing) instead of internal job state which races with operator retries, replace bare grpc assert with retried wait_for_cluster_grpc_removal - helpers: work around hypershift bug where capi-provider-agent controller is killed during HostedCluster teardown before it can remove the AgentCluster deprovision finalizer, causing infinite deletion deadlock. Force-remove orphaned finalizers during wait_for_cluster_deletion poll. Co-Authored-By: Claude Code --- tests/core/helpers.py | 44 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index c058f8b6ad..365f6dab53 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -2,7 +2,7 @@ from tests.core.grpc_client import GRPCClient from tests.core.k8s_client import K8sClient -from tests.core.runner import poll_until +from tests.core.runner import poll_until, run_unchecked def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: @@ -159,8 +159,20 @@ def _check_phase() -> str: def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: + # HACK: Hypershift has a bug where the capi-provider-agent controller (which runs + # inside the hosted control plane namespace) is killed during teardown before it can + # remove the AgentCluster deprovision finalizer. This orphaned finalizer blocks + # namespace termination, which blocks hypershift's own finalizer removal, deadlocking + # deletion indefinitely. The same class of bug was already fixed for Karpenter: + # https://github.com/openshift/hypershift/blob/main/hypershift-operator/controllers/hostedcluster/karpenter.go#L88 + # (resolveKarpenterFinalizer). Until hypershift applies the same fix for CAPI + # infrastructure CRs, we force-remove the orphaned finalizer on every poll iteration. + def _check_deleted() -> bool: + _force_cleanup_agentcluster_finalizers(k8s=k8s, name=name) + return not k8s.is_present(resource="clusterorder", name=name) + poll_until( - fn=lambda: not k8s.is_present(resource="clusterorder", name=name), + fn=_check_deleted, until=lambda v: v is True, retries=120, delay=10, @@ -168,6 +180,34 @@ def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: ) +def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None: + # HCP namespace: {osac-ns}-{co-name}-{hc-name}, where hc-name == co-name + hc_ns = f"{k8s.namespace}-{name}" + cp_ns = f"{hc_ns}-{name}" + finalizer = "agentclustercapi-provider.agent-install.openshift.io/deprovision" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, "get", "agentclusters.capi-provider.agent-install.openshift.io", + "-n", cp_ns, "-o", f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", + ) + if rc != 0 or not output.strip(): + return + for ac_name in output.strip().split(): + finalizers_json, rc = run_unchecked( + *base_args, "get", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", cp_ns, "-o", "jsonpath={.metadata.finalizers}", + ) + if rc != 0 or finalizer not in finalizers_json: + continue + import json + idx = json.loads(finalizers_json).index(finalizer) + run_unchecked( + *base_args, "patch", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", cp_ns, "--type=json", + f'-p=[{{"op": "remove", "path": "/metadata/finalizers/{idx}"}}]', + ) + + def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: poll_until( fn=lambda: uuid not in grpc.list_cluster_ids(), From c1f306162076e4a05e8b941e964a454f768741ff Mon Sep 17 00:00:00 2001 From: Juan Hernandez Date: Fri, 29 May 2026 12:40:24 +0200 Subject: [PATCH 015/112] NO-ISSUE: Ensure tenant organizations exist before E2E tests Fulfillment-service pull request #603 adds a foreign key constraint requiring every resource's tenant to reference an existing organization. Create the required organizations (shared, tenant1, tenant2) at session start so that tests creating resources via the API continue to work. Related: https://github.com/osac-project/fulfillment-service/pull/603 Assisted-by: Claude Code Signed-off-by: Juan Hernandez --- tests/conftest.py | 6 ++++++ tests/core/grpc_client.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 1aca9ff641..257069bde5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,12 @@ def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPC return GRPCClient(address=fulfillment_address, token=token) +@pytest.fixture(scope="session", autouse=True) +def ensure_organizations(grpc: GRPCClient) -> None: + for name in ("shared", "tenant1", "tenant2"): + grpc.ensure_organization(name=name) + + @pytest.fixture(scope="session") def k8s_hub_client(namespace: str) -> K8sClient: return K8sClient(namespace=namespace) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index dc79a4e15b..b7380d7ed3 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import subprocess from typing import Any from tests.core.runner import run @@ -112,3 +113,15 @@ def list_security_group_ids(self) -> list[str]: def delete_security_group(self, *, sg_id: str) -> None: self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) + + # Organization operations + + def ensure_organization(self, *, name: str) -> None: + try: + self.call( + service=f"{PRIVATE_API}.Organizations/Create", + data={"object": {"metadata": {"name": name}}}, + ) + except subprocess.CalledProcessError as e: + if "AlreadyExists" not in (e.stdout or "") and "AlreadyExists" not in (e.stderr or ""): + raise From 36d5acad540663abcc70dc5b60ce4437e906de6f Mon Sep 17 00:00:00 2001 From: Juan Hernandez Date: Fri, 29 May 2026 12:46:08 +0200 Subject: [PATCH 016/112] Remove 'shared' from the list of organizations to create The 'shared' organization is built-in and doesn't need to be created. Signed-off-by: Juan Hernandez Assisted-by: Claude Code Signed-off-by: Juan Hernandez --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 257069bde5..5fd1ea8682 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,7 @@ def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPC @pytest.fixture(scope="session", autouse=True) def ensure_organizations(grpc: GRPCClient) -> None: - for name in ("shared", "tenant1", "tenant2"): + for name in ("tenant1", "tenant2"): grpc.ensure_organization(name=name) From 2346d721c6348ed4e98cfa227f1845f6d6b76738 Mon Sep 17 00:00:00 2001 From: Juan Hernandez Date: Fri, 29 May 2026 13:49:23 +0200 Subject: [PATCH 017/112] Match grpcurl status code precisely in ensure_organization Check for 'Code: AlreadyExists' instead of a bare 'AlreadyExists' substring so that errors whose message happens to contain that string are not silently swallowed. Assisted-by: Claude Code Signed-off-by: Juan Hernandez --- tests/core/grpc_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index b7380d7ed3..2046f99e11 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re import subprocess from typing import Any @@ -123,5 +124,6 @@ def ensure_organization(self, *, name: str) -> None: data={"object": {"metadata": {"name": name}}}, ) except subprocess.CalledProcessError as e: - if "AlreadyExists" not in (e.stdout or "") and "AlreadyExists" not in (e.stderr or ""): - raise + output = (e.stdout or "") + (e.stderr or "") + if not re.search(r"Code:\s*AlreadyExists", output): + raise RuntimeError(f"Failed to create organization '{name}': {output}") from None From 4decc398e976b1e9cee2ad658ade85295c12b7a6 Mon Sep 17 00:00:00 2001 From: Omer Vishlitzky <22615781+omer-vishlitzky@users.noreply.github.com> Date: Sat, 30 May 2026 16:38:07 +0300 Subject: [PATCH 018/112] Revert "NO-ISSUE: Ensure tenant organizations exist before E2E tests" --- tests/conftest.py | 6 ------ tests/core/grpc_client.py | 15 --------------- 2 files changed, 21 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5fd1ea8682..1aca9ff641 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,12 +37,6 @@ def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPC return GRPCClient(address=fulfillment_address, token=token) -@pytest.fixture(scope="session", autouse=True) -def ensure_organizations(grpc: GRPCClient) -> None: - for name in ("tenant1", "tenant2"): - grpc.ensure_organization(name=name) - - @pytest.fixture(scope="session") def k8s_hub_client(namespace: str) -> K8sClient: return K8sClient(namespace=namespace) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 2046f99e11..dc79a4e15b 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -1,8 +1,6 @@ from __future__ import annotations import json -import re -import subprocess from typing import Any from tests.core.runner import run @@ -114,16 +112,3 @@ def list_security_group_ids(self) -> list[str]: def delete_security_group(self, *, sg_id: str) -> None: self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) - - # Organization operations - - def ensure_organization(self, *, name: str) -> None: - try: - self.call( - service=f"{PRIVATE_API}.Organizations/Create", - data={"object": {"metadata": {"name": name}}}, - ) - except subprocess.CalledProcessError as e: - output = (e.stdout or "") + (e.stderr or "") - if not re.search(r"Code:\s*AlreadyExists", output): - raise RuntimeError(f"Failed to create organization '{name}': {output}") from None From 3a6c696d3982346f8c7fb04537aced9248e68249 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Mon, 1 Jun 2026 23:20:01 +0300 Subject: [PATCH 019/112] OSAC-1169: force-clear orphaned agent labels during cluster deletion The HyperShift CAPI provider sometimes fails to clear the clusterdeployment-namespace label from agents after HostedCluster deletion. The delete playbook's detach_and_unlabel_all_removed_agents skips agents with this label set, leaving the clusterorder label stuck and blocking agent reuse for all subsequent tests. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 365f6dab53..a52723c3a7 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -159,16 +159,20 @@ def _check_phase() -> str: def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: - # HACK: Hypershift has a bug where the capi-provider-agent controller (which runs - # inside the hosted control plane namespace) is killed during teardown before it can - # remove the AgentCluster deprovision finalizer. This orphaned finalizer blocks - # namespace termination, which blocks hypershift's own finalizer removal, deadlocking - # deletion indefinitely. The same class of bug was already fixed for Karpenter: - # https://github.com/openshift/hypershift/blob/main/hypershift-operator/controllers/hostedcluster/karpenter.go#L88 - # (resolveKarpenterFinalizer). Until hypershift applies the same fix for CAPI - # infrastructure CRs, we force-remove the orphaned finalizer on every poll iteration. + # HACK: HyperShift has multiple teardown bugs where controllers leave orphaned state + # that deadlocks HostedCluster deletion. We force-clean on every poll iteration: + # + # 1. AgentCluster deprovision finalizer: capi-provider-agent is killed during teardown + # before removing its finalizer, blocking namespace termination. + # https://github.com/openshift/hypershift/blob/main/hypershift-operator/controllers/hostedcluster/karpenter.go#L88 + # + # 2. Agent labels: the CAPI provider sometimes fails to clear + # clusterdeployment-namespace from agents after HostedCluster deletion. The delete + # playbook's detach_and_unlabel skips agents that still have this label set, leaving + # the clusterorder label stuck and blocking agent reuse for subsequent tests. def _check_deleted() -> bool: _force_cleanup_agentcluster_finalizers(k8s=k8s, name=name) + _force_cleanup_agent_labels(k8s=k8s, name=name) return not k8s.is_present(resource="clusterorder", name=name) poll_until( @@ -208,6 +212,25 @@ def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None ) +def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: + agent_ns = "hardware-inventory" + clusterorder_label = "osac.openshift.io/clusterorder" + clusterdeployment_ns_label = "agent-install.openshift.io/clusterdeployment-namespace" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, "get", "agents.agent-install.openshift.io", + "-n", agent_ns, "-l", f"{clusterorder_label}={name}", + "-o", "jsonpath={.items[*].metadata.name}", + ) + if rc != 0 or not output.strip(): + return + for agent_name in output.strip().split(): + run_unchecked( + *base_args, "label", f"agents.agent-install.openshift.io/{agent_name}", + "-n", agent_ns, f"{clusterorder_label}-", f"{clusterdeployment_ns_label}-", + ) + + def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: poll_until( fn=lambda: uuid not in grpc.list_cluster_ids(), From 4f7cb549bf083e9eeafb98fd755a7c7ac0c87c2a Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 2 Jun 2026 01:53:16 +0300 Subject: [PATCH 020/112] OSAC-1169: use non-blocking delete in CaaS test fixtures Two CaaS test fixtures call k8s.delete() which runs kubectl delete without --wait=false. This blocks for up to 600 seconds waiting for finalizers to be removed. The wait_for_cluster_deletion() function (which contains the HyperShift finalizer and agent label workarounds) is the next call after k8s.delete() and never gets to run. Switch these fixtures to k8s.delete(wait=False) so kubectl returns immediately after setting deletionTimestamp, allowing wait_for_cluster_deletion() to execute and clean up orphaned finalizers and agent labels. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/k8s_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 8b68fae71c..20cebfa40d 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -42,8 +42,11 @@ def apply(self, *, manifest: str) -> None: args: list[str] = [*self._base(), "apply", "-f", "-"] subprocess.run(args, input=manifest, capture_output=True, text=True, check=True) - def delete(self, *, resource: str, name: str) -> None: - run(*self._base(), "delete", resource, name, "-n", self.namespace, timeout=600) + def delete(self, *, resource: str, name: str, wait: bool = True) -> None: + args = [*self._base(), "delete", resource, name, "-n", self.namespace] + if not wait: + args.append("--wait=false") + run(*args, timeout=600) def is_present(self, *, resource: str, name: str) -> bool: _, rc = run_unchecked(*self._base(), "get", resource, name, "-n", self.namespace) From 0fa855b99ed824c29fcd70fec9fa53413db373a0 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 3 Jun 2026 11:59:51 +0300 Subject: [PATCH 021/112] Ensure tenant organizations exist before E2E tests fulfillment-service PR#603 added a foreign key constraint requiring every resource's tenant column to reference an existing organization. The JWT tests authenticate as tenant1/tenant2 users, but nothing creates those organizations, causing InvalidArgument errors. This combines the approach from PR#53 (ensure_organization method) with the routing fix from PR#55, using the internal API Route URL instead of ClusterIP to reach the private gRPC endpoint from the CI test pod. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/conftest.py | 19 +++++++++++++++++++ tests/core/grpc_client.py | 13 +++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 1aca9ff641..f2676c9362 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,11 @@ def fulfillment_address(namespace: str, cluster_domain: str) -> str: return env("OSAC_FULFILLMENT_ADDRESS", f"fulfillment-api-{namespace}.{cluster_domain}:443") +@pytest.fixture(scope="session") +def fulfillment_private_address(namespace: str, cluster_domain: str) -> str: + return env("OSAC_FULFILLMENT_PRIVATE_ADDRESS", f"fulfillment-internal-api-{namespace}.{cluster_domain}:443") + + @pytest.fixture(scope="session") def service_account() -> str: return env("OSAC_SERVICE_ACCOUNT", "admin") @@ -37,6 +42,20 @@ def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPC return GRPCClient(address=fulfillment_address, token=token) +@pytest.fixture(scope="session") +def private_grpc(fulfillment_private_address: str, namespace: str, service_account: str) -> GRPCClient: + token: str = run( + "oc", "create", "token", service_account, "-n", namespace, "--duration", "1h", "--as", "system:admin" + ) + return GRPCClient(address=fulfillment_private_address, token=token) + + +@pytest.fixture(scope="session", autouse=True) +def ensure_organizations(private_grpc: GRPCClient) -> None: + for name in ("tenant1", "tenant2"): + private_grpc.ensure_organization(name=name) + + @pytest.fixture(scope="session") def k8s_hub_client(namespace: str) -> K8sClient: return K8sClient(namespace=namespace) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index dc79a4e15b..f109221b1c 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import re +import subprocess from typing import Any from tests.core.runner import run @@ -112,3 +114,14 @@ def list_security_group_ids(self) -> list[str]: def delete_security_group(self, *, sg_id: str) -> None: self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) + + def ensure_organization(self, *, name: str) -> None: + try: + self.call( + service=f"{PRIVATE_API}.Organizations/Create", + data={"object": {"metadata": {"name": name}}}, + ) + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if not re.search(r"Code:\s*AlreadyExists", output): + raise RuntimeError(f"Failed to create organization '{name}': {output}") from None From a0cd4496717342eb5e108db6a84abbb1a2e4469d Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 3 Jun 2026 12:30:16 +0300 Subject: [PATCH 022/112] Preserve exception chain in ensure_organization Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/grpc_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index f109221b1c..3d41c41238 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -124,4 +124,4 @@ def ensure_organization(self, *, name: str) -> None: except subprocess.CalledProcessError as e: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): - raise RuntimeError(f"Failed to create organization '{name}': {output}") from None + raise RuntimeError(f"Failed to create organization '{name}': {output}") from e From b0722e7665e887c188f1f6f97c60f8e6416357ae Mon Sep 17 00:00:00 2001 From: Ori Amizur <60868946+ori-amizur@users.noreply.github.com> Date: Wed, 3 Jun 2026 15:26:49 +0300 Subject: [PATCH 023/112] OSAC-768: Add network attachment support for VMaaS tests (#45) Update all compute instance tests to use explicit network attachments, ensuring VMs are provisioned on OSAC-managed subnets instead of the default pod network. Changes: 1. Add network_attachments parameter to OsacCLI.create_compute_instance() - Supports subnet and security-groups configuration - Builds --network-attachment CLI flags - Validates attachment structure 2. Add networking fixtures in tests/vmaas/conftest.py: - test_run_id: Unique ID per test run to avoid resource conflicts - default_networking: Creates VirtualNetwork + Subnet with cleanup - default_subnet: Returns subnet ID for CLI/gRPC usage - default_subnet_ref: Returns subnet CR name for K8s API usage 3. Update all compute instance tests to use network attachments: - test_compute_instance_api_fields: Use default_subnet_ref - test_compute_instance_cli_fields: Use default_subnet - test_compute_instance_creation: Use default_subnet - test_compute_instance_delete_during_provision: Use default_subnet - test_compute_instance_restart: Use default_subnet - test_compute_instance_restart_negative: Use default_subnet This ensures VMs are created with proper tenant network isolation, IP management, and security group support rather than using the default Kubernetes pod network (masquerade). Co-authored-by: Claude Sonnet 4.5 --- tests/core/osac_cli.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index b88f27196f..75b326ed89 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +from typing import Any from tests.core.runner import run, run_unchecked @@ -23,6 +24,7 @@ def create_compute_instance( self, *, template: str, + network_attachments: list[dict[str, Any]] | None = None, cores: int = 2, memory_gib: int = 4, boot_disk_size: int = 20, @@ -50,6 +52,30 @@ def create_compute_instance( "--run-strategy", run_strategy, ] + + # Add network attachments + if network_attachments is not None: + for idx, attachment in enumerate(network_attachments): + subnet = attachment.get("subnet") + if not subnet or not isinstance(subnet, str): + raise ValueError(f"network_attachments[{idx}]: 'subnet' must be a non-empty string, got {subnet!r}") + + security_groups = attachment.get("security_groups", []) + if not isinstance(security_groups, list): + raise ValueError(f"network_attachments[{idx}]: 'security_groups' must be a list, got {type(security_groups).__name__}") + + if security_groups and not all(isinstance(sg, str) and sg for sg in security_groups): + raise ValueError(f"network_attachments[{idx}]: all security_groups must be non-empty strings") + + # Build network-attachment flag value + # Format: subnet=,security-groups=, + parts = [f"subnet={subnet}"] + if security_groups: + sg_list = ",".join(security_groups) + parts.append(f"security-groups={sg_list}") + + args.extend(["--network-attachment", ",".join(parts)]) + if user_data_secret_ref is not None: args.extend(["--user-data", user_data_secret_ref]) From 8eab40951a95e847fcbca69d7e9e9522a29e9e82 Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Tue, 5 May 2026 10:00:10 -0400 Subject: [PATCH 024/112] OSAC-206: Scaffold out publicip pool creation and deletion using grpc apis --- tests/core/grpc_client.py | 48 +++++++++++++++++++++++++++ tests/core/helpers.py | 70 +++++++++++++++++++++++++++++++++++++++ tests/core/k8s_client.py | 66 ++++++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 3d41c41238..7257423c82 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -125,3 +125,51 @@ def ensure_organization(self, *, name: str) -> None: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): raise RuntimeError(f"Failed to create organization '{name}': {output}") from e + + # PublicIPPool operations (private API only) + + def create_public_ip_pool( + self, + *, + name: str, + cidrs: list[str], + ip_family: str = "IP_FAMILY_IPV4", + implementation_strategy: str = "metallb-l2", + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.PublicIPPools/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "cidrs": cidrs, + "ip_family": ip_family, + "implementation_strategy": implementation_strategy, + }, + } + }, + ) + return response["object"]["id"] + + def list_public_ip_pool_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.PublicIPPools/List") + return [item["id"] for item in response.get("items", [])] + + def delete_public_ip_pool(self, *, pool_id: str) -> None: + self.call(service=f"{PRIVATE_API}.PublicIPPools/Delete", data={"id": pool_id}) + + # PublicIP operations (public API) + + def create_public_ip(self, *, name: str, pool: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.PublicIPs/Create", + data={"object": {"metadata": {"name": name}, "spec": {"pool": pool}}}, + ) + return response["object"]["id"] + + def list_public_ip_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.PublicIPs/List") + return [item["id"] for item in response.get("items", [])] + + def delete_public_ip(self, *, public_ip_id: str) -> None: + self.call(service=f"{PUBLIC_API}.PublicIPs/Delete", data={"id": public_ip_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index a52723c3a7..564e208dc4 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -137,6 +137,76 @@ def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: ) +def wait_for_public_ip_pool_uuid(*, k8s: K8sClient, name: str) -> str: + return poll_until( + fn=lambda: k8s.get_public_ip_pool_uuid(name=name, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"PublicIPPool UUID label for {name}", + ) + + +def wait_for_public_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_public_ip_pool_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"PublicIPPool CR for {uuid}", + ) + + +def wait_for_public_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_public_ip_pool_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} PublicIPPool Ready", + ) + + +def wait_for_public_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="publicippool", name=name), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{name} PublicIPPool deletion", + ) + + +def wait_for_public_ip_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_public_ip_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"PublicIP CR for {uuid}", + ) + + +def wait_for_public_ip_allocated(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_public_ip_state(name=name, checked=False), + until=lambda v: v == "Allocated", + retries=60, + delay=5, + description=f"{name} PublicIP Allocated", + ) + + +def wait_for_public_ip_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="publicip", name=name), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{name} PublicIP deletion", + ) + + def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_cluster_order_name(uuid=uuid, checked=False), diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 20cebfa40d..9c978ec446 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -197,6 +197,72 @@ def get_vm_run_strategy(self, *, name: str, vm_namespace: str) -> str: *self._base(), "get", "virtualmachine", name, "-n", vm_namespace, "-o", "jsonpath={.spec.runStrategy}" ) + # PublicIPPool queries + + def patch_public_ip_pool_implementation_strategy( + self, *, name: str, strategy: str = "metallb-l2" + ) -> tuple[str, int]: + return self.patch( + resource="publicippool", + name=name, + patch=json.dumps({"spec": {"implementationStrategy": strategy}}), + ) + + def get_public_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "publicippool", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/publicippool-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_public_ip_pool_uuid(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "publicippool", name, "-n", self.namespace, + "-o", "jsonpath={.metadata.labels.osac\\.openshift\\.io/publicippool-uuid}", checked=checked + ) + return output if rc == 0 else "" + + def get_public_ip_pool_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "publicippool", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # PublicIP queries + + def get_public_ip_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "publicip", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/publicip-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_public_ip_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "publicip", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_public_ip_state(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "publicip", name, "-n", self.namespace, "-o", "jsonpath={.status.state}", checked=checked + ) + return output if rc == 0 else "" + # ClusterOrder queries def get_cluster_order_name(self, *, uuid: str, checked: bool = True) -> str: From 656afc02650efad1bf7d1c3c3a261b993dc084a6 Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Wed, 13 May 2026 10:04:08 -0400 Subject: [PATCH 025/112] OSAC-206: Write capacity tests for public ips, group tests in subdir --- tests/core/grpc_client.py | 6 ++++++ tests/core/helpers.py | 10 ---------- tests/core/k8s_client.py | 22 ---------------------- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 7257423c82..22ae555c02 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -151,6 +151,9 @@ def create_public_ip_pool( ) return response["object"]["id"] + def get_public_ip_pool(self, *, pool_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.PublicIPPools/Get", data={"id": pool_id}) + def list_public_ip_pool_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.PublicIPPools/List") return [item["id"] for item in response.get("items", [])] @@ -167,6 +170,9 @@ def create_public_ip(self, *, name: str, pool: str) -> str: ) return response["object"]["id"] + def get_public_ip(self, *, public_ip_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.PublicIPs/Get", data={"id": public_ip_id}) + def list_public_ip_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.PublicIPs/List") return [item["id"] for item in response.get("items", [])] diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 564e208dc4..218af58a52 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -137,16 +137,6 @@ def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: ) -def wait_for_public_ip_pool_uuid(*, k8s: K8sClient, name: str) -> str: - return poll_until( - fn=lambda: k8s.get_public_ip_pool_uuid(name=name, checked=False), - until=lambda v: v != "", - retries=30, - delay=2, - description=f"PublicIPPool UUID label for {name}", - ) - - def wait_for_public_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_public_ip_pool_name(uuid=uuid, checked=False), diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 9c978ec446..9e0480317d 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -199,15 +199,6 @@ def get_vm_run_strategy(self, *, name: str, vm_namespace: str) -> str: # PublicIPPool queries - def patch_public_ip_pool_implementation_strategy( - self, *, name: str, strategy: str = "metallb-l2" - ) -> tuple[str, int]: - return self.patch( - resource="publicippool", - name=name, - patch=json.dumps({"spec": {"implementationStrategy": strategy}}), - ) - def get_public_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: output, rc = self._get( "get", @@ -222,13 +213,6 @@ def get_public_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: ) return output if rc == 0 else "" - def get_public_ip_pool_uuid(self, *, name: str, checked: bool = True) -> str: - output, rc = self._get( - "get", "publicippool", name, "-n", self.namespace, - "-o", "jsonpath={.metadata.labels.osac\\.openshift\\.io/publicippool-uuid}", checked=checked - ) - return output if rc == 0 else "" - def get_public_ip_pool_phase(self, *, name: str, checked: bool = True) -> str: output, rc = self._get( "get", "publicippool", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked @@ -251,12 +235,6 @@ def get_public_ip_name(self, *, uuid: str, checked: bool = True) -> str: ) return output if rc == 0 else "" - def get_public_ip_phase(self, *, name: str, checked: bool = True) -> str: - output, rc = self._get( - "get", "publicip", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked - ) - return output if rc == 0 else "" - def get_public_ip_state(self, *, name: str, checked: bool = True) -> str: output, rc = self._get( "get", "publicip", name, "-n", self.namespace, "-o", "jsonpath={.status.state}", checked=checked From 3e87f2fb217a11c19ca5cc0d2df1921d550ecdf6 Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Mon, 1 Jun 2026 15:03:41 -0400 Subject: [PATCH 026/112] OSAC-206: Reduce poll tretries --- tests/core/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 218af58a52..db0b54beec 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -161,7 +161,7 @@ def wait_for_public_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicippool", name=name), until=lambda v: v is True, - retries=60, + retries=18, delay=5, description=f"{name} PublicIPPool deletion", ) @@ -191,7 +191,7 @@ def wait_for_public_ip_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicip", name=name), until=lambda v: v is True, - retries=60, + retries=18, delay=5, description=f"{name} PublicIP deletion", ) From d8739660ae64a744b86c605a418e44158b9c8b11 Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Tue, 2 Jun 2026 16:13:45 -0400 Subject: [PATCH 027/112] OSAC-206: Refactor capacity tests to share state but greatly reduce runtime by cutting ip create/delete operations --- tests/core/helpers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index db0b54beec..852fe570a6 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -142,7 +142,7 @@ def wait_for_public_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: fn=lambda: k8s.get_public_ip_pool_name(uuid=uuid, checked=False), until=lambda v: v != "", retries=30, - delay=2, + delay=1, description=f"PublicIPPool CR for {uuid}", ) @@ -152,7 +152,7 @@ def wait_for_public_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: fn=lambda: k8s.get_public_ip_pool_phase(name=name, checked=False), until=lambda v: v == "Ready", retries=60, - delay=5, + delay=2, description=f"{name} PublicIPPool Ready", ) @@ -161,8 +161,8 @@ def wait_for_public_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicippool", name=name), until=lambda v: v is True, - retries=18, - delay=5, + retries=30, + delay=2, description=f"{name} PublicIPPool deletion", ) @@ -172,7 +172,7 @@ def wait_for_public_ip_cr(*, k8s: K8sClient, uuid: str) -> str: fn=lambda: k8s.get_public_ip_name(uuid=uuid, checked=False), until=lambda v: v != "", retries=30, - delay=2, + delay=1, description=f"PublicIP CR for {uuid}", ) @@ -182,7 +182,7 @@ def wait_for_public_ip_allocated(*, k8s: K8sClient, name: str) -> None: fn=lambda: k8s.get_public_ip_state(name=name, checked=False), until=lambda v: v == "Allocated", retries=60, - delay=5, + delay=2, description=f"{name} PublicIP Allocated", ) @@ -191,8 +191,8 @@ def wait_for_public_ip_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicip", name=name), until=lambda v: v is True, - retries=18, - delay=5, + retries=30, + delay=2, description=f"{name} PublicIP deletion", ) From add6d8fe3ee31991107f2a139a8df173392e5fb6 Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Wed, 3 Jun 2026 11:40:36 -0400 Subject: [PATCH 028/112] OSAC-206: Bump delete wait timeouts to be consistent with other delete ops --- tests/core/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 852fe570a6..3f519a13f2 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -161,8 +161,8 @@ def wait_for_public_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicippool", name=name), until=lambda v: v is True, - retries=30, - delay=2, + retries=120, + delay=5, description=f"{name} PublicIPPool deletion", ) @@ -191,8 +191,8 @@ def wait_for_public_ip_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicip", name=name), until=lambda v: v is True, - retries=30, - delay=2, + retries=120, + delay=5, description=f"{name} PublicIP deletion", ) From e460d6e6eb707bdb22fca2ba5f5fba11a102f515 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Fri, 5 Jun 2026 14:53:45 +0300 Subject: [PATCH 029/112] Increase PublicIP allocation timeout from 120s to 300s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AAP controller-task pod rollouts can cause in-flight provisioning jobs to be reaped. The operator retries and succeeds, but recovery takes ~2.5 minutes — exceeding the previous 120s timeout. This was observed in 3/23 PR #42 CI runs and 1/4 stress test clusters. Other wait functions in the same file already use 300s-600s timeouts. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 3f519a13f2..91c6e4d8e1 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -182,7 +182,7 @@ def wait_for_public_ip_allocated(*, k8s: K8sClient, name: str) -> None: fn=lambda: k8s.get_public_ip_state(name=name, checked=False), until=lambda v: v == "Allocated", retries=60, - delay=2, + delay=5, description=f"{name} PublicIP Allocated", ) From a2abfc0a3c703874ab39832c22c4dd6475becdd2 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Fri, 5 Jun 2026 14:53:45 +0300 Subject: [PATCH 030/112] Increase PublicIP allocation timeout from 120s to 300s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AAP controller-task pod rollouts can cause in-flight provisioning jobs to be reaped. The operator retries and succeeds, but recovery takes ~2.5 minutes — exceeding the previous 120s timeout. This was observed in 3/23 PR #42 CI runs and 1/4 stress test clusters. Other wait functions in the same file already use 300s-600s timeouts. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 3f519a13f2..f5d5f4cc98 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -152,7 +152,7 @@ def wait_for_public_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: fn=lambda: k8s.get_public_ip_pool_phase(name=name, checked=False), until=lambda v: v == "Ready", retries=60, - delay=2, + delay=5, description=f"{name} PublicIPPool Ready", ) @@ -182,7 +182,7 @@ def wait_for_public_ip_allocated(*, k8s: K8sClient, name: str) -> None: fn=lambda: k8s.get_public_ip_state(name=name, checked=False), until=lambda v: v == "Allocated", retries=60, - delay=2, + delay=5, description=f"{name} PublicIP Allocated", ) From 755e6ab3892a82f967c2a4ccd5a050fdb5a631c3 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Sun, 7 Jun 2026 01:50:25 +0300 Subject: [PATCH 031/112] NO_ISSUE: don't treat Failed phase as terminal in wait_for_cluster_ready MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operator retries provision with backoff after a failed AAP job (e.g. "0 agents available" during agent reclaim, or "Job reaped due to instance shutdown" during controller-task rollout). The phase goes Failed → Progressing → Ready on retry. The assert was killing the test immediately on Failed, before the operator's 2-minute backoff retry could kick in. This caused cascading test failures even though the operator would have recovered. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index f5d5f4cc98..f21bb6c6d2 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -208,13 +208,12 @@ def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: - def _check_phase() -> str: - phase: str = k8s.get_cluster_order_phase(name=name, checked=False) - assert phase != "Failed", f"{name} ClusterOrder entered Failed phase" - return phase - poll_until( - fn=_check_phase, until=lambda v: v == "Ready", retries=120, delay=15, description=f"{name} ClusterOrder Ready" + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=120, + delay=15, + description=f"{name} ClusterOrder Ready", ) From 54dd86dcb651bad6826c803f461dff4be7ccea15 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 10 Jun 2026 12:13:17 +0300 Subject: [PATCH 032/112] OSAC-1383: force-cleanup orphaned CAPI Machine pre-terminate hooks during teardown The capi-provider-agent controller sets a pre-terminate hook annotation on CAPI Machines but gets killed during CP namespace teardown before removing it. This leaves the Machine stuck in Deleting with condition WaitingExternalHook, blocking the entire deletion cascade. Add _force_cleanup_machine_preterminate_hooks() to the existing wait_for_cluster_deletion polling loop alongside the AgentCluster finalizer and Agent label workarounds. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index f21bb6c6d2..146b19b822 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -229,9 +229,15 @@ def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: # clusterdeployment-namespace from agents after HostedCluster deletion. The delete # playbook's detach_and_unlabel skips agents that still have this label set, leaving # the clusterorder label stuck and blocking agent reuse for subsequent tests. + # + # 3. Machine pre-terminate hooks: the CAPI provider sets a pre-terminate hook + # annotation on Machines, but is killed before removing it. The CAPI Machine + # controller waits forever for the annotation to be removed, blocking the entire + # deletion cascade (Machine → MachineSet → CAPI Cluster → HostedCluster). def _check_deleted() -> bool: _force_cleanup_agentcluster_finalizers(k8s=k8s, name=name) _force_cleanup_agent_labels(k8s=k8s, name=name) + _force_cleanup_machine_preterminate_hooks(k8s=k8s, name=name) return not k8s.is_present(resource="clusterorder", name=name) poll_until( @@ -290,6 +296,23 @@ def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: ) +def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> None: + cp_ns = f"{k8s.namespace}-{name}-{name}" + hook = "pre-terminate.delete.hook.machine.cluster.x-k8s.io/agentmachine" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, "get", "machines.cluster.x-k8s.io", + "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}", + ) + if rc != 0 or not output.strip(): + return + for machine_name in output.strip().split(): + run_unchecked( + *base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", + "-n", cp_ns, f"{hook}-", + ) + + def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: poll_until( fn=lambda: uuid not in grpc.list_cluster_ids(), From 98e97fade8a908f3a545cb4d7cad39c1f7b56c2a Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Tue, 9 Jun 2026 15:42:24 -0400 Subject: [PATCH 033/112] OSAC-839: E2E integration tests for PublicIPAttachment lifecycle Assisted-by: Claude --- tests/core/grpc_client.py | 24 +++++++++++++++++++ tests/core/helpers.py | 49 +++++++++++++++++++++++++++++++++++++++ tests/core/k8s_client.py | 22 ++++++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 22ae555c02..6aeac54e97 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -179,3 +179,27 @@ def list_public_ip_ids(self) -> list[str]: def delete_public_ip(self, *, public_ip_id: str) -> None: self.call(service=f"{PUBLIC_API}.PublicIPs/Delete", data={"id": public_ip_id}) + + # PublicIPAttachment operations (public API) + + def create_public_ip_attachment(self, *, name: str, public_ip: str, compute_instance: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.PublicIPAttachments/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"public_ip": public_ip, "compute_instance": compute_instance}, + } + }, + ) + return response["object"]["id"] + + def get_public_ip_attachment(self, *, attachment_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Get", data={"id": attachment_id}) + + def list_public_ip_attachment_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.PublicIPAttachments/List") + return [item["id"] for item in response.get("items", [])] + + def delete_public_ip_attachment(self, *, attachment_id: str) -> None: + self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Delete", data={"id": attachment_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 146b19b822..85ec32eae0 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -1,10 +1,24 @@ from __future__ import annotations +import re +import subprocess + +import pytest + from tests.core.grpc_client import GRPCClient from tests.core.k8s_client import K8sClient from tests.core.runner import poll_until, run_unchecked +def assert_grpc_rejected( + exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], + code: str, +) -> None: + exc = exc_info.value + combined: str = (exc.stderr or "") + (exc.stdout or "") + assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" + + def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_compute_instance_name(uuid=uuid, checked=False), @@ -197,6 +211,41 @@ def wait_for_public_ip_deletion(*, k8s: K8sClient, name: str) -> None: ) +def wait_for_public_ip_attachment_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_public_ip_attachment_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"PublicIPAttachment CR for {uuid}", + ) + + +def wait_for_public_ip_attachment_ready(*, k8s: K8sClient, name: str) -> None: + def _check_phase() -> str: + phase: str = k8s.get_public_ip_attachment_phase(name=name, checked=False) + assert phase != "Failed", f"{name} PublicIPAttachment entered Failed phase" + return phase + + poll_until( + fn=_check_phase, + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} PublicIPAttachment Ready", + ) + + +def wait_for_public_ip_attachment_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="publicipattachment", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} PublicIPAttachment deletion", + ) + + def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_cluster_order_name(uuid=uuid, checked=False), diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 9e0480317d..0cb6eea2c8 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -241,6 +241,28 @@ def get_public_ip_state(self, *, name: str, checked: bool = True) -> str: ) return output if rc == 0 else "" + # PublicIPAttachment queries + + def get_public_ip_attachment_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "publicipattachment", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/publicipattachment-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_public_ip_attachment_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "publicipattachment", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + # ClusterOrder queries def get_cluster_order_name(self, *, uuid: str, checked: bool = True) -> str: From b872d357fd18fd20b4b21bcebe942313a68b1039 Mon Sep 17 00:00:00 2001 From: Elad Tabak Date: Wed, 20 May 2026 12:38:38 +0300 Subject: [PATCH 034/112] OSAC-706: add E2E tests for catalog item lifecycle Add 5 E2E tests covering catalog item CRUD, visibility filtering, cluster creation via catalog item, unpublished rejection, and delete-when-referenced protection. Extend GRPCClient with catalog item operations (create, get, list, delete) and call_unchecked for negative test cases. Add CLI helper for creating clusters with --catalog-item flag. Generated with [Claude Code](https://claude.com/claude-code) --- tests/core/grpc_client.py | 30 +++++++++++++++++++++++++++++- tests/core/osac_cli.py | 6 ++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 6aeac54e97..8b43c0d527 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -5,7 +5,7 @@ import subprocess from typing import Any -from tests.core.runner import run +from tests.core.runner import run, run_unchecked PUBLIC_API: str = "osac.public.v1" PRIVATE_API: str = "osac.private.v1" @@ -87,6 +87,13 @@ def list_subnet_ids(self) -> list[str]: def delete_subnet(self, *, subnet_id: str) -> None: self.call(service=f"{PUBLIC_API}.Subnets/Delete", data={"id": subnet_id}) + def call_unchecked(self, *, service: str, data: dict[str, Any] | None = None) -> tuple[str, int]: + args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] + if data is not None: + args.extend(["-d", json.dumps(data)]) + args.extend([self.address, service]) + return run_unchecked(*args) + # Cluster operations def list_cluster_ids(self) -> list[str]: @@ -203,3 +210,24 @@ def list_public_ip_attachment_ids(self) -> list[str]: def delete_public_ip_attachment(self, *, attachment_id: str) -> None: self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Delete", data={"id": attachment_id}) + + # ClusterCatalogItem operations + + def create_cluster_catalog_item( + self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None + ) -> str: + obj: dict[str, Any] = {"metadata": {"name": name}, "title": name, "template": template, "published": published} + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Create", data={"object": obj}) + return response["object"]["id"] + + def get_cluster_catalog_item(self, *, catalog_item_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/Get", data={"id": catalog_item_id}) + + def list_cluster_catalog_item_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/List") + return [item["id"] for item in response.get("items", [])] + + def delete_cluster_catalog_item(self, *, catalog_item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Delete", data={"id": catalog_item_id}) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 75b326ed89..44220b4e60 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -128,5 +128,11 @@ def get_cluster_credential(self, credential: str, *, uuid: str) -> str: def get_unchecked(self, resource: str) -> tuple[str, int]: return run_unchecked(self.binary, "get", resource) + def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str) -> str: + stdout: str = run(self.binary, "create", "cluster", "--catalog-item", catalog_item, "--name", name) + match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) + assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" + return match.group(1) + def delete_cluster(self, *, uuid: str) -> None: run(self.binary, "delete", "cluster", uuid) From 1db95d08c4dedc81c0cc7264e4548551ba0c6efe Mon Sep 17 00:00:00 2001 From: Elad Tabak Date: Tue, 9 Jun 2026 11:15:28 +0300 Subject: [PATCH 035/112] OSAC-706: address code review findings - Extract _build_args in GRPCClient to prevent checked/unchecked drift - Extract _parse_uuid in OsacCLI to deduplicate UUID parsing - Use pytest.raises for expected-failure assertion - Wait for cluster removal before deleting catalog item in teardown Generated with [Claude Code](https://claude.com/claude-code) --- tests/core/grpc_client.py | 13 ++++++------- tests/core/osac_cli.py | 21 +++++++++------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 8b43c0d527..a7eacf263d 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -16,12 +16,15 @@ def __init__(self, *, address: str, token: str) -> None: self.address: str = address self.token: str = token - def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: + def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> list[str]: args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] if data is not None: args.extend(["-d", json.dumps(data)]) args.extend([self.address, service]) - return json.loads(run(*args)) + return args + + def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: + return json.loads(run(*self._build_args(service=service, data=data))) def list_compute_instance_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/List") @@ -88,11 +91,7 @@ def delete_subnet(self, *, subnet_id: str) -> None: self.call(service=f"{PUBLIC_API}.Subnets/Delete", data={"id": subnet_id}) def call_unchecked(self, *, service: str, data: dict[str, Any] | None = None) -> tuple[str, int]: - args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] - if data is not None: - args.extend(["-d", json.dumps(data)]) - args.extend([self.address, service]) - return run_unchecked(*args) + return run_unchecked(*self._build_args(service=service, data=data)) # Cluster operations diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 44220b4e60..768ae23d2e 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -17,6 +17,12 @@ def __init__(self, *, binary: str, address: str, token_script: str, namespace: s def relogin(self) -> None: run(self.binary, "login", "--address", self._address, "--insecure", "--token-script", self._token_script) + @staticmethod + def _parse_uuid(stdout: str) -> str: + match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) + assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" + return match.group(1) + def create_hub(self, *, hub_id: str, kubeconfig: str) -> None: run(self.binary, "create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) @@ -79,10 +85,7 @@ def create_compute_instance( if user_data_secret_ref is not None: args.extend(["--user-data", user_data_secret_ref]) - stdout: str = run(*args) - match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) - assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" - return match.group(1) + return self._parse_uuid(run(*args)) def delete_compute_instance(self, *, uuid: str) -> None: run(self.binary, "delete", "computeinstance", uuid) @@ -111,10 +114,7 @@ def create_cluster( for key, path in template_parameter_files.items(): args.extend(["-f", f"{key}={path}"]) - stdout: str = run(*args) - match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) - assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" - return match.group(1) + return self._parse_uuid(run(*args)) def get(self, resource: str, *, output: str | None = None) -> str: args: list[str] = [self.binary, "get", resource] @@ -129,10 +129,7 @@ def get_unchecked(self, resource: str) -> tuple[str, int]: return run_unchecked(self.binary, "get", resource) def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str) -> str: - stdout: str = run(self.binary, "create", "cluster", "--catalog-item", catalog_item, "--name", name) - match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) - assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" - return match.group(1) + return self._parse_uuid(run(self.binary, "create", "cluster", "--catalog-item", catalog_item, "--name", name)) def delete_cluster(self, *, uuid: str) -> None: run(self.binary, "delete", "cluster", uuid) From 36a27fb6e9c6f7e358fa4c1c55543957fc88f7a6 Mon Sep 17 00:00:00 2001 From: Alberto Losada Date: Tue, 16 Jun 2026 08:51:35 +0200 Subject: [PATCH 036/112] OSAC-1548: add E2E tests for ComputeInstanceCatalogItem lifecycle Five tests mirroring ClusterCatalogItem coverage from PR #44: - CRUD lifecycle (create, get, list, delete) - Unpublished items filtered from public API - Compute instance creation via catalog item - Unpublished catalog item rejection - Referential integrity on catalog item deletion Uses gRPC for compute instance creation instead of osac CLI due to CLI bug OSAC-1533 (--network-attachment ignored with --catalog-item). Adds networking fixtures (VirtualNetwork + Subnet) for tests that create compute instances. Assisted-by: Claude Code Signed-off-by: Alberto Losada --- tests/core/grpc_client.py | 34 ++++++++++++++++++++++++++++++++++ tests/core/osac_cli.py | 6 ++++++ 2 files changed, 40 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index a7eacf263d..4f0053eab8 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -26,6 +26,17 @@ def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> li def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: return json.loads(run(*self._build_args(service=service, data=data))) + def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str]) -> str: + attachments = [{"subnet": sid} for sid in subnet_ids] + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ComputeInstances/Create", + data={"object": {"spec": {"catalog_item": catalog_item, "network_attachments": attachments}}}, + ) + return response["object"]["id"] + + def delete_compute_instance(self, *, ci_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ComputeInstances/Delete", data={"id": ci_id}) + def list_compute_instance_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/List") return [item["id"] for item in response.get("items", [])] @@ -230,3 +241,26 @@ def list_cluster_catalog_item_ids(self) -> list[str]: def delete_cluster_catalog_item(self, *, catalog_item_id: str) -> None: self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Delete", data={"id": catalog_item_id}) + + # ComputeInstanceCatalogItem operations + + def create_compute_instance_catalog_item( + self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None + ) -> str: + obj: dict[str, Any] = {"metadata": {"name": name}, "title": name, "template": template, "published": published} + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Create", data={"object": obj} + ) + return response["object"]["id"] + + def get_compute_instance_catalog_item(self, *, catalog_item_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/Get", data={"id": catalog_item_id}) + + def list_compute_instance_catalog_item_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/List") + return [item["id"] for item in response.get("items", [])] + + def delete_compute_instance_catalog_item(self, *, catalog_item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Delete", data={"id": catalog_item_id}) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 768ae23d2e..3080d13d19 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -131,5 +131,11 @@ def get_unchecked(self, resource: str) -> tuple[str, int]: def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str) -> str: return self._parse_uuid(run(self.binary, "create", "cluster", "--catalog-item", catalog_item, "--name", name)) + def create_compute_instance_with_catalog_item(self, *, catalog_item: str, subnet: str | None = None) -> str: + args: list[str] = [self.binary, "create", "computeinstance", "--catalog-item", catalog_item] + if subnet is not None: + args.extend(["--network-attachment", f"subnet={subnet}"]) + return self._parse_uuid(run(*args)) + def delete_cluster(self, *, uuid: str) -> None: run(self.binary, "delete", "cluster", uuid) From 57eee2e26289f9c4b91b8743120f3e1954aeb48b Mon Sep 17 00:00:00 2001 From: Alberto Losada Date: Fri, 19 Jun 2026 11:19:16 +0200 Subject: [PATCH 037/112] Add update, unpublish transition, and field_definitions E2E tests Address Ygal's review: add Update step to CRUD tests, verify Get after Delete, add unpublish transition test, replace pytest.raises with call_unchecked for error message verification, and add field_definitions persistence tests for both cluster and compute instance catalog items. Assisted-by: Claude Code Signed-off-by: Alberto Losada --- tests/core/grpc_client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 4f0053eab8..9634b04fac 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -239,6 +239,13 @@ def list_cluster_catalog_item_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/List") return [item["id"] for item in response.get("items", [])] + def update_cluster_catalog_item(self, *, catalog_item_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_cluster_catalog_item requires at least one field to update") + obj: dict[str, Any] = {"id": catalog_item_id, **fields} + data: dict[str, Any] = {"object": obj, "update_mask": {"paths": list(fields.keys())}} + return self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Update", data=data) + def delete_cluster_catalog_item(self, *, catalog_item_id: str) -> None: self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Delete", data={"id": catalog_item_id}) @@ -262,5 +269,12 @@ def list_compute_instance_catalog_item_ids(self) -> list[str]: response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/List") return [item["id"] for item in response.get("items", [])] + def update_compute_instance_catalog_item(self, *, catalog_item_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_compute_instance_catalog_item requires at least one field to update") + obj: dict[str, Any] = {"id": catalog_item_id, **fields} + data: dict[str, Any] = {"object": obj, "update_mask": {"paths": list(fields.keys())}} + return self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Update", data=data) + def delete_compute_instance_catalog_item(self, *, catalog_item_id: str) -> None: self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Delete", data={"id": catalog_item_id}) From deadbc86f6b3ebbf378bc0acc4d67af794408641 Mon Sep 17 00:00:00 2001 From: Ilya Skornyakov Date: Wed, 24 Jun 2026 19:58:10 +0300 Subject: [PATCH 038/112] OSAC-1593: add console e2e tests Add WebSocket and gRPC console session tests covering serial console connectivity, ticket single-use (JTI) enforcement, concurrent session rejection, ticket expiry, nonexistent VM handling, and invalid ticket rejection. Assisted-by: Claude --- tests/core/grpc_client.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 6aeac54e97..634b6edf13 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -115,12 +115,24 @@ def list_security_group_ids(self) -> list[str]: def delete_security_group(self, *, sg_id: str) -> None: self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) + # Console operations + + def create_console_session( + self, *, resource_type: str, resource_id: str, console_type: str, client_id: str = "" + ) -> dict[str, Any]: + data: dict[str, Any] = { + "object": {"resourceType": resource_type, "resourceId": resource_id, "type": console_type} + } + if client_id: + data["object"]["clientId"] = client_id + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ConsoleSessions/Create", data=data) + return response["object"] + + # Organization operations + def ensure_organization(self, *, name: str) -> None: try: - self.call( - service=f"{PRIVATE_API}.Organizations/Create", - data={"object": {"metadata": {"name": name}}}, - ) + self.call(service=f"{PRIVATE_API}.Organizations/Create", data={"object": {"metadata": {"name": name}}}) except subprocess.CalledProcessError as e: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): From cfad1651371a62ce615e25db67d1d6fa9cc60f43 Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Fri, 26 Jun 2026 09:09:38 +0200 Subject: [PATCH 039/112] OSAC-77: Add E2E test for tenant storage onboarding lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new tests/storage/ suite that validates the full storage controller lifecycle: namespace + Tenant CR creation → Stage 1 (StorageBackendReady) → Stage 2 (ClusterStorageReady) → ordered teardown with finalizer release. The suite auto-detects whether the storage controller is enabled by checking the operator deployment's envFrom secrets and skips automatically when storage is not configured. Test infrastructure additions: - K8sClient: Tenant queries (phase, conditions, storageClasses, finalizers) and cluster-scoped StorageClass/Secret label queries - Helpers: wait_for_tenant_cr, wait_for_tenant_condition (with Failed phase early-exit), wait_for_tenant_deletion, and storage resource waits - GRPCClient: Organization CRUD methods for future test use - Makefile: test-storage target Tested on edge-17 SNO (OCP 4.18) with mock VMS server and OSAC deployed via Helm. Full provisioning + teardown cycle completes in ~110 seconds. Assisted-by: Claude Code Signed-off-by: Zoltan Szabo --- tests/core/grpc_client.py | 20 ++++- tests/core/helpers.py | 153 +++++++++++++++++++++++++++++--------- tests/core/k8s_client.py | 77 +++++++++++++++++++ 3 files changed, 210 insertions(+), 40 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 6aeac54e97..bf832a8768 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -117,10 +117,7 @@ def delete_security_group(self, *, sg_id: str) -> None: def ensure_organization(self, *, name: str) -> None: try: - self.call( - service=f"{PRIVATE_API}.Organizations/Create", - data={"object": {"metadata": {"name": name}}}, - ) + self.call(service=f"{PRIVATE_API}.Organizations/Create", data={"object": {"metadata": {"name": name}}}) except subprocess.CalledProcessError as e: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): @@ -203,3 +200,18 @@ def list_public_ip_attachment_ids(self) -> list[str]: def delete_public_ip_attachment(self, *, attachment_id: str) -> None: self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Delete", data={"id": attachment_id}) + + # Organization operations (private API) + + def create_organization(self, *, name: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.Organizations/Create", data={"object": {"metadata": {"name": name}}} + ) + return response["object"]["id"] + + def list_organization_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.Organizations/List") + return [item["id"] for item in response.get("items", [])] + + def delete_organization(self, *, org_id: str) -> None: + self.call(service=f"{PRIVATE_API}.Organizations/Delete", data={"id": org_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 85ec32eae0..329f44fa46 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -10,10 +10,7 @@ from tests.core.runner import poll_until, run_unchecked -def assert_grpc_rejected( - exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], - code: str, -) -> None: +def assert_grpc_rejected(exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], code: str) -> None: exc = exc_info.value combined: str = (exc.stderr or "") + (exc.stdout or "") assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" @@ -33,9 +30,7 @@ def wait_for_provision(*, k8s: K8sClient, name: str) -> None: def _check_provisioned() -> str: phase: str = k8s.get_compute_instance_phase(name=name, checked=False) assert phase != "Failed", f"{name} entered Failed phase before Provisioned=True" - return k8s.get_compute_instance_condition_status( - name=name, condition_type="Provisioned", checked=False - ) + return k8s.get_compute_instance_condition_status(name=name, condition_type="Provisioned", checked=False) poll_until( fn=_check_provisioned, @@ -52,13 +47,7 @@ def _check_phase() -> str: assert phase != "Failed", f"{name} entered Failed phase" return phase - poll_until( - fn=_check_phase, - until=lambda v: v == "Running", - retries=90, - delay=10, - description=f"{name} Running", - ) + poll_until(fn=_check_phase, until=lambda v: v == "Running", retries=90, delay=10, description=f"{name} Running") def wait_for_restart(*, k8s: K8sClient, name: str, initial: str, restart_ts: str) -> None: @@ -290,11 +279,7 @@ def _check_deleted() -> bool: return not k8s.is_present(resource="clusterorder", name=name) poll_until( - fn=_check_deleted, - until=lambda v: v is True, - retries=120, - delay=10, - description=f"{name} ClusterOrder deletion", + fn=_check_deleted, until=lambda v: v is True, retries=120, delay=10, description=f"{name} ClusterOrder deletion" ) @@ -305,23 +290,38 @@ def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None finalizer = "agentclustercapi-provider.agent-install.openshift.io/deprovision" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "agentclusters.capi-provider.agent-install.openshift.io", - "-n", cp_ns, "-o", f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", + *base_args, + "get", + "agentclusters.capi-provider.agent-install.openshift.io", + "-n", + cp_ns, + "-o", + f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", ) if rc != 0 or not output.strip(): return for ac_name in output.strip().split(): finalizers_json, rc = run_unchecked( - *base_args, "get", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", - "-n", cp_ns, "-o", "jsonpath={.metadata.finalizers}", + *base_args, + "get", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "-o", + "jsonpath={.metadata.finalizers}", ) if rc != 0 or finalizer not in finalizers_json: continue import json + idx = json.loads(finalizers_json).index(finalizer) run_unchecked( - *base_args, "patch", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", - "-n", cp_ns, "--type=json", + *base_args, + "patch", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "--type=json", f'-p=[{{"op": "remove", "path": "/metadata/finalizers/{idx}"}}]', ) @@ -332,16 +332,27 @@ def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: clusterdeployment_ns_label = "agent-install.openshift.io/clusterdeployment-namespace" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "agents.agent-install.openshift.io", - "-n", agent_ns, "-l", f"{clusterorder_label}={name}", - "-o", "jsonpath={.items[*].metadata.name}", + *base_args, + "get", + "agents.agent-install.openshift.io", + "-n", + agent_ns, + "-l", + f"{clusterorder_label}={name}", + "-o", + "jsonpath={.items[*].metadata.name}", ) if rc != 0 or not output.strip(): return for agent_name in output.strip().split(): run_unchecked( - *base_args, "label", f"agents.agent-install.openshift.io/{agent_name}", - "-n", agent_ns, f"{clusterorder_label}-", f"{clusterdeployment_ns_label}-", + *base_args, + "label", + f"agents.agent-install.openshift.io/{agent_name}", + "-n", + agent_ns, + f"{clusterorder_label}-", + f"{clusterdeployment_ns_label}-", ) @@ -350,16 +361,12 @@ def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> N hook = "pre-terminate.delete.hook.machine.cluster.x-k8s.io/agentmachine" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "machines.cluster.x-k8s.io", - "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}", + *base_args, "get", "machines.cluster.x-k8s.io", "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}" ) if rc != 0 or not output.strip(): return for machine_name in output.strip().split(): - run_unchecked( - *base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", - "-n", cp_ns, f"{hook}-", - ) + run_unchecked(*base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", "-n", cp_ns, f"{hook}-") def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: @@ -400,3 +407,77 @@ def wait_for_security_group_deletion(*, k8s: K8sClient, name: str) -> None: delay=5, description=f"{name} SecurityGroup deletion", ) + + +# Tenant helpers + + +def wait_for_tenant_cr(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.is_present(resource="tenant", name=name), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"Tenant CR {name}", + ) + + +def wait_for_tenant_condition(*, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True") -> None: + def _check() -> str: + phase: str = k8s.get_tenant_phase(name=name, checked=False) + if phase == "Failed": + cond_status = k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) + if cond_status != expected_status: + raise AssertionError(f"Tenant {name} entered Failed phase before {condition_type}={expected_status}") + return k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) + + poll_until( + fn=_check, + until=lambda v: v == expected_status, + retries=120, + delay=5, + description=f"Tenant {name} {condition_type}={expected_status}", + ) + + +def wait_for_tenant_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="tenant", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"Tenant {name} deletion", + ) + + +# Storage resource helpers + + +def wait_for_storage_classes_by_tenant(*, k8s: K8sClient, tenant_name: str, min_count: int = 1) -> list[str]: + return poll_until( + fn=lambda: k8s.list_storage_class_names_by_tenant(tenant_name=tenant_name), + until=lambda v: len(v) >= min_count, + retries=120, + delay=5, + description=f"StorageClasses for tenant {tenant_name} (>= {min_count})", + ) + + +def wait_for_storage_classes_removed(*, k8s: K8sClient, tenant_name: str) -> None: + poll_until( + fn=lambda: k8s.count_storage_classes_by_tenant(tenant_name=tenant_name), + until=lambda v: v == 0, + retries=120, + delay=5, + description=f"StorageClasses for tenant {tenant_name} removed", + ) + + +def wait_for_secrets_removed(*, k8s: K8sClient, tenant_name: str, namespace: str) -> None: + poll_until( + fn=lambda: k8s.count_secrets_by_tenant(tenant_name=tenant_name, namespace=namespace), + until=lambda v: v == 0, + retries=120, + delay=5, + description=f"Secrets for tenant {tenant_name} in {namespace} removed", + ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 0cb6eea2c8..0554df22eb 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -319,6 +319,83 @@ def get_cluster_order_spec(self, *, name: str) -> dict[str, Any]: output = self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.spec}") return json.loads(output) if output else {} + # Tenant queries + + def get_tenant_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "tenant", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_tenant_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_tenant_storage_classes(self, *, name: str, checked: bool = True) -> list[dict[str, str]]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("status", {}).get("storageClasses", []) + + def get_tenant_finalizers(self, *, name: str, checked: bool = True) -> list[str]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("metadata", {}).get("finalizers", []) + + # Cluster-scoped storage resource queries (no -n flag) + + def count_storage_classes_by_tenant(self, *, tenant_name: str) -> int: + output, rc = self._get( + "get", "storageclass", "-l", f"osac.openshift.io/tenant={tenant_name}", "--no-headers", checked=False + ) + if rc != 0 or not output.strip(): + return 0 + return len(output.strip().splitlines()) + + def list_storage_class_names_by_tenant(self, *, tenant_name: str) -> list[str]: + output, rc = self._get( + "get", + "storageclass", + "-l", + f"osac.openshift.io/tenant={tenant_name}", + "-o", + "jsonpath={.items[*].metadata.name}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return output.strip().split() + + def get_storage_class_labels(self, *, name: str) -> dict[str, str]: + output, rc = self._get("get", "storageclass", name, "-o", "json", checked=False) + if rc != 0: + return {} + return json.loads(output).get("metadata", {}).get("labels", {}) + + # Namespaced secret queries with explicit namespace (for cross-namespace lookups) + + def count_secrets_by_tenant(self, *, tenant_name: str, namespace: str) -> int: + output, rc = self._get( + "get", + "secret", + "-n", + namespace, + "-l", + f"osac.openshift.io/tenant={tenant_name}", + "--no-headers", + checked=False, + ) + if rc != 0 or not output.strip(): + return 0 + return len(output.strip().splitlines()) + # SecurityGroup queries def get_security_group_name(self, *, uuid: str, checked: bool = True) -> str: From 2120e01e8f1c58f85ad4b2822d13c33a124e4538 Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Fri, 26 Jun 2026 09:23:35 +0200 Subject: [PATCH 040/112] Revert formatting-only changes to existing helper/client code Keep the diff focused on new storage test additions only. Assisted-by: Claude Code Signed-off-by: Zoltan Szabo --- tests/core/grpc_client.py | 5 ++- tests/core/helpers.py | 79 ++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index bf832a8768..e3cc813da2 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -117,7 +117,10 @@ def delete_security_group(self, *, sg_id: str) -> None: def ensure_organization(self, *, name: str) -> None: try: - self.call(service=f"{PRIVATE_API}.Organizations/Create", data={"object": {"metadata": {"name": name}}}) + self.call( + service=f"{PRIVATE_API}.Organizations/Create", + data={"object": {"metadata": {"name": name}}}, + ) except subprocess.CalledProcessError as e: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 329f44fa46..85937e3473 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -10,7 +10,10 @@ from tests.core.runner import poll_until, run_unchecked -def assert_grpc_rejected(exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], code: str) -> None: +def assert_grpc_rejected( + exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], + code: str, +) -> None: exc = exc_info.value combined: str = (exc.stderr or "") + (exc.stdout or "") assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" @@ -30,7 +33,9 @@ def wait_for_provision(*, k8s: K8sClient, name: str) -> None: def _check_provisioned() -> str: phase: str = k8s.get_compute_instance_phase(name=name, checked=False) assert phase != "Failed", f"{name} entered Failed phase before Provisioned=True" - return k8s.get_compute_instance_condition_status(name=name, condition_type="Provisioned", checked=False) + return k8s.get_compute_instance_condition_status( + name=name, condition_type="Provisioned", checked=False + ) poll_until( fn=_check_provisioned, @@ -47,7 +52,13 @@ def _check_phase() -> str: assert phase != "Failed", f"{name} entered Failed phase" return phase - poll_until(fn=_check_phase, until=lambda v: v == "Running", retries=90, delay=10, description=f"{name} Running") + poll_until( + fn=_check_phase, + until=lambda v: v == "Running", + retries=90, + delay=10, + description=f"{name} Running", + ) def wait_for_restart(*, k8s: K8sClient, name: str, initial: str, restart_ts: str) -> None: @@ -279,7 +290,11 @@ def _check_deleted() -> bool: return not k8s.is_present(resource="clusterorder", name=name) poll_until( - fn=_check_deleted, until=lambda v: v is True, retries=120, delay=10, description=f"{name} ClusterOrder deletion" + fn=_check_deleted, + until=lambda v: v is True, + retries=120, + delay=10, + description=f"{name} ClusterOrder deletion", ) @@ -290,38 +305,23 @@ def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None finalizer = "agentclustercapi-provider.agent-install.openshift.io/deprovision" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, - "get", - "agentclusters.capi-provider.agent-install.openshift.io", - "-n", - cp_ns, - "-o", - f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", + *base_args, "get", "agentclusters.capi-provider.agent-install.openshift.io", + "-n", cp_ns, "-o", f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", ) if rc != 0 or not output.strip(): return for ac_name in output.strip().split(): finalizers_json, rc = run_unchecked( - *base_args, - "get", - f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", - "-n", - cp_ns, - "-o", - "jsonpath={.metadata.finalizers}", + *base_args, "get", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", cp_ns, "-o", "jsonpath={.metadata.finalizers}", ) if rc != 0 or finalizer not in finalizers_json: continue import json - idx = json.loads(finalizers_json).index(finalizer) run_unchecked( - *base_args, - "patch", - f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", - "-n", - cp_ns, - "--type=json", + *base_args, "patch", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", cp_ns, "--type=json", f'-p=[{{"op": "remove", "path": "/metadata/finalizers/{idx}"}}]', ) @@ -332,27 +332,16 @@ def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: clusterdeployment_ns_label = "agent-install.openshift.io/clusterdeployment-namespace" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, - "get", - "agents.agent-install.openshift.io", - "-n", - agent_ns, - "-l", - f"{clusterorder_label}={name}", - "-o", - "jsonpath={.items[*].metadata.name}", + *base_args, "get", "agents.agent-install.openshift.io", + "-n", agent_ns, "-l", f"{clusterorder_label}={name}", + "-o", "jsonpath={.items[*].metadata.name}", ) if rc != 0 or not output.strip(): return for agent_name in output.strip().split(): run_unchecked( - *base_args, - "label", - f"agents.agent-install.openshift.io/{agent_name}", - "-n", - agent_ns, - f"{clusterorder_label}-", - f"{clusterdeployment_ns_label}-", + *base_args, "label", f"agents.agent-install.openshift.io/{agent_name}", + "-n", agent_ns, f"{clusterorder_label}-", f"{clusterdeployment_ns_label}-", ) @@ -361,12 +350,16 @@ def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> N hook = "pre-terminate.delete.hook.machine.cluster.x-k8s.io/agentmachine" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "machines.cluster.x-k8s.io", "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}" + *base_args, "get", "machines.cluster.x-k8s.io", + "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}", ) if rc != 0 or not output.strip(): return for machine_name in output.strip().split(): - run_unchecked(*base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", "-n", cp_ns, f"{hook}-") + run_unchecked( + *base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", + "-n", cp_ns, f"{hook}-", + ) def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: From 4535a3ebf4197bc5ddbd6c1cbf16990bd859f96f Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Fri, 26 Jun 2026 09:27:51 +0200 Subject: [PATCH 041/112] Remove unused Organization CRUD methods from GRPCClient These were added during development but the storage test creates Tenant CRs directly via K8s API instead. Assisted-by: Claude Code Signed-off-by: Zoltan Szabo --- tests/core/grpc_client.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index e3cc813da2..6aeac54e97 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -203,18 +203,3 @@ def list_public_ip_attachment_ids(self) -> list[str]: def delete_public_ip_attachment(self, *, attachment_id: str) -> None: self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Delete", data={"id": attachment_id}) - - # Organization operations (private API) - - def create_organization(self, *, name: str) -> str: - response: dict[str, Any] = self.call( - service=f"{PRIVATE_API}.Organizations/Create", data={"object": {"metadata": {"name": name}}} - ) - return response["object"]["id"] - - def list_organization_ids(self) -> list[str]: - response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.Organizations/List") - return [item["id"] for item in response.get("items", [])] - - def delete_organization(self, *, org_id: str) -> None: - self.call(service=f"{PRIVATE_API}.Organizations/Delete", data={"id": org_id}) From 0055a97e01fc5b045ce327fadffc2abcadf7832e Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Fri, 26 Jun 2026 09:33:42 +0200 Subject: [PATCH 042/112] Address CodeRabbit review findings - Add OSAC_ENABLE_STORAGE_CONTROLLER to direct env-var detection in conftest, consistent with the envFrom branch - Fail fast in wait_for_tenant_condition if the Tenant CR disappears during polling instead of burning the full 600s timeout - Remove unused storage_config_namespace parameter from _verify_teardown Assisted-by: Claude Code Signed-off-by: Zoltan Szabo --- tests/core/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 85937e3473..284ee18052 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -417,6 +417,8 @@ def wait_for_tenant_cr(*, k8s: K8sClient, name: str) -> None: def wait_for_tenant_condition(*, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True") -> None: def _check() -> str: + if not k8s.is_present(resource="tenant", name=name): + raise AssertionError(f"Tenant {name} disappeared before {condition_type}={expected_status}") phase: str = k8s.get_tenant_phase(name=name, checked=False) if phase == "Failed": cond_status = k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) From 3bdddfb259cdb109295c51b1a17a243133900ea7 Mon Sep 17 00:00:00 2001 From: Dakota Crowder Date: Fri, 26 Jun 2026 09:49:53 -0400 Subject: [PATCH 043/112] OSAC-1532: update E2E test client to use Tenants API Rename ensure_organization to ensure_tenant and switch the gRPC call from Organizations/Create to Tenants/Create to match the API rename. Assisted-by: Claude Code Signed-off-by: Dakota Crowder --- tests/conftest.py | 4 ++-- tests/core/grpc_client.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f2676c9362..ea4a043b38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,9 +51,9 @@ def private_grpc(fulfillment_private_address: str, namespace: str, service_accou @pytest.fixture(scope="session", autouse=True) -def ensure_organizations(private_grpc: GRPCClient) -> None: +def ensure_tenants(private_grpc: GRPCClient) -> None: for name in ("tenant1", "tenant2"): - private_grpc.ensure_organization(name=name) + private_grpc.ensure_tenant(name=name) @pytest.fixture(scope="session") diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 634b6edf13..42144fe621 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -128,15 +128,18 @@ def create_console_session( response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ConsoleSessions/Create", data=data) return response["object"] - # Organization operations + # Tenant operations - def ensure_organization(self, *, name: str) -> None: + def ensure_tenant(self, *, name: str) -> None: try: - self.call(service=f"{PRIVATE_API}.Organizations/Create", data={"object": {"metadata": {"name": name}}}) + self.call( + service=f"{PRIVATE_API}.Tenants/Create", + data={"object": {"metadata": {"name": name}}}, + ) except subprocess.CalledProcessError as e: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): - raise RuntimeError(f"Failed to create organization '{name}': {output}") from e + raise RuntimeError(f"Failed to create tenant '{name}': {output}") from e # PublicIPPool operations (private API only) From eecd7dd68c33991eccbb851bc2838832d50214f1 Mon Sep 17 00:00:00 2001 From: Ilya Skornyakov Date: Wed, 24 Jun 2026 19:55:58 +0300 Subject: [PATCH 044/112] NO-ISSUE: parallelize test execution with pytest-xdist Add pytest-xdist with 4 workers and loadfile distribution. Refactor OsacCLI to use per-instance config directories so parallel workers don't overwrite each other's login credentials. Convert CLI fixtures to generators with cleanup. Assisted-by: Claude --- tests/conftest.py | 20 ++++++++++++----- tests/core/osac_cli.py | 51 ++++++++++++++++++++++++++++-------------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f2676c9362..852725273f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,7 @@ from __future__ import annotations +from collections.abc import Iterator + import pytest from tests.core.grpc_client import GRPCClient @@ -62,13 +64,15 @@ def k8s_hub_client(namespace: str) -> K8sClient: @pytest.fixture(scope="session") -def cli(namespace: str, fulfillment_address: str, service_account: str) -> OsacCLI: - return OsacCLI( +def cli(namespace: str, fulfillment_address: str, service_account: str) -> Iterator[OsacCLI]: + instance = OsacCLI( binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", token_script=f"oc create token -n {namespace} {service_account} --as system:admin", namespace=namespace, ) + yield instance + instance.close() @pytest.fixture(scope="session") @@ -91,23 +95,27 @@ def _make_jwt_token_script(keycloak_url: str, username: str, password: str) -> s @pytest.fixture(scope="session") -def jwt_cli_user(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> OsacCLI: - return OsacCLI( +def jwt_cli_user(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: + instance = OsacCLI( binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", token_script=_make_jwt_token_script(keycloak_url, "my_user", jwt_password), namespace=namespace, ) + yield instance + instance.close() @pytest.fixture(scope="session") -def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> OsacCLI: - return OsacCLI( +def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: + instance = OsacCLI( binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", token_script=_make_jwt_token_script(keycloak_url, "tenant1_admin", jwt_password), namespace=namespace, ) + yield instance + instance.close() @pytest.fixture(scope="session") diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 3080d13d19..175c74927b 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -1,6 +1,8 @@ from __future__ import annotations import re +import shutil +import tempfile from typing import Any from tests.core.runner import run, run_unchecked @@ -12,10 +14,23 @@ def __init__(self, *, binary: str, address: str, token_script: str, namespace: s self.namespace: str = namespace self._address: str = address self._token_script: str = token_script - run(binary, "login", "--address", address, "--insecure", "--token-script", token_script) + # Each OsacCLI instance gets its own config directory so that parallel + # xdist workers (or multiple CLI fixtures) don't overwrite each other's + # login credentials via the shared ~/.config/osac/config.json. + self._config_dir: str = tempfile.mkdtemp(prefix="osac-config-") + self._run("login", "--address", address, "--insecure", "--token-script", token_script) + + def close(self) -> None: + shutil.rmtree(self._config_dir, ignore_errors=True) + + def _run(self, *args: str, timeout: int = 300) -> str: + return run(self.binary, "--config", self._config_dir, *args, timeout=timeout) + + def _run_unchecked(self, *args: str, timeout: int = 300) -> tuple[str, int]: + return run_unchecked(self.binary, "--config", self._config_dir, *args, timeout=timeout) def relogin(self) -> None: - run(self.binary, "login", "--address", self._address, "--insecure", "--token-script", self._token_script) + self._run("login", "--address", self._address, "--insecure", "--token-script", self._token_script) @staticmethod def _parse_uuid(stdout: str) -> str: @@ -24,7 +39,7 @@ def _parse_uuid(stdout: str) -> str: return match.group(1) def create_hub(self, *, hub_id: str, kubeconfig: str) -> None: - run(self.binary, "create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) + self._run("create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) def create_compute_instance( self, @@ -40,7 +55,6 @@ def create_compute_instance( user_data_secret_ref: str | None = None, ) -> str: args: list[str] = [ - self.binary, "create", "computeinstance", "--template", @@ -68,7 +82,10 @@ def create_compute_instance( security_groups = attachment.get("security_groups", []) if not isinstance(security_groups, list): - raise ValueError(f"network_attachments[{idx}]: 'security_groups' must be a list, got {type(security_groups).__name__}") + raise ValueError( + f"network_attachments[{idx}]: 'security_groups' must be a list," + f" got {type(security_groups).__name__}" + ) if security_groups and not all(isinstance(sg, str) and sg for sg in security_groups): raise ValueError(f"network_attachments[{idx}]: all security_groups must be non-empty strings") @@ -85,10 +102,10 @@ def create_compute_instance( if user_data_secret_ref is not None: args.extend(["--user-data", user_data_secret_ref]) - return self._parse_uuid(run(*args)) + return self._parse_uuid(self._run(*args)) def delete_compute_instance(self, *, uuid: str) -> None: - run(self.binary, "delete", "computeinstance", uuid) + self._run("delete", "computeinstance", uuid) def create_cluster( self, @@ -100,7 +117,7 @@ def create_cluster( template_parameters: dict[str, str] | None = None, template_parameter_files: dict[str, str] | None = None, ) -> str: - args: list[str] = [self.binary, "create", "cluster", "--template", template] + args: list[str] = ["create", "cluster", "--template", template] if name is not None: args.extend(["--name", name]) if pull_secret_file is not None: @@ -114,28 +131,28 @@ def create_cluster( for key, path in template_parameter_files.items(): args.extend(["-f", f"{key}={path}"]) - return self._parse_uuid(run(*args)) + return self._parse_uuid(self._run(*args)) def get(self, resource: str, *, output: str | None = None) -> str: - args: list[str] = [self.binary, "get", resource] + args: list[str] = ["get", resource] if output is not None: args.extend(["-o", output]) - return run(*args) + return self._run(*args) def get_cluster_credential(self, credential: str, *, uuid: str) -> str: - return run(self.binary, "get", credential, uuid) + return self._run("get", credential, uuid) def get_unchecked(self, resource: str) -> tuple[str, int]: - return run_unchecked(self.binary, "get", resource) + return self._run_unchecked("get", resource) def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str) -> str: - return self._parse_uuid(run(self.binary, "create", "cluster", "--catalog-item", catalog_item, "--name", name)) + return self._parse_uuid(self._run("create", "cluster", "--catalog-item", catalog_item, "--name", name)) def create_compute_instance_with_catalog_item(self, *, catalog_item: str, subnet: str | None = None) -> str: - args: list[str] = [self.binary, "create", "computeinstance", "--catalog-item", catalog_item] + args: list[str] = ["create", "computeinstance", "--catalog-item", catalog_item] if subnet is not None: args.extend(["--network-attachment", f"subnet={subnet}"]) - return self._parse_uuid(run(*args)) + return self._parse_uuid(self._run(*args)) def delete_cluster(self, *, uuid: str) -> None: - run(self.binary, "delete", "cluster", uuid) + self._run("delete", "cluster", uuid) From 76a6410620474ee8bfb97eca8b3f828b89a801e6 Mon Sep 17 00:00:00 2001 From: Omer Vishlitzky <22615781+omer-vishlitzky@users.noreply.github.com> Date: Sun, 28 Jun 2026 20:22:28 +0300 Subject: [PATCH 045/112] NO_ISSUE: don't treat Failed phase as terminal in wait helpers (#111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as 755e6ab (wait_for_cluster_ready) applied to the ComputeInstance and PublicIPAttachment wait helpers. The operator re-evaluates the phase on every reconcile from the current KubeVirt PrintableStatus. ErrorUnschedulable maps to Failed but is recoverable — the scheduler retries and the phase transitions Failed → Starting → Running. The assert was killing the test before recovery could complete. Removes the assert from wait_for_provision, wait_for_running, and wait_for_public_ip_attachment_ready to match every other wait helper in the file, which all poll until the target state and let the timeout be the safety net. Co-authored-by: Claude Opus 4.6 (1M context) --- tests/core/helpers.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 85ec32eae0..3e65491fb2 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -30,15 +30,10 @@ def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_provision(*, k8s: K8sClient, name: str) -> None: - def _check_provisioned() -> str: - phase: str = k8s.get_compute_instance_phase(name=name, checked=False) - assert phase != "Failed", f"{name} entered Failed phase before Provisioned=True" - return k8s.get_compute_instance_condition_status( - name=name, condition_type="Provisioned", checked=False - ) - poll_until( - fn=_check_provisioned, + fn=lambda: k8s.get_compute_instance_condition_status( + name=name, condition_type="Provisioned", checked=False + ), until=lambda v: v == "True", retries=120, delay=5, @@ -47,13 +42,8 @@ def _check_provisioned() -> str: def wait_for_running(*, k8s: K8sClient, name: str) -> None: - def _check_phase() -> str: - phase: str = k8s.get_compute_instance_phase(name=name, checked=False) - assert phase != "Failed", f"{name} entered Failed phase" - return phase - poll_until( - fn=_check_phase, + fn=lambda: k8s.get_compute_instance_phase(name=name, checked=False), until=lambda v: v == "Running", retries=90, delay=10, @@ -222,13 +212,8 @@ def wait_for_public_ip_attachment_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_public_ip_attachment_ready(*, k8s: K8sClient, name: str) -> None: - def _check_phase() -> str: - phase: str = k8s.get_public_ip_attachment_phase(name=name, checked=False) - assert phase != "Failed", f"{name} PublicIPAttachment entered Failed phase" - return phase - poll_until( - fn=_check_phase, + fn=lambda: k8s.get_public_ip_attachment_phase(name=name, checked=False), until=lambda v: v == "Ready", retries=60, delay=5, From 1e00b6dcb5af6a894ccb2a8590364e3c26acc72d Mon Sep 17 00:00:00 2001 From: CrystalChun Date: Mon, 29 Jun 2026 16:17:18 -0500 Subject: [PATCH 046/112] NO-ISSUE: Request organization scope in JWT tokens for E2E tests After the organizations-to-tenants API rename in fulfillment-service, the authorization interceptor expects JWT tokens to include organization membership claims to determine assignable tenants. Without the "organization" scope, Keycloak returns an empty organization claim {}, which prevents the fallback to groups and causes "failed to determine assignable tenants" errors. This change updates both the get_jwt() function and _make_jwt_token_script() to request "scope=openid organization" instead of just "scope=openid", ensuring that Keycloak includes the user's organization memberships in the token. Fixes the following E2E test failures: - test_jwt_virtual_network_lifecycle - test_jwt_security_group_lifecycle - test_jwt_tenant_isolation Co-Authored-By: Claude Sonnet 4.5 --- tests/conftest.py | 2 +- tests/core/keycloak.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 63835f1aaf..376c0ad37d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -89,7 +89,7 @@ def _make_jwt_token_script(keycloak_url: str, username: str, password: str) -> s return ( f"curl -sk -X POST {keycloak_url}/realms/osac/protocol/openid-connect/token" f" -d grant_type=password -d client_id=osac-cli" - f" -d username={username} -d password={password} -d scope=openid" + f" -d username={username} -d password={password} -d 'scope=openid organization'" " | python3 -c \"import sys,json;print(json.load(sys.stdin)['access_token'])\"" ) diff --git a/tests/core/keycloak.py b/tests/core/keycloak.py index 428c0021f9..b1fdbac7ff 100644 --- a/tests/core/keycloak.py +++ b/tests/core/keycloak.py @@ -23,7 +23,7 @@ def get_jwt(*, keycloak_url: str, realm: str, client_id: str, username: str, pas "-d", f"password={password}", "-d", - "scope=openid", + "scope=openid organization", ) response: dict[str, str] = json.loads(stdout) token: str | None = response.get("access_token") From 4006769f15826a4f928784e976445a1dde9a60a7 Mon Sep 17 00:00:00 2001 From: CrystalChun Date: Mon, 29 Jun 2026 16:40:01 -0500 Subject: [PATCH 047/112] NO-ISSUE: Add users to Keycloak organizations during E2E test setup After the organizations-to-tenants rename, users need to be members of Keycloak organizations (not just groups) for the organization claim to be populated in JWT tokens. Without this, the authorization interceptor cannot determine assignable tenants. This change adds: - tests/core/keycloak_admin.py: Helper module for Keycloak admin API calls (similar to integration test's addUsersToKeycloakOrganizations function) - setup_organization_memberships fixture: Autouse session fixture that runs after ensure_tenants to: 1. Wait for tenant1/tenant2 organizations to sync from Tenant resources 2. Add tenant1_user/tenant1_admin to tenant1 organization 3. Add tenant2_user/tenant2_admin to tenant2 organization 4. Create /members groups in each organization 5. Add users to their organization's /members group This ensures that when JWT tokens are requested with the "organization" scope, Keycloak includes the user's organization memberships in the token, allowing the authorization interceptor to determine assignable tenants. Fixes the same E2E test failures addressed by the scope change: - test_jwt_virtual_network_lifecycle - test_jwt_security_group_lifecycle - test_jwt_tenant_isolation Co-Authored-By: Claude Sonnet 4.5 --- tests/conftest.py | 66 +++++++++++ tests/core/keycloak_admin.py | 219 +++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 tests/core/keycloak_admin.py diff --git a/tests/conftest.py b/tests/conftest.py index 376c0ad37d..5387d6a187 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,14 @@ from tests.core.grpc_client import GRPCClient from tests.core.k8s_client import K8sClient from tests.core.keycloak import get_jwt +from tests.core.keycloak_admin import ( + add_user_to_organization, + add_user_to_organization_group, + ensure_organization_group, + get_admin_token, + get_user_id, + wait_for_organization, +) from tests.core.osac_cli import OsacCLI from tests.core.runner import env, run @@ -58,6 +66,64 @@ def ensure_tenants(private_grpc: GRPCClient) -> None: private_grpc.ensure_tenant(name=name) +@pytest.fixture(scope="session") +def keycloak_admin_password() -> str: + return env("OSAC_KEYCLOAK_ADMIN_PASSWORD", "admin") + + +@pytest.fixture(scope="session", autouse=True) +def setup_organization_memberships( + ensure_tenants: None, keycloak_url: str, keycloak_admin_password: str +) -> None: + """ + Add test users to their corresponding Keycloak organizations. + This runs after ensure_tenants creates the Tenant resources, which the + tenant controller syncs to Keycloak as organizations. + """ + # Get admin token for Keycloak admin API + admin_token = get_admin_token(keycloak_url=keycloak_url, username="admin", password=keycloak_admin_password) + + # Map of organization name -> list of usernames + org_users = { + "tenant1": ["tenant1_user", "tenant1_admin"], + "tenant2": ["tenant2_user", "tenant2_admin"], + } + + for org_name, usernames in org_users.items(): + # Wait for the organization to be synced to Keycloak by the tenant controller + org_id = wait_for_organization(keycloak_url=keycloak_url, admin_token=admin_token, org_name=org_name) + + # Add each user to the organization + for username in usernames: + user_id = get_user_id(keycloak_url=keycloak_url, admin_token=admin_token, username=username) + add_user_to_organization( + keycloak_url=keycloak_url, + admin_token=admin_token, + org_id=org_id, + user_id=user_id, + username=username, + org_name=org_name, + ) + + # Create /members group in the organization and add all users to it + # This is required for the organization scope to include the organization in the JWT token + group_id = ensure_organization_group( + keycloak_url=keycloak_url, admin_token=admin_token, org_id=org_id, org_name=org_name + ) + + for username in usernames: + user_id = get_user_id(keycloak_url=keycloak_url, admin_token=admin_token, username=username) + add_user_to_organization_group( + keycloak_url=keycloak_url, + admin_token=admin_token, + org_id=org_id, + group_id=group_id, + user_id=user_id, + username=username, + org_name=org_name, + ) + + @pytest.fixture(scope="session") def k8s_hub_client(namespace: str) -> K8sClient: return K8sClient(namespace=namespace) diff --git a/tests/core/keycloak_admin.py b/tests/core/keycloak_admin.py new file mode 100644 index 0000000000..9358b22693 --- /dev/null +++ b/tests/core/keycloak_admin.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +import time +from typing import Any +from urllib.parse import urlencode + +from tests.core.runner import run + + +def get_admin_token(*, keycloak_url: str, username: str, password: str) -> str: + """Get an admin access token for Keycloak admin API calls.""" + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + stdout: str = run( + "curl", + "-sk", + "--fail-with-body", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + "client_id=admin-cli", + "-d", + f"username={username}", + "-d", + f"password={password}", + ) + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get admin token from Keycloak: {error}") + return token + + +def keycloak_admin_request( + *, keycloak_url: str, admin_token: str, method: str, path: str, data: Any = None +) -> tuple[int, bytes]: + """ + Make an authenticated request to the Keycloak admin API for the 'osac' realm. + The path is relative to /admin/realms/osac (e.g., "/organizations", "/users/{id}"). + Returns (status_code, response_body). + """ + url = f"{keycloak_url}/admin/realms/osac{path}" + args = [ + "curl", + "-sk", + "-w", + "\n%{http_code}", + "-X", + method, + "-H", + f"Authorization: Bearer {admin_token}", + "-H", + "Content-Type: application/json", + ] + if data is not None: + if isinstance(data, str): + args.extend(["-d", data]) + else: + args.extend(["-d", json.dumps(data)]) + + args.append(url) + + output: str = run(*args) + lines = output.strip().split("\n") + status_code = int(lines[-1]) + body = "\n".join(lines[:-1]).encode("utf-8") + + return status_code, body + + +def wait_for_organization( + *, keycloak_url: str, admin_token: str, org_name: str, timeout_seconds: int = 60 +) -> str: + """ + Wait for an organization to be synced to Keycloak and return its ID. + Polls with exponential backoff until the organization exists or timeout is reached. + """ + start_time = time.time() + interval = 1.0 + max_interval = 10.0 + + while time.time() - start_time < timeout_seconds: + query = urlencode({"exact": "true", "search": org_name}) + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="GET", + path=f"/organizations?{query}", + ) + + if status != 200: + raise RuntimeError(f"Failed to query organizations: status={status} body={body.decode()}") + + orgs: list[dict[str, Any]] = json.loads(body) + if len(orgs) > 0: + org_id: str = orgs[0]["id"] + return org_id + + time.sleep(interval) + interval = min(interval * 2, max_interval) + + raise RuntimeError(f"Organization '{org_name}' not found in Keycloak after {timeout_seconds}s") + + +def get_user_id(*, keycloak_url: str, admin_token: str, username: str) -> str: + """Get a user's ID by username.""" + query = urlencode({"username": username, "exact": "true"}) + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/users?{query}" + ) + + if status != 200: + raise RuntimeError(f"Failed to get user '{username}': status={status} body={body.decode()}") + + users: list[dict[str, Any]] = json.loads(body) + if len(users) == 0: + raise RuntimeError(f"User '{username}' not found in Keycloak") + + return users[0]["id"] + + +def add_user_to_organization( + *, keycloak_url: str, admin_token: str, org_id: str, user_id: str, username: str, org_name: str +) -> None: + """Add a user to a Keycloak organization.""" + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="POST", + path=f"/organizations/{org_id}/members", + data=user_id, + ) + + # 201 Created, 204 No Content, or 409 Conflict (already a member) are all acceptable + # 400 with "Duplicate resource error" is also acceptable (Keycloak returns this when user is already a member) + if status == 400: + error_msg = body.decode().lower() + if "duplicate" not in error_msg: + raise RuntimeError( + f"Failed to add user '{username}' to organization '{org_name}': status={status} body={body.decode()}" + ) + elif status not in (201, 204, 409): + raise RuntimeError( + f"Failed to add user '{username}' to organization '{org_name}': status={status} body={body.decode()}" + ) + + +def ensure_organization_group(*, keycloak_url: str, admin_token: str, org_id: str, org_name: str) -> str: + """ + Ensure a /members group exists in the organization and return its ID. + Creates the group if it doesn't exist. + """ + group_name = "/members" + group_payload = {"name": group_name} + + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="POST", + path=f"/organizations/{org_id}/groups", + data=group_payload, + ) + + # 201 Created or 409 Conflict (already exists) are acceptable + if status == 201: + group_resp: dict[str, Any] = json.loads(body) + return group_resp["id"] + elif status == 409: + # Group already exists, need to fetch it + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/organizations/{org_id}/groups" + ) + + if status != 200: + raise RuntimeError( + f"Failed to get groups for organization '{org_name}': status={status} body={body.decode()}" + ) + + groups: list[dict[str, Any]] = json.loads(body) + for g in groups: + if g.get("name") == group_name: + return g["id"] + + raise RuntimeError(f"Failed to find group '{group_name}' in organization '{org_name}'") + else: + raise RuntimeError( + f"Failed to create group '{group_name}' in organization '{org_name}': status={status} body={body.decode()}" + ) + + +def add_user_to_organization_group( + *, keycloak_url: str, admin_token: str, org_id: str, group_id: str, user_id: str, username: str, org_name: str +) -> None: + """Add a user to a group within a Keycloak organization.""" + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="PUT", + path=f"/organizations/{org_id}/groups/{group_id}/members/{user_id}", + ) + + # 200 OK, 201 Created, 204 No Content, or 409 Conflict are all acceptable + # 400 with "Duplicate resource error" is also acceptable (Keycloak returns this when user is already a member) + if status == 400: + error_msg = body.decode().lower() + if "duplicate" not in error_msg: + raise RuntimeError( + f"Failed to add user '{username}' to group in organization '{org_name}': " + f"status={status} body={body.decode()}" + ) + elif status not in (200, 201, 204, 409): + raise RuntimeError( + f"Failed to add user '{username}' to group in organization '{org_name}': " + f"status={status} body={body.decode()}" + ) From 45d07438a4f7dd59abcb592b3f3f6e567c37bf2a Mon Sep 17 00:00:00 2001 From: Elior Erez Date: Tue, 30 Jun 2026 23:58:58 -0400 Subject: [PATCH 048/112] Fix PublicIPPool ready race between K8s CR and gRPC server The make_pool fixture waits for the K8s CR to report Ready, but the fulfillment-service database may still show PENDING due to the controller feedback loop lag. This caused test_validation_rejections to fail with FailedPrecondition when its public_ip fixture tried to create a PublicIP immediately after pool creation. Add wait_for_public_ip_pool_grpc_ready() that polls the private gRPC API to confirm the pool state is READY in the fulfillment-service database before returning from make_pool. --- tests/core/helpers.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index a564363868..4e13b2659f 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -9,6 +9,8 @@ from tests.core.k8s_client import K8sClient from tests.core.runner import poll_until, run_unchecked +_POOL_READY_STATE = "PUBLIC_IP_POOL_STATE_READY" + def assert_grpc_rejected( exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], @@ -161,6 +163,30 @@ def wait_for_public_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: ) +def wait_for_public_ip_pool_grpc_ready(*, private_grpc: GRPCClient, pool_id: str) -> None: + """Poll the private gRPC API until the pool state is READY. + + The K8s CR status may report Ready before the fulfillment-service database + has been updated by the controller feedback loop. Polling via gRPC closes + this race so that subsequent PublicIP creation does not hit + FailedPrecondition. + """ + def _state() -> str: + try: + pool = private_grpc.get_public_ip_pool(pool_id=pool_id) + except subprocess.CalledProcessError: + return "" + return pool.get("object", {}).get("status", {}).get("state", "") + + poll_until( + fn=_state, + until=lambda v: v == _POOL_READY_STATE, + retries=30, + delay=2, + description=f"PublicIPPool {pool_id} gRPC READY", + ) + + def wait_for_public_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: not k8s.is_present(resource="publicippool", name=name), From d9cfd7cd7b5d9281455f9508a9a248f1c2dd2311 Mon Sep 17 00:00:00 2001 From: Crystal Date: Wed, 1 Jul 2026 09:26:11 -0500 Subject: [PATCH 049/112] NO-ISSUE: Update tests to use valid user with valid tenant (#147) --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5387d6a187..c482ddbe72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -165,7 +165,7 @@ def jwt_cli_user(namespace: str, fulfillment_address: str, keycloak_url: str, jw instance = OsacCLI( binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", - token_script=_make_jwt_token_script(keycloak_url, "my_user", jwt_password), + token_script=_make_jwt_token_script(keycloak_url, "tenant1_user", jwt_password), namespace=namespace, ) yield instance From ac190b11f786187d7fd77435ec94b7d695736d16 Mon Sep 17 00:00:00 2001 From: Ygal Blum Date: Mon, 6 Jul 2026 08:56:40 -0400 Subject: [PATCH 050/112] OSAC-1221: add InstanceType E2E tests (#104) Add end-to-end tests for the InstanceType resource lifecycle and ComputeInstance integration with instance types. - Add InstanceType CRUD operations to GRPCClient (private API) - Add InstanceType CLI methods and instance_type parameter to OsacCLI - Add InstanceType lifecycle E2E test (create, describe, get, state transitions, delete) - Add ComputeInstance with instance_type E2E tests (happy path with reconciler expansion, deletion protection, deprecated warning, nonexistent type rejection, obsolete type rejection) Assisted-by: Claude Code Signed-off-by: Ygal Blum --- tests/core/grpc_client.py | 44 +++++++++++++++++++++++++++++++++++++++ tests/core/osac_cli.py | 32 ++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 7cd9a98269..a9c8be3393 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -293,3 +293,47 @@ def update_compute_instance_catalog_item(self, *, catalog_item_id: str, **fields def delete_compute_instance_catalog_item(self, *, catalog_item_id: str) -> None: self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Delete", data={"id": catalog_item_id}) + + # InstanceType operations (private API only) + + def create_instance_type( + self, + *, + name: str, + cores: int, + memory_gib: int, + description: str = "", + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.InstanceTypes/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "cores": cores, + "memory_gib": memory_gib, + "description": description, + }, + } + }, + ) + return response["object"]["id"] + + def get_instance_type(self, *, name: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.InstanceTypes/Get", data={"id": name}) + + def list_instance_type_names(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.InstanceTypes/List") + return [item["metadata"]["name"] for item in response.get("items", [])] + + def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]: + return self.call( + service=f"{PRIVATE_API}.InstanceTypes/Update", + data={ + "object": {"id": name, "spec": {"state": state}}, + "updateMask": {"paths": ["spec.state"]}, + }, + ) + + def delete_instance_type(self, *, name: str) -> None: + self.call(service=f"{PRIVATE_API}.InstanceTypes/Delete", data={"id": name}) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 175c74927b..d82f32da0f 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -23,6 +23,10 @@ def __init__(self, *, binary: str, address: str, token_script: str, namespace: s def close(self) -> None: shutil.rmtree(self._config_dir, ignore_errors=True) + @property + def config_dir(self) -> str: + return self._config_dir + def _run(self, *args: str, timeout: int = 300) -> str: return run(self.binary, "--config", self._config_dir, *args, timeout=timeout) @@ -46,23 +50,20 @@ def create_compute_instance( *, template: str, network_attachments: list[dict[str, Any]] | None = None, - cores: int = 2, - memory_gib: int = 4, + cores: int | None = None, + memory_gib: int | None = None, boot_disk_size: int = 20, image: str = "quay.io/containerdisks/fedora:latest", image_source_type: str = "registry", run_strategy: str = "Always", user_data_secret_ref: str | None = None, + instance_type: str | None = None, ) -> str: args: list[str] = [ "create", "computeinstance", "--template", template, - "--cores", - str(cores), - "--memory-gib", - str(memory_gib), "--boot-disk-size", str(boot_disk_size), "--image", @@ -73,6 +74,13 @@ def create_compute_instance( run_strategy, ] + if instance_type is not None: + if cores is not None or memory_gib is not None: + raise ValueError("Cannot specify cores/memory_gib together with instance_type") + args.extend(["--instance-type", instance_type]) + else: + args.extend(["--cores", str(2 if cores is None else cores), "--memory-gib", str(4 if memory_gib is None else memory_gib)]) + # Add network attachments if network_attachments is not None: for idx, attachment in enumerate(network_attachments): @@ -107,6 +115,18 @@ def create_compute_instance( def delete_compute_instance(self, *, uuid: str) -> None: self._run("delete", "computeinstance", uuid) + def create_instance_type(self, *, name: str, cores: int, memory_gib: int, description: str = "") -> str: + args: list[str] = ["create", "instancetype", "--name", name, "--cores", str(cores), "--memory-gib", str(memory_gib)] + if description: + args.extend(["--description", description]) + return self._parse_uuid(self._run(*args)) + + def describe_instance_type(self, *, name: str) -> str: + return self._run("describe", "instancetype", name) + + def delete_instance_type(self, *, name: str) -> None: + self._run("delete", "instancetype", name) + def create_cluster( self, *, From cc4b571324c7429077ae74fbcec8567fd481bc2c Mon Sep 17 00:00:00 2001 From: Ygal Blum Date: Tue, 7 Jul 2026 19:45:23 -0400 Subject: [PATCH 051/112] OSAC-2116: migrate E2E tests from cores/memory_gib to instance_type Replace direct cores/memory_gib parameters with instance_type across all E2E test infrastructure, preparing for the breaking API removal in Phase 5. - Add default_instance_type to OsacCLI with explicit None-check fallback - Remove cores/memory_gib params from create_compute_instance (clean break) - Add session-scoped default_instance_type fixture in vmaas/conftest.py - Migrate CLI and API field tests to use instance_type path - Replace cores/memoryGiB immutability tests with instanceType immutability - Migrate CatalogItem field_definitions from cpu_cores/memory_gb to instance_type Assisted-by: Claude Code Signed-off-by: Ygal Blum --- tests/core/osac_cli.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index d82f32da0f..c67755d961 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -9,11 +9,20 @@ class OsacCLI: - def __init__(self, *, binary: str, address: str, token_script: str, namespace: str) -> None: + def __init__( + self, + *, + binary: str, + address: str, + token_script: str, + namespace: str, + default_instance_type: str | None = None, + ) -> None: self.binary: str = binary self.namespace: str = namespace self._address: str = address self._token_script: str = token_script + self.default_instance_type: str | None = default_instance_type # Each OsacCLI instance gets its own config directory so that parallel # xdist workers (or multiple CLI fixtures) don't overwrite each other's # login credentials via the shared ~/.config/osac/config.json. @@ -50,8 +59,6 @@ def create_compute_instance( *, template: str, network_attachments: list[dict[str, Any]] | None = None, - cores: int | None = None, - memory_gib: int | None = None, boot_disk_size: int = 20, image: str = "quay.io/containerdisks/fedora:latest", image_source_type: str = "registry", @@ -74,12 +81,11 @@ def create_compute_instance( run_strategy, ] - if instance_type is not None: - if cores is not None or memory_gib is not None: - raise ValueError("Cannot specify cores/memory_gib together with instance_type") - args.extend(["--instance-type", instance_type]) + effective_instance_type = instance_type if instance_type is not None else self.default_instance_type + if effective_instance_type is not None: + args.extend(["--instance-type", effective_instance_type]) else: - args.extend(["--cores", str(2 if cores is None else cores), "--memory-gib", str(4 if memory_gib is None else memory_gib)]) + raise ValueError("instance_type or default_instance_type must be set") # Add network attachments if network_attachments is not None: From d0a17528f980588c26d6f75e9c01449f74d40133 Mon Sep 17 00:00:00 2001 From: Haim Tayrie Date: Thu, 9 Jul 2026 15:39:17 +0300 Subject: [PATCH 052/112] Add retry logic to get_jwt() for Keycloak startup flakiness The E2E tests were failing when Keycloak wasn't ready at test startup. The get_jwt() function had no retry logic, causing immediate failures with curl exit code 22 when Keycloak HTTP requests failed during initialization. This change adds retry logic using the existing poll_until() helper: - Retries up to 24 times with 5-second intervals (total 2 minutes) - Uses run_unchecked() to handle transient failures gracefully - Returns None on failure/malformed JSON to trigger retry - Leverages poll_until() which provides built-in timeout and logging Benefits of using poll_until(): - Consistent retry pattern with the rest of the codebase - Automatic timeout error messages with context - Cleaner code compared to manual retry loops - No need for manual exponential backoff implementation Fixes: flaky E2E test failures when Keycloak is not ready Assisted-by: Claude Code --- tests/core/keycloak.py | 67 +++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/tests/core/keycloak.py b/tests/core/keycloak.py index b1fdbac7ff..3de7ac9827 100644 --- a/tests/core/keycloak.py +++ b/tests/core/keycloak.py @@ -2,32 +2,51 @@ import json -from tests.core.runner import run +from tests.core.runner import poll_until, run_unchecked def get_jwt(*, keycloak_url: str, realm: str, client_id: str, username: str, password: str) -> str: + """Get JWT token from Keycloak with retry logic for startup flakiness.""" token_url = f"{keycloak_url}/realms/{realm}/protocol/openid-connect/token" - stdout: str = run( - "curl", - "-sk", - "--fail-with-body", - "-X", - "POST", - token_url, - "-d", - "grant_type=password", - "-d", - f"client_id={client_id}", - "-d", - f"username={username}", - "-d", - f"password={password}", - "-d", - "scope=openid organization", + + def try_get_token() -> str | None: + stdout, returncode = run_unchecked( + "curl", + "-sk", + "--fail-with-body", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + f"client_id={client_id}", + "-d", + f"username={username}", + "-d", + f"password={password}", + "-d", + "scope=openid organization", + ) + + if returncode != 0: + return None + + try: + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get JWT from Keycloak for user '{username}': {error}") + return token + except json.JSONDecodeError: + # Malformed response - Keycloak might still be starting + return None + + return poll_until( + fn=try_get_token, + until=lambda token: token is not None, + retries=24, + delay=5, + description=f"Keycloak JWT token for user '{username}'", ) - response: dict[str, str] = json.loads(stdout) - token: str | None = response.get("access_token") - if not token: - error: str = response.get("error_description", response.get("error", "unknown error")) - raise RuntimeError(f"Failed to get JWT from Keycloak for user '{username}': {error}") - return token From baacc8215552f410495c168842ceb056b83636cf Mon Sep 17 00:00:00 2001 From: Haim Tayrie Date: Thu, 9 Jul 2026 15:52:37 +0300 Subject: [PATCH 053/112] Address security and timeout concerns in get_jwt() CodeRabbit review identified two issues: 1. Security: Removed PII leakage from error messages - Removed username from RuntimeError messages - Removed username from poll_until description - Keep only safe metadata (error description from Keycloak) 2. Timeout: Add curl-level timeout to prevent hung requests - Added --max-time 10 to curl invocation - Each retry attempt now fails fast (10s max) - Ensures retry/backoff stays within intended window Fixes: CodeRabbit security and timeout findings Assisted-by: Claude Code Signed-off-by: Haim Tayrie --- tests/core/keycloak.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/core/keycloak.py b/tests/core/keycloak.py index 3de7ac9827..55a3859706 100644 --- a/tests/core/keycloak.py +++ b/tests/core/keycloak.py @@ -14,6 +14,8 @@ def try_get_token() -> str | None: "curl", "-sk", "--fail-with-body", + "--max-time", + "10", "-X", "POST", token_url, @@ -37,16 +39,12 @@ def try_get_token() -> str | None: token: str | None = response.get("access_token") if not token: error: str = response.get("error_description", response.get("error", "unknown error")) - raise RuntimeError(f"Failed to get JWT from Keycloak for user '{username}': {error}") + raise RuntimeError(f"Failed to get JWT from Keycloak: {error}") return token except json.JSONDecodeError: # Malformed response - Keycloak might still be starting return None return poll_until( - fn=try_get_token, - until=lambda token: token is not None, - retries=24, - delay=5, - description=f"Keycloak JWT token for user '{username}'", + fn=try_get_token, until=lambda token: token is not None, retries=24, delay=5, description="Keycloak JWT token" ) From 6714a8498c1fa2860cc386b8a4e9135cbbfe0064 Mon Sep 17 00:00:00 2001 From: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Date: Mon, 29 Jun 2026 23:43:47 -0400 Subject: [PATCH 054/112] OSAC-1123: add CaaS cluster storage E2E test Adds an E2E test for the CaaS storage path (Stage 3) of the Storage Controller. The test validates the full lifecycle: Tenant creation, VMaaS storage readiness, ClusterOrder provisioning with tenant annotation, CaaS storage provisioning (finalizer, ClusterStorageReady condition, Tenant clusterStorage status), and teardown cleanup. New k8s_client methods query ClusterOrder conditions, finalizers, and Tenant clusterStorage status. New wait helpers poll for CaaS-specific state transitions. The storage conftest skip logic is fixed (was inverted, skipping tests when the controller IS configured) and extended to skip CaaS tests when OSAC_PULL_SECRET_PATH is not set, since CaaS requires HyperShift cluster provisioning infrastructure. Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Assisted-by: Cursor/Claude Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> --- tests/core/helpers.py | 131 ++++++++++++++++++++++++++++++--------- tests/core/k8s_client.py | 22 +++++++ 2 files changed, 124 insertions(+), 29 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 4e13b2659f..f53de4cfc6 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -2,6 +2,7 @@ import re import subprocess +from typing import Any import pytest @@ -12,10 +13,7 @@ _POOL_READY_STATE = "PUBLIC_IP_POOL_STATE_READY" -def assert_grpc_rejected( - exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], - code: str, -) -> None: +def assert_grpc_rejected(exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], code: str) -> None: exc = exc_info.value combined: str = (exc.stderr or "") + (exc.stdout or "") assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" @@ -33,9 +31,7 @@ def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_provision(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: k8s.get_compute_instance_condition_status( - name=name, condition_type="Provisioned", checked=False - ), + fn=lambda: k8s.get_compute_instance_condition_status(name=name, condition_type="Provisioned", checked=False), until=lambda v: v == "True", retries=120, delay=5, @@ -301,11 +297,7 @@ def _check_deleted() -> bool: return not k8s.is_present(resource="clusterorder", name=name) poll_until( - fn=_check_deleted, - until=lambda v: v is True, - retries=120, - delay=10, - description=f"{name} ClusterOrder deletion", + fn=_check_deleted, until=lambda v: v is True, retries=120, delay=10, description=f"{name} ClusterOrder deletion" ) @@ -316,23 +308,38 @@ def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None finalizer = "agentclustercapi-provider.agent-install.openshift.io/deprovision" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "agentclusters.capi-provider.agent-install.openshift.io", - "-n", cp_ns, "-o", f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", + *base_args, + "get", + "agentclusters.capi-provider.agent-install.openshift.io", + "-n", + cp_ns, + "-o", + f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", ) if rc != 0 or not output.strip(): return for ac_name in output.strip().split(): finalizers_json, rc = run_unchecked( - *base_args, "get", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", - "-n", cp_ns, "-o", "jsonpath={.metadata.finalizers}", + *base_args, + "get", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "-o", + "jsonpath={.metadata.finalizers}", ) if rc != 0 or finalizer not in finalizers_json: continue import json + idx = json.loads(finalizers_json).index(finalizer) run_unchecked( - *base_args, "patch", f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", - "-n", cp_ns, "--type=json", + *base_args, + "patch", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "--type=json", f'-p=[{{"op": "remove", "path": "/metadata/finalizers/{idx}"}}]', ) @@ -343,16 +350,27 @@ def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: clusterdeployment_ns_label = "agent-install.openshift.io/clusterdeployment-namespace" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "agents.agent-install.openshift.io", - "-n", agent_ns, "-l", f"{clusterorder_label}={name}", - "-o", "jsonpath={.items[*].metadata.name}", + *base_args, + "get", + "agents.agent-install.openshift.io", + "-n", + agent_ns, + "-l", + f"{clusterorder_label}={name}", + "-o", + "jsonpath={.items[*].metadata.name}", ) if rc != 0 or not output.strip(): return for agent_name in output.strip().split(): run_unchecked( - *base_args, "label", f"agents.agent-install.openshift.io/{agent_name}", - "-n", agent_ns, f"{clusterorder_label}-", f"{clusterdeployment_ns_label}-", + *base_args, + "label", + f"agents.agent-install.openshift.io/{agent_name}", + "-n", + agent_ns, + f"{clusterorder_label}-", + f"{clusterdeployment_ns_label}-", ) @@ -361,16 +379,12 @@ def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> N hook = "pre-terminate.delete.hook.machine.cluster.x-k8s.io/agentmachine" base_args = [*k8s._base(), "--as", "system:admin"] output, rc = run_unchecked( - *base_args, "get", "machines.cluster.x-k8s.io", - "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}", + *base_args, "get", "machines.cluster.x-k8s.io", "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}" ) if rc != 0 or not output.strip(): return for machine_name in output.strip().split(): - run_unchecked( - *base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", - "-n", cp_ns, f"{hook}-", - ) + run_unchecked(*base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", "-n", cp_ns, f"{hook}-") def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: @@ -456,6 +470,65 @@ def wait_for_tenant_deletion(*, k8s: K8sClient, name: str) -> None: ) +# CaaS cluster storage helpers + + +def wait_for_cluster_order_condition( + *, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True" +) -> None: + def _check() -> str: + if not k8s.is_present(resource="clusterorder", name=name): + raise AssertionError(f"ClusterOrder {name} disappeared before {condition_type}={expected_status}") + phase: str = k8s.get_cluster_order_phase(name=name, checked=False) + if phase == "Failed": + cond_status = k8s.get_cluster_order_condition_status( + name=name, condition_type=condition_type, checked=False + ) + if cond_status != expected_status: + raise AssertionError( + f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}" + ) + return k8s.get_cluster_order_condition_status(name=name, condition_type=condition_type, checked=False) + + poll_until( + fn=_check, + until=lambda v: v == expected_status, + retries=120, + delay=10, + description=f"ClusterOrder {name} {condition_type}={expected_status}", + ) + + +def wait_for_tenant_cluster_storage_entry(*, k8s: K8sClient, tenant_name: str, cluster_name: str) -> dict[str, Any]: + def _check() -> dict[str, Any] | None: + entries = k8s.get_tenant_cluster_storage(name=tenant_name, checked=False) + for entry in entries: + if entry.get("clusterName") == cluster_name and entry.get("ready") is True: + return entry + return None + + return poll_until( + fn=_check, + until=lambda v: v is not None, + retries=60, + delay=10, + description=f"Tenant {tenant_name} clusterStorage entry for {cluster_name}", + ) + + +def wait_for_tenant_cluster_storage_entry_removed(*, k8s: K8sClient, tenant_name: str, cluster_name: str) -> None: + poll_until( + fn=lambda: all( + entry.get("clusterName") != cluster_name + for entry in k8s.get_tenant_cluster_storage(name=tenant_name, checked=False) + ), + until=lambda v: v is True, + retries=60, + delay=10, + description=f"Tenant {tenant_name} clusterStorage entry for {cluster_name} removed", + ) + + # Storage resource helpers diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 0554df22eb..a15f472201 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -319,6 +319,22 @@ def get_cluster_order_spec(self, *, name: str) -> dict[str, Any]: output = self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.spec}") return json.loads(output) if output else {} + def get_cluster_order_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_cluster_order_finalizers(self, *, name: str, checked: bool = True) -> list[str]: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("metadata", {}).get("finalizers", []) + # Tenant queries def get_tenant_phase(self, *, name: str, checked: bool = True) -> str: @@ -349,6 +365,12 @@ def get_tenant_finalizers(self, *, name: str, checked: bool = True) -> list[str] return [] return json.loads(output).get("metadata", {}).get("finalizers", []) + def get_tenant_cluster_storage(self, *, name: str, checked: bool = True) -> list[dict[str, Any]]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("status", {}).get("clusterStorage", []) + # Cluster-scoped storage resource queries (no -n flag) def count_storage_classes_by_tenant(self, *, tenant_name: str) -> int: From 311b5e740ea3323bb7317194bd9ddf27ea40b85e Mon Sep 17 00:00:00 2001 From: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Date: Wed, 8 Jul 2026 00:42:07 -0400 Subject: [PATCH 055/112] OSAC-1123: increase cluster ready timeout to 60 minutes The CaaS E2E test timed out at 30 minutes on the SE-Lab SNO cluster while waiting for the network operator to become available on the hosted cluster. The worker node installed and rebooted successfully, but the network operator needed more time on resource-constrained lab hardware. This increases the wait_for_cluster_ready timeout from 30 minutes (120 retries x 15s) to 60 minutes (240 retries x 15s). All four tests that use this helper share the same cluster provisioning path, so the higher timeout benefits all of them. Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Assisted-by: Cursor/Claude Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> --- tests/core/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index f53de4cfc6..9206755fc7 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -267,7 +267,7 @@ def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: poll_until( fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), until=lambda v: v == "Ready", - retries=120, + retries=240, delay=15, description=f"{name} ClusterOrder Ready", ) From ed51b61637ea8408a32c9e6585b6aafeee2c112b Mon Sep 17 00:00:00 2001 From: Ori Amizur Date: Wed, 8 Jul 2026 13:31:33 +0300 Subject: [PATCH 056/112] OSAC-2092: Rename public-ip E2E tests to external-ip Rename all public-ip test infrastructure to use the external-ip API, matching the ExternalIP resource rename tracked under OSAC-1442. - Rename gRPC service calls, K8s resources, labels, methods, and helpers from PublicIP to ExternalIP across grpc_client.py, k8s_client.py, helpers.py - Rename tests/vmaas/public_ip/ to tests/vmaas/external_ip/ with updated test files (test_external_ip_pool_lifecycle.py, test_external_ip_pool_capacity.py) - Fix make_pool teardown to clean up child resources (attachments, IPs) before deleting the pool, preventing orphaned resources on the cluster - Update test_validation_rejections to expect FailedPrecondition for duplicate attachment, matching ExternalIP API behavior - Update JWT smoke test to use externalips instead of publicips Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/grpc_client.py | 56 +++++++++++++++++----------------- tests/core/helpers.py | 64 +++++++++++++++++++-------------------- tests/core/k8s_client.py | 36 +++++++++++----------- 3 files changed, 78 insertions(+), 78 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index a9c8be3393..841cb6f831 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -158,9 +158,9 @@ def ensure_tenant(self, *, name: str) -> None: if not re.search(r"Code:\s*AlreadyExists", output): raise RuntimeError(f"Failed to create tenant '{name}': {output}") from e - # PublicIPPool operations (private API only) + # ExternalIPPool operations (private API only) - def create_public_ip_pool( + def create_external_ip_pool( self, *, name: str, @@ -169,7 +169,7 @@ def create_public_ip_pool( implementation_strategy: str = "metallb-l2", ) -> str: response: dict[str, Any] = self.call( - service=f"{PRIVATE_API}.PublicIPPools/Create", + service=f"{PRIVATE_API}.ExternalIPPools/Create", data={ "object": { "metadata": {"name": name}, @@ -183,58 +183,58 @@ def create_public_ip_pool( ) return response["object"]["id"] - def get_public_ip_pool(self, *, pool_id: str) -> dict[str, Any]: - return self.call(service=f"{PRIVATE_API}.PublicIPPools/Get", data={"id": pool_id}) + def get_external_ip_pool(self, *, pool_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ExternalIPPools/Get", data={"id": pool_id}) - def list_public_ip_pool_ids(self) -> list[str]: - response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.PublicIPPools/List") + def list_external_ip_pool_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ExternalIPPools/List") return [item["id"] for item in response.get("items", [])] - def delete_public_ip_pool(self, *, pool_id: str) -> None: - self.call(service=f"{PRIVATE_API}.PublicIPPools/Delete", data={"id": pool_id}) + def delete_external_ip_pool(self, *, pool_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ExternalIPPools/Delete", data={"id": pool_id}) - # PublicIP operations (public API) + # ExternalIP operations (public API) - def create_public_ip(self, *, name: str, pool: str) -> str: + def create_external_ip(self, *, name: str, pool: str) -> str: response: dict[str, Any] = self.call( - service=f"{PUBLIC_API}.PublicIPs/Create", + service=f"{PUBLIC_API}.ExternalIPs/Create", data={"object": {"metadata": {"name": name}, "spec": {"pool": pool}}}, ) return response["object"]["id"] - def get_public_ip(self, *, public_ip_id: str) -> dict[str, Any]: - return self.call(service=f"{PUBLIC_API}.PublicIPs/Get", data={"id": public_ip_id}) + def get_external_ip(self, *, external_ip_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ExternalIPs/Get", data={"id": external_ip_id}) - def list_public_ip_ids(self) -> list[str]: - response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.PublicIPs/List") + def list_external_ip_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ExternalIPs/List") return [item["id"] for item in response.get("items", [])] - def delete_public_ip(self, *, public_ip_id: str) -> None: - self.call(service=f"{PUBLIC_API}.PublicIPs/Delete", data={"id": public_ip_id}) + def delete_external_ip(self, *, external_ip_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ExternalIPs/Delete", data={"id": external_ip_id}) - # PublicIPAttachment operations (public API) + # ExternalIPAttachment operations (public API) - def create_public_ip_attachment(self, *, name: str, public_ip: str, compute_instance: str) -> str: + def create_external_ip_attachment(self, *, name: str, external_ip: str, compute_instance: str) -> str: response: dict[str, Any] = self.call( - service=f"{PUBLIC_API}.PublicIPAttachments/Create", + service=f"{PUBLIC_API}.ExternalIPAttachments/Create", data={ "object": { "metadata": {"name": name}, - "spec": {"public_ip": public_ip, "compute_instance": compute_instance}, + "spec": {"external_ip": external_ip, "compute_instance": compute_instance}, } }, ) return response["object"]["id"] - def get_public_ip_attachment(self, *, attachment_id: str) -> dict[str, Any]: - return self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Get", data={"id": attachment_id}) + def get_external_ip_attachment(self, *, attachment_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Get", data={"id": attachment_id}) - def list_public_ip_attachment_ids(self) -> list[str]: - response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.PublicIPAttachments/List") + def list_external_ip_attachment_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/List") return [item["id"] for item in response.get("items", [])] - def delete_public_ip_attachment(self, *, attachment_id: str) -> None: - self.call(service=f"{PUBLIC_API}.PublicIPAttachments/Delete", data={"id": attachment_id}) + def delete_external_ip_attachment(self, *, attachment_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Delete", data={"id": attachment_id}) # ClusterCatalogItem operations diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 4e13b2659f..2797b02565 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -9,7 +9,7 @@ from tests.core.k8s_client import K8sClient from tests.core.runner import poll_until, run_unchecked -_POOL_READY_STATE = "PUBLIC_IP_POOL_STATE_READY" +_POOL_READY_STATE = "EXTERNAL_IP_POOL_STATE_READY" def assert_grpc_rejected( @@ -143,37 +143,37 @@ def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: ) -def wait_for_public_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: +def wait_for_external_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( - fn=lambda: k8s.get_public_ip_pool_name(uuid=uuid, checked=False), + fn=lambda: k8s.get_external_ip_pool_name(uuid=uuid, checked=False), until=lambda v: v != "", retries=30, delay=1, - description=f"PublicIPPool CR for {uuid}", + description=f"ExternalIPPool CR for {uuid}", ) -def wait_for_public_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: +def wait_for_external_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: k8s.get_public_ip_pool_phase(name=name, checked=False), + fn=lambda: k8s.get_external_ip_pool_phase(name=name, checked=False), until=lambda v: v == "Ready", retries=60, delay=5, - description=f"{name} PublicIPPool Ready", + description=f"{name} ExternalIPPool Ready", ) -def wait_for_public_ip_pool_grpc_ready(*, private_grpc: GRPCClient, pool_id: str) -> None: +def wait_for_external_ip_pool_grpc_ready(*, private_grpc: GRPCClient, pool_id: str) -> None: """Poll the private gRPC API until the pool state is READY. The K8s CR status may report Ready before the fulfillment-service database has been updated by the controller feedback loop. Polling via gRPC closes - this race so that subsequent PublicIP creation does not hit + this race so that subsequent ExternalIP creation does not hit FailedPrecondition. """ def _state() -> str: try: - pool = private_grpc.get_public_ip_pool(pool_id=pool_id) + pool = private_grpc.get_external_ip_pool(pool_id=pool_id) except subprocess.CalledProcessError: return "" return pool.get("object", {}).get("status", {}).get("state", "") @@ -183,77 +183,77 @@ def _state() -> str: until=lambda v: v == _POOL_READY_STATE, retries=30, delay=2, - description=f"PublicIPPool {pool_id} gRPC READY", + description=f"ExternalIPPool {pool_id} gRPC READY", ) -def wait_for_public_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: +def wait_for_external_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: not k8s.is_present(resource="publicippool", name=name), + fn=lambda: not k8s.is_present(resource="externalippool", name=name), until=lambda v: v is True, retries=120, delay=5, - description=f"{name} PublicIPPool deletion", + description=f"{name} ExternalIPPool deletion", ) -def wait_for_public_ip_cr(*, k8s: K8sClient, uuid: str) -> str: +def wait_for_external_ip_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( - fn=lambda: k8s.get_public_ip_name(uuid=uuid, checked=False), + fn=lambda: k8s.get_external_ip_name(uuid=uuid, checked=False), until=lambda v: v != "", retries=30, delay=1, - description=f"PublicIP CR for {uuid}", + description=f"ExternalIP CR for {uuid}", ) -def wait_for_public_ip_allocated(*, k8s: K8sClient, name: str) -> None: +def wait_for_external_ip_allocated(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: k8s.get_public_ip_state(name=name, checked=False), + fn=lambda: k8s.get_external_ip_state(name=name, checked=False), until=lambda v: v == "Allocated", retries=60, delay=5, - description=f"{name} PublicIP Allocated", + description=f"{name} ExternalIP Allocated", ) -def wait_for_public_ip_deletion(*, k8s: K8sClient, name: str) -> None: +def wait_for_external_ip_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: not k8s.is_present(resource="publicip", name=name), + fn=lambda: not k8s.is_present(resource="externalip", name=name), until=lambda v: v is True, retries=120, delay=5, - description=f"{name} PublicIP deletion", + description=f"{name} ExternalIP deletion", ) -def wait_for_public_ip_attachment_cr(*, k8s: K8sClient, uuid: str) -> str: +def wait_for_external_ip_attachment_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( - fn=lambda: k8s.get_public_ip_attachment_name(uuid=uuid, checked=False), + fn=lambda: k8s.get_external_ip_attachment_name(uuid=uuid, checked=False), until=lambda v: v != "", retries=30, delay=1, - description=f"PublicIPAttachment CR for {uuid}", + description=f"ExternalIPAttachment CR for {uuid}", ) -def wait_for_public_ip_attachment_ready(*, k8s: K8sClient, name: str) -> None: +def wait_for_external_ip_attachment_ready(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: k8s.get_public_ip_attachment_phase(name=name, checked=False), + fn=lambda: k8s.get_external_ip_attachment_phase(name=name, checked=False), until=lambda v: v == "Ready", retries=60, delay=5, - description=f"{name} PublicIPAttachment Ready", + description=f"{name} ExternalIPAttachment Ready", ) -def wait_for_public_ip_attachment_deletion(*, k8s: K8sClient, name: str) -> None: +def wait_for_external_ip_attachment_deletion(*, k8s: K8sClient, name: str) -> None: poll_until( - fn=lambda: not k8s.is_present(resource="publicipattachment", name=name), + fn=lambda: not k8s.is_present(resource="externalipattachment", name=name), until=lambda v: v is True, retries=120, delay=5, - description=f"{name} PublicIPAttachment deletion", + description=f"{name} ExternalIPAttachment deletion", ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 0554df22eb..a6ed4fa6e0 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -197,69 +197,69 @@ def get_vm_run_strategy(self, *, name: str, vm_namespace: str) -> str: *self._base(), "get", "virtualmachine", name, "-n", vm_namespace, "-o", "jsonpath={.spec.runStrategy}" ) - # PublicIPPool queries + # ExternalIPPool queries - def get_public_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: + def get_external_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: output, rc = self._get( "get", - "publicippool", + "externalippool", "-n", self.namespace, "-l", - f"osac.openshift.io/publicippool-uuid={uuid}", + f"osac.openshift.io/externalippool-uuid={uuid}", "-o", "jsonpath={.items[0].metadata.name}", checked=checked, ) return output if rc == 0 else "" - def get_public_ip_pool_phase(self, *, name: str, checked: bool = True) -> str: + def get_external_ip_pool_phase(self, *, name: str, checked: bool = True) -> str: output, rc = self._get( - "get", "publicippool", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + "get", "externalippool", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked ) return output if rc == 0 else "" - # PublicIP queries + # ExternalIP queries - def get_public_ip_name(self, *, uuid: str, checked: bool = True) -> str: + def get_external_ip_name(self, *, uuid: str, checked: bool = True) -> str: output, rc = self._get( "get", - "publicip", + "externalip", "-n", self.namespace, "-l", - f"osac.openshift.io/publicip-uuid={uuid}", + f"osac.openshift.io/externalip-uuid={uuid}", "-o", "jsonpath={.items[0].metadata.name}", checked=checked, ) return output if rc == 0 else "" - def get_public_ip_state(self, *, name: str, checked: bool = True) -> str: + def get_external_ip_state(self, *, name: str, checked: bool = True) -> str: output, rc = self._get( - "get", "publicip", name, "-n", self.namespace, "-o", "jsonpath={.status.state}", checked=checked + "get", "externalip", name, "-n", self.namespace, "-o", "jsonpath={.status.state}", checked=checked ) return output if rc == 0 else "" - # PublicIPAttachment queries + # ExternalIPAttachment queries - def get_public_ip_attachment_name(self, *, uuid: str, checked: bool = True) -> str: + def get_external_ip_attachment_name(self, *, uuid: str, checked: bool = True) -> str: output, rc = self._get( "get", - "publicipattachment", + "externalipattachment", "-n", self.namespace, "-l", - f"osac.openshift.io/publicipattachment-uuid={uuid}", + f"osac.openshift.io/externalipattachment-uuid={uuid}", "-o", "jsonpath={.items[0].metadata.name}", checked=checked, ) return output if rc == 0 else "" - def get_public_ip_attachment_phase(self, *, name: str, checked: bool = True) -> str: + def get_external_ip_attachment_phase(self, *, name: str, checked: bool = True) -> str: output, rc = self._get( - "get", "publicipattachment", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + "get", "externalipattachment", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked ) return output if rc == 0 else "" From d488d16c0d425d3f74bdbd1eb5ef8fb79640903a Mon Sep 17 00:00:00 2001 From: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:21:59 -0400 Subject: [PATCH 057/112] OSAC-1123: address review feedback on CaaS storage E2E test - Remove unused storage_config_namespace fixture parameter - Assert patch return code when annotating ClusterOrder with tenant - Cache condition status to avoid redundant API call in wait_for_cluster_order_condition Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> Assisted-by: Cursor/Claude Signed-off-by: akshaynadkarni <25892229+akshaynadkarni@users.noreply.github.com> --- tests/core/helpers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 9206755fc7..81c997cb43 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -480,15 +480,14 @@ def _check() -> str: if not k8s.is_present(resource="clusterorder", name=name): raise AssertionError(f"ClusterOrder {name} disappeared before {condition_type}={expected_status}") phase: str = k8s.get_cluster_order_phase(name=name, checked=False) - if phase == "Failed": - cond_status = k8s.get_cluster_order_condition_status( - name=name, condition_type=condition_type, checked=False + cond_status = k8s.get_cluster_order_condition_status( + name=name, condition_type=condition_type, checked=False + ) + if phase == "Failed" and cond_status != expected_status: + raise AssertionError( + f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}" ) - if cond_status != expected_status: - raise AssertionError( - f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}" - ) - return k8s.get_cluster_order_condition_status(name=name, condition_type=condition_type, checked=False) + return cond_status poll_until( fn=_check, From 891dfdd0eadd0c3ab072d1d738eb4773cf777db1 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Fri, 26 Jun 2026 13:51:52 -0400 Subject: [PATCH 058/112] OSAC-1561: add BareMetalInstance and BareMetalHost E2E test utilities Assisted-by: Claude Code Signed-off-by: Nick Carboni --- tests/core/grpc_client.py | 53 ++++++++++++++++++++++++++ tests/core/helpers.py | 78 +++++++++++++++++++++++++++++++++++++++ tests/core/k8s_client.py | 58 +++++++++++++++++++++++++++++ tests/core/osac_cli.py | 13 +++++++ 4 files changed, 202 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 841cb6f831..fc9ccde9ca 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -337,3 +337,56 @@ def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]: def delete_instance_type(self, *, name: str) -> None: self.call(service=f"{PRIVATE_API}.InstanceTypes/Delete", data={"id": name}) + + # BareMetalInstance operations (public API) + + def list_baremetal_instance_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.BareMetalInstances/List") + return [item["id"] for item in response.get("items", [])] + + def get_baremetal_instance(self, *, bmi_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.BareMetalInstances/Get", data={"id": bmi_id}) + + def get_baremetal_instance_state(self, *, bmi_id: str) -> str: + response: dict[str, Any] = self.get_baremetal_instance(bmi_id=bmi_id) + return response.get("object", {}).get("status", {}).get("state", "") + + def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"run_strategy": run_strategy}}, + "updateMask": {"paths": ["spec.run_strategy"]}, + }, + ) + + def delete_baremetal_instance(self, *, bmi_id: str) -> None: + self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id}) + + # BareMetalInstanceCatalogItem operations (private API for admin setup) + + def create_baremetal_instance_catalog_item( + self, + *, + name: str, + title: str, + description: str, + template: str, + field_definitions: list[dict[str, Any]] | None = None, + ) -> str: + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": title, + "description": description, + "template": template, + "published": True, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Create", data={"object": obj} + ) + return response["object"]["id"] + + def delete_baremetal_instance_catalog_item(self, *, item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Delete", data={"id": item_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index c81c4d8a41..1a1933b812 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -559,3 +559,81 @@ def wait_for_secrets_removed(*, k8s: K8sClient, tenant_name: str, namespace: str delay=5, description=f"Secrets for tenant {tenant_name} in {namespace} removed", ) + + +# BareMetalInstance helpers + + +def wait_for_bmi_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_baremetal_instance_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"BareMetalInstance CR for {uuid}", + ) + + +def wait_for_bmi_running(*, grpc: GRPCClient, bmi_id: str) -> None: + def _check_state() -> str: + state: str = grpc.get_baremetal_instance_state(bmi_id=bmi_id) + assert "FAILED" not in state, f"BareMetalInstance {bmi_id} entered {state}" + return state + + poll_until( + fn=_check_state, + until=lambda v: v == "BARE_METAL_INSTANCE_STATE_RUNNING", + retries=120, + delay=10, + description=f"{bmi_id} RUNNING", + ) + + +def wait_for_bmi_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="baremetalinstance", name=name), + until=lambda v: v is True, + retries=120, + delay=10, + description=f"{name} BareMetalInstance deletion", + ) + + +def wait_for_bmi_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_baremetal_instance_ids(), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{uuid} removed from gRPC BareMetalInstance list", + ) + + +def wait_for_bmh_provisioned(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None: + def _check() -> str: + state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace) + assert state != "error", f"BMH {name} entered error state" + return state + + poll_until( + fn=_check, + until=lambda v: v == "provisioned", + retries=120, + delay=10, + description=f"{name} BMH provisioned", + ) + + +def wait_for_bmh_available(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None: + def _check() -> str: + state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace) + assert state != "error", f"BMH {name} entered error state" + return state + + poll_until( + fn=_check, + until=lambda v: v in ("available", "ready"), + retries=120, + delay=10, + description=f"{name} BMH available", + ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 22e3c29893..e0e2662602 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -439,3 +439,61 @@ def get_security_group_phase(self, *, name: str, checked: bool = True) -> str: "get", "securitygroup", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked ) return output if rc == 0 else "" + + # BareMetalInstance queries + + def get_baremetal_instance_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "baremetalinstance", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/baremetalinstance-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_baremetal_instance_external_host_id(self, *, name: str) -> str: + return self.get_jsonpath(resource="baremetalinstance", name=name, jsonpath="{.spec.externalHostID}") + + # BareMetalHost queries (explicit namespace — BMHs live in a different namespace) + + def get_bmh_provisioning_state(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", + "baremetalhost", + name, + "-n", + bmh_namespace, + "-o", + "jsonpath={.status.provisioning.state}", + checked=False, + ) + return output if rc == 0 else "" + + def get_bmh_image_url(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.image.url}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_consumer_ref(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.consumerRef.name}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_online(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.online}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_powered_on(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.status.poweredOn}", checked=False + ) + return output if rc == 0 else "" diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index c67755d961..c1b3d85855 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -182,3 +182,16 @@ def create_compute_instance_with_catalog_item(self, *, catalog_item: str, subnet def delete_cluster(self, *, uuid: str) -> None: self._run("delete", "cluster", uuid) + + def create_baremetal_instance( + self, *, name: str, catalog_item: str, ssh_key: str | None = None, user_data: str | None = None + ) -> str: + args: list[str] = ["create", "baremetalinstance", "--name", name, "--catalog-item", catalog_item] + if ssh_key is not None: + args.extend(["--ssh-key", ssh_key]) + if user_data is not None: + args.extend(["--user-data", user_data]) + return self._parse_uuid(self._run(*args)) + + def delete_baremetal_instance(self, *, uuid: str) -> None: + self._run("delete", "baremetalinstance", uuid) From e0634bda1fa4f7f378b03099294a593260b32e89 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Thu, 25 Jun 2026 14:25:57 -0400 Subject: [PATCH 059/112] OSAC-1561: add BMaaS E2E lifecycle and power management tests Add tests/bmaas/ with: - conftest: session-scoped catalog item fixture (template via OSAC_BMI_TEMPLATE env var), BMH namespace config - test_baremetal_instance_lifecycle: full provision/deprovision cycle verifying API states and BMH infrastructure (image, consumerRef) - test_baremetal_instance_power_management: run_strategy HALTED/ALWAYS reflected in BMH spec.online Assisted-by: Claude Code Signed-off-by: Nick Carboni --- tests/bmaas/__init__.py | 0 tests/bmaas/conftest.py | 62 ++++++++++++ .../test_baremetal_instance_lifecycle.py | 98 +++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 tests/bmaas/__init__.py create mode 100644 tests/bmaas/conftest.py create mode 100644 tests/bmaas/test_baremetal_instance_lifecycle.py diff --git a/tests/bmaas/__init__.py b/tests/bmaas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/bmaas/conftest.py b/tests/bmaas/conftest.py new file mode 100644 index 0000000000..3499d6cbe9 --- /dev/null +++ b/tests/bmaas/conftest.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import subprocess +import tempfile +import uuid +from collections.abc import Generator +from pathlib import Path + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.runner import env + + +@pytest.fixture(scope="session") +def bmi_template() -> str: + return env("OSAC_BMI_TEMPLATE", "osac.templates.bm_host_provisioning") + + +@pytest.fixture(scope="session") +def bmh_namespace() -> str: + return env("OSAC_BMH_NAMESPACE", "host-inventory") + + +@pytest.fixture(scope="session") +def test_run_id() -> str: + return str(uuid.uuid4())[:8] + + +@pytest.fixture(scope="session") +def ssh_public_key() -> Generator[str, None, None]: + with tempfile.TemporaryDirectory() as tmpdir: + key_path = Path(tmpdir) / "bmi-test-key" + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-f", str(key_path), "-N", "", "-C", "bmi-e2e-test"], + capture_output=True, + check=True, + ) + yield (key_path.with_suffix(".pub")).read_text().strip() + + +@pytest.fixture(scope="session") +def catalog_item(private_grpc: GRPCClient, bmi_template: str, test_run_id: str) -> Generator[str, None, None]: + name = f"e2e-bmaas-{test_run_id}" + print(f"\nCreating BareMetalInstanceCatalogItem: {name}") + item_id: str = private_grpc.create_baremetal_instance_catalog_item( + name=name, + title=f"E2E BMaaS Test ({test_run_id})", + description="Temporary catalog item for BMaaS E2E tests", + template=bmi_template, + field_definitions=[{"path": "ssh_public_key", "display_name": "SSH Public Key", "editable": True}], + ) + print(f"CatalogItem created: {item_id}") + + yield item_id + + try: + print(f"\nDeleting BareMetalInstanceCatalogItem {item_id}...") + private_grpc.delete_baremetal_instance_catalog_item(item_id=item_id) + print(f"CatalogItem {item_id} deleted") + except Exception as e: + print(f"WARNING: Failed to delete catalog item {item_id}: {e}") diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py new file mode 100644 index 0000000000..648dc6b378 --- /dev/null +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_bmh_available, + wait_for_bmh_provisioned, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + + +def test_baremetal_instance_lifecycle( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name = f"e2e-bmi-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + # Verify provisioning + wait_for_bmh_provisioned(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + + image_url: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url != "", f"BMH {bmh_name} has no image URL after provisioning" + + consumer_ref: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref != "", f"BMH {bmh_name} has no consumerRef after allocation" + + online: str = k8s_hub_client.get_bmh_online(name=bmh_name, bmh_namespace=bmh_ns) + assert online == "true", f"BMH {bmh_name} should be online after provisioning, got: {online}" + + # Power off + halted = "BARE_METAL_INSTANCE_RUN_STRATEGY_HALTED" + grpc.update_baremetal_instance_run_strategy(bmi_id=bmi_id, run_strategy=halted) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "false", + retries=60, + delay=5, + description=f"{bmh_name} powered off", + ) + + # Power on + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_ALWAYS" + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on", + ) + + # Deprovision + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + + image_url_after: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url_after == "", f"BMH {bmh_name} image not cleared after deprovision: {image_url_after}" + + consumer_ref_after: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref_after == "", f"BMH {bmh_name} consumerRef not cleared: {consumer_ref_after}" + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + except Exception: + pass + raise From 39ea9f39332d427d57e0c7d7f551396c38ee2f08 Mon Sep 17 00:00:00 2001 From: Elior Erez Date: Tue, 21 Jul 2026 09:35:06 -0400 Subject: [PATCH 060/112] OSAC-2801: Add CaaS/Netris e2e job Adds the real CaaS/Netris e2e test job to e2e-caas-netris.yml/ e2e-caas-netris-caller.yml, plus hardening found necessary by real GitHub Actions dispatches on cold ephemeral hardware (validated via 11+ real dispatches against osac-project/osac-test-infra directly -- self-hosted runners require the branch to exist there, not just on a fork): - poll_until gains an opt-in retry_on_error flag (default off, so the 60+ existing checked call sites keep failing fast); only wait_for_cluster_grpc_removal opts in, since grpc calls can transiently 401 right after a token refresh. - Several ansible retry budgets (MCE-Available wait, Keycloak readiness, FRR install on a fresh ISP VM) were too tight for real first-boot timing; widened based on what real dispatches showed. - Disambiguated two identically-named tasks in lab_deploy. - Added MetalLB and kube-apiserver Service diagnostics to gather-caas, so a first-attempt HostedCluster creation stall (a known, self-healing but currently ~30-80min-costly issue) is diagnosable from CI artifacts alone next time it happens. Root cause: deploy-osac restores a pre-baked golden snapshot with OSAC/AAP already installed rather than installing fresh, so nothing in this flow picks up current osac-aap content until a periodic in-cluster config-as-code job re-syncs AAP's Project config -- the first HostedCluster-creation attempt after a fresh restore usually races that resync and loses. - Explicit least-privilege `permissions: contents: read` on both workflow files (neither declared one before). - Branch-type dispatch inputs (osac-aap-branch, osac-installer-branch, fulfillment-service-branch) are now validated with `git check-ref-format --branch` instead of the shared character whitelist, which allowed colons and other git-invalid characters. - Fixed a get_url checksum default that could error on non-looped get_url tasks, an apt-get failure that could be silently swallowed by a `| tail` pipeline (no pipefail), and a misleadingly-named registered variable. - Added a periodic schedule (8x/day, offset from the hour to avoid GitHub's shared cron congestion) so regressions get caught without someone having to remember to dispatch manually. --- tests/conftest.py | 8 ++++-- tests/core/helpers.py | 10 ++++++- tests/core/runner.py | 64 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c482ddbe72..48ffa9bd4c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,8 +46,12 @@ def service_account() -> str: @pytest.fixture(scope="session") def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPCClient: + # wait_for_cluster_ready's own budget alone can run up to 120min on cold + # EC2 hardware, plus deletion/verification steps after it -- a token this + # short can expire mid-session, failing every subsequent grpcurl call + # with UNAUTHENTICATED. Stay safely above the worst-case session length. token: str = run( - "oc", "create", "token", service_account, "-n", namespace, "--duration", "1h", "--as", "system:admin" + "oc", "create", "token", service_account, "-n", namespace, "--duration", "4h", "--as", "system:admin" ) return GRPCClient(address=fulfillment_address, token=token) @@ -55,7 +59,7 @@ def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPC @pytest.fixture(scope="session") def private_grpc(fulfillment_private_address: str, namespace: str, service_account: str) -> GRPCClient: token: str = run( - "oc", "create", "token", service_account, "-n", namespace, "--duration", "1h", "--as", "system:admin" + "oc", "create", "token", service_account, "-n", namespace, "--duration", "4h", "--as", "system:admin" ) return GRPCClient(address=fulfillment_private_address, token=token) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 1a1933b812..7639f320dd 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -264,10 +264,14 @@ def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: + # Must stay safely above osac-aap's own wait_for_clusteroperators_retries + # budget (60 min) plus earlier steps in the same AAP job (create hosted + # cluster, retrieve kubeconfig, etc.), or this times out first with a + # less useful error while the ClusterOrder is still legitimately Progressing. poll_until( fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), until=lambda v: v == "Ready", - retries=240, + retries=480, delay=15, description=f"{name} ClusterOrder Ready", ) @@ -388,12 +392,16 @@ def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> N def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + # retry_on_error=True: a flaky grpcurl call hitting a momentarily-busy + # route right after heavy cluster-deletion activity shouldn't fail the + # whole test on the first hiccup. poll_until( fn=lambda: uuid not in grpc.list_cluster_ids(), until=lambda v: v is True, retries=60, delay=5, description=f"{uuid} removed from gRPC cluster list", + retry_on_error=True, ) diff --git a/tests/core/runner.py b/tests/core/runner.py index 51786d091b..1d4f16aa7a 100644 --- a/tests/core/runner.py +++ b/tests/core/runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import subprocess import time @@ -8,6 +9,14 @@ T = TypeVar("T") +logger = logging.getLogger(__name__) + +# How often to log a "still waiting" progress line during a poll_until call, +# regardless of its own retry/delay settings -- long polls (e.g. waiting up +# to ~60 minutes for a bare-metal cluster to become Ready) would otherwise +# print nothing until they finish or time out. +_PROGRESS_LOG_INTERVAL_S = 30 + def run(*args: str, timeout: int = 300) -> str: result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=True) @@ -21,14 +30,59 @@ def run_unchecked(*args: str, timeout: int = 300) -> tuple[str, int]: def poll_until( - *, fn: Callable[[], T], until: Callable[[T], bool], retries: int = 60, delay: int = 5, description: str + *, + fn: Callable[[], T], + until: Callable[[T], bool], + retries: int = 60, + delay: int = 5, + description: str, + retry_on_error: bool = False, ) -> T: value: T | None = None - for _ in range(retries): - value = fn() - if until(value): - return value + last_error: subprocess.CalledProcessError | None = None + start = time.monotonic() + last_logged = start + logger.info("Waiting for %s...", description) + for attempt in range(retries): + # retry_on_error is opt-in: most callers' fn() raising CalledProcessError + # indicates a real bug (bad namespace, typo'd resource, auth + # misconfig) and should fail fast, not be swallowed for the whole + # retry budget. Only pollers built on a checked (raising) subprocess + # call where a transient hiccup is expected -- e.g. a flaky grpcurl + # call hitting a momentarily-busy route -- should set this. + if retry_on_error: + try: + value = fn() + except subprocess.CalledProcessError as exc: + last_error = exc + value = None + else: + last_error = None + if until(value): + logger.info("%s — done after %.0fs", description, time.monotonic() - start) + return value + else: + value = fn() + if until(value): + logger.info("%s — done after %.0fs", description, time.monotonic() - start) + return value + now = time.monotonic() + if now - last_logged >= _PROGRESS_LOG_INTERVAL_S: + logger.info( + "Still waiting for %s (attempt %d/%d, %.0fs elapsed, last value: %r%s)", + description, + attempt + 1, + retries, + now - start, + value, + f", last error: {last_error}" if last_error else "", + ) + last_logged = now time.sleep(delay) + if last_error is not None: + raise TimeoutError( + f"{description} — timeout after {retries * delay}s, last call failed: {last_error}" + ) from last_error raise TimeoutError(f"{description} — timeout after {retries * delay}s, last value: {value!r}") From fa2cd7cef29a2d8b4c636c3c472ae860c7df19fe Mon Sep 17 00:00:00 2001 From: Vladik Romanovsky Date: Wed, 15 Jul 2026 19:04:05 -0400 Subject: [PATCH 061/112] OSAC-1586: add E2E test for cluster deletion feedback Verify that deleting a cluster transitions through DELETING state in both the K8s CR phase and the fulfillment-service gRPC API before being archived. Add wait_for_cluster_deleting and wait_for_cluster_grpc_state helpers to support polling for intermediate deletion states. Assisted-by: Claude Code Signed-off-by: Vladik Romanovsky --- tests/core/helpers.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 7639f320dd..efee090da1 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -391,6 +391,33 @@ def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> N run_unchecked(*base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", "-n", cp_ns, f"{hook}-") +def wait_for_cluster_deleting(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Deleting", + retries=30, + delay=5, + description=f"{name} ClusterOrder Deleting phase", + ) + + +def wait_for_cluster_grpc_state(*, grpc: GRPCClient, uuid: str, state: str) -> None: + def _get_state() -> str: + try: + cluster = grpc.get_cluster(cluster_id=uuid) + return cluster.get("object", {}).get("status", {}).get("state", "") + except Exception: + return "" + + poll_until( + fn=_get_state, + until=lambda v: v == state, + retries=30, + delay=5, + description=f"{uuid} cluster gRPC state {state}", + ) + + def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: # retry_on_error=True: a flaky grpcurl call hitting a momentarily-busy # route right after heavy cluster-deletion activity shouldn't fail the From 7ff4a0c1376e1df938b6103951ff7fa9082d416e Mon Sep 17 00:00:00 2001 From: Vladik Romanovsky Date: Wed, 15 Jul 2026 20:30:23 -0400 Subject: [PATCH 062/112] OSAC-1586: add lightweight E2E test for cluster deletion feedback Add a kind-compatible variant of the cluster deletion feedback test that skips full provisioning (no HyperShift needed). Creates a cluster, waits for Progressing phase, deletes it, and verifies the DELETING state appears in both the K8s CR and the gRPC API. Add wait_for_cluster_progressing helper for polling the Progressing phase with short timeouts. Assisted-by: Claude Code Signed-off-by: Vladik Romanovsky --- tests/core/helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index efee090da1..257b78f867 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -263,6 +263,16 @@ def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: ) +def wait_for_cluster_progressing(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Progressing", + retries=30, + delay=2, + description=f"{name} ClusterOrder Progressing phase", + ) + + def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: # Must stay safely above osac-aap's own wait_for_clusteroperators_retries # budget (60 min) plus earlier steps in the same AAP job (create hosted From 3a3c10b3ca5489dda6ecdd372dd9a1baa9ed64df Mon Sep 17 00:00:00 2001 From: Vladik Romanovsky Date: Thu, 23 Jul 2026 08:04:19 -0400 Subject: [PATCH 063/112] OSAC-1586: improve cleanup and error handling in E2E tests - Move try/finally to cover cluster order CR discovery so cleanup runs even if wait_for_cluster_order_cr fails - Narrow exception handling to subprocess.CalledProcessError in cleanup and wait_for_cluster_grpc_state helper - Simplify cleanup to UUID-based delete (no CR name needed) Assisted-by: Claude Code Signed-off-by: Vladik Romanovsky --- tests/core/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 257b78f867..13bbd87ee7 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -415,9 +415,9 @@ def wait_for_cluster_grpc_state(*, grpc: GRPCClient, uuid: str, state: str) -> N def _get_state() -> str: try: cluster = grpc.get_cluster(cluster_id=uuid) - return cluster.get("object", {}).get("status", {}).get("state", "") - except Exception: + except subprocess.CalledProcessError: return "" + return cluster.get("object", {}).get("status", {}).get("state", "") poll_until( fn=_get_state, From adb3c8a628a74e684aa7b26f09235ab132fcbe7b Mon Sep 17 00:00:00 2001 From: Vladik Romanovsky Date: Tue, 28 Jul 2026 21:36:39 -0400 Subject: [PATCH 064/112] OSAC-1586: consolidate deletion feedback test and handle timing race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback: - Consolidate full deletion test into existing test_cluster_create.py instead of a separate test file (trewest) - Handle the DELETING→archived race: the CLUSTER_STATE_DELETING window can be sub-second when the feedback finalizer is the last one, so wait_for_cluster_grpc_deleting_or_archived accepts either state (trewest) - Use contextlib.suppress instead of try/except/pass (CodeRabbit) Assisted-by: Claude Code Signed-off-by: Vladik Romanovsky --- tests/core/helpers.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 13bbd87ee7..29ac704d47 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -411,20 +411,28 @@ def wait_for_cluster_deleting(*, k8s: K8sClient, name: str) -> None: ) -def wait_for_cluster_grpc_state(*, grpc: GRPCClient, uuid: str, state: str) -> None: - def _get_state() -> str: +def wait_for_cluster_grpc_deleting_or_archived(*, grpc: GRPCClient, uuid: str) -> None: + """Succeed if we catch CLUSTER_STATE_DELETING or if the cluster is already archived. + + The DELETING window in the fulfillment-service is extremely short (one + Update + Signal round-trip). Polling for the exact state is racey; accepting + either DELETING or 'already gone' makes the assertion reliable. + """ + + def _done() -> bool: try: cluster = grpc.get_cluster(cluster_id=uuid) + state = cluster.get("object", {}).get("status", {}).get("state", "") + return state == "CLUSTER_STATE_DELETING" except subprocess.CalledProcessError: - return "" - return cluster.get("object", {}).get("status", {}).get("state", "") + return True poll_until( - fn=_get_state, - until=lambda v: v == state, + fn=_done, + until=lambda v: v is True, retries=30, - delay=5, - description=f"{uuid} cluster gRPC state {state}", + delay=2, + description=f"{uuid} gRPC DELETING or already archived", ) From aeeba03eaf24001a7f120bc40ec3f3e117091d4b Mon Sep 17 00:00:00 2001 From: Ameya Sathe Date: Wed, 29 Jul 2026 14:53:24 +0530 Subject: [PATCH 065/112] fix(OSAC-2789): assign per-worker log files and merge sorted e2e logs pytest-xdist workers interleave records when sharing a single log_file, and can truncate each other's output. Configure each worker to write to its own e2e_.log, and update gather-osac-logs.sh to merge all e2e*.log files sorted by timestamp into the final e2e.log artifact. Assisted-by: Claude Code Signed-off-by: Ameya Sathe rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- tests/conftest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 48ffa9bd4c..6ef06175d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ from __future__ import annotations +import os from collections.abc import Iterator +from pathlib import Path import pytest @@ -19,6 +21,20 @@ from tests.core.runner import env, run +def pytest_configure(config: pytest.Config) -> None: + """Give each xdist worker its own log_file so records stay chronologically ordered. + + A shared log_file is interleaved across workers as they report at their own + pace; gather-osac-logs.sh merges the per-worker files back into one sorted + e2e.log artifact. + """ + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + if worker_id is not None: + log_dir = Path(config.getini("log_file")).parent + log_dir.mkdir(parents=True, exist_ok=True) + config.option.log_file = str(log_dir / f"e2e_{worker_id}.log") + + @pytest.fixture(scope="session") def namespace() -> str: return env("OSAC_NAMESPACE", "osac-devel") From 49e1d5b0d0e06b26a42d0bb9022b3bf239c0321b Mon Sep 17 00:00:00 2001 From: Elior Erez Date: Sun, 2 Aug 2026 15:25:17 -0400 Subject: [PATCH 066/112] OSAC-3499: Bump BareMetalInstance deletion timeout past the operator's max backoff wait_for_bmi_deletion's 1200s (20min) window is shorter than osac-operator's shared pkg/provisioning retry ceiling for a failed deprovision job (BackoffMaxDelay = 30min). When the AAP deprovision job fails (e.g. the AAP job-pod attach flakiness tracked in OSAC-3499) and needs a couple of retries before succeeding, the operator is still correctly retrying -- just slower than this test's patience -- and the test times out on a resource that would have deleted successfully given more time. Confirmed via 3 real e2e-bmaas failures today with this exact signature, spanning 2 different runner hosts. bare-metal-fulfillment-operator's BareMetalInstance deprovisioning calls osac-operator's shared pkg/provisioning.RunDeprovisioningLifecycle directly (github.com/osac-project/osac-operator is a real Go module dependency, not a separate reimplementation), so this is the identical mechanism OSAC-3499 already documented for VirtualNetwork, not a distinct BMF-specific bug. Bumped to 2700s (45min): 15 minutes of margin over the 30-minute max backoff, accounting for the retried job's own execution time on top of the wait. A genuinely stuck resource still fails the test, just later. --- tests/core/helpers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 29ac704d47..5f0ea3e4b2 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -643,10 +643,18 @@ def _check_state() -> str: def wait_for_bmi_deletion(*, k8s: K8sClient, name: str) -> None: + # 2700s (45min), not the old 1200s (20min): the deprovision AAP job this + # blocks on retries with exponential backoff up to a 30-minute ceiling + # (osac-operator's shared pkg/provisioning, BackoffMaxDelay) when it fails + # -- e.g. under the AAP job-pod attach flakiness tracked in OSAC-3499. A + # 20-minute window can time out here while the operator is still correctly + # retrying and would have succeeded; 45 minutes gives it room for a worst-case + # backoff wait plus job execution time, without silently swallowing an + # actually-stuck deletion (still fails, just later). poll_until( fn=lambda: not k8s.is_present(resource="baremetalinstance", name=name), until=lambda v: v is True, - retries=120, + retries=270, delay=10, description=f"{name} BareMetalInstance deletion", ) From 0b69429245661a886ae6ff5ae7a52552b81844c0 Mon Sep 17 00:00:00 2001 From: CrystalChun Date: Mon, 3 Aug 2026 13:19:41 -0500 Subject: [PATCH 067/112] OSAC-3263: Update all objects in e2e to have a name Assisted-by: Claude Code --- tests/core/grpc_client.py | 7 +++++-- tests/core/osac_cli.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index fc9ccde9ca..222a3f2fb7 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -26,11 +26,14 @@ def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> li def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: return json.loads(run(*self._build_args(service=service, data=data))) - def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str]) -> str: + def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str], name: str | None = None) -> str: attachments = [{"subnet": sid} for sid in subnet_ids] + obj: dict[str, Any] = {"spec": {"catalog_item": catalog_item, "network_attachments": attachments}} + if name is not None: + obj["metadata"] = {"name": name} response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.ComputeInstances/Create", - data={"object": {"spec": {"catalog_item": catalog_item, "network_attachments": attachments}}}, + data={"object": obj}, ) return response["object"]["id"] diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index c1b3d85855..00eaa5f6b9 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -58,6 +58,7 @@ def create_compute_instance( self, *, template: str, + name: str | None = None, network_attachments: list[dict[str, Any]] | None = None, boot_disk_size: int = 20, image: str = "quay.io/containerdisks/fedora:latest", @@ -80,6 +81,8 @@ def create_compute_instance( "--run-strategy", run_strategy, ] + if name is not None: + args.extend(["--name", name]) effective_instance_type = instance_type if instance_type is not None else self.default_instance_type if effective_instance_type is not None: @@ -174,8 +177,12 @@ def get_unchecked(self, resource: str) -> tuple[str, int]: def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str) -> str: return self._parse_uuid(self._run("create", "cluster", "--catalog-item", catalog_item, "--name", name)) - def create_compute_instance_with_catalog_item(self, *, catalog_item: str, subnet: str | None = None) -> str: + def create_compute_instance_with_catalog_item( + self, *, catalog_item: str, name: str | None = None, subnet: str | None = None + ) -> str: args: list[str] = ["create", "computeinstance", "--catalog-item", catalog_item] + if name is not None: + args.extend(["--name", name]) if subnet is not None: args.extend(["--network-attachment", f"subnet={subnet}"]) return self._parse_uuid(self._run(*args)) From 6b8e0a730064c448362a86b9b322ee8e22dac740 Mon Sep 17 00:00:00 2001 From: Tzif Date: Wed, 5 Aug 2026 09:26:37 +0300 Subject: [PATCH 068/112] OSAC-3161: add gpu parameter to GRPCClient.create_instance_type Assisted-by: Claude Code Signed-off-by: Tzif --- tests/core/grpc_client.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 222a3f2fb7..230e647d11 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -306,19 +306,18 @@ def create_instance_type( cores: int, memory_gib: int, description: str = "", + gpu: dict[str, Any] | None = None, ) -> str: + spec: dict[str, Any] = { + "cores": cores, + "memory_gib": memory_gib, + "description": description, + } + if gpu is not None: + spec["gpu"] = gpu response: dict[str, Any] = self.call( service=f"{PRIVATE_API}.InstanceTypes/Create", - data={ - "object": { - "metadata": {"name": name}, - "spec": { - "cores": cores, - "memory_gib": memory_gib, - "description": description, - }, - } - }, + data={"object": {"metadata": {"name": name}, "spec": spec}}, ) return response["object"]["id"] From 8118afdb6a4924181607c20ae06902bf6767fdd1 Mon Sep 17 00:00:00 2001 From: htayrie-rh Date: Wed, 5 Aug 2026 19:25:59 +0300 Subject: [PATCH 069/112] OSAC-1330: Update E2E tests for typed resource references (#296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update all gRPC client calls and test fixtures to use typed reference objects (e.g., {"id": value} or {"name": value}) instead of raw strings, matching the fulfillment-service API changes for type-safe resource references. - Update grpc_client.py to pass typed reference dicts for subnet, virtual_network, pool, external_ip, compute_instance, catalog_item, and template fields - Update default template/network_class values from underscore to hyphenated names (e.g., osac.templates.ocp_virt_vm → ocp-virt-vm) - Update test assertions to access nested reference fields - Update AGENTS.md and CLAUDE.md documentation Depends on osac-project/osac#85 Assisted-by: Claude Code Signed-off-by: Haim Tayrie Co-authored-by: Haim Tayrie --- tests/bmaas/conftest.py | 2 +- tests/core/grpc_client.py | 35 ++++++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/bmaas/conftest.py b/tests/bmaas/conftest.py index 3499d6cbe9..f2dd30c676 100644 --- a/tests/bmaas/conftest.py +++ b/tests/bmaas/conftest.py @@ -14,7 +14,7 @@ @pytest.fixture(scope="session") def bmi_template() -> str: - return env("OSAC_BMI_TEMPLATE", "osac.templates.bm_host_provisioning") + return env("OSAC_BMI_TEMPLATE", "bm-host-provisioning") @pytest.fixture(scope="session") diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 222a3f2fb7..bb3c115956 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -27,8 +27,8 @@ def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, return json.loads(run(*self._build_args(service=service, data=data))) def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str], name: str | None = None) -> str: - attachments = [{"subnet": sid} for sid in subnet_ids] - obj: dict[str, Any] = {"spec": {"catalog_item": catalog_item, "network_attachments": attachments}} + attachments = [{"subnet": {"id": sid}} for sid in subnet_ids] + obj: dict[str, Any] = {"spec": {"catalog_item": {"id": catalog_item}, "network_attachments": attachments}} if name is not None: obj["metadata"] = {"name": name} response: dict[str, Any] = self.call( @@ -54,7 +54,7 @@ def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[st return self.call( service=f"{PUBLIC_API}.ComputeInstances/Update", data={ - "object": {"id": uuid, "spec": {"template": template, "restart_requested_at": timestamp}}, + "object": {"id": uuid, "spec": {"template": {"name": template}, "restart_requested_at": timestamp}}, "updateMask": {"paths": ["spec.restart_requested_at"]}, }, ) @@ -65,7 +65,10 @@ def create_virtual_network(self, *, name: str, network_class: str, ipv4_cidr: st response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.VirtualNetworks/Create", data={ - "object": {"metadata": {"name": name}, "spec": {"network_class": network_class, "ipv4_cidr": ipv4_cidr}} + "object": { + "metadata": {"name": name}, + "spec": {"network_class": {"name": network_class}, "ipv4_cidr": ipv4_cidr}, + } }, ) return response["object"]["id"] @@ -88,7 +91,7 @@ def create_subnet(self, *, name: str, virtual_network: str, ipv4_cidr: str) -> s data={ "object": { "metadata": {"name": name}, - "spec": {"virtual_network": virtual_network, "ipv4_cidr": ipv4_cidr}, + "spec": {"virtual_network": {"id": virtual_network}, "ipv4_cidr": ipv4_cidr}, } }, ) @@ -121,7 +124,7 @@ def get_cluster(self, *, cluster_id: str) -> dict[str, Any]: def create_security_group(self, *, name: str, virtual_network: str) -> str: response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.SecurityGroups/Create", - data={"object": {"metadata": {"name": name}, "spec": {"virtual_network": virtual_network}}}, + data={"object": {"metadata": {"name": name}, "spec": {"virtual_network": {"id": virtual_network}}}}, ) return response["object"]["id"] @@ -201,7 +204,7 @@ def delete_external_ip_pool(self, *, pool_id: str) -> None: def create_external_ip(self, *, name: str, pool: str) -> str: response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.ExternalIPs/Create", - data={"object": {"metadata": {"name": name}, "spec": {"pool": pool}}}, + data={"object": {"metadata": {"name": name}, "spec": {"pool": {"id": pool}}}}, ) return response["object"]["id"] @@ -223,7 +226,7 @@ def create_external_ip_attachment(self, *, name: str, external_ip: str, compute_ data={ "object": { "metadata": {"name": name}, - "spec": {"external_ip": external_ip, "compute_instance": compute_instance}, + "spec": {"external_ip": {"id": external_ip}, "compute_instance": {"id": compute_instance}}, } }, ) @@ -244,7 +247,12 @@ def delete_external_ip_attachment(self, *, attachment_id: str) -> None: def create_cluster_catalog_item( self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None ) -> str: - obj: dict[str, Any] = {"metadata": {"name": name}, "title": name, "template": template, "published": published} + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": name, + "template": {"name": template}, + "published": published, + } if field_definitions is not None: obj["field_definitions"] = field_definitions response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Create", data={"object": obj}) @@ -272,7 +280,12 @@ def delete_cluster_catalog_item(self, *, catalog_item_id: str) -> None: def create_compute_instance_catalog_item( self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None ) -> str: - obj: dict[str, Any] = {"metadata": {"name": name}, "title": name, "template": template, "published": published} + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": name, + "template": {"name": template}, + "published": published, + } if field_definitions is not None: obj["field_definitions"] = field_definitions response: dict[str, Any] = self.call( @@ -381,7 +394,7 @@ def create_baremetal_instance_catalog_item( "metadata": {"name": name}, "title": title, "description": description, - "template": template, + "template": {"name": template}, "published": True, } if field_definitions is not None: From 2ad4d164216e344342cb76ee6df98b78ff73352d Mon Sep 17 00:00:00 2001 From: Menny Aboush Date: Tue, 11 Aug 2026 10:38:31 +0300 Subject: [PATCH 070/112] OSAC-3402: Add E2E test for bare metal instance restart (#302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * OSAC-3402: Add E2E test for bare metal instance restart Add a dedicated test for BMI restart (power cycle) that verifies the full restart_trigger flow: increment spec.restart_trigger via Update RPC, observe RESTART_IN_PROGRESS condition, wait for status.restart_trigger echo, confirm host comes back online, and verify BMI returns to RUNNING state. Also adds update_baremetal_instance_restart_trigger() helper to GRPCClient for field-masked restart_trigger updates. Note: This test depends on OSAC-3548 (fulfillment controller premature trigger echo race condition) being fixed to pass. Assisted-by: Claude Code Signed-off-by: MENNY ABOUSH * OSAC-3402: Remove transient RESTART_IN_PROGRESS poll The RESTART_IN_PROGRESS condition is transient and cannot be reliably observed — it may flip TRUE→FALSE between poll intervals. Use status.restart_trigger echo as the authoritative restart completion signal instead. Assisted-by: Claude Code Signed-off-by: MENNY ABOUSH * OSAC-3402: Address review — observe RESTART_IN_PROGRESS, remove redundant assertion Add back poll for RESTART_IN_PROGRESS == CONDITION_STATUS_TRUE. With the IsRestartComplete fix (osac/PR #129) the operator now waits for poweredOn=true before completing, so the TRUE window is wide enough (~10-20s) to reliably observe. Remove the redundant final_trigger assertion — the trigger echo poll already guarantees status.restart_trigger == new_trigger. Assisted-by: Claude Code Signed-off-by: MENNY ABOUSH --------- Signed-off-by: MENNY ABOUSH Co-authored-by: MENNY ABOUSH --- .../bmaas/test_baremetal_instance_restart.py | 108 ++++++++++++++++++ tests/core/grpc_client.py | 9 ++ 2 files changed, 117 insertions(+) create mode 100644 tests/bmaas/test_baremetal_instance_restart.py diff --git a/tests/bmaas/test_baremetal_instance_restart.py b/tests/bmaas/test_baremetal_instance_restart.py new file mode 100644 index 0000000000..147ba7faf0 --- /dev/null +++ b/tests/bmaas/test_baremetal_instance_restart.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import logging +from typing import Any + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import wait_for_bmi_cr, wait_for_bmi_deletion, wait_for_bmi_grpc_removal, wait_for_bmi_running +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +logger = logging.getLogger(__name__) + +_RESTART_IN_PROGRESS: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_IN_PROGRESS" +_RESTART_FAILED: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_FAILED" + + +def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + for condition in response.get("object", {}).get("status", {}).get("conditions", []): + if condition.get("type") == condition_type: + return condition.get("status", "") + return "" + + +def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) + + +def test_baremetal_instance_restart( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name: str = f"e2e-bmi-restart-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + initial_trigger: int = _get_status_restart_trigger(grpc, bmi_id) + new_trigger: int = initial_trigger + 1 + logger.info("Incrementing restart_trigger from %d to %d", initial_trigger, new_trigger) + + grpc.update_baremetal_instance_restart_trigger(bmi_id=bmi_id, restart_trigger=new_trigger) + + poll_until( + fn=lambda: _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS), + until=lambda v: v == "CONDITION_STATUS_TRUE", + retries=60, + delay=2, + description=f"{bmi_id} RESTART_IN_PROGRESS condition appears", + ) + + poll_until( + fn=lambda: _get_status_restart_trigger(grpc, bmi_id), + until=lambda v: v == new_trigger, + retries=120, + delay=10, + description=f"{bmi_id} status.restart_trigger echoes {new_trigger}", + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on after restart", + ) + + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + restart_in_progress: str = _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS) + assert restart_in_progress in ("", "CONDITION_STATUS_FALSE"), ( + f"RESTART_IN_PROGRESS should have cleared after restart, got: {restart_in_progress}" + ) + + restart_failed: str = _get_condition_status(grpc, bmi_id, _RESTART_FAILED) + assert restart_failed in ("", "CONDITION_STATUS_FALSE"), ( + f"Unexpected RESTART_FAILED condition: {restart_failed}" + ) + + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + except Exception: + logger.exception("Failed to delete BMI %s during cleanup", bmi_id) + raise diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 7d2411014e..12efed18b0 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -375,6 +375,15 @@ def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: s }, ) + def update_baremetal_instance_restart_trigger(self, *, bmi_id: str, restart_trigger: int) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"restart_trigger": restart_trigger}}, + "updateMask": {"paths": ["spec.restart_trigger"]}, + }, + ) + def delete_baremetal_instance(self, *, bmi_id: str) -> None: self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id}) From 5f3506d363f310c88706f9a5e8c83480490e955a Mon Sep 17 00:00:00 2001 From: Omer Vishlitzky <22615781+omer-vishlitzky@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:21:09 +0300 Subject: [PATCH 071/112] OSAC-3434: Add VMaaS metering lifecycle E2E tests (#301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * OSAC-3434: Add cross-cutting concern test pattern and metering infrastructure Establish the composable fixture pattern for testing features that span multiple domains (metering, storage, networking, catalog): - Verifier class in tests/core/.py - Fixture + marker + skip logic in tests/conftest.py - Tests annotate with @pytest.mark. and inject fixture Metering implementation: - kafka-python-ng dependency for Kafka consumption - MeteringCollector: background Kafka consumer with expect/verify. Validates CloudEvent structure (specversion, source, tenant, billing dimensions, schema version) on every matched event. - metering fixture: starts collector, yields, verifies + stops - @pytest.mark.metering marker, auto-skip without KAFKA_BOOTSTRAP_SERVERS - test-metering Makefile target (pytest -m metering) Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Add metering verification to VMaaS lifecycle tests Annotate existing VMaaS tests with @pytest.mark.metering and inject the metering fixture. No new test files — metering verification is composable on top of existing lifecycle tests. Annotated tests: - test_compute_instance_lifecycle: created.v1 + started.v1 + deleted.v1 - test_compute_instance_delete_during_provision: created.v1 + deleted.v1 - test_compute_instance_restart: created.v1 + started.v1 When KAFKA_BOOTSTRAP_SERVERS is not set, these tests run without metering verification (marker auto-skips the fixture). Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Fix consumer race condition and billing_dimensions assertion - Wait for Kafka partition assignment before returning from start(). First poll() triggers partition assignment; threading.Event gates start() until consumer is ready. Prevents missing early events. - Assert billing_dimensions contains instance_type, image_ref, and boot_disk_size_gib per OSAC-3434 AC, not just key existence. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Add missing metering ACs — restart events and short-lived VM - Restart test: add suspended.v1 + started.v1 expectations during the restart cycle (VM stops then starts — both transitions must produce metering events) - Short-lived VM test (CAP-4): create and immediately delete a VM without waiting for Running. Verify created.v1 + deleted.v1 appear on Kafka — validates sub-minute billing granularity Note: AC2 checks suspended.v1 event type but not previous_state field (always null in Phase 1, no state tracking). Full AC2 coverage in Phase 2 when previous_state is populated. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Fail hard when metering tests run without Kafka If @pytest.mark.metering tests are collected but KAFKA_BOOTSTRAP_SERVERS is not set, fail immediately instead of silently skipping. Missing Kafka when metering is expected is a configuration bug, not a graceful degradation. Use -m 'not metering' to explicitly exclude metering tests. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Fix review findings — event dedup, consumer robustness, docstring - Track matched events: _matched set prevents the same Kafka event from satisfying multiple expectations. Restart test expecting two started.v1 events now requires two distinct events. - Propagate consumer errors: if KafkaConsumer constructor fails (bad creds, TLS, unreachable), the exception is saved and re-raised from start() instead of a generic timeout message. - Wait for partition assignment: loop on consumer.assignment() with bounded retries instead of a single poll(timeout_ms=0). - Guard leaked thread: fixture wraps start() in try/except, calls stop() on failure before re-raising. - Fix displaced docstring in pytest_configure. - Fix unused loop variable (records.values() not records.items()). Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Signal ready on both success and failure paths - Set _ready on exception path so start() surfaces the error immediately instead of waiting 30s timeout - Check assignment after poll loop — fail if no partitions assigned instead of silently proceeding with no assignment Assisted-by: Claude Code Signed-off-by: omer-vishlitzky OSAC-3434: Pass Kafka credentials to VMaaS E2E test container Extract Kafka bootstrap address, SASL password, and CA cert from the deployed cluster and pass them to the test container. Without these, the MeteringCollector can't connect to Kafka and metering tests fail at collection time. Only set when Kafka is deployed (kafka cluster exists in osac-kafka namespace). SASL password masked in workflow logs. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix(pr-review): switch to test adapter HTTP API, add heartbeat test Replace direct Kafka consumer with HTTP client querying the metering test adapter's /events endpoint. The test adapter runs inside the cluster — CI creates an OpenShift Route to make it externally accessible. Eliminates the cluster-internal DNS resolution issue that caused all metering tests to ERROR on teardown. MeteringCollector: - HTTP-based: queries GET /events?type=...&resource_id=...&since=... - No background thread, no Kafka dependency, no SSL config - Per-expectation timeout (default 60s, heartbeat uses 120s) - Enhanced structure validation: numeric boot_disk_size_gib, project_id presence, RFC3339 transition_time, previous_state/duration_seconds for state transition events Tests: - New: test_compute_instance_heartbeat — verifies heartbeat events appear for a RUNNING VM within 120s (Phase 2 heartbeat generator) - Fix: restart test expects resumed.v1 (not started.v1) for STOPPED→RUNNING transition - Rename: test_metering_short_lived_vm → test_compute_instance_short_lived_metering (AGENTS.md naming convention) CI: - Replace Kafka credential extraction with oc expose route for test adapter; single METERING_ADAPTER_URL env var replaces 5 Kafka vars - Remove kafka-python-ng dependency Review feedback (masayag): - Remove unused config param from pytest_collection_modifyitems - Fix AGENTS.md stale path (tests/concerns/conftest.py → tests/conftest.py) Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix(pr-review): HTTP resilience, stop-VM test, structure validation HTTP resilience: - Wrap _fetch_events in try/except inside poll_until find() closure. Transient HTTP errors (URLError, JSONDecodeError, OSError) are logged and retried instead of crashing the test immediately. Stop-VM metering test (AC 2): - New test_compute_instance_stop_metering: creates VM, waits for RUNNING, patches runStrategy to Halted, waits for Stopped, verifies suspended.v1 event. Covers the "stop VM" acceptance criterion that was missing (only restart exercised suspended.v1 before). Structure validation: - Assert osacresourceid extension matches expected resource_id (previously only used for matching, not validated) Heartbeat test: - Call metering.verify() explicitly before deleting VM to ensure heartbeat assertion is checked while VM is still RUNNING Cleanup: - Remove phantom kafka-python-ng from uv.lock - Restore grpcurl health check explanatory comments in CI workflow - Add name=unique_name() to short-lived test for consistency Assisted-by: Claude Code Signed-off-by: omer-vishlitzky feat: comprehensive metering E2E coverage — heartbeat stop, reconciliation, deep validation Metering collector enhancements: - assert_no_events: negative assertion for verifying events do NOT appear within a time window (heartbeat stops after non-billable) - get_event: returns event data for value assertions beyond structure - verify() clears expectations after completion for multi-phase tests - suspended.v1 deep validation: previous_state must be RUNNING, duration_seconds must be positive (closes a billing interval) New tests: - test_compute_instance_stop_metering: enhanced with heartbeat-stops verification — after stopping VM, asserts no heartbeat events appear within 90s (design doc line 815) - test_compute_instance_reconciliation_metering: restarts metering pod, waits for readiness (startup reconciliation), verifies heartbeats resume for still-running VM (design doc CAP-15, line 839) CI workflow: - Remove conditional metering exclusion — metering is mandatory for vmaas, missing adapter is an infra bug that should fail immediately. Always pass METERING_ADAPTER_URL to test container. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix(pr-review): contract mismatches, validation depth, robustness Contract fixes (blocking): - Skip transition_time assertion for heartbeat events (heartbeatData has no transition_time field — generator.go:179) - Accept "osac-metering/reconciler" as valid source (synthetic heartbeats from reconciler use different source — reconciler.go:389) Validation depth: - osacresourcetype asserts exact value "compute_instance" (was presence) - resumed.v1 previous_state asserts value in (STOPPED, PAUSED) - assert_no_events uses datetime parsing instead of string comparison (Z vs +00:00 format difference) - assert_no_events error includes event IDs and timestamps - verify() warns when called with no expectations Robustness: - Workflow: check route host non-empty before setting URL; emit ::error:: if adapter unavailable; quote METERING_ADAPTER_URL - Heartbeat timeout 180s (was 120s) — 3x interval for margin - Reconciliation test VM cleanup in try/finally block Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix: get_event after verify, billing dimensions truthiness get_event() after verify() was dead code — verify() added matched event IDs to _matched, then get_event() skipped them and timed out. The billing dimensions value check in test_compute_instance_lifecycle never executed. Fix: verify() now stores matched events in _verified dict; get_event() returns from cache if available. billing_dimensions: instance_type and image_ref now checked for truthiness (not just key presence), catching empty string values. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix: enable metering test adapter in CI install Add --set metering.testAdapter.enabled=true to Helm install args. The test adapter pod is required for metering E2E tests — without it, the route creation step fails and pytest_collection_modifyitems correctly aborts the session (metering is mandatory for vmaas). Also fix get_event after verify (verified events cached in _verified dict) and billing_dimensions truthiness checks. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky Revert "fix: enable metering test adapter in CI install" This reverts commit 2d0d014f71ab1a37d78c97b50ad62b750676f330. fix: suspended.v1 previous_state accepts transient states, K8sClient API fix suspended.v1 validation: - Accept RUNNING, STOPPING, STARTING as valid previous_state values (was: RUNNING only). A VM goes RUNNING→STOPPING→STOPPED, so the suspended.v1 event may capture STOPPING as previous_state. Reconciliation test: - Use runner.run_unchecked instead of K8sClient.run (which doesn't exist) for pod deletion and readiness polling. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix: allow null duration_seconds in suspended.v1 When a VM transitions RUNNING→STOPPING→STOPPED, the billing interval closes on the RUNNING→STOPPING transition (updated.v1 with duration). The subsequent STOPPING→STOPPED transition emits suspended.v1 with null duration_seconds because no billing interval was open. This is correct behavior — the assertion should accept null OR positive. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix: relax instance_type value check (proto mismatch), stop previous_state instance_type value check removed from lifecycle test — metering service proto needs regeneration after PR #85 changed instance_type from optional string to InstanceTypeReference. The generic _validate_structure still checks instance_type is present and non-empty. Value check can be restored after metering proto is regenerated. Stop test: accept STOPPING as valid previous_state alongside RUNNING. VM transitions RUNNING→STOPPING→STOPPED; the suspended.v1 event may capture either as previous_state depending on timing. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky Revert "fix: relax instance_type value check (proto mismatch), stop previous_state" This reverts commit 1e7aae7b1c9cf0dbf11bfd75b68e493731237266. fix: resolve echo-adapter Service by label, fail fast if missing oc expose svc/osac-metering-test-adapter targets a name that no longer exists after the test-adapter to echo-adapter rename — the Service never gets exposed, and the swallowed errors (2>/dev/null || true, || echo "") let the job limp forward with an empty METERING_ADAPTER_URL instead of failing at the source. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix: match resource_id field from echo-adapter response The echo-adapter returns resource_id at the top level (extracted from the osacresourceid extension). Add it as first match check alongside osacresourceid and data.resource_id. Depends on osac-project/osac#250 Assisted-by: Claude Code Signed-off-by: omer-vishlitzky fix: heartbeat race in stop metering test, ensure cleanup on failure Wait one heartbeat interval after suspended.v1 before asserting no more heartbeats — the projection DB lags behind Kafka, so one stale heartbeat sweep can fire after the stop is confirmed. Wrap post-stop assertions in try/finally so the compute instance is always cleaned up, matching test_compute_instance_reconciliation_metering. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky * fix: remove reconciliation test, restore stop_metering original timing Remove test_compute_instance_reconciliation_metering — it restarts the metering pod, which disrupts all other metering tests running in parallel via xdist (Watch stream replay causes spurious resumed.v1 events on stopped VMs in other workers). Will re-enable once a disruptive test isolation pattern is established. Tracked in OSAC-3918. Restore test_compute_instance_stop_metering to original form — the heartbeat race was caused by the reconciliation test's pod restart, not by projection lag. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky * fix: wait for heartbeat drain before asserting silence after VM stop The heartbeat generator queries the projection DB, which lags behind Kafka publication (publishAndUpsert publishes first, updates DB second). One more heartbeat sweep can fire against the stale projection after suspended.v1 is confirmed. Wait 90s (> heartbeat interval) to let in-flight heartbeats drain before starting the no-heartbeat assertion window. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky * fix: stop VM via gRPC API instead of K8s CR patch The test patched the ComputeInstance K8s CR directly to set runStrategy=Halted. The fulfillment-service API record still had run_strategy=Always, so the controller reconciled the discrepancy and restarted the VM — causing a spurious resumed.v1 event and legitimate heartbeats that failed the assert_no_events check. Use the private gRPC UpdateComputeInstance RPC with an update mask on spec.run_strategy instead. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --------- Signed-off-by: omer-vishlitzky --- tests/conftest.py | 32 +++++++ tests/core/grpc_client.py | 9 ++ tests/core/metering.py | 197 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 tests/core/metering.py diff --git a/tests/conftest.py b/tests/conftest.py index 6ef06175d7..1af0ebd9c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ get_user_id, wait_for_organization, ) +from tests.core.metering import MeteringCollector from tests.core.osac_cli import OsacCLI from tests.core.runner import env, run @@ -28,6 +29,7 @@ def pytest_configure(config: pytest.Config) -> None: pace; gather-osac-logs.sh merges the per-worker files back into one sorted e2e.log artifact. """ + config.addinivalue_line("markers", "metering: test verifies metering events via the test adapter HTTP API") worker_id = os.environ.get("PYTEST_XDIST_WORKER") if worker_id is not None: log_dir = Path(config.getini("log_file")).parent @@ -218,3 +220,33 @@ def jwt_grpc_tenant2(fulfillment_address: str, keycloak_url: str, jwt_password: keycloak_url=keycloak_url, realm="osac", client_id="osac-cli", username="tenant2_user", password=jwt_password ) return GRPCClient(address=fulfillment_address, token=token) + + +# --- Cross-cutting concern: Metering --- + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + metering_tests = [item for item in items if item.get_closest_marker("metering")] + if metering_tests and not os.environ.get("METERING_ADAPTER_URL"): + pytest.fail( + f"METERING_ADAPTER_URL is not set but {len(metering_tests)} test(s) require metering. " + "Set the env var or run with -m 'not metering' to exclude metering tests.", + pytrace=False, + ) + + +@pytest.fixture +def metering() -> Iterator[MeteringCollector]: + """Composable metering verifier. Inject into any test to verify + that lifecycle events appear via the test adapter HTTP API. + + Records a start timestamp on setup, verifies all expectations + (including CloudEvent structure validation) on teardown. + """ + collector = MeteringCollector(base_url=env("METERING_ADAPTER_URL")) + collector.start() + yield collector + try: + collector.verify() + finally: + collector.stop() diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 12efed18b0..9d60605f39 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -37,6 +37,15 @@ def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str], n ) return response["object"]["id"] + def update_compute_instance_run_strategy(self, *, ci_id: str, run_strategy: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.ComputeInstances/Update", + data={ + "object": {"id": ci_id, "spec": {"run_strategy": run_strategy}}, + "updateMask": {"paths": ["spec.run_strategy"]}, + }, + ) + def delete_compute_instance(self, *, ci_id: str) -> None: self.call(service=f"{PUBLIC_API}.ComputeInstances/Delete", data={"id": ci_id}) diff --git a/tests/core/metering.py b/tests/core/metering.py new file mode 100644 index 0000000000..f4e68fc04c --- /dev/null +++ b/tests/core/metering.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import json +import logging +import time +import urllib.error +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from tests.core.runner import poll_until + +logger = logging.getLogger(__name__) + + +def _parse_time(s: str) -> datetime: + if s.endswith("Z"): + s = s[:-1] + "+00:00" + return datetime.fromisoformat(s) + + +@dataclass +class ExpectedEvent: + event_type: str + resource_id: str + timeout: int = 60 + + +class MeteringCollector: + """Collects CloudEvents from the metering test adapter HTTP API and verifies expectations. + + The test adapter runs inside the Kubernetes cluster and exposes + ``GET /events?type={event_type}&resource_id={resource_id}&since={RFC3339}`` + which returns a JSON array of CloudEvents. + + Usage in tests:: + + metering.expect("osac.resource.created.v1", resource_id=uuid) + # ... test actions ... + # metering.verify() runs automatically on fixture teardown + """ + + def __init__(self, *, base_url: str) -> None: + self._base_url = base_url.rstrip("/") + self._expectations: list[ExpectedEvent] = [] + self._matched: set[str] = set() + self._verified: dict[tuple[str, str], dict[str, Any]] = {} + self._start_time: str = "" + + def start(self) -> None: + """Record the start time so event queries only return events after this point.""" + self._start_time = datetime.now(UTC).isoformat() + + def stop(self) -> None: + """No-op -- no background resources to clean up.""" + + def expect(self, event_type: str, resource_id: str, *, timeout: int = 60) -> None: + self._expectations.append(ExpectedEvent(event_type=event_type, resource_id=resource_id, timeout=timeout)) + + def verify(self) -> None: + if not self._expectations: + logger.warning("verify() called with no expectations — did you forget to call expect()?") + return + for exp in self._expectations: + event = self._poll_for_event(exp) + self._validate_structure(event, exp) + self._verified[(exp.event_type, exp.resource_id)] = event + self._expectations.clear() + + def get_event(self, event_type: str, resource_id: str, *, timeout: int = 60) -> dict[str, Any]: + """Return a verified event, or poll for one if not yet verified.""" + key = (event_type, resource_id) + if key in self._verified: + return self._verified[key] + exp = ExpectedEvent(event_type=event_type, resource_id=resource_id, timeout=timeout) + return self._poll_for_event(exp) + + def assert_no_events(self, event_type: str, resource_id: str, *, since: str, within: int = 60) -> None: + """Assert that no events of the given type appear for the resource within a time window.""" + time.sleep(within) + events = self._fetch_events(event_type, resource_id) + since_dt = datetime.fromisoformat(since) + matching = [ + ev for ev in events + if ev.get("type") == event_type + and (ev.get("resource_id") == resource_id + or ev.get("osacresourceid") == resource_id + or ev.get("data", {}).get("resource_id") == resource_id) + and _parse_time(ev.get("time", "")) >= since_dt + ] + assert not matching, ( + f"Expected no {event_type} events for {resource_id} after {since}, " + f"but found {len(matching)}: {[{'id': e.get('id'), 'time': e.get('time')} for e in matching]}" + ) + + def _fetch_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: + params = urlencode({ + "type": event_type, + "resource_id": resource_id, + "since": self._start_time, + }) + url = f"{self._base_url}/events?{params}" + req = Request(url) + with urlopen(req, timeout=10) as resp: # noqa: S310 + return json.loads(resp.read().decode("utf-8")) + + def _poll_for_event(self, expected: ExpectedEvent) -> dict[str, Any]: + result: list[dict[str, Any]] = [] + + def find() -> bool: + try: + events = self._fetch_events(expected.event_type, expected.resource_id) + except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc: + logger.warning("Transient error fetching metering events, will retry: %s", exc) + return False + for ev in events: + ev_id = ev.get("id", "") + if ev_id and ev_id in self._matched: + continue + if ev.get("type") == expected.event_type and ( + ev.get("resource_id") == expected.resource_id + or ev.get("osacresourceid") == expected.resource_id + or ev.get("data", {}).get("resource_id") == expected.resource_id + ): + if ev_id: + self._matched.add(ev_id) + result.append(ev) + return True + return False + + poll_until( + fn=find, + until=lambda found: found is True, + retries=expected.timeout // 2, + delay=2, + description=f"metering event type={expected.event_type} resource_id={expected.resource_id}", + ) + return result[0] + + @staticmethod + def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: + assert event.get("specversion") == "1.0", f"Wrong specversion: {event.get('specversion')}" + assert event.get("source") in ("osac-metering", "osac-metering/reconciler"), \ + f"Wrong source: {event.get('source')}" + assert event.get("id"), "Missing event id" + assert event.get("time"), "Missing event time" + assert event.get("osacresourcetype") == "compute_instance", \ + f"Wrong osacresourcetype: {event.get('osacresourcetype')}" + assert event.get("osacresourceid") == expected.resource_id, \ + f"Wrong osacresourceid: {event.get('osacresourceid')}" + assert event.get("osactenant"), "Missing osactenant" + + data = event.get("data", {}) + assert data.get("resource_id") == expected.resource_id, f"Wrong resource_id in data: {data.get('resource_id')}" + assert data.get("resource_type"), "Missing resource_type in data" + assert data.get("tenant_id"), "Missing tenant_id in data" + assert data.get("schema_version") == "v1", f"Wrong schema_version: {data.get('schema_version')}" + + bd = data.get("billing_dimensions", {}) + assert bd.get("instance_type"), "Missing or empty instance_type in billing_dimensions" + assert bd.get("image_ref"), "Missing or empty image_ref in billing_dimensions" + assert "boot_disk_size_gib" in bd, "Missing boot_disk_size_gib in billing_dimensions" + assert isinstance( + bd["boot_disk_size_gib"], (int, float) + ), f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" + + assert "project_id" in data, "Missing project_id in data" + + if expected.event_type != "osac.resource.heartbeat.v1": + assert data.get("transition_time"), "Missing transition_time in data" + try: + datetime.fromisoformat(data["transition_time"]) + except (ValueError, TypeError) as exc: + raise AssertionError(f"Invalid RFC3339 transition_time: {data.get('transition_time')}") from exc + + transition_types = { + "osac.resource.started.v1", + "osac.resource.suspended.v1", + "osac.resource.resumed.v1", + } + if expected.event_type in transition_types: + assert "previous_state" in data, f"Missing previous_state in {expected.event_type}" + assert "duration_seconds" in data, f"Missing duration_seconds in {expected.event_type}" + + if expected.event_type == "osac.resource.suspended.v1": + valid_suspended_previous = ("RUNNING", "STOPPING", "STARTING") + assert data.get("previous_state") in valid_suspended_previous, ( + f"suspended.v1 previous_state should be one of {valid_suspended_previous}, " + f"got {data.get('previous_state')!r}" + ) + + if expected.event_type == "osac.resource.resumed.v1": + assert data.get("previous_state") in ("STOPPED", "PAUSED"), ( + f"resumed.v1 should have previous_state in (STOPPED, PAUSED), got {data.get('previous_state')}" + ) From 19035c3ad06318ecbc970273678ff63a649d324e Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 12 Aug 2026 03:59:31 +0300 Subject: [PATCH 072/112] =?UTF-8?q?NO-ISSUE:=20fix=20E2E=20flakes=20?= =?UTF-8?q?=E2=80=94=20retry=20metering=20HTTP,=20serialize=20BMaaS=20test?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add retry logic to MeteringCollector._fetch_events to handle transient OpenShift router connection drops during HAProxy reloads. Serialize BMaaS tests with xdist_group to avoid concurrent Ironic provisioning through single-threaded sushy-emulator, which caused ~19% timeout failures. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/bmaas/test_baremetal_instance_lifecycle.py | 3 +++ tests/bmaas/test_baremetal_instance_restart.py | 3 +++ tests/core/metering.py | 14 +++++++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 648dc6b378..259f8e9eb0 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from tests.core.grpc_client import GRPCClient from tests.core.helpers import ( wait_for_bmh_available, @@ -14,6 +16,7 @@ from tests.core.runner import poll_until +@pytest.mark.xdist_group("bmaas") def test_baremetal_instance_lifecycle( cli: OsacCLI, grpc: GRPCClient, diff --git a/tests/bmaas/test_baremetal_instance_restart.py b/tests/bmaas/test_baremetal_instance_restart.py index 147ba7faf0..20e643a612 100644 --- a/tests/bmaas/test_baremetal_instance_restart.py +++ b/tests/bmaas/test_baremetal_instance_restart.py @@ -3,6 +3,8 @@ import logging from typing import Any +import pytest + from tests.core.grpc_client import GRPCClient from tests.core.helpers import wait_for_bmi_cr, wait_for_bmi_deletion, wait_for_bmi_grpc_removal, wait_for_bmi_running from tests.core.k8s_client import K8sClient @@ -28,6 +30,7 @@ def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) +@pytest.mark.xdist_group("bmaas") def test_baremetal_instance_restart( cli: OsacCLI, grpc: GRPCClient, diff --git a/tests/core/metering.py b/tests/core/metering.py index f4e68fc04c..f27c5e9ee4 100644 --- a/tests/core/metering.py +++ b/tests/core/metering.py @@ -102,9 +102,17 @@ def _fetch_events(self, event_type: str, resource_id: str) -> list[dict[str, Any "since": self._start_time, }) url = f"{self._base_url}/events?{params}" - req = Request(url) - with urlopen(req, timeout=10) as resp: # noqa: S310 - return json.loads(resp.read().decode("utf-8")) + last_exc: OSError | None = None + for attempt in range(3): + try: + req = Request(url) + with urlopen(req, timeout=10) as resp: # noqa: S310 + return json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, OSError) as exc: + last_exc = exc + logger.warning("Transient HTTP error fetching events (attempt %d/3): %s", attempt + 1, exc) + time.sleep(2) + raise last_exc # type: ignore[misc] def _poll_for_event(self, expected: ExpectedEvent) -> dict[str, Any]: result: list[dict[str, Any]] = [] From 52c4f2cae46b742569679458e811a104eaebfcc0 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 12 Aug 2026 09:34:27 +0300 Subject: [PATCH 073/112] =?UTF-8?q?NO-ISSUE:=20revert=20bmaas=20xdist=5Fgr?= =?UTF-8?q?oup=20=E2=80=94=20loadfile=20ignores=20group=20markers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/bmaas/test_baremetal_instance_lifecycle.py | 3 --- tests/bmaas/test_baremetal_instance_restart.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 259f8e9eb0..648dc6b378 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -1,7 +1,5 @@ from __future__ import annotations -import pytest - from tests.core.grpc_client import GRPCClient from tests.core.helpers import ( wait_for_bmh_available, @@ -16,7 +14,6 @@ from tests.core.runner import poll_until -@pytest.mark.xdist_group("bmaas") def test_baremetal_instance_lifecycle( cli: OsacCLI, grpc: GRPCClient, diff --git a/tests/bmaas/test_baremetal_instance_restart.py b/tests/bmaas/test_baremetal_instance_restart.py index 20e643a612..147ba7faf0 100644 --- a/tests/bmaas/test_baremetal_instance_restart.py +++ b/tests/bmaas/test_baremetal_instance_restart.py @@ -3,8 +3,6 @@ import logging from typing import Any -import pytest - from tests.core.grpc_client import GRPCClient from tests.core.helpers import wait_for_bmi_cr, wait_for_bmi_deletion, wait_for_bmi_grpc_removal, wait_for_bmi_running from tests.core.k8s_client import K8sClient @@ -30,7 +28,6 @@ def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) -@pytest.mark.xdist_group("bmaas") def test_baremetal_instance_restart( cli: OsacCLI, grpc: GRPCClient, From 0b84c38bae6cd7a294cbfbf6f10f1788d20f406b Mon Sep 17 00:00:00 2001 From: Omer Vishlitzky <22615781+omer-vishlitzky@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:36:04 +0300 Subject: [PATCH 074/112] Revert "OSAC-3402: Add E2E test for bare metal instance restart (#302)" This reverts commit 2ad4d164216e344342cb76ee6df98b78ff73352d. --- .../bmaas/test_baremetal_instance_restart.py | 108 ------------------ tests/core/grpc_client.py | 9 -- 2 files changed, 117 deletions(-) delete mode 100644 tests/bmaas/test_baremetal_instance_restart.py diff --git a/tests/bmaas/test_baremetal_instance_restart.py b/tests/bmaas/test_baremetal_instance_restart.py deleted file mode 100644 index 147ba7faf0..0000000000 --- a/tests/bmaas/test_baremetal_instance_restart.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Any - -from tests.core.grpc_client import GRPCClient -from tests.core.helpers import wait_for_bmi_cr, wait_for_bmi_deletion, wait_for_bmi_grpc_removal, wait_for_bmi_running -from tests.core.k8s_client import K8sClient -from tests.core.osac_cli import OsacCLI -from tests.core.runner import poll_until - -logger = logging.getLogger(__name__) - -_RESTART_IN_PROGRESS: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_IN_PROGRESS" -_RESTART_FAILED: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_FAILED" - - -def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: - response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) - for condition in response.get("object", {}).get("status", {}).get("conditions", []): - if condition.get("type") == condition_type: - return condition.get("status", "") - return "" - - -def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: - response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) - return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) - - -def test_baremetal_instance_restart( - cli: OsacCLI, - grpc: GRPCClient, - k8s_hub_client: K8sClient, - catalog_item: str, - bmh_namespace: str, - test_run_id: str, - ssh_public_key: str, -) -> None: - name: str = f"e2e-bmi-restart-{test_run_id}" - bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) - - try: - assert bmi_id in grpc.list_baremetal_instance_ids() - - bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) - wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) - - external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) - assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" - bmh_ns, bmh_name = external_host_id.split("/", 1) - assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" - - initial_trigger: int = _get_status_restart_trigger(grpc, bmi_id) - new_trigger: int = initial_trigger + 1 - logger.info("Incrementing restart_trigger from %d to %d", initial_trigger, new_trigger) - - grpc.update_baremetal_instance_restart_trigger(bmi_id=bmi_id, restart_trigger=new_trigger) - - poll_until( - fn=lambda: _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS), - until=lambda v: v == "CONDITION_STATUS_TRUE", - retries=60, - delay=2, - description=f"{bmi_id} RESTART_IN_PROGRESS condition appears", - ) - - poll_until( - fn=lambda: _get_status_restart_trigger(grpc, bmi_id), - until=lambda v: v == new_trigger, - retries=120, - delay=10, - description=f"{bmi_id} status.restart_trigger echoes {new_trigger}", - ) - - poll_until( - fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), - until=lambda v: v == "true", - retries=60, - delay=5, - description=f"{bmh_name} powered on after restart", - ) - - wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) - - restart_in_progress: str = _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS) - assert restart_in_progress in ("", "CONDITION_STATUS_FALSE"), ( - f"RESTART_IN_PROGRESS should have cleared after restart, got: {restart_in_progress}" - ) - - restart_failed: str = _get_condition_status(grpc, bmi_id, _RESTART_FAILED) - assert restart_failed in ("", "CONDITION_STATUS_FALSE"), ( - f"Unexpected RESTART_FAILED condition: {restart_failed}" - ) - - cli.delete_baremetal_instance(uuid=bmi_id) - wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) - wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) - except BaseException: - bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) - if bmi_cr: - try: - cli.delete_baremetal_instance(uuid=bmi_id) - wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) - wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) - except Exception: - logger.exception("Failed to delete BMI %s during cleanup", bmi_id) - raise diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 9d60605f39..a6a431971e 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -384,15 +384,6 @@ def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: s }, ) - def update_baremetal_instance_restart_trigger(self, *, bmi_id: str, restart_trigger: int) -> dict[str, Any]: - return self.call( - service=f"{PUBLIC_API}.BareMetalInstances/Update", - data={ - "object": {"id": bmi_id, "spec": {"restart_trigger": restart_trigger}}, - "updateMask": {"paths": ["spec.restart_trigger"]}, - }, - ) - def delete_baremetal_instance(self, *, bmi_id: str) -> None: self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id}) From 19cecf5d53f11f636fe0148e09d100f57ff7b8bb Mon Sep 17 00:00:00 2001 From: Haim Tayrie Date: Thu, 6 Aug 2026 17:46:53 +0300 Subject: [PATCH 075/112] OSAC-1330: add references E2E test suite infrastructure Add shared fixtures, helper methods, and Makefile target for the typed resource reference E2E test suite (OSAC-3095/3100/3105/3110/3114). - tests/references/conftest.py: session-scoped fixtures for network class, virtual network, subnet, and security group with cleanup - tests/core/helpers.py: assert_grpc_field_violation for reference validation error assertions - tests/core/grpc_client.py: methods for NATGateway, RoleBinding, ProjectMembership CRUD and generic filtered list - Makefile: test-references target Assisted-by: Claude Code Signed-off-by: Haim Tayrie --- tests/core/grpc_client.py | 97 ++++++++++++++++++++++++++++----------- tests/core/helpers.py | 26 ++++++----- 2 files changed, 86 insertions(+), 37 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index a6a431971e..8bc1d13c10 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -31,10 +31,7 @@ def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str], n obj: dict[str, Any] = {"spec": {"catalog_item": {"id": catalog_item}, "network_attachments": attachments}} if name is not None: obj["metadata"] = {"name": name} - response: dict[str, Any] = self.call( - service=f"{PUBLIC_API}.ComputeInstances/Create", - data={"object": obj}, - ) + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/Create", data={"object": obj}) return response["object"]["id"] def update_compute_instance_run_strategy(self, *, ci_id: str, run_strategy: str) -> dict[str, Any]: @@ -164,10 +161,7 @@ def create_console_session( def ensure_tenant(self, *, name: str) -> None: try: - self.call( - service=f"{PRIVATE_API}.Tenants/Create", - data={"object": {"metadata": {"name": name}}}, - ) + self.call(service=f"{PRIVATE_API}.Tenants/Create", data={"object": {"metadata": {"name": name}}}) except subprocess.CalledProcessError as e: output = (e.stdout or "") + (e.stderr or "") if not re.search(r"Code:\s*AlreadyExists", output): @@ -322,24 +316,13 @@ def delete_compute_instance_catalog_item(self, *, catalog_item_id: str) -> None: # InstanceType operations (private API only) def create_instance_type( - self, - *, - name: str, - cores: int, - memory_gib: int, - description: str = "", - gpu: dict[str, Any] | None = None, + self, *, name: str, cores: int, memory_gib: int, description: str = "", gpu: dict[str, Any] | None = None ) -> str: - spec: dict[str, Any] = { - "cores": cores, - "memory_gib": memory_gib, - "description": description, - } + spec: dict[str, Any] = {"cores": cores, "memory_gib": memory_gib, "description": description} if gpu is not None: spec["gpu"] = gpu response: dict[str, Any] = self.call( - service=f"{PRIVATE_API}.InstanceTypes/Create", - data={"object": {"metadata": {"name": name}, "spec": spec}}, + service=f"{PRIVATE_API}.InstanceTypes/Create", data={"object": {"metadata": {"name": name}, "spec": spec}} ) return response["object"]["id"] @@ -353,10 +336,7 @@ def list_instance_type_names(self) -> list[str]: def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]: return self.call( service=f"{PRIVATE_API}.InstanceTypes/Update", - data={ - "object": {"id": name, "spec": {"state": state}}, - "updateMask": {"paths": ["spec.state"]}, - }, + data={"object": {"id": name, "spec": {"state": state}}, "updateMask": {"paths": ["spec.state"]}}, ) def delete_instance_type(self, *, name: str) -> None: @@ -414,3 +394,68 @@ def create_baremetal_instance_catalog_item( def delete_baremetal_instance_catalog_item(self, *, item_id: str) -> None: self.call(service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Delete", data={"id": item_id}) + + # Generic filtered list + + def list_with_filter(self, *, service: str, filter_expr: str) -> list[dict[str, Any]]: + response: dict[str, Any] = self.call(service=service, data={"filter": filter_expr}) + return response.get("items", []) + + # NATGateway operations (public API) + + def create_nat_gateway(self, *, name: str, virtual_network_name: str, external_ip_name: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.NATGateways/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "virtual_network": {"name": virtual_network_name}, + "external_ip": {"name": external_ip_name}, + }, + } + }, + ) + return response["object"]["id"] + + def delete_nat_gateway(self, *, nat_gateway_id: str) -> None: + self.call(service=f"{PUBLIC_API}.NATGateways/Delete", data={"id": nat_gateway_id}) + + # RoleBinding operations (public API) + + def create_role_binding(self, *, name: str, role_name: str, user_names: list[str]) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.RoleBindings/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"role": {"name": role_name}, "users": [{"name": u} for u in user_names]}, + } + }, + ) + return response["object"]["id"] + + def get_role_binding(self, *, role_binding_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.RoleBindings/Get", data={"id": role_binding_id}) + + def delete_role_binding(self, *, role_binding_id: str) -> None: + self.call(service=f"{PUBLIC_API}.RoleBindings/Delete", data={"id": role_binding_id}) + + # ProjectMembership operations (public API) + + def create_project_membership( + self, *, name: str, project_name: str, user_name: str, role: str = "PROJECT_MEMBERSHIP_ROLE_VIEWER" + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ProjectMemberships/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"project": {"name": project_name}, "user": {"name": user_name}, "role": role}, + } + }, + ) + return response["object"]["id"] + + def delete_project_membership(self, *, membership_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ProjectMemberships/Delete", data={"id": membership_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 5f0ea3e4b2..13cc11695c 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -19,6 +19,17 @@ def assert_grpc_rejected(exc_info: pytest.ExceptionInfo[subprocess.CalledProcess assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" +def assert_grpc_field_violation( + exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], *, field_path: str +) -> None: + assert_grpc_rejected(exc_info, "InvalidArgument") + exc = exc_info.value + combined: str = (exc.stderr or "") + (exc.stdout or "") + assert field_path in combined, ( + f"Expected FieldViolation containing '{field_path}' in error output, got: {combined.strip()}" + ) + + def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_compute_instance_name(uuid=uuid, checked=False), @@ -167,6 +178,7 @@ def wait_for_external_ip_pool_grpc_ready(*, private_grpc: GRPCClient, pool_id: s this race so that subsequent ExternalIP creation does not hit FailedPrecondition. """ + def _state() -> str: try: pool = private_grpc.get_external_ip_pool(pool_id=pool_id) @@ -533,13 +545,9 @@ def _check() -> str: if not k8s.is_present(resource="clusterorder", name=name): raise AssertionError(f"ClusterOrder {name} disappeared before {condition_type}={expected_status}") phase: str = k8s.get_cluster_order_phase(name=name, checked=False) - cond_status = k8s.get_cluster_order_condition_status( - name=name, condition_type=condition_type, checked=False - ) + cond_status = k8s.get_cluster_order_condition_status(name=name, condition_type=condition_type, checked=False) if phase == "Failed" and cond_status != expected_status: - raise AssertionError( - f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}" - ) + raise AssertionError(f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}") return cond_status poll_until( @@ -677,11 +685,7 @@ def _check() -> str: return state poll_until( - fn=_check, - until=lambda v: v == "provisioned", - retries=120, - delay=10, - description=f"{name} BMH provisioned", + fn=_check, until=lambda v: v == "provisioned", retries=120, delay=10, description=f"{name} BMH provisioned" ) From 98c8db628caa3cc841be309496bedd77b64ce058 Mon Sep 17 00:00:00 2001 From: Haim Tayrie Date: Thu, 6 Aug 2026 17:56:33 +0300 Subject: [PATCH 076/112] OSAC-3114: add E2E tests for IAM resource references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test RoleBinding→Role+Users by name with reference resolution, ProjectMembership→Users by name, invalid role name error reporting, and multi-user role binding reference resolution. Fix create_project_membership to match proto spec (role enum + users list). Assisted-by: Claude Code Signed-off-by: Haim Tayrie --- tests/core/grpc_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 8bc1d13c10..9a6b4e5d55 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -444,14 +444,14 @@ def delete_role_binding(self, *, role_binding_id: str) -> None: # ProjectMembership operations (public API) def create_project_membership( - self, *, name: str, project_name: str, user_name: str, role: str = "PROJECT_MEMBERSHIP_ROLE_VIEWER" + self, *, name: str, user_names: list[str], role: str = "PROJECT_MEMBERSHIP_ROLE_VIEWER" ) -> str: response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.ProjectMemberships/Create", data={ "object": { "metadata": {"name": name}, - "spec": {"project": {"name": project_name}, "user": {"name": user_name}, "role": role}, + "spec": {"role": role, "users": [{"name": u} for u in user_names]}, } }, ) From 612ad640dfc12b5908f76005832a5a61a4ead9ea Mon Sep 17 00:00:00 2001 From: Daniel Erez Date: Mon, 10 Aug 2026 15:38:13 +0300 Subject: [PATCH 077/112] OSAC-1330: discover cluster version from environment for cluster creation tests Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/core/osac_cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 00eaa5f6b9..2fa39ddfa0 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -174,8 +174,13 @@ def get_cluster_credential(self, credential: str, *, uuid: str) -> str: def get_unchecked(self, resource: str) -> tuple[str, int]: return self._run_unchecked("get", resource) - def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str) -> str: - return self._parse_uuid(self._run("create", "cluster", "--catalog-item", catalog_item, "--name", name)) + def create_cluster_with_catalog_item( + self, *, catalog_item: str, name: str, version: str | None = None + ) -> str: + args = ["create", "cluster", "--catalog-item", catalog_item, "--name", name] + if version is not None: + args.extend(["--version", version]) + return self._parse_uuid(self._run(*args)) def create_compute_instance_with_catalog_item( self, *, catalog_item: str, name: str | None = None, subnet: str | None = None From c30e00d70921f9a1d4897134f1e15815cad4dde2 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 11 Aug 2026 20:39:40 +0300 Subject: [PATCH 078/112] OSAC-3438: Add CaaS metering lifecycle E2E tests - Generalize MeteringCollector._validate_structure for both compute_instance and cluster_order resource types - Extract validate_vmaas_billing / validate_caas_billing helpers for resource-type-specific billing dimension checks - Add get_all_events() for N+1 record count assertions Tests: - test_cluster_metering_lifecycle: full lifecycle (create, started, heartbeat N+1 decomposition, delete) with CaaS billing validation - test_cluster_metering_event_structure: CloudEvent fields, resource type, cluster_template in billing dimensions CI: - Add echo-adapter route + METERING_ADAPTER_URL to CaaS workflow Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/core/metering.py | 49 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/tests/core/metering.py b/tests/core/metering.py index f27c5e9ee4..3158ed84e5 100644 --- a/tests/core/metering.py +++ b/tests/core/metering.py @@ -95,6 +95,10 @@ def assert_no_events(self, event_type: str, resource_id: str, *, since: str, wit f"but found {len(matching)}: {[{'id': e.get('id'), 'time': e.get('time')} for e in matching]}" ) + def get_all_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: + """Return all events matching type and resource_id (for N+1 count assertions).""" + return self._fetch_events(event_type, resource_id) + def _fetch_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: params = urlencode({ "type": event_type, @@ -154,10 +158,10 @@ def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: f"Wrong source: {event.get('source')}" assert event.get("id"), "Missing event id" assert event.get("time"), "Missing event time" - assert event.get("osacresourcetype") == "compute_instance", \ - f"Wrong osacresourcetype: {event.get('osacresourcetype')}" assert event.get("osacresourceid") == expected.resource_id, \ f"Wrong osacresourceid: {event.get('osacresourceid')}" + assert event.get("osacresourcetype") in ("compute_instance", "cluster_order"), \ + f"Wrong osacresourcetype: {event.get('osacresourcetype')}" assert event.get("osactenant"), "Missing osactenant" data = event.get("data", {}) @@ -165,16 +169,8 @@ def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: assert data.get("resource_type"), "Missing resource_type in data" assert data.get("tenant_id"), "Missing tenant_id in data" assert data.get("schema_version") == "v1", f"Wrong schema_version: {data.get('schema_version')}" - - bd = data.get("billing_dimensions", {}) - assert bd.get("instance_type"), "Missing or empty instance_type in billing_dimensions" - assert bd.get("image_ref"), "Missing or empty image_ref in billing_dimensions" - assert "boot_disk_size_gib" in bd, "Missing boot_disk_size_gib in billing_dimensions" - assert isinstance( - bd["boot_disk_size_gib"], (int, float) - ), f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" - assert "project_id" in data, "Missing project_id in data" + assert "billing_dimensions" in data, "Missing billing_dimensions in data" if expected.event_type != "osac.resource.heartbeat.v1": assert data.get("transition_time"), "Missing transition_time in data" @@ -192,14 +188,23 @@ def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: assert "previous_state" in data, f"Missing previous_state in {expected.event_type}" assert "duration_seconds" in data, f"Missing duration_seconds in {expected.event_type}" - if expected.event_type == "osac.resource.suspended.v1": - valid_suspended_previous = ("RUNNING", "STOPPING", "STARTING") - assert data.get("previous_state") in valid_suspended_previous, ( - f"suspended.v1 previous_state should be one of {valid_suspended_previous}, " - f"got {data.get('previous_state')!r}" - ) - - if expected.event_type == "osac.resource.resumed.v1": - assert data.get("previous_state") in ("STOPPED", "PAUSED"), ( - f"resumed.v1 should have previous_state in (STOPPED, PAUSED), got {data.get('previous_state')}" - ) + +def validate_vmaas_billing(event: dict[str, Any]) -> None: + bd = event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("instance_type"), "Missing or empty instance_type in billing_dimensions" + assert bd.get("image_ref"), "Missing or empty image_ref in billing_dimensions" + assert "boot_disk_size_gib" in bd, "Missing boot_disk_size_gib in billing_dimensions" + assert isinstance( + bd["boot_disk_size_gib"], (int, float) + ), f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" + + +def validate_caas_billing(event: dict[str, Any]) -> None: + bd = event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("cluster_template"), "Missing cluster_template in billing_dimensions" + assert bd.get("component") in ("control_plane", "worker"), \ + f"component should be control_plane or worker, got {bd.get('component')!r}" + assert bd.get("node_set"), "Missing node_set in billing_dimensions" + assert bd.get("host_type"), "Missing host_type in billing_dimensions" + assert isinstance(bd.get("node_count"), (int, float)) and bd["node_count"] > 0, \ + f"node_count should be positive number, got {bd.get('node_count')!r}" From da3b4fe600ff4fc3f74e7b6377bbec65c362dd2d Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 11 Aug 2026 23:15:46 +0300 Subject: [PATCH 079/112] fix: restore billing validation, merge CaaS tests into existing lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore previous_state value validation in _validate_structure (suspended.v1 and resumed.v1 allowed values) - Dispatch billing dimension validation by resource type inside _validate_structure — fixes VMaaS billing regression where instance_type/image_ref/boot_disk_size_gib checks were dropped - Remove standalone CaaS metering test — add metering fixture and expectations to existing test_cluster_create instead (saves one full cluster provision cycle ~30min) Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/core/metering.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/core/metering.py b/tests/core/metering.py index 3158ed84e5..326143dfae 100644 --- a/tests/core/metering.py +++ b/tests/core/metering.py @@ -188,8 +188,28 @@ def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: assert "previous_state" in data, f"Missing previous_state in {expected.event_type}" assert "duration_seconds" in data, f"Missing duration_seconds in {expected.event_type}" - -def validate_vmaas_billing(event: dict[str, Any]) -> None: + if expected.event_type == "osac.resource.suspended.v1": + valid = ("RUNNING", "STOPPING", "STARTING", "PROGRESSING", "READY") + assert data.get("previous_state") in valid, ( + f"suspended.v1 previous_state should be one of {valid}, " + f"got {data.get('previous_state')!r}" + ) + + if expected.event_type == "osac.resource.resumed.v1": + valid = ("STOPPED", "PAUSED", "FAILED", "DELETE_FAILED") + assert data.get("previous_state") in valid, ( + f"resumed.v1 previous_state should be one of {valid}, " + f"got {data.get('previous_state')!r}" + ) + + resource_type = event.get("osacresourcetype") + if resource_type == "compute_instance": + _validate_vmaas_billing(event) + elif resource_type == "cluster_order": + _validate_caas_billing(event) + + +def _validate_vmaas_billing(event: dict[str, Any]) -> None: bd = event.get("data", {}).get("billing_dimensions", {}) assert bd.get("instance_type"), "Missing or empty instance_type in billing_dimensions" assert bd.get("image_ref"), "Missing or empty image_ref in billing_dimensions" @@ -199,7 +219,7 @@ def validate_vmaas_billing(event: dict[str, Any]) -> None: ), f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" -def validate_caas_billing(event: dict[str, Any]) -> None: +def _validate_caas_billing(event: dict[str, Any]) -> None: bd = event.get("data", {}).get("billing_dimensions", {}) assert bd.get("cluster_template"), "Missing cluster_template in billing_dimensions" assert bd.get("component") in ("control_plane", "worker"), \ From dd6b9fad741f85a60f7b602029ed14f82171c0e3 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Tue, 11 Aug 2026 23:20:36 +0300 Subject: [PATCH 080/112] fix: validate topLevelDims for CaaS created/deleted events created.v1 and deleted.v1 use topLevelDims (cluster_template + release_image, no per-component fields). Validate both fields instead of returning early after cluster_template. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/core/metering.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/core/metering.py b/tests/core/metering.py index 326143dfae..7491be8017 100644 --- a/tests/core/metering.py +++ b/tests/core/metering.py @@ -206,7 +206,7 @@ def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: if resource_type == "compute_instance": _validate_vmaas_billing(event) elif resource_type == "cluster_order": - _validate_caas_billing(event) + _validate_caas_billing(event, expected.event_type) def _validate_vmaas_billing(event: dict[str, Any]) -> None: @@ -219,9 +219,14 @@ def _validate_vmaas_billing(event: dict[str, Any]) -> None: ), f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" -def _validate_caas_billing(event: dict[str, Any]) -> None: +def _validate_caas_billing(event: dict[str, Any], event_type: str) -> None: bd = event.get("data", {}).get("billing_dimensions", {}) assert bd.get("cluster_template"), "Missing cluster_template in billing_dimensions" + # created.v1 and deleted.v1 use topLevelDims (cluster_template + + # release_image only, no per-component decomposition) + if event_type in ("osac.resource.created.v1", "osac.resource.deleted.v1"): + assert bd.get("release_image"), "Missing release_image in billing_dimensions" + return assert bd.get("component") in ("control_plane", "worker"), \ f"component should be control_plane or worker, got {bd.get('component')!r}" assert bd.get("node_set"), "Missing node_set in billing_dimensions" From 8c76bbe47a805cbfc5b83e7f4bb7f8d5b548be29 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 12 Aug 2026 15:48:54 +0300 Subject: [PATCH 081/112] fix: add scaling test with updated.v1 assertion Add scale_cluster() CLI wrapper and scaling verification to test_cluster_create: scale a worker node set up by 1, verify osac.resource.updated.v1 carries correct node_set and node_count in billing_dimensions, then scale back before deletion. Covers OSAC-3438 AC: "Scale a worker node set, verify osac.resource.updated.v1 event is emitted." Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/core/osac_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 2fa39ddfa0..817ba62614 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -192,6 +192,9 @@ def create_compute_instance_with_catalog_item( args.extend(["--network-attachment", f"subnet={subnet}"]) return self._parse_uuid(self._run(*args)) + def scale_cluster(self, *, uuid: str, node_set: str, size: int) -> None: + self._run("scale", "cluster", uuid, "--node-set", node_set, "--size", str(size)) + def delete_cluster(self, *, uuid: str) -> None: self._run("delete", "cluster", uuid) From f2bfde0b66069a540974257d922adca73bcf373d Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Wed, 12 Aug 2026 22:00:59 +0300 Subject: [PATCH 082/112] fix: suppress spurious warning on fixture teardown verify() Only warn when verify() is called with no expectations AND nothing was previously verified. A second verify() after all expectations were already checked is normal (fixture teardown safety net). Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/core/metering.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/metering.py b/tests/core/metering.py index 7491be8017..c7f398e254 100644 --- a/tests/core/metering.py +++ b/tests/core/metering.py @@ -61,7 +61,8 @@ def expect(self, event_type: str, resource_id: str, *, timeout: int = 60) -> Non def verify(self) -> None: if not self._expectations: - logger.warning("verify() called with no expectations — did you forget to call expect()?") + if not self._verified: + logger.warning("verify() called with no expectations — did you forget to call expect()?") return for exp in self._expectations: event = self._poll_for_event(exp) From 94c6e940c612ddef5162b3194f2cb7eb053616f8 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Thu, 13 Aug 2026 21:31:05 +0300 Subject: [PATCH 083/112] fix JWT token expiry in gRPC test clients GRPCClient now accepts an optional token_factory callable that refreshes the JWT on every request instead of using a static token acquired once at session start. The jwt_grpc_tenant1/2 fixtures use this to get fresh tokens from Keycloak, eliminating "token expired N seconds ago" flakes during long test sessions. Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/conftest.py | 24 ++++++++++++++++++------ tests/core/grpc_client.py | 12 ++++++++++-- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1af0ebd9c3..7a2da9e4c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -208,18 +208,30 @@ def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, j @pytest.fixture(scope="session") def jwt_grpc_tenant1(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: - token: str = get_jwt( - keycloak_url=keycloak_url, realm="osac", client_id="osac-cli", username="tenant1_user", password=jwt_password + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_user", + password=jwt_password, + ), ) - return GRPCClient(address=fulfillment_address, token=token) @pytest.fixture(scope="session") def jwt_grpc_tenant2(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: - token: str = get_jwt( - keycloak_url=keycloak_url, realm="osac", client_id="osac-cli", username="tenant2_user", password=jwt_password + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant2_user", + password=jwt_password, + ), ) - return GRPCClient(address=fulfillment_address, token=token) # --- Cross-cutting concern: Metering --- diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 9a6b4e5d55..b6fef2e857 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -3,6 +3,7 @@ import json import re import subprocess +from collections.abc import Callable from typing import Any from tests.core.runner import run, run_unchecked @@ -12,9 +13,16 @@ class GRPCClient: - def __init__(self, *, address: str, token: str) -> None: + def __init__(self, *, address: str, token: str = "", token_factory: Callable[[], str] | None = None) -> None: self.address: str = address - self.token: str = token + self._static_token: str = token + self._token_factory: Callable[[], str] | None = token_factory + + @property + def token(self) -> str: + if self._token_factory is not None: + return self._token_factory() + return self._static_token def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> list[str]: args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] From 91511bbd96d4ae9181e0ce2f1875e889d2b5b236 Mon Sep 17 00:00:00 2001 From: omer-vishlitzky Date: Thu, 13 Aug 2026 22:47:37 +0300 Subject: [PATCH 084/112] fix review findings: TTL cache, pipefail, SHELL directive Address code review findings: - Add 60s TTL cache to GRPCClient token_factory to avoid hitting Keycloak on every gRPC call during polling loops - Add set -o pipefail to bmaas and e2e-vmaas.yml deploy steps so curl failures in pipes propagate instead of being silently swallowed - Add SHELL ["/bin/bash", "-o", "pipefail", "-c"] to Containerfile so curl | tar pipe failures are caught during image build Assisted-by: Claude Code Signed-off-by: omer-vishlitzky --- tests/core/grpc_client.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index b6fef2e857..7b7499f287 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -3,6 +3,7 @@ import json import re import subprocess +import time from collections.abc import Callable from typing import Any @@ -13,15 +14,23 @@ class GRPCClient: + _TOKEN_TTL: float = 60.0 + def __init__(self, *, address: str, token: str = "", token_factory: Callable[[], str] | None = None) -> None: self.address: str = address self._static_token: str = token self._token_factory: Callable[[], str] | None = token_factory + self._cached_token: str = "" + self._cached_at: float = 0.0 @property def token(self) -> str: if self._token_factory is not None: - return self._token_factory() + now = time.monotonic() + if not self._cached_token or (now - self._cached_at) >= self._TOKEN_TTL: + self._cached_token = self._token_factory() + self._cached_at = now + return self._cached_token return self._static_token def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> list[str]: From 1db55f2f0418372b059cfc78c75186b80f05b156 Mon Sep 17 00:00:00 2001 From: Ilya Skornyakov Date: Thu, 6 Aug 2026 17:46:50 +0300 Subject: [PATCH 085/112] OSAC-2166: E2E tests for version-based cluster provisioning Assisted-by: Claude --- tests/core/grpc_client.py | 62 +++++++++++++++++++++++++++++++++++++++ tests/core/osac_cli.py | 3 ++ 2 files changed, 65 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 7b7499f287..95b4f71fb4 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -359,6 +359,68 @@ def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]: def delete_instance_type(self, *, name: str) -> None: self.call(service=f"{PRIVATE_API}.InstanceTypes/Delete", data={"id": name}) + # ClusterVersion operations (private API only) + + def create_cluster_version( + self, + *, + version: str, + image: str, + enabled: bool = True, + is_default: bool = False, + state: str = "CLUSTER_VERSION_STATE_ACTIVE", + ) -> dict[str, str]: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ClusterVersions/Create", + data={ + "object": { + "spec": { + "version": version, + "image": image, + "enabled": enabled, + "is_default": is_default, + "state": state, + } + } + }, + ) + cluster_version: dict[str, Any] = response["object"] + return {"id": cluster_version["id"], "name": cluster_version["metadata"]["name"]} + + def get_cluster_version(self, *, version_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ClusterVersions/Get", data={"id": version_id}) + + def list_cluster_version_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterVersions/List") + return [item["id"] for item in response.get("items", [])] + + def update_cluster_version(self, *, version_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_cluster_version requires at least one field to update") + return self.call( + service=f"{PRIVATE_API}.ClusterVersions/Update", + data={"object": {"id": version_id, "spec": dict(fields)}, "updateMask": {"paths": [f"spec.{k}" for k in fields]}}, + ) + + def delete_cluster_version(self, *, version_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ClusterVersions/Delete", data={"id": version_id}) + + def ensure_cluster_version(self, *, version: str, image: str) -> dict[str, str]: + """Create a ClusterVersion, tolerating AlreadyExists left behind by a prior failed run. + + Returns {"id": ..., "name": ...} for the resolved ClusterVersion.""" + try: + return self.create_cluster_version(version=version, image=image) + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if not re.search(r"Code:\s*AlreadyExists", output): + raise RuntimeError(f"Failed to create cluster version '{version}': {output}") from e + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterVersions/List") + for item in response.get("items", []): + if item.get("spec", {}).get("version") == version: + return {"id": item["id"], "name": item["metadata"]["name"]} + raise RuntimeError(f"Cluster version '{version}' reported AlreadyExists but not found in list") + # BareMetalInstance operations (public API) def list_baremetal_instance_ids(self) -> list[str]: diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 817ba62614..433688a5e4 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -143,6 +143,7 @@ def create_cluster( name: str | None = None, pull_secret_file: str | None = None, ssh_public_key_file: str | None = None, + version: str | None = None, template_parameters: dict[str, str] | None = None, template_parameter_files: dict[str, str] | None = None, ) -> str: @@ -153,6 +154,8 @@ def create_cluster( args.extend(["--pull-secret-file", pull_secret_file]) if ssh_public_key_file is not None: args.extend(["--ssh-public-key-file", ssh_public_key_file]) + if version is not None: + args.extend(["--version", version]) if template_parameters is not None: for key, value in template_parameters.items(): args.extend(["-p", f"{key}={value}"]) From 0b5340ee53a6e0ea37edaf4f82faabff14099d41 Mon Sep 17 00:00:00 2001 From: Tzif Date: Sun, 9 Aug 2026 21:31:30 +0300 Subject: [PATCH 086/112] OSAC-3837: add GPU flags to E2E CLI wrapper and test CLI instance type creation Add gpu_pci_device_selector, gpu_resource_name, and gpu_count parameters to OsacCLI.create_instance_type(). Add test_create_instance_type_via_cli to exercise the full CLI-to-server path for GPU instance type creation. Assisted-by: Claude Code Signed-off-by: Tzif --- tests/core/osac_cli.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 433688a5e4..7e38084593 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -124,10 +124,28 @@ def create_compute_instance( def delete_compute_instance(self, *, uuid: str) -> None: self._run("delete", "computeinstance", uuid) - def create_instance_type(self, *, name: str, cores: int, memory_gib: int, description: str = "") -> str: - args: list[str] = ["create", "instancetype", "--name", name, "--cores", str(cores), "--memory-gib", str(memory_gib)] + def create_instance_type( + self, + *, + name: str, + cores: int, + memory_gib: int, + description: str = "", + gpu_pci_device_selector: str = "", + gpu_resource_name: str = "", + gpu_count: int = 0, + ) -> str: + args: list[str] = [ + "create", "instancetype", "--name", name, "--cores", str(cores), "--memory-gib", str(memory_gib), + ] if description: args.extend(["--description", description]) + if gpu_pci_device_selector: + args.extend(["--gpu-pci-device-selector", gpu_pci_device_selector]) + if gpu_resource_name: + args.extend(["--gpu-resource-name", gpu_resource_name]) + if gpu_count: + args.extend(["--gpu-count", str(gpu_count)]) return self._parse_uuid(self._run(*args)) def describe_instance_type(self, *, name: str) -> str: From ecf3987a536bd4cc708453300c8138a07ff859b0 Mon Sep 17 00:00:00 2001 From: Tzif Date: Sun, 16 Aug 2026 16:32:41 +0300 Subject: [PATCH 087/112] OSAC-3837: add private CLI support for E2E instance type tests The CLI's create instancetype command uses the private API (osac.private.v1.InstanceTypes), which requires logging in with --private to the internal fulfillment endpoint. Without this, the CLI hits the public gateway and gets Unimplemented. - Pass --private on all CLI logins to unlock private API packages - Add private_cli fixture connecting to the internal address - Use private_cli in test_create_instance_type_via_cli Assisted-by: Claude Code Signed-off-by: Tzif --- tests/conftest.py | 13 +++++++++++++ tests/core/osac_cli.py | 12 ++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7a2da9e4c0..047f02c39c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -163,6 +163,19 @@ def cli(namespace: str, fulfillment_address: str, service_account: str) -> Itera instance.close() +@pytest.fixture(scope="session") +def private_cli(namespace: str, fulfillment_private_address: str, service_account: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_private_address.rsplit(':', 1)[0]}", + token_script=f"oc create token -n {namespace} {service_account} --as system:admin", + namespace=namespace, + private=True, + ) + yield instance + instance.close() + + @pytest.fixture(scope="session") def keycloak_url(cluster_domain: str) -> str: return env("OSAC_KEYCLOAK_URL", f"https://keycloak-keycloak.{cluster_domain}") diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 7e38084593..dc3df4e7d3 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -17,17 +17,22 @@ def __init__( token_script: str, namespace: str, default_instance_type: str | None = None, + private: bool = False, ) -> None: self.binary: str = binary self.namespace: str = namespace self._address: str = address self._token_script: str = token_script + self._private: bool = private self.default_instance_type: str | None = default_instance_type # Each OsacCLI instance gets its own config directory so that parallel # xdist workers (or multiple CLI fixtures) don't overwrite each other's # login credentials via the shared ~/.config/osac/config.json. self._config_dir: str = tempfile.mkdtemp(prefix="osac-config-") - self._run("login", "--address", address, "--insecure", "--token-script", token_script) + login_args = ["login", "--address", address, "--insecure", "--token-script", token_script] + if self._private: + login_args.append("--private") + self._run(*login_args) def close(self) -> None: shutil.rmtree(self._config_dir, ignore_errors=True) @@ -43,7 +48,10 @@ def _run_unchecked(self, *args: str, timeout: int = 300) -> tuple[str, int]: return run_unchecked(self.binary, "--config", self._config_dir, *args, timeout=timeout) def relogin(self) -> None: - self._run("login", "--address", self._address, "--insecure", "--token-script", self._token_script) + login_args = ["login", "--address", self._address, "--insecure", "--token-script", self._token_script] + if self._private: + login_args.append("--private") + self._run(*login_args) @staticmethod def _parse_uuid(stdout: str) -> str: From 78886762bb4f90089c096680a3823bdc2d2da531 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Fri, 14 Aug 2026 14:31:35 +0200 Subject: [PATCH 088/112] OSAC-3721: add jwt_grpc_tenant1_admin gRPC fixture Adds a session-scoped GRPCClient fixture authenticated as tenant1_admin via Keycloak JWT. Enables E2E tests that distinguish Tenant Admin from Tenant User at the gRPC level (e.g., DiskImage AC-2). Signed-off-by: Marc Sluiter Assisted-by: Claude Code Signed-off-by: Marc Sluiter --- tests/conftest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 047f02c39c..cffeb82113 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -219,6 +219,20 @@ def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, j instance.close() +@pytest.fixture(scope="session") +def jwt_grpc_tenant1_admin(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_admin", + password=jwt_password, + ), + ) + + @pytest.fixture(scope="session") def jwt_grpc_tenant1(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: return GRPCClient( From 186249c9ce7780b310075f3153210deb5762500d Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Fri, 14 Aug 2026 14:31:45 +0200 Subject: [PATCH 089/112] OSAC-3721: add DiskImage CRUD methods to GRPCClient Adds create, get, list (with optional CEL filter), update lifecycle, and delete methods for the DiskImages public API. Follows existing resource method patterns in grpc_client.py. Signed-off-by: Marc Sluiter Assisted-by: Claude Code Signed-off-by: Marc Sluiter --- tests/core/grpc_client.py | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 95b4f71fb4..64c5c5b7e1 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -474,6 +474,49 @@ def create_baremetal_instance_catalog_item( def delete_baremetal_instance_catalog_item(self, *, item_id: str) -> None: self.call(service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Delete", data={"id": item_id}) + # DiskImage operations (public API) + + def create_disk_image( + self, + *, + source_ref: str, + architecture: list[str] | None = None, + source_type: str = "SOURCE_TYPE_REGISTRY", + guest_os_family: str = "GUEST_OS_FAMILY_LINUX", + name: str | None = None, + ) -> str: + spec: dict[str, Any] = { + "source_type": source_type, + "source_ref": source_ref, + "guest_os_family": guest_os_family, + "architecture": architecture or ["ARCHITECTURE_AMD64"], + } + obj: dict[str, Any] = {"spec": spec} + if name is not None: + obj["metadata"] = {"name": name} + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.DiskImages/Create", data={"object": obj}) + return response["object"]["id"] + + def get_disk_image(self, *, disk_image_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.DiskImages/Get", data={"id": disk_image_id}) + + def list_disk_image_ids(self, *, filter_expr: str | None = None) -> list[str]: + data: dict[str, Any] | None = {"filter": filter_expr} if filter_expr else None + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.DiskImages/List", data=data) + return [item["id"] for item in response.get("items", [])] + + def update_disk_image_lifecycle(self, *, disk_image_id: str, lifecycle: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.DiskImages/Update", + data={ + "object": {"id": disk_image_id, "spec": {"lifecycle": lifecycle}}, + "updateMask": {"paths": ["spec.lifecycle"]}, + }, + ) + + def delete_disk_image(self, *, disk_image_id: str) -> None: + self.call(service=f"{PUBLIC_API}.DiskImages/Delete", data={"id": disk_image_id}) + # Generic filtered list def list_with_filter(self, *, service: str, filter_expr: str) -> list[dict[str, Any]]: From a46e47d0b18bc04a3833eef4644a17560abe2ca1 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Fri, 14 Aug 2026 14:34:09 -0400 Subject: [PATCH 090/112] OSAC-3553: gate bmaas e2e on gRPC readiness + retry first tenant create The autouse ensure_tenants fixture makes the suite's first gRPC call (Tenants/Create) right after `helm --wait` returns. `helm --wait` only confirms the pod is Ready, not that the Service/route endpoints have converged, so the call can briefly hit a not-yet-routable backend and fail with code = Unavailable ("connection refused"), aborting the whole bmaas run (3 confirmed instances in one day). bmaas was the only sibling full-install workflow missing the gRPC health-check gate that OSAC-2528 (vmaas) and OSAC-3505 (caas) added. Port the same guarded grpcurl install + Health/Check-SERVING gate into e2e-bmaas-full-install.yml, and add defense-in-depth retry on transient Unavailable in GRPCClient.ensure_tenant so any entry path that skips the workflow gate is still covered. Assisted-by: Claude Code Signed-off-by: Nick Carboni --- tests/core/grpc_client.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 64c5c5b7e1..efe6c0ca54 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -176,12 +176,25 @@ def create_console_session( # Tenant operations - def ensure_tenant(self, *, name: str) -> None: - try: - self.call(service=f"{PRIVATE_API}.Tenants/Create", data={"object": {"metadata": {"name": name}}}) - except subprocess.CalledProcessError as e: - output = (e.stdout or "") + (e.stderr or "") - if not re.search(r"Code:\s*AlreadyExists", output): + def ensure_tenant(self, *, name: str, retries: int = 10, delay: int = 5) -> None: + # OSAC-3553: retry the transient `code = Unavailable` / connection-refused + # transport failure the first call can hit after `helm --wait` returns, + # before the route converges. AlreadyExists is idempotent success; any + # other error still fails fast. Note the two error shapes are matched + # differently: a completed RPC prints grpcurl's `Code: ` block + # (AlreadyExists), while a transport failure prints the Go status string + # `code = Unavailable desc = ...` -- see tests/vmaas/external_ip/conftest.py. + for attempt in range(retries): + try: + self.call(service=f"{PRIVATE_API}.Tenants/Create", data={"object": {"metadata": {"name": name}}}) + return + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if re.search(r"Code:\s*AlreadyExists", output): + return + if attempt < retries - 1 and ("Unavailable" in output or "connection refused" in output.lower()): + time.sleep(delay) + continue raise RuntimeError(f"Failed to create tenant '{name}': {output}") from e # ExternalIPPool operations (private API only) From 069ea09fec2bd42f72e0b3971c2a0252c7e18cc2 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Mon, 17 Aug 2026 21:06:48 +0200 Subject: [PATCH 091/112] OSAC-3725: add ComputeInstance DiskImage integration E2E tests Five gRPC test functions covering DiskImage-ComputeInstance integration: - AC-1: Create CI with DiskImage, full provisioning, verify CRD fields - AC-2: OBSOLETE DiskImage blocks CI creation (FailedPrecondition) - AC-3: DEPRECATED DiskImage allows creation with warning - AC-4: Template disk_image default propagation - AC-5: Deletion protection lifecycle Adds grpc_client methods: create_compute_instance_with_disk_image, create_compute_instance_template, delete_compute_instance_template. Also migrates the existing CLI-based ComputeInstance tests off the removed `--image`/`--image-source-type` flags (deleted from the osac CLI in OSAC-3714) to the new `--disk-image ` flag, so the suite keeps building against osac main: - osac_cli.create_compute_instance drops the image params and gains a disk_image param with a default_disk_image fallback, mirroring the existing default_instance_type wiring. - New session-scoped default_disk_image fixture (+ autouse CLI wiring) provisions one shared AVAILABLE Linux DiskImage via the public API, so the scaffolding tests need no per-test change. - Drops the now-redundant image.sourceRef assertion in the CLI field-parity test (that mapping is owned by the AC-1 gRPC test) to avoid duplicated coverage. Assisted-by: Claude Code Signed-off-by: Marc Sluiter --- tests/core/grpc_client.py | 40 +++++++++++++++++++++++++++++++++++++++ tests/core/osac_cli.py | 15 +++++++++------ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index efe6c0ca54..d472206bb4 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -530,6 +530,46 @@ def update_disk_image_lifecycle(self, *, disk_image_id: str, lifecycle: str) -> def delete_disk_image(self, *, disk_image_id: str) -> None: self.call(service=f"{PUBLIC_API}.DiskImages/Delete", data={"id": disk_image_id}) + # ComputeInstance creation with explicit DiskImage (public API) + + def create_compute_instance_with_disk_image( + self, + *, + template: str, + disk_image_name: str, + subnet_ids: list[str], + instance_type: str | None = None, + name: str | None = None, + ) -> dict[str, Any]: + attachments = [{"subnet": {"id": sid}} for sid in subnet_ids] + spec: dict[str, Any] = { + "template": {"name": template}, + "disk_image": {"name": disk_image_name}, + "network_attachments": attachments, + } + if instance_type is not None: + spec["instance_type"] = {"name": instance_type} + obj: dict[str, Any] = {"spec": spec} + if name is not None: + obj["metadata"] = {"name": name} + return self.call(service=f"{PUBLIC_API}.ComputeInstances/Create", data={"object": obj}) + + # ComputeInstanceTemplate operations (private API) + + def create_compute_instance_template( + self, *, name: str, title: str, description: str, spec_defaults: dict[str, Any] | None = None + ) -> str: + obj: dict[str, Any] = {"metadata": {"name": name}, "title": title, "description": description} + if spec_defaults is not None: + obj["spec_defaults"] = spec_defaults + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ComputeInstanceTemplates/Create", data={"object": obj} + ) + return response["object"]["id"] + + def delete_compute_instance_template(self, *, template_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ComputeInstanceTemplates/Delete", data={"id": template_id}) + # Generic filtered list def list_with_filter(self, *, service: str, filter_expr: str) -> list[dict[str, Any]]: diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index dc3df4e7d3..20eaa7afb1 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -17,6 +17,7 @@ def __init__( token_script: str, namespace: str, default_instance_type: str | None = None, + default_disk_image: str | None = None, private: bool = False, ) -> None: self.binary: str = binary @@ -25,6 +26,7 @@ def __init__( self._token_script: str = token_script self._private: bool = private self.default_instance_type: str | None = default_instance_type + self.default_disk_image: str | None = default_disk_image # Each OsacCLI instance gets its own config directory so that parallel # xdist workers (or multiple CLI fixtures) don't overwrite each other's # login credentials via the shared ~/.config/osac/config.json. @@ -69,8 +71,7 @@ def create_compute_instance( name: str | None = None, network_attachments: list[dict[str, Any]] | None = None, boot_disk_size: int = 20, - image: str = "quay.io/containerdisks/fedora:latest", - image_source_type: str = "registry", + disk_image: str | None = None, run_strategy: str = "Always", user_data_secret_ref: str | None = None, instance_type: str | None = None, @@ -82,10 +83,6 @@ def create_compute_instance( template, "--boot-disk-size", str(boot_disk_size), - "--image", - image, - "--image-source-type", - image_source_type, "--run-strategy", run_strategy, ] @@ -98,6 +95,12 @@ def create_compute_instance( else: raise ValueError("instance_type or default_instance_type must be set") + effective_disk_image = disk_image if disk_image is not None else self.default_disk_image + if effective_disk_image is not None: + args.extend(["--disk-image", effective_disk_image]) + else: + raise ValueError("disk_image or default_disk_image must be set") + # Add network attachments if network_attachments is not None: for idx, attachment in enumerate(network_attachments): From 7e9432cd977be0b2f13e1494b79d2fc88fa8fb52 Mon Sep 17 00:00:00 2001 From: MENNY ABOUSH Date: Thu, 13 Aug 2026 14:10:20 +0300 Subject: [PATCH 092/112] OSAC-3402: Add BMI restart E2E test in lifecycle file for sequential execution Move test_baremetal_instance_restart into test_baremetal_instance_lifecycle.py so that --dist loadfile keeps both tests on the same xdist worker, running them sequentially. This avoids concurrent Ironic provisioning through the single-threaded sushy-emulator, which caused ~76% CI failure rate when the tests ran in parallel from separate files. Also adds update_baremetal_instance_restart_trigger() helper to GRPCClient for field-masked restart_trigger updates. Assisted-by: Claude Code Signed-off-by: MENNY ABOUSH --- .../test_baremetal_instance_lifecycle.py | 107 ++++++++++++++++++ tests/core/grpc_client.py | 9 ++ 2 files changed, 116 insertions(+) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 648dc6b378..06c77fe4dd 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -1,5 +1,8 @@ from __future__ import annotations +import logging +from typing import Any + from tests.core.grpc_client import GRPCClient from tests.core.helpers import ( wait_for_bmh_available, @@ -13,6 +16,24 @@ from tests.core.osac_cli import OsacCLI from tests.core.runner import poll_until +logger = logging.getLogger(__name__) + +_RESTART_IN_PROGRESS: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_IN_PROGRESS" +_RESTART_FAILED: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_FAILED" + + +def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + for condition in response.get("object", {}).get("status", {}).get("conditions", []): + if condition.get("type") == condition_type: + return condition.get("status", "") + return "" + + +def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) + def test_baremetal_instance_lifecycle( cli: OsacCLI, @@ -25,6 +46,8 @@ def test_baremetal_instance_lifecycle( ) -> None: name = f"e2e-bmi-{test_run_id}" bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + bmh_ns = "" + bmh_name = "" try: assert bmi_id in grpc.list_baremetal_instance_ids() @@ -93,6 +116,90 @@ def test_baremetal_instance_lifecycle( cli.delete_baremetal_instance(uuid=bmi_id) wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + if bmh_name: + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) except Exception: pass raise + + +def test_baremetal_instance_restart( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name: str = f"e2e-bmi-restart-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + initial_trigger: int = _get_status_restart_trigger(grpc, bmi_id) + new_trigger: int = initial_trigger + 1 + logger.info("Incrementing restart_trigger from %d to %d", initial_trigger, new_trigger) + + grpc.update_baremetal_instance_restart_trigger(bmi_id=bmi_id, restart_trigger=new_trigger) + + poll_until( + fn=lambda: _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS), + until=lambda v: v == "CONDITION_STATUS_TRUE", + retries=60, + delay=2, + description=f"{bmi_id} RESTART_IN_PROGRESS condition appears", + ) + + poll_until( + fn=lambda: _get_status_restart_trigger(grpc, bmi_id), + until=lambda v: v == new_trigger, + retries=120, + delay=10, + description=f"{bmi_id} status.restart_trigger echoes {new_trigger}", + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on after restart", + ) + + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + restart_in_progress: str = _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS) + assert restart_in_progress in ("", "CONDITION_STATUS_FALSE"), ( + f"RESTART_IN_PROGRESS should have cleared after restart, got: {restart_in_progress}" + ) + + restart_failed: str = _get_condition_status(grpc, bmi_id, _RESTART_FAILED) + assert restart_failed in ("", "CONDITION_STATUS_FALSE"), ( + f"Unexpected RESTART_FAILED condition: {restart_failed}" + ) + + # Deprovision + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + except Exception: + logger.exception("Failed to delete BMI %s during cleanup", bmi_id) + raise diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index d472206bb4..338c6c3a6a 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -456,6 +456,15 @@ def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: s }, ) + def update_baremetal_instance_restart_trigger(self, *, bmi_id: str, restart_trigger: int) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"restart_trigger": restart_trigger}}, + "updateMask": {"paths": ["spec.restart_trigger"]}, + }, + ) + def delete_baremetal_instance(self, *, bmi_id: str) -> None: self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id}) From 8d76ffcf16347116e9acb1b232ab0a05a46a5cf8 Mon Sep 17 00:00:00 2001 From: Daniel Erez Date: Tue, 18 Aug 2026 23:45:23 +0300 Subject: [PATCH 093/112] OSAC-1060: Switch grpc and cli fixtures from SA token to JWT The grpc and cli fixtures authenticate using a ServiceAccount token (oc create token), which resolves to the 'shared' tenant in the fulfillment service. PR osac-project/osac#352 restricts resource creation in shared and system tenants for tenant-scoped resources, causing all E2E tests that create VirtualNetworks, Subnets, Clusters, BareMetalInstances, etc. via these fixtures to fail with PermissionDenied. Switch both fixtures to use Keycloak JWT tokens for 'tenant1_admin', which resolves to the 'tenant1' tenant. This is a regular tenant where resource creation is allowed. The change is backwards-compatible: creating resources in tenant1 works on current main as well. Pre-create JWT users via private_grpc before any JWT-authenticated calls to avoid a JIT provisioning race condition when xdist workers make concurrent first requests. The private_grpc and private_cli fixtures remain on SA tokens since they operate on the private API, where platform-scoped servers (NetworkClasses, ExternalIPPools, CatalogItems, etc.) explicitly opt in to the shared tenant via AddAllowedTenants. Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/conftest.py | 48 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cffeb82113..dafcbcac43 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,14 @@ from __future__ import annotations +import contextlib import os +import subprocess from collections.abc import Iterator from pathlib import Path import pytest -from tests.core.grpc_client import GRPCClient +from tests.core.grpc_client import PRIVATE_API, GRPCClient from tests.core.k8s_client import K8sClient from tests.core.keycloak import get_jwt from tests.core.keycloak_admin import ( @@ -63,15 +65,17 @@ def service_account() -> str: @pytest.fixture(scope="session") -def grpc(fulfillment_address: str, namespace: str, service_account: str) -> GRPCClient: - # wait_for_cluster_ready's own budget alone can run up to 120min on cold - # EC2 hardware, plus deletion/verification steps after it -- a token this - # short can expire mid-session, failing every subsequent grpcurl call - # with UNAUTHENTICATED. Stay safely above the worst-case session length. - token: str = run( - "oc", "create", "token", service_account, "-n", namespace, "--duration", "4h", "--as", "system:admin" +def grpc(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_admin", + password=jwt_password, + ), ) - return GRPCClient(address=fulfillment_address, token=token) @pytest.fixture(scope="session") @@ -88,6 +92,28 @@ def ensure_tenants(private_grpc: GRPCClient) -> None: private_grpc.ensure_tenant(name=name) +@pytest.fixture(scope="session", autouse=True) +def ensure_jwt_users(ensure_tenants: None, private_grpc: GRPCClient) -> None: + """Pre-create users that JWT fixtures authenticate as, so JIT provisioning + doesn't race on concurrent first requests from xdist workers.""" + for username, tenant in [ + ("tenant1_admin", "tenant1"), + ("tenant1_user", "tenant1"), + ("tenant2_user", "tenant2"), + ("tenant2_admin", "tenant2"), + ]: + with contextlib.suppress(subprocess.CalledProcessError): + private_grpc.call( + service=f"{PRIVATE_API}.Users/Create", + data={ + "object": { + "metadata": {"name": username.replace("_", "-"), "tenant": tenant}, + "spec": {"username": username, "enabled": True}, + } + }, + ) + + @pytest.fixture(scope="session") def keycloak_admin_password() -> str: return env("OSAC_KEYCLOAK_ADMIN_PASSWORD", "admin") @@ -152,11 +178,11 @@ def k8s_hub_client(namespace: str) -> K8sClient: @pytest.fixture(scope="session") -def cli(namespace: str, fulfillment_address: str, service_account: str) -> Iterator[OsacCLI]: +def cli(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: instance = OsacCLI( binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", - token_script=f"oc create token -n {namespace} {service_account} --as system:admin", + token_script=_make_jwt_token_script(keycloak_url, "tenant1_admin", jwt_password), namespace=namespace, ) yield instance From a79551877297cb39339b395749ebf0bc1a0466dd Mon Sep 17 00:00:00 2001 From: Daniel Erez Date: Thu, 20 Aug 2026 10:29:34 +0300 Subject: [PATCH 094/112] OSAC-1060: Make JWT realm, client-id, and username configurable Add OSAC_KEYCLOAK_REALM, OSAC_KEYCLOAK_CLIENT_ID, and OSAC_JWT_USERNAME env vars to configure the grpc and cli fixtures, rather than hardcoding these values. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- tests/conftest.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index dafcbcac43..d08b602a8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,14 +65,36 @@ def service_account() -> str: @pytest.fixture(scope="session") -def grpc(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: +def keycloak_realm() -> str: + return env("OSAC_KEYCLOAK_REALM", "osac") + + +@pytest.fixture(scope="session") +def keycloak_client_id() -> str: + return env("OSAC_KEYCLOAK_CLIENT_ID", "osac-cli") + + +@pytest.fixture(scope="session") +def jwt_username() -> str: + return env("OSAC_JWT_USERNAME", "tenant1_admin") + + +@pytest.fixture(scope="session") +def grpc( + fulfillment_address: str, + keycloak_url: str, + keycloak_realm: str, + keycloak_client_id: str, + jwt_username: str, + jwt_password: str, +) -> GRPCClient: return GRPCClient( address=fulfillment_address, token_factory=lambda: get_jwt( keycloak_url=keycloak_url, - realm="osac", - client_id="osac-cli", - username="tenant1_admin", + realm=keycloak_realm, + client_id=keycloak_client_id, + username=jwt_username, password=jwt_password, ), ) @@ -178,11 +200,11 @@ def k8s_hub_client(namespace: str) -> K8sClient: @pytest.fixture(scope="session") -def cli(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: +def cli(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_username: str, jwt_password: str) -> Iterator[OsacCLI]: # noqa: E501 instance = OsacCLI( binary=env("OSAC_CLI_PATH", "osac"), address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", - token_script=_make_jwt_token_script(keycloak_url, "tenant1_admin", jwt_password), + token_script=_make_jwt_token_script(keycloak_url, jwt_username, jwt_password), namespace=namespace, ) yield instance From 992b1744ff6cda6e8df5c2ed0a95d050dfc8daad Mon Sep 17 00:00:00 2001 From: Siddarth R Date: Wed, 19 Aug 2026 11:23:20 -0400 Subject: [PATCH 095/112] OSAC-1980: update E2E tests for public NetworkClass removal The fulfillment-service public API (osac-project/osac PR for OSAC-1980) removes the network_class field from VirtualNetworkSpec and deletes the public NetworkClasses gRPC service and CLI command entirely; VirtualNetwork creation now always falls back to the deployment's single default NetworkClass (published via osac-aap config-as-code). Drop network_class from GRPCClient.create_virtual_network() and all callers/fixtures, remove the network_class fixture that queried the deleted NetworkClasses/List RPC, and drop "networkclasses" from the JWT CLI list-access smoke test. Update docs referencing the removed OSAC_NETWORK_CLASS env var and network_class fixture. Assisted-by: Cursor Co-authored-by: Cursor --- tests/core/grpc_client.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 338c6c3a6a..2dda0106b4 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -84,15 +84,10 @@ def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[st # VirtualNetwork operations - def create_virtual_network(self, *, name: str, network_class: str, ipv4_cidr: str) -> str: + def create_virtual_network(self, *, name: str, ipv4_cidr: str) -> str: response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.VirtualNetworks/Create", - data={ - "object": { - "metadata": {"name": name}, - "spec": {"network_class": {"name": network_class}, "ipv4_cidr": ipv4_cidr}, - } - }, + data={"object": {"metadata": {"name": name}, "spec": {"ipv4_cidr": ipv4_cidr}}}, ) return response["object"]["id"] From c273b97f27cb66e1d274f2b414703f9dac0988f0 Mon Sep 17 00:00:00 2001 From: Siddarth R Date: Wed, 19 Aug 2026 15:42:02 -0400 Subject: [PATCH 096/112] OSAC-1980: keep network_class optional until backend removal lands The public network_class field/NetworkClasses service are still present on osac's deployed main (OSAC-1980 hasn't merged there yet), and several existing tests -- tests/vmaas/test_metadata_name_validation.py and the tests/references/ suite's ref_virtual_network fixture -- still legitimately need to pass an explicit network_class to build valid requests. Removing the parameter entirely from GRPCClient.create_virtual_network() broke those tests in CI. Make network_class an optional kwarg instead of deleting it, and restore a network_class fixture in tests/vmaas/conftest.py (mirroring tests/references/conftest.py's ref_network_class) for tests that still need one. Fixtures/tests intentionally updated to omit it (default_networking, catalog_networking, the lifecycle and JWT smoke tests) are unaffected since the deployment already defaults sensibly when it's omitted. Assisted-by: Cursor Co-authored-by: Cursor --- tests/core/grpc_client.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 2dda0106b4..a315a43239 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -84,10 +84,16 @@ def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[st # VirtualNetwork operations - def create_virtual_network(self, *, name: str, ipv4_cidr: str) -> str: + def create_virtual_network(self, *, name: str, ipv4_cidr: str, network_class: str | None = None) -> str: + spec: dict[str, Any] = {"ipv4_cidr": ipv4_cidr} + if network_class is not None: + # NetworkClass remains part of the currently deployed public API (pending removal in + # OSAC-1980); accept it as optional so callers that still need an explicit class (e.g. + # reference/typed-field tests) keep working, while defaulting callers omit it entirely. + spec["network_class"] = {"name": network_class} response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.VirtualNetworks/Create", - data={"object": {"metadata": {"name": name}, "spec": {"ipv4_cidr": ipv4_cidr}}}, + data={"object": {"metadata": {"name": name}, "spec": spec}}, ) return response["object"]["id"] From 29ab7afd327c09dfdba24597d84c92a3b4b72700 Mon Sep 17 00:00:00 2001 From: Marc Sluiter Date: Fri, 21 Aug 2026 12:08:24 +0200 Subject: [PATCH 097/112] OSAC-4234: require explicit template id and assert template-side disk_image default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_template_disk_image_default previously created a ComputeInstance from a custom ComputeInstanceTemplate to prove the template's disk_image default was inherited. That path is not e2e-testable: spec.templateID is written verbatim into the ComputeInstance CR and used by AAP as the Ansible role name to run, and only AAP-published templates (e.g. osac.templates.ocp_virt_vm) have a backing role. A custom template's id has no role, so both the provision and delete AAP jobs fail ("the role '...' was not found"); because the fulfillment object is finalizer-coupled to the CR, the stuck CR also keeps the gRPC object alive, timing out teardown. (Reusing a real published id instead just collides with the platform template.) Rewrite the test to assert the default where it is actually applied server-side: the reference-validator interceptor resolves spec_defaults.disk_image.name to an id and backfills it before storage. The test now creates the disk image + template, Gets the template back, and checks specDefaults.diskImage.id — no ComputeInstance, no VM, no teardown race. Add a get_compute_instance_template helper to grpc_client for the read-back. Make template_id a mandatory keyword arg on create_compute_instance_template (it will be validated server-side soon) and have every caller pass an AAP-publisher-shaped id. This also fixes test_disk_image_deletion_protection_template (added on main), which called the helper without a template_id. Provisioning a VM from a custom template default remains untestable in e2e until the backend supports provisionable custom templates. Assisted-by: Claude Code Signed-off-by: Marc Sluiter --- tests/core/grpc_client.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index a315a43239..ccfcae263e 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -567,9 +567,21 @@ def create_compute_instance_with_disk_image( # ComputeInstanceTemplate operations (private API) def create_compute_instance_template( - self, *, name: str, title: str, description: str, spec_defaults: dict[str, Any] | None = None + self, *, template_id: str, name: str, title: str, description: str, spec_defaults: dict[str, Any] | None = None ) -> str: - obj: dict[str, Any] = {"metadata": {"name": name}, "title": title, "description": description} + # template_id is mandatory on purpose: it is written verbatim into the + # osac-operator ComputeInstance CR's spec.templateID, which the CRD + # validates against ^[a-zA-Z_][a-zA-Z0-9._]*$. Leaving it empty makes the + # server assign a UUIDv7 (hyphens + leading digit) that the CRD rejects at + # admission, so the CR is never created. Real templates are published by + # AAP with a dotted/underscored id (e.g. "osac.templates.ocp_virt_vm"); + # callers must follow that pattern. + obj: dict[str, Any] = { + "id": template_id, + "metadata": {"name": name}, + "title": title, + "description": description, + } if spec_defaults is not None: obj["spec_defaults"] = spec_defaults response: dict[str, Any] = self.call( @@ -580,6 +592,9 @@ def create_compute_instance_template( def delete_compute_instance_template(self, *, template_id: str) -> None: self.call(service=f"{PRIVATE_API}.ComputeInstanceTemplates/Delete", data={"id": template_id}) + def get_compute_instance_template(self, *, template_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ComputeInstanceTemplates/Get", data={"id": template_id}) + # Generic filtered list def list_with_filter(self, *, service: str, filter_expr: str) -> list[dict[str, Any]]: From 031318366f2c045e74108021104e71e299affbf3 Mon Sep 17 00:00:00 2001 From: Siddarth R Date: Fri, 21 Aug 2026 09:02:18 -0400 Subject: [PATCH 098/112] OSAC-1980: drop remaining public NetworkClass E2E usage The fulfillment-service public API no longer exposes NetworkClasses or VirtualNetworkSpec.network_class. Remove the fixtures that called NetworkClasses/List, stop sending network_class on VirtualNetwork create/ update, delete NetworkClass-only reference tests, and drop the already- skipped public NetworkClass uniqueness suite. Assisted-by: Cursor Co-authored-by: Cursor --- tests/core/grpc_client.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index ccfcae263e..59ac84ceef 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -84,16 +84,10 @@ def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[st # VirtualNetwork operations - def create_virtual_network(self, *, name: str, ipv4_cidr: str, network_class: str | None = None) -> str: - spec: dict[str, Any] = {"ipv4_cidr": ipv4_cidr} - if network_class is not None: - # NetworkClass remains part of the currently deployed public API (pending removal in - # OSAC-1980); accept it as optional so callers that still need an explicit class (e.g. - # reference/typed-field tests) keep working, while defaulting callers omit it entirely. - spec["network_class"] = {"name": network_class} + def create_virtual_network(self, *, name: str, ipv4_cidr: str) -> str: response: dict[str, Any] = self.call( service=f"{PUBLIC_API}.VirtualNetworks/Create", - data={"object": {"metadata": {"name": name}, "spec": spec}}, + data={"object": {"metadata": {"name": name}, "spec": {"ipv4_cidr": ipv4_cidr}}}, ) return response["object"]["id"] @@ -413,7 +407,10 @@ def update_cluster_version(self, *, version_id: str, **fields: Any) -> dict[str, raise ValueError("update_cluster_version requires at least one field to update") return self.call( service=f"{PRIVATE_API}.ClusterVersions/Update", - data={"object": {"id": version_id, "spec": dict(fields)}, "updateMask": {"paths": [f"spec.{k}" for k in fields]}}, + data={ + "object": {"id": version_id, "spec": dict(fields)}, + "updateMask": {"paths": [f"spec.{k}" for k in fields]}, + }, ) def delete_cluster_version(self, *, version_id: str) -> None: From bb558f4ffbe9d40de50e27c1204fc3474ed49a1c Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 09:30:38 +0200 Subject: [PATCH 099/112] OSAC-4205: Add NIC metadata E2E assertions to BMaaS lifecycle tests - Add describe_baremetal_instance() to OsacCLI helper - Assert status.hardware.nics populated and MACs valid after RUNNING - Assert osac describe output shows Network Interfaces section with MACs - Add test_baremetal_instance_nic_invariant: all RUNNING BMIs have NICs - Add test_baremetal_instance_tenant_isolation: cross-tenant 403 + admin cross-tenant read (skipif OSAC_TENANT2_BMI_ID not set) Assisted-by: Claude Code --- .../test_baremetal_instance_lifecycle.py | 62 +++++++++++++++++++ tests/core/osac_cli.py | 16 +++-- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 06c77fe4dd..352a2819b5 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -1,10 +1,16 @@ from __future__ import annotations import logging +import os +import re +import subprocess from typing import Any +import pytest + from tests.core.grpc_client import GRPCClient from tests.core.helpers import ( + assert_grpc_rejected, wait_for_bmh_available, wait_for_bmh_provisioned, wait_for_bmi_cr, @@ -20,6 +26,25 @@ _RESTART_IN_PROGRESS: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_IN_PROGRESS" _RESTART_FAILED: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_FAILED" +_MAC_PATTERN: re.Pattern[str] = re.compile(r"^([0-9a-f]{2}:){5}[0-9a-f]{2}$") + + +def _assert_nic_metadata(*, grpc: GRPCClient, bmi_id: str, cli: OsacCLI, bmi_cr_name: str) -> None: + """Assert NIC metadata is populated and valid in both API and CLI output.""" + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + nics: list[dict[str, Any]] = response.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) + assert nics, f"Expected non-empty status.hardware.nics for BMI {bmi_id}" + for nic in nics: + mac = nic.get("mac", "") + assert _MAC_PATTERN.match(mac), f"MAC '{mac}' does not match expected lowercase colon-separated format" + + describe_output: str = cli.describe_baremetal_instance(name=bmi_cr_name) + assert "Network Interfaces:" in describe_output, ( + "osac describe baremetalinstance output missing 'Network Interfaces:' section" + ) + assert _MAC_PATTERN.search(describe_output), ( + "osac describe baremetalinstance 'Network Interfaces:' section contains no valid MAC address" + ) def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: @@ -55,6 +80,9 @@ def test_baremetal_instance_lifecycle( bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + # Verify NIC metadata is populated (OSAC-3254) + _assert_nic_metadata(grpc=grpc, bmi_id=bmi_id, cli=cli, bmi_cr_name=bmi_cr_name) + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" bmh_ns, bmh_name = external_host_id.split("/", 1) @@ -203,3 +231,37 @@ def test_baremetal_instance_restart( except Exception: logger.exception("Failed to delete BMI %s during cleanup", bmi_id) raise + + +def test_baremetal_instance_nic_invariant(grpc: GRPCClient) -> None: + """All RUNNING BareMetalInstances must have status.hardware.nics populated (OSAC-3254).""" + bmi_ids = grpc.list_baremetal_instance_ids() + for bmi_id in bmi_ids: + data: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + state: str = data.get("object", {}).get("status", {}).get("state", "") + if state != "BARE_METAL_INSTANCE_STATE_RUNNING": + continue + nics: list[dict[str, Any]] = data.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) + assert nics, f"Running BMI {bmi_id} has no status.hardware.nics" + for nic in nics: + mac = nic.get("mac", "") + assert _MAC_PATTERN.match(mac), f"BMI {bmi_id} has invalid MAC format: '{mac}'" + + +@pytest.mark.skipif( + not os.getenv("OSAC_TENANT2_BMI_ID"), + reason="Requires OSAC_TENANT2_BMI_ID env var pointing to a BMI owned by tenant2", +) +def test_baremetal_instance_tenant_isolation(grpc: GRPCClient, jwt_grpc_tenant1: GRPCClient) -> None: + """Tenant 1 user cannot read a BMI belonging to tenant 2; admin can read NICs cross-tenant.""" + tenant2_bmi_id = os.environ["OSAC_TENANT2_BMI_ID"] + + # Admin can read the BMI and see NICs + bmi_data: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=tenant2_bmi_id) + nics: list[dict[str, Any]] = bmi_data.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) + assert nics, f"Admin expected non-empty status.hardware.nics on tenant2 BMI {tenant2_bmi_id}" + + # Tenant 1 user must be denied access to tenant 2's BMI + with pytest.raises(subprocess.CalledProcessError) as exc_info: + jwt_grpc_tenant1.get_baremetal_instance(bmi_id=tenant2_bmi_id) + assert_grpc_rejected(exc_info, "PermissionDenied") diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index 20eaa7afb1..d5d3ac2398 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -147,7 +147,14 @@ def create_instance_type( gpu_count: int = 0, ) -> str: args: list[str] = [ - "create", "instancetype", "--name", name, "--cores", str(cores), "--memory-gib", str(memory_gib), + "create", + "instancetype", + "--name", + name, + "--cores", + str(cores), + "--memory-gib", + str(memory_gib), ] if description: args.extend(["--description", description]) @@ -206,9 +213,7 @@ def get_cluster_credential(self, credential: str, *, uuid: str) -> str: def get_unchecked(self, resource: str) -> tuple[str, int]: return self._run_unchecked("get", resource) - def create_cluster_with_catalog_item( - self, *, catalog_item: str, name: str, version: str | None = None - ) -> str: + def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str, version: str | None = None) -> str: args = ["create", "cluster", "--catalog-item", catalog_item, "--name", name] if version is not None: args.extend(["--version", version]) @@ -240,5 +245,8 @@ def create_baremetal_instance( args.extend(["--user-data", user_data]) return self._parse_uuid(self._run(*args)) + def describe_baremetal_instance(self, *, name: str) -> str: + return self._run("describe", "baremetalinstance", name) + def delete_baremetal_instance(self, *, uuid: str) -> None: self._run("delete", "baremetalinstance", uuid) From 32ea39c74079496664e919efcf648dafca48796d Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 09:41:55 +0200 Subject: [PATCH 100/112] OSAC-4205: cross-check BMI MACs against BareMetalHost hardware inventory Add K8sClient.get_bmh_hardware_nics to read BMH status.hardware.nics. Assert that the MAC set reported in status.hardware.nics matches the MAC set from the BMH hardware inspection data, not just format alone. Assisted-by: Claude Code --- .../test_baremetal_instance_lifecycle.py | 37 ++++++++++++++++--- tests/core/k8s_client.py | 16 ++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 352a2819b5..e77bab4b32 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -29,14 +29,33 @@ _MAC_PATTERN: re.Pattern[str] = re.compile(r"^([0-9a-f]{2}:){5}[0-9a-f]{2}$") -def _assert_nic_metadata(*, grpc: GRPCClient, bmi_id: str, cli: OsacCLI, bmi_cr_name: str) -> None: - """Assert NIC metadata is populated and valid in both API and CLI output.""" +def _assert_nic_metadata( + *, + grpc: GRPCClient, + bmi_id: str, + cli: OsacCLI, + bmi_cr_name: str, + k8s: K8sClient, + bmh_name: str, + bmh_namespace: str, +) -> None: + """Assert NIC metadata is populated, valid, and matches the BMH hardware inventory.""" response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) nics: list[dict[str, Any]] = response.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) assert nics, f"Expected non-empty status.hardware.nics for BMI {bmi_id}" + bmi_macs: set[str] = set() for nic in nics: mac = nic.get("mac", "") assert _MAC_PATTERN.match(mac), f"MAC '{mac}' does not match expected lowercase colon-separated format" + bmi_macs.add(mac) + + # Cross-check: BMI MACs must match the BareMetalHost hardware inspection data + bmh_macs: set[str] = set(k8s.get_bmh_hardware_nics(name=bmh_name, bmh_namespace=bmh_namespace)) + assert bmh_macs, f"BareMetalHost {bmh_name} has no hardware.nics — inspection may not have completed" + assert bmi_macs == bmh_macs, ( + f"BMI status.hardware.nics {sorted(bmi_macs)} does not match " + f"BareMetalHost hardware.nics {sorted(bmh_macs)}" + ) describe_output: str = cli.describe_baremetal_instance(name=bmi_cr_name) assert "Network Interfaces:" in describe_output, ( @@ -80,14 +99,22 @@ def test_baremetal_instance_lifecycle( bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) - # Verify NIC metadata is populated (OSAC-3254) - _assert_nic_metadata(grpc=grpc, bmi_id=bmi_id, cli=cli, bmi_cr_name=bmi_cr_name) - external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" bmh_ns, bmh_name = external_host_id.split("/", 1) assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + # Verify NIC metadata matches the BMH hardware inventory (OSAC-3254) + _assert_nic_metadata( + grpc=grpc, + bmi_id=bmi_id, + cli=cli, + bmi_cr_name=bmi_cr_name, + k8s=k8s_hub_client, + bmh_name=bmh_name, + bmh_namespace=bmh_ns, + ) + # Verify provisioning wait_for_bmh_provisioned(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index e0e2662602..8072b42627 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -497,3 +497,19 @@ def get_bmh_powered_on(self, *, name: str, bmh_namespace: str) -> str: "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.status.poweredOn}", checked=False ) return output if rc == 0 else "" + + def get_bmh_hardware_nics(self, *, name: str, bmh_namespace: str) -> list[str]: + """Return lowercased MAC addresses from BareMetalHost hardware inspection data.""" + output, rc = self._get( + "get", + "baremetalhost", + name, + "-n", + bmh_namespace, + "-o", + "jsonpath={.status.hardware.nics[*].mac}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return [mac.lower() for mac in output.split()] From 1babeffa74b6edd0e2c566823c5251431293de3e Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 09:44:10 +0200 Subject: [PATCH 101/112] OSAC-4205: drop test_baremetal_instance_tenant_isolation Tenant isolation is covered by existing OPA policy tests and not specific to NIC metadata. Remove unused imports alongside. Assisted-by: Claude Code --- .../test_baremetal_instance_lifecycle.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index e77bab4b32..011ca65ba6 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -1,16 +1,11 @@ from __future__ import annotations import logging -import os import re -import subprocess from typing import Any -import pytest - from tests.core.grpc_client import GRPCClient from tests.core.helpers import ( - assert_grpc_rejected, wait_for_bmh_available, wait_for_bmh_provisioned, wait_for_bmi_cr, @@ -275,20 +270,3 @@ def test_baremetal_instance_nic_invariant(grpc: GRPCClient) -> None: assert _MAC_PATTERN.match(mac), f"BMI {bmi_id} has invalid MAC format: '{mac}'" -@pytest.mark.skipif( - not os.getenv("OSAC_TENANT2_BMI_ID"), - reason="Requires OSAC_TENANT2_BMI_ID env var pointing to a BMI owned by tenant2", -) -def test_baremetal_instance_tenant_isolation(grpc: GRPCClient, jwt_grpc_tenant1: GRPCClient) -> None: - """Tenant 1 user cannot read a BMI belonging to tenant 2; admin can read NICs cross-tenant.""" - tenant2_bmi_id = os.environ["OSAC_TENANT2_BMI_ID"] - - # Admin can read the BMI and see NICs - bmi_data: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=tenant2_bmi_id) - nics: list[dict[str, Any]] = bmi_data.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) - assert nics, f"Admin expected non-empty status.hardware.nics on tenant2 BMI {tenant2_bmi_id}" - - # Tenant 1 user must be denied access to tenant 2's BMI - with pytest.raises(subprocess.CalledProcessError) as exc_info: - jwt_grpc_tenant1.get_baremetal_instance(bmi_id=tenant2_bmi_id) - assert_grpc_rejected(exc_info, "PermissionDenied") From a53647c1b8212835d719f61c37d81151dee095c2 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 09:46:35 +0200 Subject: [PATCH 102/112] OSAC-4205: check BMI CR MACs in _assert_nic_metadata, drop invariant test Add K8sClient.get_bmi_hardware_nics to read BMI CR status.hardware.nics. _assert_nic_metadata now verifies the full chain: API response == BMI CR status == BMH hardware inspection data. Remove test_baremetal_instance_nic_invariant. Assisted-by: Claude Code --- .../test_baremetal_instance_lifecycle.py | 21 +++++++------------ tests/core/k8s_client.py | 16 ++++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 011ca65ba6..54b93ade6e 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -44,6 +44,14 @@ def _assert_nic_metadata( assert _MAC_PATTERN.match(mac), f"MAC '{mac}' does not match expected lowercase colon-separated format" bmi_macs.add(mac) + # Cross-check: BMI CR status.hardware.nics must match the API response + cr_macs: set[str] = set(k8s.get_bmi_hardware_nics(name=bmi_cr_name)) + assert cr_macs, f"BareMetalInstance CR {bmi_cr_name} has no status.hardware.nics" + assert bmi_macs == cr_macs, ( + f"API status.hardware.nics {sorted(bmi_macs)} does not match " + f"BMI CR status.hardware.nics {sorted(cr_macs)}" + ) + # Cross-check: BMI MACs must match the BareMetalHost hardware inspection data bmh_macs: set[str] = set(k8s.get_bmh_hardware_nics(name=bmh_name, bmh_namespace=bmh_namespace)) assert bmh_macs, f"BareMetalHost {bmh_name} has no hardware.nics — inspection may not have completed" @@ -255,18 +263,5 @@ def test_baremetal_instance_restart( raise -def test_baremetal_instance_nic_invariant(grpc: GRPCClient) -> None: - """All RUNNING BareMetalInstances must have status.hardware.nics populated (OSAC-3254).""" - bmi_ids = grpc.list_baremetal_instance_ids() - for bmi_id in bmi_ids: - data: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) - state: str = data.get("object", {}).get("status", {}).get("state", "") - if state != "BARE_METAL_INSTANCE_STATE_RUNNING": - continue - nics: list[dict[str, Any]] = data.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) - assert nics, f"Running BMI {bmi_id} has no status.hardware.nics" - for nic in nics: - mac = nic.get("mac", "") - assert _MAC_PATTERN.match(mac), f"BMI {bmi_id} has invalid MAC format: '{mac}'" diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 8072b42627..96dfb405bf 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -459,6 +459,22 @@ def get_baremetal_instance_name(self, *, uuid: str, checked: bool = True) -> str def get_baremetal_instance_external_host_id(self, *, name: str) -> str: return self.get_jsonpath(resource="baremetalinstance", name=name, jsonpath="{.spec.externalHostID}") + def get_bmi_hardware_nics(self, *, name: str) -> list[str]: + """Return lowercased MAC addresses from BareMetalInstance CR status.hardware.nics.""" + output, rc = self._get( + "get", + "baremetalinstance", + name, + "-n", + self.namespace, + "-o", + "jsonpath={.status.hardware.nics[*].mac}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return [mac.lower() for mac in output.split()] + # BareMetalHost queries (explicit namespace — BMHs live in a different namespace) def get_bmh_provisioning_state(self, *, name: str, bmh_namespace: str) -> str: From 77026261789be15dc6bba90509cdb27b4dd3014e Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 09:49:50 +0200 Subject: [PATCH 103/112] =?UTF-8?q?OSAC-4205:=20verify=20NIC=20chain=20BMH?= =?UTF-8?q?=20=E2=86=92=20BMI=20CR=20=E2=86=92=20gRPC=20API=20=E2=86=92=20?= =?UTF-8?q?CLI=20in=20order?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _assert_nic_metadata now reads from source of truth outward: 1. BMH hardware.nics (ground truth) 2. BMI CR status.hardware.nics == BMH 3. gRPC API status.hardware.nics == BMH 4. CLI describe lists all BMH MACs Assisted-by: Claude Code --- .../test_baremetal_instance_lifecycle.py | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 54b93ade6e..227cd3defe 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -34,39 +34,40 @@ def _assert_nic_metadata( bmh_name: str, bmh_namespace: str, ) -> None: - """Assert NIC metadata is populated, valid, and matches the BMH hardware inventory.""" - response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) - nics: list[dict[str, Any]] = response.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) - assert nics, f"Expected non-empty status.hardware.nics for BMI {bmi_id}" - bmi_macs: set[str] = set() - for nic in nics: - mac = nic.get("mac", "") - assert _MAC_PATTERN.match(mac), f"MAC '{mac}' does not match expected lowercase colon-separated format" - bmi_macs.add(mac) - - # Cross-check: BMI CR status.hardware.nics must match the API response + """Verify NIC MAC addresses propagate correctly from BMH → BMI CR → gRPC API → CLI.""" + # 1. BMH is the source of truth — hardware inspection provides the authoritative MAC list + bmh_macs: set[str] = set(k8s.get_bmh_hardware_nics(name=bmh_name, bmh_namespace=bmh_namespace)) + assert bmh_macs, f"BareMetalHost {bmh_name} has no hardware.nics — inspection may not have completed" + for mac in bmh_macs: + assert _MAC_PATTERN.match(mac), f"BMH MAC '{mac}' does not match expected lowercase colon-separated format" + + # 2. BMI CR status.hardware.nics must match the BMH cr_macs: set[str] = set(k8s.get_bmi_hardware_nics(name=bmi_cr_name)) assert cr_macs, f"BareMetalInstance CR {bmi_cr_name} has no status.hardware.nics" - assert bmi_macs == cr_macs, ( - f"API status.hardware.nics {sorted(bmi_macs)} does not match " - f"BMI CR status.hardware.nics {sorted(cr_macs)}" + assert cr_macs == bmh_macs, ( + f"BMI CR status.hardware.nics {sorted(cr_macs)} does not match " + f"BareMetalHost hardware.nics {sorted(bmh_macs)}" ) - # Cross-check: BMI MACs must match the BareMetalHost hardware inspection data - bmh_macs: set[str] = set(k8s.get_bmh_hardware_nics(name=bmh_name, bmh_namespace=bmh_namespace)) - assert bmh_macs, f"BareMetalHost {bmh_name} has no hardware.nics — inspection may not have completed" - assert bmi_macs == bmh_macs, ( - f"BMI status.hardware.nics {sorted(bmi_macs)} does not match " + # 3. gRPC API response must match the BMI CR + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + nics: list[dict[str, Any]] = response.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) + assert nics, f"gRPC API returned no status.hardware.nics for BMI {bmi_id}" + api_macs: set[str] = {nic.get("mac", "") for nic in nics} + assert api_macs == bmh_macs, ( + f"gRPC API status.hardware.nics {sorted(api_macs)} does not match " f"BareMetalHost hardware.nics {sorted(bmh_macs)}" ) + # 4. CLI describe output must list all BMH MACs under the Network Interfaces section describe_output: str = cli.describe_baremetal_instance(name=bmi_cr_name) assert "Network Interfaces:" in describe_output, ( "osac describe baremetalinstance output missing 'Network Interfaces:' section" ) - assert _MAC_PATTERN.search(describe_output), ( - "osac describe baremetalinstance 'Network Interfaces:' section contains no valid MAC address" - ) + for mac in bmh_macs: + assert mac in describe_output, ( + f"osac describe baremetalinstance output missing MAC '{mac}' from BMH hardware inventory" + ) def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: From 84e1a21404e7fb0ea95948435923a98cfb20272c Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 09:52:12 +0200 Subject: [PATCH 104/112] OSAC-4205: remove redundant MAC pattern check on BMH MACs get_bmh_hardware_nics already lowercases MACs; the format assertion is a no-op. Assisted-by: Claude Code --- tests/bmaas/test_baremetal_instance_lifecycle.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 227cd3defe..c24546294a 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -38,8 +38,6 @@ def _assert_nic_metadata( # 1. BMH is the source of truth — hardware inspection provides the authoritative MAC list bmh_macs: set[str] = set(k8s.get_bmh_hardware_nics(name=bmh_name, bmh_namespace=bmh_namespace)) assert bmh_macs, f"BareMetalHost {bmh_name} has no hardware.nics — inspection may not have completed" - for mac in bmh_macs: - assert _MAC_PATTERN.match(mac), f"BMH MAC '{mac}' does not match expected lowercase colon-separated format" # 2. BMI CR status.hardware.nics must match the BMH cr_macs: set[str] = set(k8s.get_bmi_hardware_nics(name=bmi_cr_name)) From 067a35ae80bc720fe0eed5ea28f75d514ef7da21 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 10:16:10 +0200 Subject: [PATCH 105/112] OSAC-4205: fix missing newline at end of file Assisted-by: Claude Code --- tests/bmaas/test_baremetal_instance_lifecycle.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index c24546294a..42f3ef0a8c 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -260,7 +260,3 @@ def test_baremetal_instance_restart( except Exception: logger.exception("Failed to delete BMI %s during cleanup", bmi_id) raise - - - - From 79acec199f87e9b3505021e83b4ae1df9c280625 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 11:11:52 +0200 Subject: [PATCH 106/112] =?UTF-8?q?OSAC-4205:=20fix=20CLI=20describe=20cal?= =?UTF-8?q?l=20=E2=80=94=20use=20user-defined=20name=20not=20K8s=20CR=20na?= =?UTF-8?q?me?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bmi_cr_name is the auto-generated CR name (bmi-{uuid}); osac describe baremetalinstance expects the user-defined name (e2e-bmi-{test_run_id}). Assisted-by: Claude Code --- tests/bmaas/test_baremetal_instance_lifecycle.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 42f3ef0a8c..675f26ac50 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -29,6 +29,7 @@ def _assert_nic_metadata( grpc: GRPCClient, bmi_id: str, cli: OsacCLI, + bmi_name: str, bmi_cr_name: str, k8s: K8sClient, bmh_name: str, @@ -58,7 +59,7 @@ def _assert_nic_metadata( ) # 4. CLI describe output must list all BMH MACs under the Network Interfaces section - describe_output: str = cli.describe_baremetal_instance(name=bmi_cr_name) + describe_output: str = cli.describe_baremetal_instance(name=bmi_name) assert "Network Interfaces:" in describe_output, ( "osac describe baremetalinstance output missing 'Network Interfaces:' section" ) @@ -111,6 +112,7 @@ def test_baremetal_instance_lifecycle( grpc=grpc, bmi_id=bmi_id, cli=cli, + bmi_name=name, bmi_cr_name=bmi_cr_name, k8s=k8s_hub_client, bmh_name=bmh_name, From e3210aedff93ca04f2b0e775541da9a7edcb7088 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 14:07:29 +0200 Subject: [PATCH 107/112] =?UTF-8?q?OSAC-4205:=20soften=20CLI=20describe=20?= =?UTF-8?q?assertion=20=E2=80=94=20log=20warning=20if=20OSAC-4204=20not=20?= =?UTF-8?q?deployed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Network Interfaces section depends on OSAC-4204. Emit a warning instead of failing when the deployed osac binary predates that change. Assisted-by: Claude Code --- .../test_baremetal_instance_lifecycle.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 675f26ac50..9b548c4e72 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -58,15 +58,20 @@ def _assert_nic_metadata( f"BareMetalHost hardware.nics {sorted(bmh_macs)}" ) - # 4. CLI describe output must list all BMH MACs under the Network Interfaces section + # 4. CLI describe output must list all BMH MACs under the Network Interfaces section. + # This assertion depends on OSAC-4204 being deployed. Skip gracefully if the deployed + # osac binary predates the Network Interfaces section. describe_output: str = cli.describe_baremetal_instance(name=bmi_name) - assert "Network Interfaces:" in describe_output, ( - "osac describe baremetalinstance output missing 'Network Interfaces:' section" - ) - for mac in bmh_macs: - assert mac in describe_output, ( - f"osac describe baremetalinstance output missing MAC '{mac}' from BMH hardware inventory" + if "Network Interfaces:" not in describe_output: + logger.warning( + "osac describe baremetalinstance output has no 'Network Interfaces:' section — " + "OSAC-4204 may not be deployed in this environment (skipping CLI assertion)" ) + else: + for mac in bmh_macs: + assert mac in describe_output, ( + f"osac describe baremetalinstance output missing MAC '{mac}' from BMH hardware inventory" + ) def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: From 9ee45716b4e363b50f7edf1b120e48509bdec500 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 14:07:54 +0200 Subject: [PATCH 108/112] =?UTF-8?q?Revert=20"OSAC-4205:=20soften=20CLI=20d?= =?UTF-8?q?escribe=20assertion=20=E2=80=94=20log=20warning=20if=20OSAC-420?= =?UTF-8?q?4=20not=20deployed"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0049ca683418676dabe49a6d93bb5f92adc581c1. --- .../test_baremetal_instance_lifecycle.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 9b548c4e72..675f26ac50 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -58,20 +58,15 @@ def _assert_nic_metadata( f"BareMetalHost hardware.nics {sorted(bmh_macs)}" ) - # 4. CLI describe output must list all BMH MACs under the Network Interfaces section. - # This assertion depends on OSAC-4204 being deployed. Skip gracefully if the deployed - # osac binary predates the Network Interfaces section. + # 4. CLI describe output must list all BMH MACs under the Network Interfaces section describe_output: str = cli.describe_baremetal_instance(name=bmi_name) - if "Network Interfaces:" not in describe_output: - logger.warning( - "osac describe baremetalinstance output has no 'Network Interfaces:' section — " - "OSAC-4204 may not be deployed in this environment (skipping CLI assertion)" + assert "Network Interfaces:" in describe_output, ( + "osac describe baremetalinstance output missing 'Network Interfaces:' section" + ) + for mac in bmh_macs: + assert mac in describe_output, ( + f"osac describe baremetalinstance output missing MAC '{mac}' from BMH hardware inventory" ) - else: - for mac in bmh_macs: - assert mac in describe_output, ( - f"osac describe baremetalinstance output missing MAC '{mac}' from BMH hardware inventory" - ) def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: From 80d074a34726de39cacf66ef70858785047bbd69 Mon Sep 17 00:00:00 2001 From: Adrien Gentil Date: Fri, 21 Aug 2026 14:52:58 +0200 Subject: [PATCH 109/112] =?UTF-8?q?OSAC-4205:=20address=20CodeRabbit=20fin?= =?UTF-8?q?dings=20=E2=80=94=20lowercase=20API=20MACs,=20scope=20CLI=20che?= =?UTF-8?q?ck=20to=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lowercase gRPC response MACs before set comparison (defensive against non-lowercased API responses). Assert MACs within the Network Interfaces section only, not the full CLI output. Assisted-by: Claude Code --- tests/bmaas/test_baremetal_instance_lifecycle.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 675f26ac50..23fb25eaac 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -52,7 +52,7 @@ def _assert_nic_metadata( response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) nics: list[dict[str, Any]] = response.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) assert nics, f"gRPC API returned no status.hardware.nics for BMI {bmi_id}" - api_macs: set[str] = {nic.get("mac", "") for nic in nics} + api_macs: set[str] = {nic.get("mac", "").lower() for nic in nics} assert api_macs == bmh_macs, ( f"gRPC API status.hardware.nics {sorted(api_macs)} does not match " f"BareMetalHost hardware.nics {sorted(bmh_macs)}" @@ -63,9 +63,10 @@ def _assert_nic_metadata( assert "Network Interfaces:" in describe_output, ( "osac describe baremetalinstance output missing 'Network Interfaces:' section" ) + ni_section = describe_output[describe_output.index("Network Interfaces:"):] for mac in bmh_macs: - assert mac in describe_output, ( - f"osac describe baremetalinstance output missing MAC '{mac}' from BMH hardware inventory" + assert mac in ni_section, ( + f"osac describe baremetalinstance 'Network Interfaces:' section missing MAC '{mac}'" ) From 487c9d9b01cfd34f3190d360f113b63603b19592 Mon Sep 17 00:00:00 2001 From: Deivanai Murugappan Rani Date: Fri, 21 Aug 2026 17:54:34 -0400 Subject: [PATCH 110/112] NO-ISSUE: BMaaS inventory exhaustion e2e + sanity/regression markers Rebased onto latest main as a single commit for merge-queue rebaseability. --- ..._baremetal_instance_inventory_exhausted.py | 144 ++++++++++++++++++ .../test_baremetal_instance_lifecycle.py | 3 + tests/core/grpc_client.py | 5 + tests/core/helpers.py | 93 +++++++++++ tests/core/k8s_client.py | 11 ++ 5 files changed, 256 insertions(+) create mode 100644 tests/bmaas/test_baremetal_instance_inventory_exhausted.py diff --git a/tests/bmaas/test_baremetal_instance_inventory_exhausted.py b/tests/bmaas/test_baremetal_instance_inventory_exhausted.py new file mode 100644 index 0000000000..70b6645d7b --- /dev/null +++ b/tests/bmaas/test_baremetal_instance_inventory_exhausted.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import re +import subprocess + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + assert_bmi_does_not_become_running, + assert_bmi_lifecycle_on_running, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, + wait_for_bmi_running_after_recovery, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +_AVAILABLE_BMH_STATES = {"available", "ready"} +_NOT_FOUND_RE = re.compile(r"Code:\s*NotFound|baremetalinstance\b.*\bnot found", re.IGNORECASE) + + +def _is_not_found(exc: subprocess.CalledProcessError) -> bool: + """Return True when delete failed because the BareMetalInstance is already gone.""" + combined = (exc.stderr or "") + (exc.stdout or "") + return bool(_NOT_FOUND_RE.search(combined)) + + +def _cleanup_bmi(*, cli: OsacCLI, grpc: GRPCClient, k8s: K8sClient, bmi_id: str) -> None: + """Delete a BareMetalInstance and wait for CR/gRPC removal. + + Only a NotFound response from delete is treated as success (already gone). + Deletion/removal timeouts and other errors are raised. + """ + bmi_cr: str = k8s.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if not bmi_cr and bmi_id not in grpc.list_baremetal_instance_ids(): + return + + try: + cli.delete_baremetal_instance(uuid=bmi_id) + except subprocess.CalledProcessError as exc: + if not _is_not_found(exc): + raise + + if bmi_cr and k8s.is_present(resource="baremetalinstance", name=bmi_cr): + wait_for_bmi_deletion(k8s=k8s, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + + +@pytest.mark.regression +def test_baremetal_instance_inventory_exhausted( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + """Exhaust BMH inventory, assert overflow stalls/fails, then recover after free capacity. + + Portable across labs/CI (e.g. 2 virtual BMHs → BMI #1, #2, then #3 overflow): + 1. Create N BMIs up front so cluster-side provisioning can overlap + 2. Wait for all N to reach RUNNING, then run full lifecycle checks on BMI #1 + 3. Create BMI N+1 and assert it does not reach RUNNING (no free BMH) + 4. Delete one claimed BMI to free a BMH + 5. Assert the overflow BMI recovers to RUNNING + 6. Cleanup remaining instances + """ + available_count: int = k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ) + assert available_count >= 1, ( + f"Need at least 1 available BMH to exercise inventory exhaustion; found {available_count}" + ) + + bmi_ids: list[str] = [] + try: + # Kick off all claim BMIs first so provisioning can proceed in parallel. + for idx in range(1, available_count + 1): + bmi_id = cli.create_baremetal_instance( + name=f"e2e-bmi-inv-{test_run_id}-{idx}", catalog_item=catalog_item, ssh_key=ssh_public_key + ) + bmi_ids.append(bmi_id) + + for bmi_id in bmi_ids: + wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + # Lifecycle checks after all claim BMIs are up (not interleaved with creates). + assert_bmi_lifecycle_on_running( + grpc=grpc, + k8s=k8s_hub_client, + bmi_id=bmi_ids[0], + bmh_namespace=bmh_namespace, + ) + + available_after_claim: int = k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ) + assert available_after_claim == 0, ( + f"Expected 0 available BMHs after claiming {available_count} hosts; found {available_after_claim}" + ) + + overflow_idx = available_count + 1 + overflow_id: str = cli.create_baremetal_instance( + name=f"e2e-bmi-inv-{test_run_id}-{overflow_idx}", catalog_item=catalog_item, ssh_key=ssh_public_key + ) + bmi_ids.append(overflow_id) + assert overflow_id in grpc.list_baremetal_instance_ids() + wait_for_bmi_cr(k8s=k8s_hub_client, uuid=overflow_id) + + overflow_state: str = assert_bmi_does_not_become_running(grpc=grpc, bmi_id=overflow_id) + assert overflow_state != "BARE_METAL_INSTANCE_STATE_RUNNING" + + available_during_overflow: int = k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ) + assert available_during_overflow == 0, ( + f"Overflow BMI must not claim a BMH; available count={available_during_overflow}" + ) + + # Free one BMH so the overflow instance can be scheduled. + released_id: str = bmi_ids[0] + _cleanup_bmi(cli=cli, grpc=grpc, k8s=k8s_hub_client, bmi_id=released_id) + bmi_ids.remove(released_id) + + poll_until( + fn=lambda: k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ), + until=lambda v: v >= 1, + retries=120, + delay=10, + description="at least one BMH available after release", + ) + + wait_for_bmi_running_after_recovery(grpc=grpc, bmi_id=overflow_id) + finally: + for bmi_id in reversed(bmi_ids): + _cleanup_bmi(cli=cli, grpc=grpc, k8s=k8s_hub_client, bmi_id=bmi_id) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py index 23fb25eaac..7b6a6407d4 100644 --- a/tests/bmaas/test_baremetal_instance_lifecycle.py +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -1,5 +1,6 @@ from __future__ import annotations +import pytest import logging import re from typing import Any @@ -83,6 +84,7 @@ def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) +@pytest.mark.sanity def test_baremetal_instance_lifecycle( cli: OsacCLI, grpc: GRPCClient, @@ -183,6 +185,7 @@ def test_baremetal_instance_lifecycle( raise +@pytest.mark.sanity def test_baremetal_instance_restart( cli: OsacCLI, grpc: GRPCClient, diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index 59ac84ceef..ee7b15a785 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -477,6 +477,11 @@ def create_baremetal_instance_catalog_item( template: str, field_definitions: list[dict[str, Any]] | None = None, ) -> str: + """Create a published BareMetalInstanceCatalogItem. + + ``template`` is sent as a typed reference ``{"name": ...}`` (OSAC-1330), + matching Cluster/ComputeInstance catalog item creates. + """ obj: dict[str, Any] = { "metadata": {"name": name}, "title": title, diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 13cc11695c..983b0b2bf6 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -2,6 +2,7 @@ import re import subprocess +import time from typing import Any import pytest @@ -650,6 +651,98 @@ def _check_state() -> str: ) +def assert_bmi_lifecycle_on_running( + *, + grpc: GRPCClient, + k8s: K8sClient, + bmi_id: str, + bmh_namespace: str, + power_cycle: bool = True, +) -> tuple[str, str]: + """Assert BMH binding/provisioning on an already-RUNNING BMI. + + Does not create or delete the instance. Inventory exhaust calls this after all + claim BMIs reach RUNNING so provisioning is not blocked by interleaved checks. + + Returns (bmi_cr_name, bmh_name). + """ + assert bmi_id in grpc.list_baremetal_instance_ids() + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s, uuid=bmi_id) + + external_host_id: str = k8s.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + wait_for_bmh_provisioned(k8s=k8s, name=bmh_name, bmh_namespace=bmh_ns) + + image_url: str = k8s.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url != "", f"BMH {bmh_name} has no image URL after provisioning" + + consumer_ref: str = k8s.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref != "", f"BMH {bmh_name} has no consumerRef after allocation" + + online: str = k8s.get_bmh_online(name=bmh_name, bmh_namespace=bmh_ns) + assert online == "true", f"BMH {bmh_name} should be online after provisioning, got: {online}" + + if power_cycle: + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_HALTED" + ) + poll_until( + fn=lambda: k8s.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "false", + retries=60, + delay=5, + description=f"{bmh_name} powered off", + ) + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_ALWAYS" + ) + poll_until( + fn=lambda: k8s.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on", + ) + + return bmi_cr_name, bmh_name + + +def assert_bmi_does_not_become_running(*, grpc: GRPCClient, bmi_id: str, retries: int = 36, delay: int = 10) -> str: + """Observe that a BareMetalInstance never reaches RUNNING. + + Returns the last observed state. If the instance enters a FAILED state, + returns immediately (inventory / scheduling failure). + """ + last_state = "" + for _ in range(retries): + last_state = grpc.get_baremetal_instance_state(bmi_id=bmi_id) + assert last_state != "BARE_METAL_INSTANCE_STATE_RUNNING", ( + f"BareMetalInstance {bmi_id} unexpectedly reached RUNNING with no free BMH" + ) + if "FAILED" in last_state: + return last_state + time.sleep(delay) + return last_state + + +def wait_for_bmi_running_after_recovery(*, grpc: GRPCClient, bmi_id: str) -> None: + """Wait until a BMI reaches RUNNING, allowing a prior FAILED/pending state. + + Used after inventory is freed so an overflow instance can be scheduled. + Unlike wait_for_bmi_running, does not fail-fast on FAILED. + """ + poll_until( + fn=lambda: grpc.get_baremetal_instance_state(bmi_id=bmi_id), + until=lambda v: v == "BARE_METAL_INSTANCE_STATE_RUNNING", + retries=120, + delay=10, + description=f"{bmi_id} RUNNING after inventory recovery", + ) + + def wait_for_bmi_deletion(*, k8s: K8sClient, name: str) -> None: # 2700s (45min), not the old 1200s (20min): the deprovision AAP job this # blocks on retries with exponential backoff up to a 30-minute ceiling diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index 96dfb405bf..cc4f9e7553 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -514,6 +514,17 @@ def get_bmh_powered_on(self, *, name: str, bmh_namespace: str) -> str: ) return output if rc == 0 else "" + def count_bmhs_by_provisioning_state(self, *, bmh_namespace: str, states: set[str]) -> int: + """Return how many BareMetalHosts in ``bmh_namespace`` are in any of ``states``.""" + raw: str = run(*self._base(), "get", "baremetalhost", "-n", bmh_namespace, "-o", "json") + items: list[dict[str, Any]] = json.loads(raw).get("items", []) + count = 0 + for item in items: + state: str = item.get("status", {}).get("provisioning", {}).get("state", "") + if state in states: + count += 1 + return count + def get_bmh_hardware_nics(self, *, name: str, bmh_namespace: str) -> list[str]: """Return lowercased MAC addresses from BareMetalHost hardware inspection data.""" output, rc = self._get( From 618b019164f2e5833fe720f1e66dcbabedb99550 Mon Sep 17 00:00:00 2001 From: Dan Manor Date: Mon, 24 Aug 2026 19:18:19 +0300 Subject: [PATCH 111/112] WIP: NO-ISSUE: BMaaS & Netris Automations (#379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Netris backend fixes so it can deploy and test OSAC PRs (BMaaS networking) end-to-end, plus automated E2E networking tests. ## Infrastructure fixes ### 1. AAP config-as-code project git override `aap_project_git_uri` / `aap_project_git_branch` in `prep-osac`. Gap: nothing controlled `aap.configAsCode.projectGitBranch`, so AAP could run `main`'s playbooks under a PR's images/charts. ### 2. Tenant ExternalIPPool host routing `osac_external_ip_pool_cidr` (default `198.51.100.24/29`); `deploy-infra` routes + masquerades it toward the softgate. Gap: no lab prep routed a tenant pool CIDR. ### 3. prep-osac: honor each profile's operator set Dropped the blanket `cnv`/`mce`/`metallb: true` overlay. Gap: it installed CNV on BMaaS; CNV golden images filled LVMS and crash-looped AAP postgres. ### 4. setup-bmaas: host type interfaces + template host_type link `bmaas_host_type_interfaces` (eth9/eth10 fabric, eth0 lifecycle) + PATCH `host_type` onto the published template. Gap: interface validation was a no-op. ### 5. setup-bmaas: BMH osac.openshift.io/interface-macs annotation Map tenant interfaces to their Netris fabric NIC MACs (udp NICs, ordered eth9/eth10) for MAC-based IP discovery. Not `bootMACAddress` (that's bmc-net). ### 6. setup-bmaas: use all three HGX servers Add `hgx-pod00-su0-h03` to `bmaas_vm_patterns` (3 BMHs); all steps are loop-driven. ### 7. netris-lab: no default route on server mgmt NICs via DHCP `templates.go`: the mgmt-server dhcpd no longer emits `option routers` for the servers' mgmt subnet. Gap: the servers' mgmt NIC got a default route that beat the tenant NIC, so ExternalIP/DNAT replies left via mgmt → asymmetric → inbound SSH timed out. ### 8. BM parking V-Net for IPA egress Servers need internet during provisioning (IPA pulls images from quay.io). A dedicated parking V-Net with SNAT provides egress without polluting the tenant network. Iterates through several approaches (per-host DNAT → 1:1 SNAT → many-to-1 SNAT) to work around cloudsim limitations. ### 9. Vendored Netris collection fixes Fix `is not none` guard in vendored `ipam`, `vpc`, `nat` roles — Netris API returns the string `"None"` instead of null, causing delete operations to fail. ### 10. Softgate SNAT + BGP prefix list configuration Terraform configures `sgRole=snat` on softgate-2/3 and `prefixListOutbound="permit 0.0.0.0/0 le 32"` on all 4 BGP sessions — required for NATGateway egress. ### 11. BMI catalog item auto_external_ip_attachment `setup-bmaas` creates two catalog items: `ci-bm-default` (standard) and `ci-bm-auto-eip` (with `auto_external_ip_attachment` field) for auto-EIP E2E tests. ### 12. Netris site default ACL policy → deny Changed the terraform site `acldefaultpolicy` default from `permit` to `deny` for proper tenant isolation. ## BMaaS networking E2E tests (`tests/bmaas/networking/`) Automated pytest tests for the full BMaaS networking lifecycle. Run with `make test-bmaas-networking`. **Test class: `TestBmaasNetworking`** — 20 ordered tests using 3 BMHs: | Phase | Tests | What it covers | |-------|-------|---------------| | Build network | 01–04 | VirtualNetwork, 2 Subnets, SecurityGroup (SSH+ICMP), NATGateway | | Provision | 05, 05b | 3 BMIs (2 subnet A, 1 subnet B with auto-EIP), verify tenant IPs + auto ExternalIP | | Connectivity | 06–12 | L2 arping same subnet, L3 ping same/cross subnet, arping cross subnet fails, tenant isolation, NAT egress, ExternalIP ingress | | Teardown | 13–18, 14b | Delete in dependency order, verify auto-EIP garbage collected, BMHs released | **New files:** - `tests/bmaas/networking/bmi_ssh.py` — SSH connectivity helpers (virsh BMC lookup, arping, ping, curl) - `tests/bmaas/networking/conftest.py` — session fixtures (env var lookups, SSH keypair) - `tests/bmaas/networking/test_bmaas_networking.py` — test class **Core client extensions:** - `OsacCLI.create_baremetal_instance()` — `network_attachments` + `external_ip_attachment` params - `K8sClient.get_baremetal_instance_tenant_ip()` — read IP from CR status - `GRPCClient` — `create_security_group_with_rules()`, `update_security_group_rules()`, `create_external_ip_attachment_bmi()` - `helpers.py` — NATGateway wait helpers ## Verified end-to-end Full demo recorded: BMI create → networking → connectivity tests → ExternalIP ingress → teardown. All 8 parts complete successfully on the se-lab. Assisted-by: Claude Code --------- Signed-off-by: Dan Manor --- tests/bmaas/networking/__init__.py | 1 + tests/bmaas/networking/bmi_ssh.py | 106 ++++ tests/bmaas/networking/conftest.py | 55 ++ .../bmaas/networking/test_bmaas_networking.py | 473 ++++++++++++++++++ tests/core/grpc_client.py | 48 +- tests/core/helpers.py | 23 + tests/core/k8s_client.py | 5 + tests/core/osac_cli.py | 13 +- 8 files changed, 722 insertions(+), 2 deletions(-) create mode 100644 tests/bmaas/networking/__init__.py create mode 100644 tests/bmaas/networking/bmi_ssh.py create mode 100644 tests/bmaas/networking/conftest.py create mode 100644 tests/bmaas/networking/test_bmaas_networking.py diff --git a/tests/bmaas/networking/__init__.py b/tests/bmaas/networking/__init__.py new file mode 100644 index 0000000000..9d48db4f9f --- /dev/null +++ b/tests/bmaas/networking/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/bmaas/networking/bmi_ssh.py b/tests/bmaas/networking/bmi_ssh.py new file mode 100644 index 0000000000..0471260426 --- /dev/null +++ b/tests/bmaas/networking/bmi_ssh.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import logging +import subprocess + +log = logging.getLogger(__name__) + +_SSH_OPTS = [ + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + "/root/.ssh/id_rsa", +] + + +def get_bmc_ip(bmh_name: str) -> str: + domiflist = subprocess.run(["virsh", "domiflist", bmh_name], capture_output=True, text=True, timeout=10, check=True) + bmc_mac = "" + for line in domiflist.stdout.splitlines(): + if "bmc-net" in line: + bmc_mac = line.split()[4] + break + if not bmc_mac: + raise RuntimeError(f"No bmc-net interface found for VM {bmh_name}") + + leases = subprocess.run( + ["virsh", "net-dhcp-leases", "bmc-net"], capture_output=True, text=True, timeout=10, check=True + ) + for line in leases.stdout.splitlines(): + if bmc_mac in line: + ip_with_prefix = line.split()[4] + return ip_with_prefix.split("/")[0] + raise RuntimeError(f"No DHCP lease found for MAC {bmc_mac} on bmc-net") + + +def ssh_bmi(bmc_ip: str, command: str, timeout: int = 30) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["ssh", *_SSH_OPTS, f"fedora@{bmc_ip}", command], capture_output=True, text=True, timeout=timeout, check=True + ) + + +def ssh_bmi_unchecked(bmc_ip: str, command: str, timeout: int = 30) -> tuple[str, int]: + try: + result = subprocess.run( + ["ssh", *_SSH_OPTS, f"fedora@{bmc_ip}", command], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + log.warning("ssh_bmi_unchecked(%s): subprocess timed out after %ds", bmc_ip, timeout) + return f"ssh timed out after {timeout}s", 255 + return (result.stdout.strip() + "\n" + result.stderr.strip()).strip(), result.returncode + + +def arping(bmc_ip: str, target_ip: str, interface: str = "ens5", count: int = 3) -> bool: + _, rc = ssh_bmi_unchecked(bmc_ip, f"arping -c {count} -I {interface} {target_ip}", timeout=30) + return rc == 0 + + +def ping(bmc_ip: str, target_ip: str, count: int = 3, wait: int = 3) -> bool: + _, rc = ssh_bmi_unchecked(bmc_ip, f"ping -c {count} -W {wait} {target_ip}", timeout=30) + return rc == 0 + + +def curl_status(bmc_ip: str, url: str, timeout: int = 15) -> int: + output, rc = ssh_bmi_unchecked( + bmc_ip, + f"curl -s -o /dev/null -w '%{{http_code}}' --connect-timeout {timeout} {url}", + timeout=timeout + 30, + ) + log.info("curl_status(%s, %s): ssh_rc=%d, output=%r", bmc_ip, url, rc, output[-200:]) + for line in output.strip().splitlines(): + line = line.strip() + if line.isdigit() and len(line) == 3: + return int(line) + log.warning("curl_status: no HTTP status code found in output, returning 0") + return 0 + + +def ssh_via_external_ip(external_ip: str, command: str = "hostname", timeout: int = 15) -> str: + result = subprocess.run( + [ + "ssh", + "-o", + f"ConnectTimeout={timeout}", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + "/root/.ssh/id_rsa", + f"fedora@{external_ip}", + command, + ], + capture_output=True, + text=True, + timeout=timeout + 10, + check=True, + ) + return result.stdout.strip() diff --git a/tests/bmaas/networking/conftest.py b/tests/bmaas/networking/conftest.py new file mode 100644 index 0000000000..87cf447c23 --- /dev/null +++ b/tests/bmaas/networking/conftest.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import uuid +from pathlib import Path + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.runner import env + + +@pytest.fixture(scope="session") +def external_ip_pool_name() -> str: + return env("OSAC_EXTERNAL_IP_POOL", "tenant-external-pool") + + +@pytest.fixture(scope="session") +def external_ip_pool_cidr() -> str: + return env("OSAC_EXTERNAL_IP_POOL_CIDR", "198.51.100.24/29") + + +@pytest.fixture(scope="session") +def auto_eip_catalog_item_name() -> str: + return env("OSAC_BMI_AUTO_EIP_CATALOG_ITEM", "ci-bm-auto-eip") + + +@pytest.fixture(scope="session") +def mgmt_cluster_ip() -> str: + return env("OSAC_MGMT_CLUSTER_IP", "192.168.40.2") + + +@pytest.fixture(scope="session") +def net_test_run_id() -> str: + return uuid.uuid4().hex[:8] + + +@pytest.fixture(scope="session") +def bmi_template() -> str: + return env("OSAC_BMI_TEMPLATE", "bm-host-provisioning") + + +@pytest.fixture(scope="session") +def bmh_namespace() -> str: + return env("OSAC_BMH_NAMESPACE", "host-inventory") + + +@pytest.fixture(scope="session") +def catalog_item_name() -> str: + return env("OSAC_BMI_CATALOG_ITEM", "ci-bm-default") + + +@pytest.fixture(scope="session") +def net_ssh_public_key() -> str: + key_path = Path(env("OSAC_BMI_SSH_PUBLIC_KEY", "/root/.ssh/id_rsa.pub")) + return key_path.read_text().strip() diff --git a/tests/bmaas/networking/test_bmaas_networking.py b/tests/bmaas/networking/test_bmaas_networking.py new file mode 100644 index 0000000000..f63e5852a0 --- /dev/null +++ b/tests/bmaas/networking/test_bmaas_networking.py @@ -0,0 +1,473 @@ +from __future__ import annotations + +import subprocess +from typing import Any, ClassVar + +import pytest + +from tests.bmaas.networking import bmi_ssh +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_bmh_available, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, + wait_for_external_ip_allocated, + wait_for_external_ip_attachment_cr, + wait_for_external_ip_attachment_deletion, + wait_for_external_ip_attachment_ready, + wait_for_external_ip_cr, + wait_for_external_ip_deletion, + wait_for_external_ip_pool_cr, + wait_for_external_ip_pool_deletion, + wait_for_external_ip_pool_grpc_ready, + wait_for_external_ip_pool_ready, + wait_for_security_group_cr, + wait_for_security_group_deletion, + wait_for_security_group_ready, + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_subnet_ready, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + + +def _require(state: dict[str, Any], *keys: str) -> None: + missing = [k for k in keys if k not in state] + if missing: + pytest.skip(f"Prerequisite state missing: {', '.join(missing)}") + + +class TestBmaasNetworking: + state: ClassVar[dict[str, Any]] = {} + + # ── Phase 0: External IP Pool ───────────────────────────────────── + + def test_00_create_external_ip_pool( + self, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + external_ip_pool_name: str, + external_ip_pool_cidr: str, + ) -> None: + pool_id = private_grpc.create_external_ip_pool( + name=external_ip_pool_name, + cidrs=[external_ip_pool_cidr], + implementation_strategy="netris", + ) + pool_cr = wait_for_external_ip_pool_cr(k8s=k8s_hub_client, uuid=pool_id) + wait_for_external_ip_pool_ready(k8s=k8s_hub_client, name=pool_cr) + wait_for_external_ip_pool_grpc_ready(private_grpc=private_grpc, pool_id=pool_id) + + self.__class__.state.update(pool_id=pool_id, pool_cr=pool_cr) + print(f"Created ExternalIPPool {external_ip_pool_name}: {pool_id}") + + # ── Phase 1: Build the Network ────────────────────────────────────── + + def test_01_create_virtual_network( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str + ) -> None: + name = f"net-{net_test_run_id}" + vnet_id = grpc.create_virtual_network(name=name, ipv4_cidr="10.100.0.0/16") + vnet_cr = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vnet_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vnet_cr) + + self.__class__.state["vnet_id"] = vnet_id + self.__class__.state["vnet_cr"] = vnet_cr + self.__class__.state["vnet_name"] = name + + def test_02_create_subnets(self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str) -> None: + _require(self.state, "vnet_id") + + subnet_a_name = f"sub-a-{net_test_run_id}" + subnet_a_id = grpc.create_subnet( + name=subnet_a_name, virtual_network=self.state["vnet_id"], ipv4_cidr="10.100.1.0/24" + ) + subnet_a_cr = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_a_id) + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_a_cr) + + subnet_b_name = f"sub-b-{net_test_run_id}" + subnet_b_id = grpc.create_subnet( + name=subnet_b_name, virtual_network=self.state["vnet_id"], ipv4_cidr="10.100.2.0/24" + ) + subnet_b_cr = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_b_id) + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_b_cr) + + self.__class__.state.update( + subnet_a_id=subnet_a_id, subnet_a_cr=subnet_a_cr, subnet_b_id=subnet_b_id, subnet_b_cr=subnet_b_cr + ) + + def test_03_create_security_group(self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str) -> None: + _require(self.state, "vnet_id") + + sg_name = f"sg-{net_test_run_id}" + sg_id = grpc.create_security_group_with_rules( + name=sg_name, + virtual_network=self.state["vnet_id"], + ingress=[ + {"protocol": "PROTOCOL_TCP", "port_from": 22, "port_to": 22, "ipv4_cidr": "0.0.0.0/0"}, + {"protocol": "PROTOCOL_ICMP", "ipv4_cidr": "0.0.0.0/0"}, + ], + egress=[{"protocol": "PROTOCOL_ALL", "ipv4_cidr": "0.0.0.0/0"}], + ) + sg_cr = wait_for_security_group_cr(k8s=k8s_hub_client, uuid=sg_id) + wait_for_security_group_ready(k8s=k8s_hub_client, name=sg_cr) + + self.__class__.state.update(sg_id=sg_id, sg_cr=sg_cr, sg_name=sg_name) + + def test_04_create_nat_gateway( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str + ) -> None: + _require(self.state, "vnet_name", "pool_id") + + nat_eip_name = f"nat-eip-{net_test_run_id}" + nat_eip_id = grpc.create_external_ip(name=nat_eip_name, pool=self.state["pool_id"]) + nat_eip_cr = wait_for_external_ip_cr(k8s=k8s_hub_client, uuid=nat_eip_id) + wait_for_external_ip_allocated(k8s=k8s_hub_client, name=nat_eip_cr) + + nat_name = f"nat-{net_test_run_id}" + nat_id = grpc.create_nat_gateway( + name=nat_name, virtual_network_name=self.state["vnet_name"], external_ip_name=nat_eip_name + ) + poll_until( + fn=lambda: ( + grpc.call(service="osac.public.v1.NATGateways/Get", data={"id": nat_id}) + .get("object", {}) + .get("status", {}) + .get("state", "") + ), + until=lambda s: s in ("NAT_GATEWAY_STATE_READY", "Ready"), + retries=30, + delay=5, + description=f"NATGateway {nat_name} to become Ready", + ) + + self.__class__.state.update( + nat_eip_id=nat_eip_id, nat_eip_cr=nat_eip_cr, nat_eip_name=nat_eip_name, nat_id=nat_id, nat_name=nat_name + ) + + # ── Phase 2: Provision Servers ────────────────────────────────────── + + def test_05_create_three_bmis( + self, + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item_name: str, + auto_eip_catalog_item_name: str, + net_ssh_public_key: str, + bmh_namespace: str, + net_test_run_id: str, + ) -> None: + _require(self.state, "subnet_a_id", "subnet_b_id", "sg_id") + + subnet_a = self.state["subnet_a_id"] + subnet_b = self.state["subnet_b_id"] + sg = self.state["sg_id"] + + bmis: list[dict[str, str]] = [] + for i, (name_suffix, subnet_id) in enumerate([("bmi1", subnet_a), ("bmi2", subnet_a), ("bmi3", subnet_b)]): + bmi_name = f"{name_suffix}-{net_test_run_id}" + is_auto_eip = i == 2 + catalog = auto_eip_catalog_item_name if is_auto_eip else catalog_item_name + bmi_id = cli.create_baremetal_instance( + name=bmi_name, + catalog_item=catalog, + ssh_key=net_ssh_public_key, + network_attachments=[f"subnet={subnet_id},interface=eth9,primary,security-groups={sg}"], + external_ip_attachment=is_auto_eip, + ) + print(f"Created BMI {bmi_name}: {bmi_id} (catalog: {catalog})") + bmis.append({"name": bmi_name, "id": bmi_id, "subnet": "a" if i < 2 else "b"}) + + for bmi in bmis: + bmi["cr"] = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi["id"]) + print(f"BMI {bmi['name']} CR: {bmi['cr']}") + + for bmi in bmis: + wait_for_bmi_running(grpc=grpc, bmi_id=bmi["id"]) + print(f"BMI {bmi['name']} is RUNNING") + + for bmi in bmis: + bmi["ip"] = poll_until( + fn=lambda b=bmi: k8s_hub_client.get_baremetal_instance_tenant_ip(name=b["cr"]), + until=lambda ip: ip != "", + retries=60, + delay=10, + description=f"BMI {bmi['name']} tenant IP assignment", + ) + + ext_host = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi["cr"]) + bmi["bmh"] = ext_host.split("/", 1)[1] + bmi["bmc_ip"] = bmi_ssh.get_bmc_ip(bmi["bmh"]) + print(f"BMI {bmi['name']}: tenant_ip={bmi['ip']}, bmh={bmi['bmh']}, bmc_ip={bmi['bmc_ip']}") + + for bmi in bmis: + if bmi["subnet"] == "a": + assert bmi["ip"].startswith("10.100.1."), f"BMI {bmi['name']} IP {bmi['ip']} not in subnet A" + else: + assert bmi["ip"].startswith("10.100.2."), f"BMI {bmi['name']} IP {bmi['ip']} not in subnet B" + + self.__class__.state["bmi1"] = bmis[0] + self.__class__.state["bmi2"] = bmis[1] + self.__class__.state["bmi3"] = bmis[2] + + def test_05b_verify_auto_eip_on_bmi3(self, grpc: GRPCClient) -> None: + _require(self.state, "bmi3") + bmi3 = self.state["bmi3"] + + def find_auto_attachment() -> dict[str, Any] | None: + attachments = grpc.call(service="osac.public.v1.ExternalIPAttachments/List") + for item in attachments.get("items", []): + bmi_ref = item.get("spec", {}).get("baremetalInstance", {}).get("id", "") + if bmi_ref == bmi3["id"]: + return item + return None + + attachment = poll_until( + fn=find_auto_attachment, + until=lambda a: a is not None, + retries=60, + delay=5, + description="auto-created ExternalIPAttachment for BMI3", + ) + + auto_attach_id = attachment["id"] + auto_eip_ref = attachment.get("spec", {}).get("externalIp", {}).get("id", "") + assert auto_eip_ref, "Auto-created attachment has no ExternalIP reference" + + eip_data = grpc.get_external_ip(external_ip_id=auto_eip_ref) + auto_ext_addr = eip_data.get("object", {}).get("status", {}).get("address", "") + assert auto_ext_addr, "Auto-created ExternalIP has no allocated address" + + self.__class__.state.update( + auto_attach_id=auto_attach_id, auto_eip_id=auto_eip_ref, auto_ext_addr=auto_ext_addr + ) + print(f"BMI3 auto EIP: {auto_ext_addr}, attachment: {auto_attach_id}") + + # ── Phase 3: Connectivity Tests ───────────────────────────────────── + + def test_06_l2_arping_same_subnet(self) -> None: + _require(self.state, "bmi1", "bmi2") + bmi1 = self.state["bmi1"] + bmi2 = self.state["bmi2"] + + poll_until( + fn=lambda: bmi_ssh.arping(bmi1["bmc_ip"], bmi2["ip"]), + until=lambda ok: ok, + retries=5, + delay=10, + description=f"L2 arping BMI1 ({bmi1['ip']}) → BMI2 ({bmi2['ip']})", + ) + + def test_07_l3_ping_same_subnet(self) -> None: + _require(self.state, "bmi1", "bmi2") + bmi1 = self.state["bmi1"] + bmi2 = self.state["bmi2"] + + poll_until( + fn=lambda: bmi_ssh.ping(bmi1["bmc_ip"], bmi2["ip"]), + until=lambda ok: ok, + retries=5, + delay=10, + description=f"L3 ping BMI1 ({bmi1['ip']}) → BMI2 ({bmi2['ip']})", + ) + + def test_08_l3_ping_cross_subnet(self) -> None: + _require(self.state, "bmi1", "bmi3") + bmi1 = self.state["bmi1"] + bmi3 = self.state["bmi3"] + + poll_until( + fn=lambda: bmi_ssh.ping(bmi1["bmc_ip"], bmi3["ip"]), + until=lambda ok: ok, + retries=5, + delay=10, + description=f"L3 ping BMI1 ({bmi1['ip']}) → BMI3 ({bmi3['ip']}) cross-subnet", + ) + + def test_09_l2_arping_cross_subnet_fails(self) -> None: + _require(self.state, "bmi1", "bmi3") + bmi1 = self.state["bmi1"] + bmi3 = self.state["bmi3"] + + assert not bmi_ssh.arping(bmi1["bmc_ip"], bmi3["ip"]), ( + f"arping from BMI1 ({bmi1['ip']}, subnet A) to BMI3 ({bmi3['ip']}, subnet B) " + f"succeeded unexpectedly — different subnets should be different broadcast domains" + ) + + def test_10_tenant_isolation(self, mgmt_cluster_ip: str) -> None: + _require(self.state, "bmi1") + bmi1 = self.state["bmi1"] + + assert not bmi_ssh.ping(bmi1["bmc_ip"], mgmt_cluster_ip), ( + f"ping from BMI1 ({bmi1['ip']}) to management cluster ({mgmt_cluster_ip}) " + f"succeeded unexpectedly — tenant isolation should prevent cross-VNet traffic" + ) + + def test_11_nat_gateway_egress(self) -> None: + _require(self.state, "bmi1") + bmi1 = self.state["bmi1"] + + poll_until( + fn=lambda: bmi_ssh.curl_status(bmi1["bmc_ip"], "https://quay.io"), + until=lambda status: status == 200, + retries=5, + delay=15, + description=f"NAT gateway egress curl quay.io (bmc_ip={bmi1['bmc_ip']}, tenant_ip={bmi1['ip']})", + ) + + def test_12_external_ip_ingress( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str + ) -> None: + _require(self.state, "bmi1", "pool_id") + bmi1 = self.state["bmi1"] + + eip_name = f"ingress-eip-{net_test_run_id}" + eip_id = grpc.create_external_ip(name=eip_name, pool=self.state["pool_id"]) + eip_cr = wait_for_external_ip_cr(k8s=k8s_hub_client, uuid=eip_id) + wait_for_external_ip_allocated(k8s=k8s_hub_client, name=eip_cr) + + attach_name = f"ingress-attach-{net_test_run_id}" + attach_id = grpc.create_external_ip_attachment_bmi( + name=attach_name, external_ip=eip_id, baremetal_instance=bmi1["id"] + ) + attach_cr = wait_for_external_ip_attachment_cr(k8s=k8s_hub_client, uuid=attach_id) + wait_for_external_ip_attachment_ready(k8s=k8s_hub_client, name=attach_cr) + + eip_data = grpc.get_external_ip(external_ip_id=eip_id) + ext_addr = eip_data.get("object", {}).get("status", {}).get("address", "") + assert ext_addr, "ExternalIP has no allocated address" + + def _try_ssh_eip() -> str: + try: + return bmi_ssh.ssh_via_external_ip(ext_addr, timeout=10) + except subprocess.CalledProcessError: + return "" + + hostname = poll_until( + fn=_try_ssh_eip, + until=lambda h: bool(h), + retries=5, + delay=15, + description=f"SSH via external IP {ext_addr}", + ) + + self.__class__.state.update( + ingress_eip_id=eip_id, + ingress_eip_cr=eip_cr, + ingress_attach_id=attach_id, + ingress_attach_cr=attach_cr, + ingress_ext_addr=ext_addr, + ) + + # ── Phase 4: Teardown ─────────────────────────────────────────────── + + def test_13_delete_external_ip_attachment(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "ingress_attach_id" not in self.state: + pytest.skip("No ExternalIPAttachment to delete") + + grpc.delete_external_ip_attachment(attachment_id=self.state["ingress_attach_id"]) + wait_for_external_ip_attachment_deletion(k8s=k8s_hub_client, name=self.state["ingress_attach_cr"]) + + grpc.delete_external_ip(external_ip_id=self.state["ingress_eip_id"]) + wait_for_external_ip_deletion(k8s=k8s_hub_client, name=self.state["ingress_eip_cr"]) + + def test_14_delete_bmis( + self, cli: OsacCLI, grpc: GRPCClient, k8s_hub_client: K8sClient, bmh_namespace: str + ) -> None: + for key in ("bmi1", "bmi2", "bmi3"): + if key not in self.state: + continue + bmi = self.state[key] + print(f"Deleting {bmi['name']}...") + cli.delete_baremetal_instance(uuid=bmi["id"]) + + for key in ("bmi1", "bmi2", "bmi3"): + if key not in self.state: + continue + bmi = self.state[key] + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi["cr"]) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi["id"]) + wait_for_bmh_available(k8s=k8s_hub_client, name=bmi["bmh"], bmh_namespace=bmh_namespace) + print(f"{bmi['name']} deprovisioned, BMH {bmi['bmh']} available") + + def test_14b_verify_auto_eip_garbage_collected(self, grpc: GRPCClient) -> None: + if "auto_attach_id" not in self.state: + pytest.skip("No auto EIP to verify") + + poll_until( + fn=lambda: self.state["auto_attach_id"] not in grpc.list_external_ip_attachment_ids(), + until=lambda gone: gone is True, + retries=30, + delay=5, + description="auto-created ExternalIPAttachment garbage collection", + ) + + poll_until( + fn=lambda: self.state["auto_eip_id"] not in grpc.list_external_ip_ids(), + until=lambda gone: gone is True, + retries=30, + delay=5, + description="auto-created ExternalIP garbage collection", + ) + print("Auto EIP and attachment garbage collected after BMI3 deletion") + + def test_15_delete_nat_gateway(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "nat_id" not in self.state: + pytest.skip("No NATGateway to delete") + + grpc.delete_nat_gateway(nat_gateway_id=self.state["nat_id"]) + poll_until( + fn=lambda: ( + self.state["nat_id"] + not in [item["id"] for item in grpc.call(service="osac.public.v1.NATGateways/List").get("items", [])] + ), + until=lambda gone: gone is True, + retries=60, + delay=5, + description=f"NATGateway {self.state['nat_name']} deletion", + ) + + grpc.delete_external_ip(external_ip_id=self.state["nat_eip_id"]) + wait_for_external_ip_deletion(k8s=k8s_hub_client, name=self.state["nat_eip_cr"]) + + def test_16_delete_security_group(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "sg_id" not in self.state: + pytest.skip("No SecurityGroup to delete") + + grpc.delete_security_group(sg_id=self.state["sg_id"]) + wait_for_security_group_deletion(k8s=k8s_hub_client, name=self.state["sg_cr"]) + + def test_17_delete_subnets(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + for key, cr_key in [("subnet_a_id", "subnet_a_cr"), ("subnet_b_id", "subnet_b_cr")]: + if key not in self.state: + continue + grpc.delete_subnet(subnet_id=self.state[key]) + wait_for_subnet_deletion(k8s=k8s_hub_client, name=self.state[cr_key]) + + def test_18_delete_virtual_network(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "vnet_id" not in self.state: + pytest.skip("No VirtualNetwork to delete") + + grpc.delete_virtual_network(vn_id=self.state["vnet_id"]) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=self.state["vnet_cr"]) + + remaining = grpc.list_virtual_network_ids() + assert self.state["vnet_id"] not in remaining, "VirtualNetwork still in API after deletion" + + def test_19_delete_external_ip_pool(self, private_grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "pool_id" not in self.state: + pytest.skip("No ExternalIPPool to delete") + + private_grpc.delete_external_ip_pool(pool_id=self.state["pool_id"]) + wait_for_external_ip_pool_deletion(k8s=k8s_hub_client, name=self.state["pool_cr"]) + + remaining = private_grpc.list_external_ip_pool_ids() + assert self.state["pool_id"] not in remaining, "ExternalIPPool still in API after deletion" diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py index ee7b15a785..fb5e82661a 100644 --- a/tests/core/grpc_client.py +++ b/tests/core/grpc_client.py @@ -156,6 +156,40 @@ def list_security_group_ids(self) -> list[str]: def delete_security_group(self, *, sg_id: str) -> None: self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) + def create_security_group_with_rules( + self, + *, + name: str, + virtual_network: str, + ingress: list[dict[str, Any]] | None = None, + egress: list[dict[str, Any]] | None = None, + ) -> str: + spec: dict[str, Any] = {"virtual_network": {"id": virtual_network}} + if ingress is not None: + spec["ingress"] = ingress + if egress is not None: + spec["egress"] = egress + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", data={"object": {"metadata": {"name": name}, "spec": spec}} + ) + return response["object"]["id"] + + def update_security_group_rules( + self, *, sg_id: str, ingress: list[dict[str, Any]] | None = None, egress: list[dict[str, Any]] | None = None + ) -> None: + spec: dict[str, Any] = {} + paths: list[str] = [] + if ingress is not None: + spec["ingress"] = ingress + paths.append("spec.ingress") + if egress is not None: + spec["egress"] = egress + paths.append("spec.egress") + self.call( + service=f"{PUBLIC_API}.SecurityGroups/Update", + data={"object": {"id": sg_id, "spec": spec}, "updateMask": {"paths": paths}}, + ) + # Console operations def create_console_session( @@ -200,7 +234,7 @@ def create_external_ip_pool( name: str, cidrs: list[str], ip_family: str = "IP_FAMILY_IPV4", - implementation_strategy: str = "metallb-l2", + implementation_strategy: str = "", ) -> str: response: dict[str, Any] = self.call( service=f"{PRIVATE_API}.ExternalIPPools/Create", @@ -270,6 +304,18 @@ def list_external_ip_attachment_ids(self) -> list[str]: def delete_external_ip_attachment(self, *, attachment_id: str) -> None: self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Delete", data={"id": attachment_id}) + def create_external_ip_attachment_bmi(self, *, name: str, external_ip: str, baremetal_instance: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ExternalIPAttachments/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"external_ip": {"id": external_ip}, "baremetal_instance": {"id": baremetal_instance}}, + } + }, + ) + return response["object"]["id"] + # ClusterCatalogItem operations def create_cluster_catalog_item( diff --git a/tests/core/helpers.py b/tests/core/helpers.py index 983b0b2bf6..4b59d94e80 100644 --- a/tests/core/helpers.py +++ b/tests/core/helpers.py @@ -266,6 +266,29 @@ def wait_for_external_ip_attachment_deletion(*, k8s: K8sClient, name: str) -> No ) +# NATGateway helpers + + +def wait_for_nat_gateway_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_jsonpath(resource="natgateway", name=name, jsonpath="{.status.state}"), + until=lambda state: state == "Ready", + retries=30, + delay=5, + description=f"NATGateway {name} to become Ready", + ) + + +def wait_for_nat_gateway_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="natgateway", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} NATGateway deletion", + ) + + def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: return poll_until( fn=lambda: k8s.get_cluster_order_name(uuid=uuid, checked=False), diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py index cc4f9e7553..998758d8c6 100644 --- a/tests/core/k8s_client.py +++ b/tests/core/k8s_client.py @@ -475,6 +475,11 @@ def get_bmi_hardware_nics(self, *, name: str) -> list[str]: return [] return [mac.lower() for mac in output.split()] + def get_baremetal_instance_tenant_ip(self, *, name: str) -> str: + return self.get_jsonpath( + resource="baremetalinstance", name=name, jsonpath="{.status.networkAttachmentStatuses[0].ipAddress}" + ) + # BareMetalHost queries (explicit namespace — BMHs live in a different namespace) def get_bmh_provisioning_state(self, *, name: str, bmh_namespace: str) -> str: diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py index d5d3ac2398..534ad49da3 100644 --- a/tests/core/osac_cli.py +++ b/tests/core/osac_cli.py @@ -236,13 +236,24 @@ def delete_cluster(self, *, uuid: str) -> None: self._run("delete", "cluster", uuid) def create_baremetal_instance( - self, *, name: str, catalog_item: str, ssh_key: str | None = None, user_data: str | None = None + self, + *, + name: str, + catalog_item: str, + ssh_key: str | None = None, + user_data: str | None = None, + network_attachments: list[str] | None = None, + external_ip_attachment: bool = False, ) -> str: args: list[str] = ["create", "baremetalinstance", "--name", name, "--catalog-item", catalog_item] if ssh_key is not None: args.extend(["--ssh-key", ssh_key]) if user_data is not None: args.extend(["--user-data", user_data]) + if external_ip_attachment: + args.extend(["--external-ip-attachment"]) + for na in network_attachments or []: + args.extend(["--network-attachment", na]) return self._parse_uuid(self._run(*args)) def describe_baremetal_instance(self, *, name: str) -> str: From e2e6d8ae2c0cfc7ea16cc2a6c8738653a4f2ba32 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Mon, 24 Aug 2026 20:01:08 +0000 Subject: [PATCH 112/112] OSAC-3593: add pyproject.toml with pytest configuration from osac-test-infra Migrates pytest config (testpaths, xdist, markers, logging), ruff and basedpyright settings from osac-project/osac-test-infra. --- pyproject.toml | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..3faefa3112 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "osac-e2e" +version = "0.1.0" +description = "OSAC end-to-end test infrastructure" +requires-python = ">=3.11" +dependencies = [ + "pytest>=8.0", + "pytest-xdist>=3.0", + "pyyaml>=6.0", + "websocket-client>=1.6", +] + +[tool.setuptools.packages.find] +include = ["tests*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-n 4 --dist loadfile" +markers = [ + "sanity: fast/essential e2e coverage for PR checks", + "regression: heavier or cross-flow e2e for periodic runs", +] +log_level = "INFO" +log_cli = true +log_cli_date_format = "%Y-%m-%dT%H:%M:%S%z" +log_cli_format = "%(asctime)s %(levelname)-8s %(message)s" +log_cli_level = "INFO" +log_file = "/tmp/test-output/e2e.log" +log_file_date_format = "%Y-%m-%dT%H:%M:%S%z" +log_file_format = "%(asctime)s %(levelname)-8s %(name)s %(message)s" +log_file_level = "INFO" +log_file_mode = "a" + +[tool.basedpyright] +reportUnusedCallResult = false +reportAny = false +reportExplicitAny = false + +[tool.ruff] +target-version = "py311" +line-length = 120 + +[tool.ruff.format] +skip-magic-trailing-comma = true + +[tool.ruff.lint.isort] +split-on-trailing-comma = false + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "ANN", "B", "SIM", "RUF"] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["ANN101", "ANN201"]