diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..3faefa3112 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "osac-e2e" +version = "0.1.0" +description = "OSAC end-to-end test infrastructure" +requires-python = ">=3.11" +dependencies = [ + "pytest>=8.0", + "pytest-xdist>=3.0", + "pyyaml>=6.0", + "websocket-client>=1.6", +] + +[tool.setuptools.packages.find] +include = ["tests*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-n 4 --dist loadfile" +markers = [ + "sanity: fast/essential e2e coverage for PR checks", + "regression: heavier or cross-flow e2e for periodic runs", +] +log_level = "INFO" +log_cli = true +log_cli_date_format = "%Y-%m-%dT%H:%M:%S%z" +log_cli_format = "%(asctime)s %(levelname)-8s %(message)s" +log_cli_level = "INFO" +log_file = "/tmp/test-output/e2e.log" +log_file_date_format = "%Y-%m-%dT%H:%M:%S%z" +log_file_format = "%(asctime)s %(levelname)-8s %(name)s %(message)s" +log_file_level = "INFO" +log_file_mode = "a" + +[tool.basedpyright] +reportUnusedCallResult = false +reportAny = false +reportExplicitAny = false + +[tool.ruff] +target-version = "py311" +line-length = 120 + +[tool.ruff.format] +skip-magic-trailing-comma = true + +[tool.ruff.lint.isort] +split-on-trailing-comma = false + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "ANN", "B", "SIM", "RUF"] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["ANN101", "ANN201"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/bmaas/__init__.py b/tests/bmaas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/bmaas/conftest.py b/tests/bmaas/conftest.py new file mode 100644 index 0000000000..f2dd30c676 --- /dev/null +++ b/tests/bmaas/conftest.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import subprocess +import tempfile +import uuid +from collections.abc import Generator +from pathlib import Path + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.runner import env + + +@pytest.fixture(scope="session") +def bmi_template() -> str: + return env("OSAC_BMI_TEMPLATE", "bm-host-provisioning") + + +@pytest.fixture(scope="session") +def bmh_namespace() -> str: + return env("OSAC_BMH_NAMESPACE", "host-inventory") + + +@pytest.fixture(scope="session") +def test_run_id() -> str: + return str(uuid.uuid4())[:8] + + +@pytest.fixture(scope="session") +def ssh_public_key() -> Generator[str, None, None]: + with tempfile.TemporaryDirectory() as tmpdir: + key_path = Path(tmpdir) / "bmi-test-key" + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-f", str(key_path), "-N", "", "-C", "bmi-e2e-test"], + capture_output=True, + check=True, + ) + yield (key_path.with_suffix(".pub")).read_text().strip() + + +@pytest.fixture(scope="session") +def catalog_item(private_grpc: GRPCClient, bmi_template: str, test_run_id: str) -> Generator[str, None, None]: + name = f"e2e-bmaas-{test_run_id}" + print(f"\nCreating BareMetalInstanceCatalogItem: {name}") + item_id: str = private_grpc.create_baremetal_instance_catalog_item( + name=name, + title=f"E2E BMaaS Test ({test_run_id})", + description="Temporary catalog item for BMaaS E2E tests", + template=bmi_template, + field_definitions=[{"path": "ssh_public_key", "display_name": "SSH Public Key", "editable": True}], + ) + print(f"CatalogItem created: {item_id}") + + yield item_id + + try: + print(f"\nDeleting BareMetalInstanceCatalogItem {item_id}...") + private_grpc.delete_baremetal_instance_catalog_item(item_id=item_id) + print(f"CatalogItem {item_id} deleted") + except Exception as e: + print(f"WARNING: Failed to delete catalog item {item_id}: {e}") diff --git a/tests/bmaas/networking/__init__.py b/tests/bmaas/networking/__init__.py new file mode 100644 index 0000000000..9d48db4f9f --- /dev/null +++ b/tests/bmaas/networking/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/bmaas/networking/bmi_ssh.py b/tests/bmaas/networking/bmi_ssh.py new file mode 100644 index 0000000000..0471260426 --- /dev/null +++ b/tests/bmaas/networking/bmi_ssh.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import logging +import subprocess + +log = logging.getLogger(__name__) + +_SSH_OPTS = [ + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + "/root/.ssh/id_rsa", +] + + +def get_bmc_ip(bmh_name: str) -> str: + domiflist = subprocess.run(["virsh", "domiflist", bmh_name], capture_output=True, text=True, timeout=10, check=True) + bmc_mac = "" + for line in domiflist.stdout.splitlines(): + if "bmc-net" in line: + bmc_mac = line.split()[4] + break + if not bmc_mac: + raise RuntimeError(f"No bmc-net interface found for VM {bmh_name}") + + leases = subprocess.run( + ["virsh", "net-dhcp-leases", "bmc-net"], capture_output=True, text=True, timeout=10, check=True + ) + for line in leases.stdout.splitlines(): + if bmc_mac in line: + ip_with_prefix = line.split()[4] + return ip_with_prefix.split("/")[0] + raise RuntimeError(f"No DHCP lease found for MAC {bmc_mac} on bmc-net") + + +def ssh_bmi(bmc_ip: str, command: str, timeout: int = 30) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["ssh", *_SSH_OPTS, f"fedora@{bmc_ip}", command], capture_output=True, text=True, timeout=timeout, check=True + ) + + +def ssh_bmi_unchecked(bmc_ip: str, command: str, timeout: int = 30) -> tuple[str, int]: + try: + result = subprocess.run( + ["ssh", *_SSH_OPTS, f"fedora@{bmc_ip}", command], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + log.warning("ssh_bmi_unchecked(%s): subprocess timed out after %ds", bmc_ip, timeout) + return f"ssh timed out after {timeout}s", 255 + return (result.stdout.strip() + "\n" + result.stderr.strip()).strip(), result.returncode + + +def arping(bmc_ip: str, target_ip: str, interface: str = "ens5", count: int = 3) -> bool: + _, rc = ssh_bmi_unchecked(bmc_ip, f"arping -c {count} -I {interface} {target_ip}", timeout=30) + return rc == 0 + + +def ping(bmc_ip: str, target_ip: str, count: int = 3, wait: int = 3) -> bool: + _, rc = ssh_bmi_unchecked(bmc_ip, f"ping -c {count} -W {wait} {target_ip}", timeout=30) + return rc == 0 + + +def curl_status(bmc_ip: str, url: str, timeout: int = 15) -> int: + output, rc = ssh_bmi_unchecked( + bmc_ip, + f"curl -s -o /dev/null -w '%{{http_code}}' --connect-timeout {timeout} {url}", + timeout=timeout + 30, + ) + log.info("curl_status(%s, %s): ssh_rc=%d, output=%r", bmc_ip, url, rc, output[-200:]) + for line in output.strip().splitlines(): + line = line.strip() + if line.isdigit() and len(line) == 3: + return int(line) + log.warning("curl_status: no HTTP status code found in output, returning 0") + return 0 + + +def ssh_via_external_ip(external_ip: str, command: str = "hostname", timeout: int = 15) -> str: + result = subprocess.run( + [ + "ssh", + "-o", + f"ConnectTimeout={timeout}", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-i", + "/root/.ssh/id_rsa", + f"fedora@{external_ip}", + command, + ], + capture_output=True, + text=True, + timeout=timeout + 10, + check=True, + ) + return result.stdout.strip() diff --git a/tests/bmaas/networking/conftest.py b/tests/bmaas/networking/conftest.py new file mode 100644 index 0000000000..87cf447c23 --- /dev/null +++ b/tests/bmaas/networking/conftest.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import uuid +from pathlib import Path + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.runner import env + + +@pytest.fixture(scope="session") +def external_ip_pool_name() -> str: + return env("OSAC_EXTERNAL_IP_POOL", "tenant-external-pool") + + +@pytest.fixture(scope="session") +def external_ip_pool_cidr() -> str: + return env("OSAC_EXTERNAL_IP_POOL_CIDR", "198.51.100.24/29") + + +@pytest.fixture(scope="session") +def auto_eip_catalog_item_name() -> str: + return env("OSAC_BMI_AUTO_EIP_CATALOG_ITEM", "ci-bm-auto-eip") + + +@pytest.fixture(scope="session") +def mgmt_cluster_ip() -> str: + return env("OSAC_MGMT_CLUSTER_IP", "192.168.40.2") + + +@pytest.fixture(scope="session") +def net_test_run_id() -> str: + return uuid.uuid4().hex[:8] + + +@pytest.fixture(scope="session") +def bmi_template() -> str: + return env("OSAC_BMI_TEMPLATE", "bm-host-provisioning") + + +@pytest.fixture(scope="session") +def bmh_namespace() -> str: + return env("OSAC_BMH_NAMESPACE", "host-inventory") + + +@pytest.fixture(scope="session") +def catalog_item_name() -> str: + return env("OSAC_BMI_CATALOG_ITEM", "ci-bm-default") + + +@pytest.fixture(scope="session") +def net_ssh_public_key() -> str: + key_path = Path(env("OSAC_BMI_SSH_PUBLIC_KEY", "/root/.ssh/id_rsa.pub")) + return key_path.read_text().strip() diff --git a/tests/bmaas/networking/test_bmaas_networking.py b/tests/bmaas/networking/test_bmaas_networking.py new file mode 100644 index 0000000000..f63e5852a0 --- /dev/null +++ b/tests/bmaas/networking/test_bmaas_networking.py @@ -0,0 +1,473 @@ +from __future__ import annotations + +import subprocess +from typing import Any, ClassVar + +import pytest + +from tests.bmaas.networking import bmi_ssh +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_bmh_available, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, + wait_for_external_ip_allocated, + wait_for_external_ip_attachment_cr, + wait_for_external_ip_attachment_deletion, + wait_for_external_ip_attachment_ready, + wait_for_external_ip_cr, + wait_for_external_ip_deletion, + wait_for_external_ip_pool_cr, + wait_for_external_ip_pool_deletion, + wait_for_external_ip_pool_grpc_ready, + wait_for_external_ip_pool_ready, + wait_for_security_group_cr, + wait_for_security_group_deletion, + wait_for_security_group_ready, + wait_for_subnet_cr, + wait_for_subnet_deletion, + wait_for_subnet_ready, + wait_for_virtual_network_cr, + wait_for_virtual_network_deletion, + wait_for_virtual_network_ready, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + + +def _require(state: dict[str, Any], *keys: str) -> None: + missing = [k for k in keys if k not in state] + if missing: + pytest.skip(f"Prerequisite state missing: {', '.join(missing)}") + + +class TestBmaasNetworking: + state: ClassVar[dict[str, Any]] = {} + + # ── Phase 0: External IP Pool ───────────────────────────────────── + + def test_00_create_external_ip_pool( + self, + private_grpc: GRPCClient, + k8s_hub_client: K8sClient, + external_ip_pool_name: str, + external_ip_pool_cidr: str, + ) -> None: + pool_id = private_grpc.create_external_ip_pool( + name=external_ip_pool_name, + cidrs=[external_ip_pool_cidr], + implementation_strategy="netris", + ) + pool_cr = wait_for_external_ip_pool_cr(k8s=k8s_hub_client, uuid=pool_id) + wait_for_external_ip_pool_ready(k8s=k8s_hub_client, name=pool_cr) + wait_for_external_ip_pool_grpc_ready(private_grpc=private_grpc, pool_id=pool_id) + + self.__class__.state.update(pool_id=pool_id, pool_cr=pool_cr) + print(f"Created ExternalIPPool {external_ip_pool_name}: {pool_id}") + + # ── Phase 1: Build the Network ────────────────────────────────────── + + def test_01_create_virtual_network( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str + ) -> None: + name = f"net-{net_test_run_id}" + vnet_id = grpc.create_virtual_network(name=name, ipv4_cidr="10.100.0.0/16") + vnet_cr = wait_for_virtual_network_cr(k8s=k8s_hub_client, uuid=vnet_id) + wait_for_virtual_network_ready(k8s=k8s_hub_client, name=vnet_cr) + + self.__class__.state["vnet_id"] = vnet_id + self.__class__.state["vnet_cr"] = vnet_cr + self.__class__.state["vnet_name"] = name + + def test_02_create_subnets(self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str) -> None: + _require(self.state, "vnet_id") + + subnet_a_name = f"sub-a-{net_test_run_id}" + subnet_a_id = grpc.create_subnet( + name=subnet_a_name, virtual_network=self.state["vnet_id"], ipv4_cidr="10.100.1.0/24" + ) + subnet_a_cr = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_a_id) + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_a_cr) + + subnet_b_name = f"sub-b-{net_test_run_id}" + subnet_b_id = grpc.create_subnet( + name=subnet_b_name, virtual_network=self.state["vnet_id"], ipv4_cidr="10.100.2.0/24" + ) + subnet_b_cr = wait_for_subnet_cr(k8s=k8s_hub_client, uuid=subnet_b_id) + wait_for_subnet_ready(k8s=k8s_hub_client, name=subnet_b_cr) + + self.__class__.state.update( + subnet_a_id=subnet_a_id, subnet_a_cr=subnet_a_cr, subnet_b_id=subnet_b_id, subnet_b_cr=subnet_b_cr + ) + + def test_03_create_security_group(self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str) -> None: + _require(self.state, "vnet_id") + + sg_name = f"sg-{net_test_run_id}" + sg_id = grpc.create_security_group_with_rules( + name=sg_name, + virtual_network=self.state["vnet_id"], + ingress=[ + {"protocol": "PROTOCOL_TCP", "port_from": 22, "port_to": 22, "ipv4_cidr": "0.0.0.0/0"}, + {"protocol": "PROTOCOL_ICMP", "ipv4_cidr": "0.0.0.0/0"}, + ], + egress=[{"protocol": "PROTOCOL_ALL", "ipv4_cidr": "0.0.0.0/0"}], + ) + sg_cr = wait_for_security_group_cr(k8s=k8s_hub_client, uuid=sg_id) + wait_for_security_group_ready(k8s=k8s_hub_client, name=sg_cr) + + self.__class__.state.update(sg_id=sg_id, sg_cr=sg_cr, sg_name=sg_name) + + def test_04_create_nat_gateway( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str + ) -> None: + _require(self.state, "vnet_name", "pool_id") + + nat_eip_name = f"nat-eip-{net_test_run_id}" + nat_eip_id = grpc.create_external_ip(name=nat_eip_name, pool=self.state["pool_id"]) + nat_eip_cr = wait_for_external_ip_cr(k8s=k8s_hub_client, uuid=nat_eip_id) + wait_for_external_ip_allocated(k8s=k8s_hub_client, name=nat_eip_cr) + + nat_name = f"nat-{net_test_run_id}" + nat_id = grpc.create_nat_gateway( + name=nat_name, virtual_network_name=self.state["vnet_name"], external_ip_name=nat_eip_name + ) + poll_until( + fn=lambda: ( + grpc.call(service="osac.public.v1.NATGateways/Get", data={"id": nat_id}) + .get("object", {}) + .get("status", {}) + .get("state", "") + ), + until=lambda s: s in ("NAT_GATEWAY_STATE_READY", "Ready"), + retries=30, + delay=5, + description=f"NATGateway {nat_name} to become Ready", + ) + + self.__class__.state.update( + nat_eip_id=nat_eip_id, nat_eip_cr=nat_eip_cr, nat_eip_name=nat_eip_name, nat_id=nat_id, nat_name=nat_name + ) + + # ── Phase 2: Provision Servers ────────────────────────────────────── + + def test_05_create_three_bmis( + self, + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item_name: str, + auto_eip_catalog_item_name: str, + net_ssh_public_key: str, + bmh_namespace: str, + net_test_run_id: str, + ) -> None: + _require(self.state, "subnet_a_id", "subnet_b_id", "sg_id") + + subnet_a = self.state["subnet_a_id"] + subnet_b = self.state["subnet_b_id"] + sg = self.state["sg_id"] + + bmis: list[dict[str, str]] = [] + for i, (name_suffix, subnet_id) in enumerate([("bmi1", subnet_a), ("bmi2", subnet_a), ("bmi3", subnet_b)]): + bmi_name = f"{name_suffix}-{net_test_run_id}" + is_auto_eip = i == 2 + catalog = auto_eip_catalog_item_name if is_auto_eip else catalog_item_name + bmi_id = cli.create_baremetal_instance( + name=bmi_name, + catalog_item=catalog, + ssh_key=net_ssh_public_key, + network_attachments=[f"subnet={subnet_id},interface=eth9,primary,security-groups={sg}"], + external_ip_attachment=is_auto_eip, + ) + print(f"Created BMI {bmi_name}: {bmi_id} (catalog: {catalog})") + bmis.append({"name": bmi_name, "id": bmi_id, "subnet": "a" if i < 2 else "b"}) + + for bmi in bmis: + bmi["cr"] = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi["id"]) + print(f"BMI {bmi['name']} CR: {bmi['cr']}") + + for bmi in bmis: + wait_for_bmi_running(grpc=grpc, bmi_id=bmi["id"]) + print(f"BMI {bmi['name']} is RUNNING") + + for bmi in bmis: + bmi["ip"] = poll_until( + fn=lambda b=bmi: k8s_hub_client.get_baremetal_instance_tenant_ip(name=b["cr"]), + until=lambda ip: ip != "", + retries=60, + delay=10, + description=f"BMI {bmi['name']} tenant IP assignment", + ) + + ext_host = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi["cr"]) + bmi["bmh"] = ext_host.split("/", 1)[1] + bmi["bmc_ip"] = bmi_ssh.get_bmc_ip(bmi["bmh"]) + print(f"BMI {bmi['name']}: tenant_ip={bmi['ip']}, bmh={bmi['bmh']}, bmc_ip={bmi['bmc_ip']}") + + for bmi in bmis: + if bmi["subnet"] == "a": + assert bmi["ip"].startswith("10.100.1."), f"BMI {bmi['name']} IP {bmi['ip']} not in subnet A" + else: + assert bmi["ip"].startswith("10.100.2."), f"BMI {bmi['name']} IP {bmi['ip']} not in subnet B" + + self.__class__.state["bmi1"] = bmis[0] + self.__class__.state["bmi2"] = bmis[1] + self.__class__.state["bmi3"] = bmis[2] + + def test_05b_verify_auto_eip_on_bmi3(self, grpc: GRPCClient) -> None: + _require(self.state, "bmi3") + bmi3 = self.state["bmi3"] + + def find_auto_attachment() -> dict[str, Any] | None: + attachments = grpc.call(service="osac.public.v1.ExternalIPAttachments/List") + for item in attachments.get("items", []): + bmi_ref = item.get("spec", {}).get("baremetalInstance", {}).get("id", "") + if bmi_ref == bmi3["id"]: + return item + return None + + attachment = poll_until( + fn=find_auto_attachment, + until=lambda a: a is not None, + retries=60, + delay=5, + description="auto-created ExternalIPAttachment for BMI3", + ) + + auto_attach_id = attachment["id"] + auto_eip_ref = attachment.get("spec", {}).get("externalIp", {}).get("id", "") + assert auto_eip_ref, "Auto-created attachment has no ExternalIP reference" + + eip_data = grpc.get_external_ip(external_ip_id=auto_eip_ref) + auto_ext_addr = eip_data.get("object", {}).get("status", {}).get("address", "") + assert auto_ext_addr, "Auto-created ExternalIP has no allocated address" + + self.__class__.state.update( + auto_attach_id=auto_attach_id, auto_eip_id=auto_eip_ref, auto_ext_addr=auto_ext_addr + ) + print(f"BMI3 auto EIP: {auto_ext_addr}, attachment: {auto_attach_id}") + + # ── Phase 3: Connectivity Tests ───────────────────────────────────── + + def test_06_l2_arping_same_subnet(self) -> None: + _require(self.state, "bmi1", "bmi2") + bmi1 = self.state["bmi1"] + bmi2 = self.state["bmi2"] + + poll_until( + fn=lambda: bmi_ssh.arping(bmi1["bmc_ip"], bmi2["ip"]), + until=lambda ok: ok, + retries=5, + delay=10, + description=f"L2 arping BMI1 ({bmi1['ip']}) → BMI2 ({bmi2['ip']})", + ) + + def test_07_l3_ping_same_subnet(self) -> None: + _require(self.state, "bmi1", "bmi2") + bmi1 = self.state["bmi1"] + bmi2 = self.state["bmi2"] + + poll_until( + fn=lambda: bmi_ssh.ping(bmi1["bmc_ip"], bmi2["ip"]), + until=lambda ok: ok, + retries=5, + delay=10, + description=f"L3 ping BMI1 ({bmi1['ip']}) → BMI2 ({bmi2['ip']})", + ) + + def test_08_l3_ping_cross_subnet(self) -> None: + _require(self.state, "bmi1", "bmi3") + bmi1 = self.state["bmi1"] + bmi3 = self.state["bmi3"] + + poll_until( + fn=lambda: bmi_ssh.ping(bmi1["bmc_ip"], bmi3["ip"]), + until=lambda ok: ok, + retries=5, + delay=10, + description=f"L3 ping BMI1 ({bmi1['ip']}) → BMI3 ({bmi3['ip']}) cross-subnet", + ) + + def test_09_l2_arping_cross_subnet_fails(self) -> None: + _require(self.state, "bmi1", "bmi3") + bmi1 = self.state["bmi1"] + bmi3 = self.state["bmi3"] + + assert not bmi_ssh.arping(bmi1["bmc_ip"], bmi3["ip"]), ( + f"arping from BMI1 ({bmi1['ip']}, subnet A) to BMI3 ({bmi3['ip']}, subnet B) " + f"succeeded unexpectedly — different subnets should be different broadcast domains" + ) + + def test_10_tenant_isolation(self, mgmt_cluster_ip: str) -> None: + _require(self.state, "bmi1") + bmi1 = self.state["bmi1"] + + assert not bmi_ssh.ping(bmi1["bmc_ip"], mgmt_cluster_ip), ( + f"ping from BMI1 ({bmi1['ip']}) to management cluster ({mgmt_cluster_ip}) " + f"succeeded unexpectedly — tenant isolation should prevent cross-VNet traffic" + ) + + def test_11_nat_gateway_egress(self) -> None: + _require(self.state, "bmi1") + bmi1 = self.state["bmi1"] + + poll_until( + fn=lambda: bmi_ssh.curl_status(bmi1["bmc_ip"], "https://quay.io"), + until=lambda status: status == 200, + retries=5, + delay=15, + description=f"NAT gateway egress curl quay.io (bmc_ip={bmi1['bmc_ip']}, tenant_ip={bmi1['ip']})", + ) + + def test_12_external_ip_ingress( + self, grpc: GRPCClient, k8s_hub_client: K8sClient, net_test_run_id: str + ) -> None: + _require(self.state, "bmi1", "pool_id") + bmi1 = self.state["bmi1"] + + eip_name = f"ingress-eip-{net_test_run_id}" + eip_id = grpc.create_external_ip(name=eip_name, pool=self.state["pool_id"]) + eip_cr = wait_for_external_ip_cr(k8s=k8s_hub_client, uuid=eip_id) + wait_for_external_ip_allocated(k8s=k8s_hub_client, name=eip_cr) + + attach_name = f"ingress-attach-{net_test_run_id}" + attach_id = grpc.create_external_ip_attachment_bmi( + name=attach_name, external_ip=eip_id, baremetal_instance=bmi1["id"] + ) + attach_cr = wait_for_external_ip_attachment_cr(k8s=k8s_hub_client, uuid=attach_id) + wait_for_external_ip_attachment_ready(k8s=k8s_hub_client, name=attach_cr) + + eip_data = grpc.get_external_ip(external_ip_id=eip_id) + ext_addr = eip_data.get("object", {}).get("status", {}).get("address", "") + assert ext_addr, "ExternalIP has no allocated address" + + def _try_ssh_eip() -> str: + try: + return bmi_ssh.ssh_via_external_ip(ext_addr, timeout=10) + except subprocess.CalledProcessError: + return "" + + hostname = poll_until( + fn=_try_ssh_eip, + until=lambda h: bool(h), + retries=5, + delay=15, + description=f"SSH via external IP {ext_addr}", + ) + + self.__class__.state.update( + ingress_eip_id=eip_id, + ingress_eip_cr=eip_cr, + ingress_attach_id=attach_id, + ingress_attach_cr=attach_cr, + ingress_ext_addr=ext_addr, + ) + + # ── Phase 4: Teardown ─────────────────────────────────────────────── + + def test_13_delete_external_ip_attachment(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "ingress_attach_id" not in self.state: + pytest.skip("No ExternalIPAttachment to delete") + + grpc.delete_external_ip_attachment(attachment_id=self.state["ingress_attach_id"]) + wait_for_external_ip_attachment_deletion(k8s=k8s_hub_client, name=self.state["ingress_attach_cr"]) + + grpc.delete_external_ip(external_ip_id=self.state["ingress_eip_id"]) + wait_for_external_ip_deletion(k8s=k8s_hub_client, name=self.state["ingress_eip_cr"]) + + def test_14_delete_bmis( + self, cli: OsacCLI, grpc: GRPCClient, k8s_hub_client: K8sClient, bmh_namespace: str + ) -> None: + for key in ("bmi1", "bmi2", "bmi3"): + if key not in self.state: + continue + bmi = self.state[key] + print(f"Deleting {bmi['name']}...") + cli.delete_baremetal_instance(uuid=bmi["id"]) + + for key in ("bmi1", "bmi2", "bmi3"): + if key not in self.state: + continue + bmi = self.state[key] + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi["cr"]) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi["id"]) + wait_for_bmh_available(k8s=k8s_hub_client, name=bmi["bmh"], bmh_namespace=bmh_namespace) + print(f"{bmi['name']} deprovisioned, BMH {bmi['bmh']} available") + + def test_14b_verify_auto_eip_garbage_collected(self, grpc: GRPCClient) -> None: + if "auto_attach_id" not in self.state: + pytest.skip("No auto EIP to verify") + + poll_until( + fn=lambda: self.state["auto_attach_id"] not in grpc.list_external_ip_attachment_ids(), + until=lambda gone: gone is True, + retries=30, + delay=5, + description="auto-created ExternalIPAttachment garbage collection", + ) + + poll_until( + fn=lambda: self.state["auto_eip_id"] not in grpc.list_external_ip_ids(), + until=lambda gone: gone is True, + retries=30, + delay=5, + description="auto-created ExternalIP garbage collection", + ) + print("Auto EIP and attachment garbage collected after BMI3 deletion") + + def test_15_delete_nat_gateway(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "nat_id" not in self.state: + pytest.skip("No NATGateway to delete") + + grpc.delete_nat_gateway(nat_gateway_id=self.state["nat_id"]) + poll_until( + fn=lambda: ( + self.state["nat_id"] + not in [item["id"] for item in grpc.call(service="osac.public.v1.NATGateways/List").get("items", [])] + ), + until=lambda gone: gone is True, + retries=60, + delay=5, + description=f"NATGateway {self.state['nat_name']} deletion", + ) + + grpc.delete_external_ip(external_ip_id=self.state["nat_eip_id"]) + wait_for_external_ip_deletion(k8s=k8s_hub_client, name=self.state["nat_eip_cr"]) + + def test_16_delete_security_group(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "sg_id" not in self.state: + pytest.skip("No SecurityGroup to delete") + + grpc.delete_security_group(sg_id=self.state["sg_id"]) + wait_for_security_group_deletion(k8s=k8s_hub_client, name=self.state["sg_cr"]) + + def test_17_delete_subnets(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + for key, cr_key in [("subnet_a_id", "subnet_a_cr"), ("subnet_b_id", "subnet_b_cr")]: + if key not in self.state: + continue + grpc.delete_subnet(subnet_id=self.state[key]) + wait_for_subnet_deletion(k8s=k8s_hub_client, name=self.state[cr_key]) + + def test_18_delete_virtual_network(self, grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "vnet_id" not in self.state: + pytest.skip("No VirtualNetwork to delete") + + grpc.delete_virtual_network(vn_id=self.state["vnet_id"]) + wait_for_virtual_network_deletion(k8s=k8s_hub_client, name=self.state["vnet_cr"]) + + remaining = grpc.list_virtual_network_ids() + assert self.state["vnet_id"] not in remaining, "VirtualNetwork still in API after deletion" + + def test_19_delete_external_ip_pool(self, private_grpc: GRPCClient, k8s_hub_client: K8sClient) -> None: + if "pool_id" not in self.state: + pytest.skip("No ExternalIPPool to delete") + + private_grpc.delete_external_ip_pool(pool_id=self.state["pool_id"]) + wait_for_external_ip_pool_deletion(k8s=k8s_hub_client, name=self.state["pool_cr"]) + + remaining = private_grpc.list_external_ip_pool_ids() + assert self.state["pool_id"] not in remaining, "ExternalIPPool still in API after deletion" diff --git a/tests/bmaas/test_baremetal_instance_inventory_exhausted.py b/tests/bmaas/test_baremetal_instance_inventory_exhausted.py new file mode 100644 index 0000000000..70b6645d7b --- /dev/null +++ b/tests/bmaas/test_baremetal_instance_inventory_exhausted.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import re +import subprocess + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + assert_bmi_does_not_become_running, + assert_bmi_lifecycle_on_running, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, + wait_for_bmi_running_after_recovery, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +_AVAILABLE_BMH_STATES = {"available", "ready"} +_NOT_FOUND_RE = re.compile(r"Code:\s*NotFound|baremetalinstance\b.*\bnot found", re.IGNORECASE) + + +def _is_not_found(exc: subprocess.CalledProcessError) -> bool: + """Return True when delete failed because the BareMetalInstance is already gone.""" + combined = (exc.stderr or "") + (exc.stdout or "") + return bool(_NOT_FOUND_RE.search(combined)) + + +def _cleanup_bmi(*, cli: OsacCLI, grpc: GRPCClient, k8s: K8sClient, bmi_id: str) -> None: + """Delete a BareMetalInstance and wait for CR/gRPC removal. + + Only a NotFound response from delete is treated as success (already gone). + Deletion/removal timeouts and other errors are raised. + """ + bmi_cr: str = k8s.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if not bmi_cr and bmi_id not in grpc.list_baremetal_instance_ids(): + return + + try: + cli.delete_baremetal_instance(uuid=bmi_id) + except subprocess.CalledProcessError as exc: + if not _is_not_found(exc): + raise + + if bmi_cr and k8s.is_present(resource="baremetalinstance", name=bmi_cr): + wait_for_bmi_deletion(k8s=k8s, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + + +@pytest.mark.regression +def test_baremetal_instance_inventory_exhausted( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + """Exhaust BMH inventory, assert overflow stalls/fails, then recover after free capacity. + + Portable across labs/CI (e.g. 2 virtual BMHs → BMI #1, #2, then #3 overflow): + 1. Create N BMIs up front so cluster-side provisioning can overlap + 2. Wait for all N to reach RUNNING, then run full lifecycle checks on BMI #1 + 3. Create BMI N+1 and assert it does not reach RUNNING (no free BMH) + 4. Delete one claimed BMI to free a BMH + 5. Assert the overflow BMI recovers to RUNNING + 6. Cleanup remaining instances + """ + available_count: int = k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ) + assert available_count >= 1, ( + f"Need at least 1 available BMH to exercise inventory exhaustion; found {available_count}" + ) + + bmi_ids: list[str] = [] + try: + # Kick off all claim BMIs first so provisioning can proceed in parallel. + for idx in range(1, available_count + 1): + bmi_id = cli.create_baremetal_instance( + name=f"e2e-bmi-inv-{test_run_id}-{idx}", catalog_item=catalog_item, ssh_key=ssh_public_key + ) + bmi_ids.append(bmi_id) + + for bmi_id in bmi_ids: + wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + # Lifecycle checks after all claim BMIs are up (not interleaved with creates). + assert_bmi_lifecycle_on_running( + grpc=grpc, + k8s=k8s_hub_client, + bmi_id=bmi_ids[0], + bmh_namespace=bmh_namespace, + ) + + available_after_claim: int = k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ) + assert available_after_claim == 0, ( + f"Expected 0 available BMHs after claiming {available_count} hosts; found {available_after_claim}" + ) + + overflow_idx = available_count + 1 + overflow_id: str = cli.create_baremetal_instance( + name=f"e2e-bmi-inv-{test_run_id}-{overflow_idx}", catalog_item=catalog_item, ssh_key=ssh_public_key + ) + bmi_ids.append(overflow_id) + assert overflow_id in grpc.list_baremetal_instance_ids() + wait_for_bmi_cr(k8s=k8s_hub_client, uuid=overflow_id) + + overflow_state: str = assert_bmi_does_not_become_running(grpc=grpc, bmi_id=overflow_id) + assert overflow_state != "BARE_METAL_INSTANCE_STATE_RUNNING" + + available_during_overflow: int = k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ) + assert available_during_overflow == 0, ( + f"Overflow BMI must not claim a BMH; available count={available_during_overflow}" + ) + + # Free one BMH so the overflow instance can be scheduled. + released_id: str = bmi_ids[0] + _cleanup_bmi(cli=cli, grpc=grpc, k8s=k8s_hub_client, bmi_id=released_id) + bmi_ids.remove(released_id) + + poll_until( + fn=lambda: k8s_hub_client.count_bmhs_by_provisioning_state( + bmh_namespace=bmh_namespace, states=_AVAILABLE_BMH_STATES + ), + until=lambda v: v >= 1, + retries=120, + delay=10, + description="at least one BMH available after release", + ) + + wait_for_bmi_running_after_recovery(grpc=grpc, bmi_id=overflow_id) + finally: + for bmi_id in reversed(bmi_ids): + _cleanup_bmi(cli=cli, grpc=grpc, k8s=k8s_hub_client, bmi_id=bmi_id) diff --git a/tests/bmaas/test_baremetal_instance_lifecycle.py b/tests/bmaas/test_baremetal_instance_lifecycle.py new file mode 100644 index 0000000000..7b6a6407d4 --- /dev/null +++ b/tests/bmaas/test_baremetal_instance_lifecycle.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import pytest +import logging +import re +from typing import Any + +from tests.core.grpc_client import GRPCClient +from tests.core.helpers import ( + wait_for_bmh_available, + wait_for_bmh_provisioned, + wait_for_bmi_cr, + wait_for_bmi_deletion, + wait_for_bmi_grpc_removal, + wait_for_bmi_running, +) +from tests.core.k8s_client import K8sClient +from tests.core.osac_cli import OsacCLI +from tests.core.runner import poll_until + +logger = logging.getLogger(__name__) + +_RESTART_IN_PROGRESS: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_IN_PROGRESS" +_RESTART_FAILED: str = "BARE_METAL_INSTANCE_CONDITION_TYPE_RESTART_FAILED" +_MAC_PATTERN: re.Pattern[str] = re.compile(r"^([0-9a-f]{2}:){5}[0-9a-f]{2}$") + + +def _assert_nic_metadata( + *, + grpc: GRPCClient, + bmi_id: str, + cli: OsacCLI, + bmi_name: str, + bmi_cr_name: str, + k8s: K8sClient, + bmh_name: str, + bmh_namespace: str, +) -> None: + """Verify NIC MAC addresses propagate correctly from BMH → BMI CR → gRPC API → CLI.""" + # 1. BMH is the source of truth — hardware inspection provides the authoritative MAC list + bmh_macs: set[str] = set(k8s.get_bmh_hardware_nics(name=bmh_name, bmh_namespace=bmh_namespace)) + assert bmh_macs, f"BareMetalHost {bmh_name} has no hardware.nics — inspection may not have completed" + + # 2. BMI CR status.hardware.nics must match the BMH + cr_macs: set[str] = set(k8s.get_bmi_hardware_nics(name=bmi_cr_name)) + assert cr_macs, f"BareMetalInstance CR {bmi_cr_name} has no status.hardware.nics" + assert cr_macs == bmh_macs, ( + f"BMI CR status.hardware.nics {sorted(cr_macs)} does not match " + f"BareMetalHost hardware.nics {sorted(bmh_macs)}" + ) + + # 3. gRPC API response must match the BMI CR + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + nics: list[dict[str, Any]] = response.get("object", {}).get("status", {}).get("hardware", {}).get("nics", []) + assert nics, f"gRPC API returned no status.hardware.nics for BMI {bmi_id}" + api_macs: set[str] = {nic.get("mac", "").lower() for nic in nics} + assert api_macs == bmh_macs, ( + f"gRPC API status.hardware.nics {sorted(api_macs)} does not match " + f"BareMetalHost hardware.nics {sorted(bmh_macs)}" + ) + + # 4. CLI describe output must list all BMH MACs under the Network Interfaces section + describe_output: str = cli.describe_baremetal_instance(name=bmi_name) + assert "Network Interfaces:" in describe_output, ( + "osac describe baremetalinstance output missing 'Network Interfaces:' section" + ) + ni_section = describe_output[describe_output.index("Network Interfaces:"):] + for mac in bmh_macs: + assert mac in ni_section, ( + f"osac describe baremetalinstance 'Network Interfaces:' section missing MAC '{mac}'" + ) + + +def _get_condition_status(grpc: GRPCClient, bmi_id: str, condition_type: str) -> str: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + for condition in response.get("object", {}).get("status", {}).get("conditions", []): + if condition.get("type") == condition_type: + return condition.get("status", "") + return "" + + +def _get_status_restart_trigger(grpc: GRPCClient, bmi_id: str) -> int: + response: dict[str, Any] = grpc.get_baremetal_instance(bmi_id=bmi_id) + return int(response.get("object", {}).get("status", {}).get("restartTrigger", "0")) + + +@pytest.mark.sanity +def test_baremetal_instance_lifecycle( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name = f"e2e-bmi-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + bmh_ns = "" + bmh_name = "" + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + # Verify NIC metadata matches the BMH hardware inventory (OSAC-3254) + _assert_nic_metadata( + grpc=grpc, + bmi_id=bmi_id, + cli=cli, + bmi_name=name, + bmi_cr_name=bmi_cr_name, + k8s=k8s_hub_client, + bmh_name=bmh_name, + bmh_namespace=bmh_ns, + ) + + # Verify provisioning + wait_for_bmh_provisioned(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + + image_url: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url != "", f"BMH {bmh_name} has no image URL after provisioning" + + consumer_ref: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref != "", f"BMH {bmh_name} has no consumerRef after allocation" + + online: str = k8s_hub_client.get_bmh_online(name=bmh_name, bmh_namespace=bmh_ns) + assert online == "true", f"BMH {bmh_name} should be online after provisioning, got: {online}" + + # Power off + halted = "BARE_METAL_INSTANCE_RUN_STRATEGY_HALTED" + grpc.update_baremetal_instance_run_strategy(bmi_id=bmi_id, run_strategy=halted) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "false", + retries=60, + delay=5, + description=f"{bmh_name} powered off", + ) + + # Power on + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_ALWAYS" + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on", + ) + + # Deprovision + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + + image_url_after: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url_after == "", f"BMH {bmh_name} image not cleared after deprovision: {image_url_after}" + + consumer_ref_after: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref_after == "", f"BMH {bmh_name} consumerRef not cleared: {consumer_ref_after}" + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + if bmh_name: + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + except Exception: + pass + raise + + +@pytest.mark.sanity +def test_baremetal_instance_restart( + cli: OsacCLI, + grpc: GRPCClient, + k8s_hub_client: K8sClient, + catalog_item: str, + bmh_namespace: str, + test_run_id: str, + ssh_public_key: str, +) -> None: + name: str = f"e2e-bmi-restart-{test_run_id}" + bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key) + + try: + assert bmi_id in grpc.list_baremetal_instance_ids() + + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id) + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + initial_trigger: int = _get_status_restart_trigger(grpc, bmi_id) + new_trigger: int = initial_trigger + 1 + logger.info("Incrementing restart_trigger from %d to %d", initial_trigger, new_trigger) + + grpc.update_baremetal_instance_restart_trigger(bmi_id=bmi_id, restart_trigger=new_trigger) + + poll_until( + fn=lambda: _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS), + until=lambda v: v == "CONDITION_STATUS_TRUE", + retries=60, + delay=2, + description=f"{bmi_id} RESTART_IN_PROGRESS condition appears", + ) + + poll_until( + fn=lambda: _get_status_restart_trigger(grpc, bmi_id), + until=lambda v: v == new_trigger, + retries=120, + delay=10, + description=f"{bmi_id} status.restart_trigger echoes {new_trigger}", + ) + + poll_until( + fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on after restart", + ) + + wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id) + + restart_in_progress: str = _get_condition_status(grpc, bmi_id, _RESTART_IN_PROGRESS) + assert restart_in_progress in ("", "CONDITION_STATUS_FALSE"), ( + f"RESTART_IN_PROGRESS should have cleared after restart, got: {restart_in_progress}" + ) + + restart_failed: str = _get_condition_status(grpc, bmi_id, _RESTART_FAILED) + assert restart_failed in ("", "CONDITION_STATUS_FALSE"), ( + f"Unexpected RESTART_FAILED condition: {restart_failed}" + ) + + # Deprovision + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns) + except BaseException: + bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False) + if bmi_cr: + try: + cli.delete_baremetal_instance(uuid=bmi_id) + wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr) + wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id) + except Exception: + logger.exception("Failed to delete BMI %s during cleanup", bmi_id) + raise diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..d08b602a8a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import contextlib +import os +import subprocess +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tests.core.grpc_client import PRIVATE_API, GRPCClient +from tests.core.k8s_client import K8sClient +from tests.core.keycloak import get_jwt +from tests.core.keycloak_admin import ( + add_user_to_organization, + add_user_to_organization_group, + ensure_organization_group, + get_admin_token, + get_user_id, + wait_for_organization, +) +from tests.core.metering import MeteringCollector +from tests.core.osac_cli import OsacCLI +from tests.core.runner import env, run + + +def pytest_configure(config: pytest.Config) -> None: + """Give each xdist worker its own log_file so records stay chronologically ordered. + + A shared log_file is interleaved across workers as they report at their own + pace; gather-osac-logs.sh merges the per-worker files back into one sorted + e2e.log artifact. + """ + config.addinivalue_line("markers", "metering: test verifies metering events via the test adapter HTTP API") + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + if worker_id is not None: + log_dir = Path(config.getini("log_file")).parent + log_dir.mkdir(parents=True, exist_ok=True) + config.option.log_file = str(log_dir / f"e2e_{worker_id}.log") + + +@pytest.fixture(scope="session") +def namespace() -> str: + return env("OSAC_NAMESPACE", "osac-devel") + + +@pytest.fixture(scope="session") +def cluster_domain() -> str: + return run("kubectl", "get", "ingress.config.openshift.io", "cluster", "-o", "jsonpath={.spec.domain}") + + +@pytest.fixture(scope="session") +def fulfillment_address(namespace: str, cluster_domain: str) -> str: + return env("OSAC_FULFILLMENT_ADDRESS", f"fulfillment-api-{namespace}.{cluster_domain}:443") + + +@pytest.fixture(scope="session") +def fulfillment_private_address(namespace: str, cluster_domain: str) -> str: + return env("OSAC_FULFILLMENT_PRIVATE_ADDRESS", f"fulfillment-internal-api-{namespace}.{cluster_domain}:443") + + +@pytest.fixture(scope="session") +def service_account() -> str: + return env("OSAC_SERVICE_ACCOUNT", "admin") + + +@pytest.fixture(scope="session") +def keycloak_realm() -> str: + return env("OSAC_KEYCLOAK_REALM", "osac") + + +@pytest.fixture(scope="session") +def keycloak_client_id() -> str: + return env("OSAC_KEYCLOAK_CLIENT_ID", "osac-cli") + + +@pytest.fixture(scope="session") +def jwt_username() -> str: + return env("OSAC_JWT_USERNAME", "tenant1_admin") + + +@pytest.fixture(scope="session") +def grpc( + fulfillment_address: str, + keycloak_url: str, + keycloak_realm: str, + keycloak_client_id: str, + jwt_username: str, + jwt_password: str, +) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm=keycloak_realm, + client_id=keycloak_client_id, + username=jwt_username, + password=jwt_password, + ), + ) + + +@pytest.fixture(scope="session") +def private_grpc(fulfillment_private_address: str, namespace: str, service_account: str) -> GRPCClient: + token: str = run( + "oc", "create", "token", service_account, "-n", namespace, "--duration", "4h", "--as", "system:admin" + ) + return GRPCClient(address=fulfillment_private_address, token=token) + + +@pytest.fixture(scope="session", autouse=True) +def ensure_tenants(private_grpc: GRPCClient) -> None: + for name in ("tenant1", "tenant2"): + private_grpc.ensure_tenant(name=name) + + +@pytest.fixture(scope="session", autouse=True) +def ensure_jwt_users(ensure_tenants: None, private_grpc: GRPCClient) -> None: + """Pre-create users that JWT fixtures authenticate as, so JIT provisioning + doesn't race on concurrent first requests from xdist workers.""" + for username, tenant in [ + ("tenant1_admin", "tenant1"), + ("tenant1_user", "tenant1"), + ("tenant2_user", "tenant2"), + ("tenant2_admin", "tenant2"), + ]: + with contextlib.suppress(subprocess.CalledProcessError): + private_grpc.call( + service=f"{PRIVATE_API}.Users/Create", + data={ + "object": { + "metadata": {"name": username.replace("_", "-"), "tenant": tenant}, + "spec": {"username": username, "enabled": True}, + } + }, + ) + + +@pytest.fixture(scope="session") +def keycloak_admin_password() -> str: + return env("OSAC_KEYCLOAK_ADMIN_PASSWORD", "admin") + + +@pytest.fixture(scope="session", autouse=True) +def setup_organization_memberships( + ensure_tenants: None, keycloak_url: str, keycloak_admin_password: str +) -> None: + """ + Add test users to their corresponding Keycloak organizations. + This runs after ensure_tenants creates the Tenant resources, which the + tenant controller syncs to Keycloak as organizations. + """ + # Get admin token for Keycloak admin API + admin_token = get_admin_token(keycloak_url=keycloak_url, username="admin", password=keycloak_admin_password) + + # Map of organization name -> list of usernames + org_users = { + "tenant1": ["tenant1_user", "tenant1_admin"], + "tenant2": ["tenant2_user", "tenant2_admin"], + } + + for org_name, usernames in org_users.items(): + # Wait for the organization to be synced to Keycloak by the tenant controller + org_id = wait_for_organization(keycloak_url=keycloak_url, admin_token=admin_token, org_name=org_name) + + # Add each user to the organization + for username in usernames: + user_id = get_user_id(keycloak_url=keycloak_url, admin_token=admin_token, username=username) + add_user_to_organization( + keycloak_url=keycloak_url, + admin_token=admin_token, + org_id=org_id, + user_id=user_id, + username=username, + org_name=org_name, + ) + + # Create /members group in the organization and add all users to it + # This is required for the organization scope to include the organization in the JWT token + group_id = ensure_organization_group( + keycloak_url=keycloak_url, admin_token=admin_token, org_id=org_id, org_name=org_name + ) + + for username in usernames: + user_id = get_user_id(keycloak_url=keycloak_url, admin_token=admin_token, username=username) + add_user_to_organization_group( + keycloak_url=keycloak_url, + admin_token=admin_token, + org_id=org_id, + group_id=group_id, + user_id=user_id, + username=username, + org_name=org_name, + ) + + +@pytest.fixture(scope="session") +def k8s_hub_client(namespace: str) -> K8sClient: + return K8sClient(namespace=namespace) + + +@pytest.fixture(scope="session") +def cli(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_username: str, jwt_password: str) -> Iterator[OsacCLI]: # noqa: E501 + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, jwt_username, jwt_password), + namespace=namespace, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def private_cli(namespace: str, fulfillment_private_address: str, service_account: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_private_address.rsplit(':', 1)[0]}", + token_script=f"oc create token -n {namespace} {service_account} --as system:admin", + namespace=namespace, + private=True, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def keycloak_url(cluster_domain: str) -> str: + return env("OSAC_KEYCLOAK_URL", f"https://keycloak-keycloak.{cluster_domain}") + + +@pytest.fixture(scope="session") +def jwt_password() -> str: + return env("OSAC_JWT_PASSWORD", "foobar") + + +def _make_jwt_token_script(keycloak_url: str, username: str, password: str) -> str: + return ( + f"curl -sk -X POST {keycloak_url}/realms/osac/protocol/openid-connect/token" + f" -d grant_type=password -d client_id=osac-cli" + f" -d username={username} -d password={password} -d 'scope=openid organization'" + " | python3 -c \"import sys,json;print(json.load(sys.stdin)['access_token'])\"" + ) + + +@pytest.fixture(scope="session") +def jwt_cli_user(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, "tenant1_user", jwt_password), + namespace=namespace, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def jwt_cli_admin(namespace: str, fulfillment_address: str, keycloak_url: str, jwt_password: str) -> Iterator[OsacCLI]: + instance = OsacCLI( + binary=env("OSAC_CLI_PATH", "osac"), + address=f"https://{fulfillment_address.rsplit(':', 1)[0]}", + token_script=_make_jwt_token_script(keycloak_url, "tenant1_admin", jwt_password), + namespace=namespace, + ) + yield instance + instance.close() + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant1_admin(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_admin", + password=jwt_password, + ), + ) + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant1(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant1_user", + password=jwt_password, + ), + ) + + +@pytest.fixture(scope="session") +def jwt_grpc_tenant2(fulfillment_address: str, keycloak_url: str, jwt_password: str) -> GRPCClient: + return GRPCClient( + address=fulfillment_address, + token_factory=lambda: get_jwt( + keycloak_url=keycloak_url, + realm="osac", + client_id="osac-cli", + username="tenant2_user", + password=jwt_password, + ), + ) + + +# --- Cross-cutting concern: Metering --- + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + metering_tests = [item for item in items if item.get_closest_marker("metering")] + if metering_tests and not os.environ.get("METERING_ADAPTER_URL"): + pytest.fail( + f"METERING_ADAPTER_URL is not set but {len(metering_tests)} test(s) require metering. " + "Set the env var or run with -m 'not metering' to exclude metering tests.", + pytrace=False, + ) + + +@pytest.fixture +def metering() -> Iterator[MeteringCollector]: + """Composable metering verifier. Inject into any test to verify + that lifecycle events appear via the test adapter HTTP API. + + Records a start timestamp on setup, verifies all expectations + (including CloudEvent structure validation) on teardown. + """ + collector = MeteringCollector(base_url=env("METERING_ADAPTER_URL")) + collector.start() + yield collector + try: + collector.verify() + finally: + collector.stop() diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/grpc_client.py b/tests/core/grpc_client.py new file mode 100644 index 0000000000..fb5e82661a --- /dev/null +++ b/tests/core/grpc_client.py @@ -0,0 +1,709 @@ +from __future__ import annotations + +import json +import re +import subprocess +import time +from collections.abc import Callable +from typing import Any + +from tests.core.runner import run, run_unchecked + +PUBLIC_API: str = "osac.public.v1" +PRIVATE_API: str = "osac.private.v1" + + +class GRPCClient: + _TOKEN_TTL: float = 60.0 + + def __init__(self, *, address: str, token: str = "", token_factory: Callable[[], str] | None = None) -> None: + self.address: str = address + self._static_token: str = token + self._token_factory: Callable[[], str] | None = token_factory + self._cached_token: str = "" + self._cached_at: float = 0.0 + + @property + def token(self) -> str: + if self._token_factory is not None: + now = time.monotonic() + if not self._cached_token or (now - self._cached_at) >= self._TOKEN_TTL: + self._cached_token = self._token_factory() + self._cached_at = now + return self._cached_token + return self._static_token + + def _build_args(self, *, service: str, data: dict[str, Any] | None = None) -> list[str]: + args: list[str] = ["grpcurl", "-insecure", "-H", f"Authorization: Bearer {self.token}"] + if data is not None: + args.extend(["-d", json.dumps(data)]) + args.extend([self.address, service]) + return args + + def call(self, *, service: str, data: dict[str, Any] | None = None) -> dict[str, Any]: + return json.loads(run(*self._build_args(service=service, data=data))) + + def create_compute_instance(self, *, catalog_item: str, subnet_ids: list[str], name: str | None = None) -> str: + attachments = [{"subnet": {"id": sid}} for sid in subnet_ids] + obj: dict[str, Any] = {"spec": {"catalog_item": {"id": catalog_item}, "network_attachments": attachments}} + if name is not None: + obj["metadata"] = {"name": name} + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/Create", data={"object": obj}) + return response["object"]["id"] + + def update_compute_instance_run_strategy(self, *, ci_id: str, run_strategy: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.ComputeInstances/Update", + data={ + "object": {"id": ci_id, "spec": {"run_strategy": run_strategy}}, + "updateMask": {"paths": ["spec.run_strategy"]}, + }, + ) + + def delete_compute_instance(self, *, ci_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ComputeInstances/Delete", data={"id": ci_id}) + + def list_compute_instance_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstances/List") + return [item["id"] for item in response.get("items", [])] + + def get_compute_instance(self, *, ci_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ComputeInstances/Get", data={"id": ci_id}) + + def get_hub(self, *, hub_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.Hubs/Get", data={"id": hub_id}) + + def update_restart(self, *, uuid: str, template: str, timestamp: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.ComputeInstances/Update", + data={ + "object": {"id": uuid, "spec": {"template": {"name": template}, "restart_requested_at": timestamp}}, + "updateMask": {"paths": ["spec.restart_requested_at"]}, + }, + ) + + # VirtualNetwork operations + + def create_virtual_network(self, *, name: str, ipv4_cidr: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.VirtualNetworks/Create", + data={"object": {"metadata": {"name": name}, "spec": {"ipv4_cidr": ipv4_cidr}}}, + ) + return response["object"]["id"] + + def get_virtual_network(self, *, vn_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.VirtualNetworks/Get", data={"id": vn_id}) + + def list_virtual_network_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.VirtualNetworks/List") + return [item["id"] for item in response.get("items", [])] + + def delete_virtual_network(self, *, vn_id: str) -> None: + self.call(service=f"{PUBLIC_API}.VirtualNetworks/Delete", data={"id": vn_id}) + + # Subnet operations + + def create_subnet(self, *, name: str, virtual_network: str, ipv4_cidr: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.Subnets/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"virtual_network": {"id": virtual_network}, "ipv4_cidr": ipv4_cidr}, + } + }, + ) + return response["object"]["id"] + + def get_subnet(self, *, subnet_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.Subnets/Get", data={"id": subnet_id}) + + def list_subnet_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Subnets/List") + return [item["id"] for item in response.get("items", [])] + + def delete_subnet(self, *, subnet_id: str) -> None: + self.call(service=f"{PUBLIC_API}.Subnets/Delete", data={"id": subnet_id}) + + def call_unchecked(self, *, service: str, data: dict[str, Any] | None = None) -> tuple[str, int]: + return run_unchecked(*self._build_args(service=service, data=data)) + + # Cluster operations + + def list_cluster_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.Clusters/List") + return [item["id"] for item in response.get("items", [])] + + def get_cluster(self, *, cluster_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.Clusters/Get", data={"id": cluster_id}) + + # SecurityGroup operations + + def create_security_group(self, *, name: str, virtual_network: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", + data={"object": {"metadata": {"name": name}, "spec": {"virtual_network": {"id": virtual_network}}}}, + ) + return response["object"]["id"] + + def get_security_group(self, *, sg_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.SecurityGroups/Get", data={"id": sg_id}) + + def list_security_group_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.SecurityGroups/List") + return [item["id"] for item in response.get("items", [])] + + def delete_security_group(self, *, sg_id: str) -> None: + self.call(service=f"{PUBLIC_API}.SecurityGroups/Delete", data={"id": sg_id}) + + def create_security_group_with_rules( + self, + *, + name: str, + virtual_network: str, + ingress: list[dict[str, Any]] | None = None, + egress: list[dict[str, Any]] | None = None, + ) -> str: + spec: dict[str, Any] = {"virtual_network": {"id": virtual_network}} + if ingress is not None: + spec["ingress"] = ingress + if egress is not None: + spec["egress"] = egress + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.SecurityGroups/Create", data={"object": {"metadata": {"name": name}, "spec": spec}} + ) + return response["object"]["id"] + + def update_security_group_rules( + self, *, sg_id: str, ingress: list[dict[str, Any]] | None = None, egress: list[dict[str, Any]] | None = None + ) -> None: + spec: dict[str, Any] = {} + paths: list[str] = [] + if ingress is not None: + spec["ingress"] = ingress + paths.append("spec.ingress") + if egress is not None: + spec["egress"] = egress + paths.append("spec.egress") + self.call( + service=f"{PUBLIC_API}.SecurityGroups/Update", + data={"object": {"id": sg_id, "spec": spec}, "updateMask": {"paths": paths}}, + ) + + # Console operations + + def create_console_session( + self, *, resource_type: str, resource_id: str, console_type: str, client_id: str = "" + ) -> dict[str, Any]: + data: dict[str, Any] = { + "object": {"resourceType": resource_type, "resourceId": resource_id, "type": console_type} + } + if client_id: + data["object"]["clientId"] = client_id + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ConsoleSessions/Create", data=data) + return response["object"] + + # Tenant operations + + def ensure_tenant(self, *, name: str, retries: int = 10, delay: int = 5) -> None: + # OSAC-3553: retry the transient `code = Unavailable` / connection-refused + # transport failure the first call can hit after `helm --wait` returns, + # before the route converges. AlreadyExists is idempotent success; any + # other error still fails fast. Note the two error shapes are matched + # differently: a completed RPC prints grpcurl's `Code: ` block + # (AlreadyExists), while a transport failure prints the Go status string + # `code = Unavailable desc = ...` -- see tests/vmaas/external_ip/conftest.py. + for attempt in range(retries): + try: + self.call(service=f"{PRIVATE_API}.Tenants/Create", data={"object": {"metadata": {"name": name}}}) + return + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if re.search(r"Code:\s*AlreadyExists", output): + return + if attempt < retries - 1 and ("Unavailable" in output or "connection refused" in output.lower()): + time.sleep(delay) + continue + raise RuntimeError(f"Failed to create tenant '{name}': {output}") from e + + # ExternalIPPool operations (private API only) + + def create_external_ip_pool( + self, + *, + name: str, + cidrs: list[str], + ip_family: str = "IP_FAMILY_IPV4", + implementation_strategy: str = "", + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ExternalIPPools/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "cidrs": cidrs, + "ip_family": ip_family, + "implementation_strategy": implementation_strategy, + }, + } + }, + ) + return response["object"]["id"] + + def get_external_ip_pool(self, *, pool_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ExternalIPPools/Get", data={"id": pool_id}) + + def list_external_ip_pool_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ExternalIPPools/List") + return [item["id"] for item in response.get("items", [])] + + def delete_external_ip_pool(self, *, pool_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ExternalIPPools/Delete", data={"id": pool_id}) + + # ExternalIP operations (public API) + + def create_external_ip(self, *, name: str, pool: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ExternalIPs/Create", + data={"object": {"metadata": {"name": name}, "spec": {"pool": {"id": pool}}}}, + ) + return response["object"]["id"] + + def get_external_ip(self, *, external_ip_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ExternalIPs/Get", data={"id": external_ip_id}) + + def list_external_ip_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ExternalIPs/List") + return [item["id"] for item in response.get("items", [])] + + def delete_external_ip(self, *, external_ip_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ExternalIPs/Delete", data={"id": external_ip_id}) + + # ExternalIPAttachment operations (public API) + + def create_external_ip_attachment(self, *, name: str, external_ip: str, compute_instance: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ExternalIPAttachments/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"external_ip": {"id": external_ip}, "compute_instance": {"id": compute_instance}}, + } + }, + ) + return response["object"]["id"] + + def get_external_ip_attachment(self, *, attachment_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Get", data={"id": attachment_id}) + + def list_external_ip_attachment_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/List") + return [item["id"] for item in response.get("items", [])] + + def delete_external_ip_attachment(self, *, attachment_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ExternalIPAttachments/Delete", data={"id": attachment_id}) + + def create_external_ip_attachment_bmi(self, *, name: str, external_ip: str, baremetal_instance: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ExternalIPAttachments/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"external_ip": {"id": external_ip}, "baremetal_instance": {"id": baremetal_instance}}, + } + }, + ) + return response["object"]["id"] + + # ClusterCatalogItem operations + + def create_cluster_catalog_item( + self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None + ) -> str: + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": name, + "template": {"name": template}, + "published": published, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Create", data={"object": obj}) + return response["object"]["id"] + + def get_cluster_catalog_item(self, *, catalog_item_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/Get", data={"id": catalog_item_id}) + + def list_cluster_catalog_item_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ClusterCatalogItems/List") + return [item["id"] for item in response.get("items", [])] + + def update_cluster_catalog_item(self, *, catalog_item_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_cluster_catalog_item requires at least one field to update") + obj: dict[str, Any] = {"id": catalog_item_id, **fields} + data: dict[str, Any] = {"object": obj, "update_mask": {"paths": list(fields.keys())}} + return self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Update", data=data) + + def delete_cluster_catalog_item(self, *, catalog_item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ClusterCatalogItems/Delete", data={"id": catalog_item_id}) + + # ComputeInstanceCatalogItem operations + + def create_compute_instance_catalog_item( + self, *, name: str, template: str, published: bool = True, field_definitions: list[dict[str, Any]] | None = None + ) -> str: + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": name, + "template": {"name": template}, + "published": published, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Create", data={"object": obj} + ) + return response["object"]["id"] + + def get_compute_instance_catalog_item(self, *, catalog_item_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/Get", data={"id": catalog_item_id}) + + def list_compute_instance_catalog_item_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.ComputeInstanceCatalogItems/List") + return [item["id"] for item in response.get("items", [])] + + def update_compute_instance_catalog_item(self, *, catalog_item_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_compute_instance_catalog_item requires at least one field to update") + obj: dict[str, Any] = {"id": catalog_item_id, **fields} + data: dict[str, Any] = {"object": obj, "update_mask": {"paths": list(fields.keys())}} + return self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Update", data=data) + + def delete_compute_instance_catalog_item(self, *, catalog_item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ComputeInstanceCatalogItems/Delete", data={"id": catalog_item_id}) + + # InstanceType operations (private API only) + + def create_instance_type( + self, *, name: str, cores: int, memory_gib: int, description: str = "", gpu: dict[str, Any] | None = None + ) -> str: + spec: dict[str, Any] = {"cores": cores, "memory_gib": memory_gib, "description": description} + if gpu is not None: + spec["gpu"] = gpu + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.InstanceTypes/Create", data={"object": {"metadata": {"name": name}, "spec": spec}} + ) + return response["object"]["id"] + + def get_instance_type(self, *, name: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.InstanceTypes/Get", data={"id": name}) + + def list_instance_type_names(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.InstanceTypes/List") + return [item["metadata"]["name"] for item in response.get("items", [])] + + def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]: + return self.call( + service=f"{PRIVATE_API}.InstanceTypes/Update", + data={"object": {"id": name, "spec": {"state": state}}, "updateMask": {"paths": ["spec.state"]}}, + ) + + def delete_instance_type(self, *, name: str) -> None: + self.call(service=f"{PRIVATE_API}.InstanceTypes/Delete", data={"id": name}) + + # ClusterVersion operations (private API only) + + def create_cluster_version( + self, + *, + version: str, + image: str, + enabled: bool = True, + is_default: bool = False, + state: str = "CLUSTER_VERSION_STATE_ACTIVE", + ) -> dict[str, str]: + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ClusterVersions/Create", + data={ + "object": { + "spec": { + "version": version, + "image": image, + "enabled": enabled, + "is_default": is_default, + "state": state, + } + } + }, + ) + cluster_version: dict[str, Any] = response["object"] + return {"id": cluster_version["id"], "name": cluster_version["metadata"]["name"]} + + def get_cluster_version(self, *, version_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ClusterVersions/Get", data={"id": version_id}) + + def list_cluster_version_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterVersions/List") + return [item["id"] for item in response.get("items", [])] + + def update_cluster_version(self, *, version_id: str, **fields: Any) -> dict[str, Any]: + if not fields: + raise ValueError("update_cluster_version requires at least one field to update") + return self.call( + service=f"{PRIVATE_API}.ClusterVersions/Update", + data={ + "object": {"id": version_id, "spec": dict(fields)}, + "updateMask": {"paths": [f"spec.{k}" for k in fields]}, + }, + ) + + def delete_cluster_version(self, *, version_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ClusterVersions/Delete", data={"id": version_id}) + + def ensure_cluster_version(self, *, version: str, image: str) -> dict[str, str]: + """Create a ClusterVersion, tolerating AlreadyExists left behind by a prior failed run. + + Returns {"id": ..., "name": ...} for the resolved ClusterVersion.""" + try: + return self.create_cluster_version(version=version, image=image) + except subprocess.CalledProcessError as e: + output = (e.stdout or "") + (e.stderr or "") + if not re.search(r"Code:\s*AlreadyExists", output): + raise RuntimeError(f"Failed to create cluster version '{version}': {output}") from e + response: dict[str, Any] = self.call(service=f"{PRIVATE_API}.ClusterVersions/List") + for item in response.get("items", []): + if item.get("spec", {}).get("version") == version: + return {"id": item["id"], "name": item["metadata"]["name"]} + raise RuntimeError(f"Cluster version '{version}' reported AlreadyExists but not found in list") + + # BareMetalInstance operations (public API) + + def list_baremetal_instance_ids(self) -> list[str]: + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.BareMetalInstances/List") + return [item["id"] for item in response.get("items", [])] + + def get_baremetal_instance(self, *, bmi_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.BareMetalInstances/Get", data={"id": bmi_id}) + + def get_baremetal_instance_state(self, *, bmi_id: str) -> str: + response: dict[str, Any] = self.get_baremetal_instance(bmi_id=bmi_id) + return response.get("object", {}).get("status", {}).get("state", "") + + def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"run_strategy": run_strategy}}, + "updateMask": {"paths": ["spec.run_strategy"]}, + }, + ) + + def update_baremetal_instance_restart_trigger(self, *, bmi_id: str, restart_trigger: int) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.BareMetalInstances/Update", + data={ + "object": {"id": bmi_id, "spec": {"restart_trigger": restart_trigger}}, + "updateMask": {"paths": ["spec.restart_trigger"]}, + }, + ) + + def delete_baremetal_instance(self, *, bmi_id: str) -> None: + self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id}) + + # BareMetalInstanceCatalogItem operations (private API for admin setup) + + def create_baremetal_instance_catalog_item( + self, + *, + name: str, + title: str, + description: str, + template: str, + field_definitions: list[dict[str, Any]] | None = None, + ) -> str: + """Create a published BareMetalInstanceCatalogItem. + + ``template`` is sent as a typed reference ``{"name": ...}`` (OSAC-1330), + matching Cluster/ComputeInstance catalog item creates. + """ + obj: dict[str, Any] = { + "metadata": {"name": name}, + "title": title, + "description": description, + "template": {"name": template}, + "published": True, + } + if field_definitions is not None: + obj["field_definitions"] = field_definitions + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Create", data={"object": obj} + ) + return response["object"]["id"] + + def delete_baremetal_instance_catalog_item(self, *, item_id: str) -> None: + self.call(service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Delete", data={"id": item_id}) + + # DiskImage operations (public API) + + def create_disk_image( + self, + *, + source_ref: str, + architecture: list[str] | None = None, + source_type: str = "SOURCE_TYPE_REGISTRY", + guest_os_family: str = "GUEST_OS_FAMILY_LINUX", + name: str | None = None, + ) -> str: + spec: dict[str, Any] = { + "source_type": source_type, + "source_ref": source_ref, + "guest_os_family": guest_os_family, + "architecture": architecture or ["ARCHITECTURE_AMD64"], + } + obj: dict[str, Any] = {"spec": spec} + if name is not None: + obj["metadata"] = {"name": name} + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.DiskImages/Create", data={"object": obj}) + return response["object"]["id"] + + def get_disk_image(self, *, disk_image_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.DiskImages/Get", data={"id": disk_image_id}) + + def list_disk_image_ids(self, *, filter_expr: str | None = None) -> list[str]: + data: dict[str, Any] | None = {"filter": filter_expr} if filter_expr else None + response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.DiskImages/List", data=data) + return [item["id"] for item in response.get("items", [])] + + def update_disk_image_lifecycle(self, *, disk_image_id: str, lifecycle: str) -> dict[str, Any]: + return self.call( + service=f"{PUBLIC_API}.DiskImages/Update", + data={ + "object": {"id": disk_image_id, "spec": {"lifecycle": lifecycle}}, + "updateMask": {"paths": ["spec.lifecycle"]}, + }, + ) + + def delete_disk_image(self, *, disk_image_id: str) -> None: + self.call(service=f"{PUBLIC_API}.DiskImages/Delete", data={"id": disk_image_id}) + + # ComputeInstance creation with explicit DiskImage (public API) + + def create_compute_instance_with_disk_image( + self, + *, + template: str, + disk_image_name: str, + subnet_ids: list[str], + instance_type: str | None = None, + name: str | None = None, + ) -> dict[str, Any]: + attachments = [{"subnet": {"id": sid}} for sid in subnet_ids] + spec: dict[str, Any] = { + "template": {"name": template}, + "disk_image": {"name": disk_image_name}, + "network_attachments": attachments, + } + if instance_type is not None: + spec["instance_type"] = {"name": instance_type} + obj: dict[str, Any] = {"spec": spec} + if name is not None: + obj["metadata"] = {"name": name} + return self.call(service=f"{PUBLIC_API}.ComputeInstances/Create", data={"object": obj}) + + # ComputeInstanceTemplate operations (private API) + + def create_compute_instance_template( + self, *, template_id: str, name: str, title: str, description: str, spec_defaults: dict[str, Any] | None = None + ) -> str: + # template_id is mandatory on purpose: it is written verbatim into the + # osac-operator ComputeInstance CR's spec.templateID, which the CRD + # validates against ^[a-zA-Z_][a-zA-Z0-9._]*$. Leaving it empty makes the + # server assign a UUIDv7 (hyphens + leading digit) that the CRD rejects at + # admission, so the CR is never created. Real templates are published by + # AAP with a dotted/underscored id (e.g. "osac.templates.ocp_virt_vm"); + # callers must follow that pattern. + obj: dict[str, Any] = { + "id": template_id, + "metadata": {"name": name}, + "title": title, + "description": description, + } + if spec_defaults is not None: + obj["spec_defaults"] = spec_defaults + response: dict[str, Any] = self.call( + service=f"{PRIVATE_API}.ComputeInstanceTemplates/Create", data={"object": obj} + ) + return response["object"]["id"] + + def delete_compute_instance_template(self, *, template_id: str) -> None: + self.call(service=f"{PRIVATE_API}.ComputeInstanceTemplates/Delete", data={"id": template_id}) + + def get_compute_instance_template(self, *, template_id: str) -> dict[str, Any]: + return self.call(service=f"{PRIVATE_API}.ComputeInstanceTemplates/Get", data={"id": template_id}) + + # Generic filtered list + + def list_with_filter(self, *, service: str, filter_expr: str) -> list[dict[str, Any]]: + response: dict[str, Any] = self.call(service=service, data={"filter": filter_expr}) + return response.get("items", []) + + # NATGateway operations (public API) + + def create_nat_gateway(self, *, name: str, virtual_network_name: str, external_ip_name: str) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.NATGateways/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": { + "virtual_network": {"name": virtual_network_name}, + "external_ip": {"name": external_ip_name}, + }, + } + }, + ) + return response["object"]["id"] + + def delete_nat_gateway(self, *, nat_gateway_id: str) -> None: + self.call(service=f"{PUBLIC_API}.NATGateways/Delete", data={"id": nat_gateway_id}) + + # RoleBinding operations (public API) + + def create_role_binding(self, *, name: str, role_name: str, user_names: list[str]) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.RoleBindings/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"role": {"name": role_name}, "users": [{"name": u} for u in user_names]}, + } + }, + ) + return response["object"]["id"] + + def get_role_binding(self, *, role_binding_id: str) -> dict[str, Any]: + return self.call(service=f"{PUBLIC_API}.RoleBindings/Get", data={"id": role_binding_id}) + + def delete_role_binding(self, *, role_binding_id: str) -> None: + self.call(service=f"{PUBLIC_API}.RoleBindings/Delete", data={"id": role_binding_id}) + + # ProjectMembership operations (public API) + + def create_project_membership( + self, *, name: str, user_names: list[str], role: str = "PROJECT_MEMBERSHIP_ROLE_VIEWER" + ) -> str: + response: dict[str, Any] = self.call( + service=f"{PUBLIC_API}.ProjectMemberships/Create", + data={ + "object": { + "metadata": {"name": name}, + "spec": {"role": role, "users": [{"name": u} for u in user_names]}, + } + }, + ) + return response["object"]["id"] + + def delete_project_membership(self, *, membership_id: str) -> None: + self.call(service=f"{PUBLIC_API}.ProjectMemberships/Delete", data={"id": membership_id}) diff --git a/tests/core/helpers.py b/tests/core/helpers.py new file mode 100644 index 0000000000..4b59d94e80 --- /dev/null +++ b/tests/core/helpers.py @@ -0,0 +1,820 @@ +from __future__ import annotations + +import re +import subprocess +import time +from typing import Any + +import pytest + +from tests.core.grpc_client import GRPCClient +from tests.core.k8s_client import K8sClient +from tests.core.runner import poll_until, run_unchecked + +_POOL_READY_STATE = "EXTERNAL_IP_POOL_STATE_READY" + + +def assert_grpc_rejected(exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], code: str) -> None: + exc = exc_info.value + combined: str = (exc.stderr or "") + (exc.stdout or "") + assert re.search(rf"Code:\s*{code}", combined), f"Expected gRPC {code}, got: {combined.strip()}" + + +def assert_grpc_field_violation( + exc_info: pytest.ExceptionInfo[subprocess.CalledProcessError], *, field_path: str +) -> None: + assert_grpc_rejected(exc_info, "InvalidArgument") + exc = exc_info.value + combined: str = (exc.stderr or "") + (exc.stdout or "") + assert field_path in combined, ( + f"Expected FieldViolation containing '{field_path}' in error output, got: {combined.strip()}" + ) + + +def wait_for_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_compute_instance_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"CR for {uuid}", + ) + + +def wait_for_provision(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_condition_status(name=name, condition_type="Provisioned", checked=False), + until=lambda v: v == "True", + retries=120, + delay=5, + description=f"{name} Provisioned condition", + ) + + +def wait_for_running(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_phase(name=name, checked=False), + until=lambda v: v == "Running", + retries=90, + delay=10, + description=f"{name} Running", + ) + + +def wait_for_restart(*, k8s: K8sClient, name: str, initial: str, restart_ts: str) -> None: + poll_until( + fn=lambda: k8s.get_compute_instance_last_restarted_at(name=name), + until=lambda v: v != "" and v != initial and v >= restart_ts, + retries=30, + delay=10, + description=f"{name} lastRestartedAt update", + ) + + +def wait_for_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="computeinstance", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} deletion", + ) + + +def wait_for_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_compute_instance_ids(), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"{uuid} removed from gRPC list", + ) + + +def wait_for_virtual_network_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_virtual_network_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"VirtualNetwork CR for {uuid}", + ) + + +def wait_for_virtual_network_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_virtual_network_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} VirtualNetwork Ready", + ) + + +def wait_for_virtual_network_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="virtualnetwork", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} VirtualNetwork deletion", + ) + + +def wait_for_subnet_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_subnet_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"Subnet CR for {uuid}", + ) + + +def wait_for_subnet_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_subnet_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} Subnet Ready", + ) + + +def wait_for_subnet_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="subnet", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} Subnet deletion", + ) + + +def wait_for_external_ip_pool_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_external_ip_pool_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"ExternalIPPool CR for {uuid}", + ) + + +def wait_for_external_ip_pool_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_external_ip_pool_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} ExternalIPPool Ready", + ) + + +def wait_for_external_ip_pool_grpc_ready(*, private_grpc: GRPCClient, pool_id: str) -> None: + """Poll the private gRPC API until the pool state is READY. + + The K8s CR status may report Ready before the fulfillment-service database + has been updated by the controller feedback loop. Polling via gRPC closes + this race so that subsequent ExternalIP creation does not hit + FailedPrecondition. + """ + + def _state() -> str: + try: + pool = private_grpc.get_external_ip_pool(pool_id=pool_id) + except subprocess.CalledProcessError: + return "" + return pool.get("object", {}).get("status", {}).get("state", "") + + poll_until( + fn=_state, + until=lambda v: v == _POOL_READY_STATE, + retries=30, + delay=2, + description=f"ExternalIPPool {pool_id} gRPC READY", + ) + + +def wait_for_external_ip_pool_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="externalippool", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} ExternalIPPool deletion", + ) + + +def wait_for_external_ip_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_external_ip_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"ExternalIP CR for {uuid}", + ) + + +def wait_for_external_ip_allocated(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_external_ip_state(name=name, checked=False), + until=lambda v: v == "Allocated", + retries=60, + delay=5, + description=f"{name} ExternalIP Allocated", + ) + + +def wait_for_external_ip_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="externalip", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} ExternalIP deletion", + ) + + +def wait_for_external_ip_attachment_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_external_ip_attachment_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=1, + description=f"ExternalIPAttachment CR for {uuid}", + ) + + +def wait_for_external_ip_attachment_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_external_ip_attachment_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} ExternalIPAttachment Ready", + ) + + +def wait_for_external_ip_attachment_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="externalipattachment", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} ExternalIPAttachment deletion", + ) + + +# NATGateway helpers + + +def wait_for_nat_gateway_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_jsonpath(resource="natgateway", name=name, jsonpath="{.status.state}"), + until=lambda state: state == "Ready", + retries=30, + delay=5, + description=f"NATGateway {name} to become Ready", + ) + + +def wait_for_nat_gateway_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="natgateway", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} NATGateway deletion", + ) + + +def wait_for_cluster_order_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_cluster_order_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"ClusterOrder CR for {uuid}", + ) + + +def wait_for_cluster_progressing(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Progressing", + retries=30, + delay=2, + description=f"{name} ClusterOrder Progressing phase", + ) + + +def wait_for_cluster_ready(*, k8s: K8sClient, name: str) -> None: + # Must stay safely above osac-aap's own wait_for_clusteroperators_retries + # budget (60 min) plus earlier steps in the same AAP job (create hosted + # cluster, retrieve kubeconfig, etc.), or this times out first with a + # less useful error while the ClusterOrder is still legitimately Progressing. + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=480, + delay=15, + description=f"{name} ClusterOrder Ready", + ) + + +def wait_for_cluster_deletion(*, k8s: K8sClient, name: str) -> None: + # HACK: HyperShift has multiple teardown bugs where controllers leave orphaned state + # that deadlocks HostedCluster deletion. We force-clean on every poll iteration: + # + # 1. AgentCluster deprovision finalizer: capi-provider-agent is killed during teardown + # before removing its finalizer, blocking namespace termination. + # https://github.com/openshift/hypershift/blob/main/hypershift-operator/controllers/hostedcluster/karpenter.go#L88 + # + # 2. Agent labels: the CAPI provider sometimes fails to clear + # clusterdeployment-namespace from agents after HostedCluster deletion. The delete + # playbook's detach_and_unlabel skips agents that still have this label set, leaving + # the clusterorder label stuck and blocking agent reuse for subsequent tests. + # + # 3. Machine pre-terminate hooks: the CAPI provider sets a pre-terminate hook + # annotation on Machines, but is killed before removing it. The CAPI Machine + # controller waits forever for the annotation to be removed, blocking the entire + # deletion cascade (Machine → MachineSet → CAPI Cluster → HostedCluster). + def _check_deleted() -> bool: + _force_cleanup_agentcluster_finalizers(k8s=k8s, name=name) + _force_cleanup_agent_labels(k8s=k8s, name=name) + _force_cleanup_machine_preterminate_hooks(k8s=k8s, name=name) + return not k8s.is_present(resource="clusterorder", name=name) + + poll_until( + fn=_check_deleted, until=lambda v: v is True, retries=120, delay=10, description=f"{name} ClusterOrder deletion" + ) + + +def _force_cleanup_agentcluster_finalizers(*, k8s: K8sClient, name: str) -> None: + # HCP namespace: {osac-ns}-{co-name}-{hc-name}, where hc-name == co-name + hc_ns = f"{k8s.namespace}-{name}" + cp_ns = f"{hc_ns}-{name}" + finalizer = "agentclustercapi-provider.agent-install.openshift.io/deprovision" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, + "get", + "agentclusters.capi-provider.agent-install.openshift.io", + "-n", + cp_ns, + "-o", + f"jsonpath={{.items[?(@.metadata.finalizers[*]=='{finalizer}')].metadata.name}}", + ) + if rc != 0 or not output.strip(): + return + for ac_name in output.strip().split(): + finalizers_json, rc = run_unchecked( + *base_args, + "get", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "-o", + "jsonpath={.metadata.finalizers}", + ) + if rc != 0 or finalizer not in finalizers_json: + continue + import json + + idx = json.loads(finalizers_json).index(finalizer) + run_unchecked( + *base_args, + "patch", + f"agentclusters.capi-provider.agent-install.openshift.io/{ac_name}", + "-n", + cp_ns, + "--type=json", + f'-p=[{{"op": "remove", "path": "/metadata/finalizers/{idx}"}}]', + ) + + +def _force_cleanup_agent_labels(*, k8s: K8sClient, name: str) -> None: + agent_ns = "hardware-inventory" + clusterorder_label = "osac.openshift.io/clusterorder" + clusterdeployment_ns_label = "agent-install.openshift.io/clusterdeployment-namespace" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, + "get", + "agents.agent-install.openshift.io", + "-n", + agent_ns, + "-l", + f"{clusterorder_label}={name}", + "-o", + "jsonpath={.items[*].metadata.name}", + ) + if rc != 0 or not output.strip(): + return + for agent_name in output.strip().split(): + run_unchecked( + *base_args, + "label", + f"agents.agent-install.openshift.io/{agent_name}", + "-n", + agent_ns, + f"{clusterorder_label}-", + f"{clusterdeployment_ns_label}-", + ) + + +def _force_cleanup_machine_preterminate_hooks(*, k8s: K8sClient, name: str) -> None: + cp_ns = f"{k8s.namespace}-{name}-{name}" + hook = "pre-terminate.delete.hook.machine.cluster.x-k8s.io/agentmachine" + base_args = [*k8s._base(), "--as", "system:admin"] + output, rc = run_unchecked( + *base_args, "get", "machines.cluster.x-k8s.io", "-n", cp_ns, "-o", "jsonpath={.items[*].metadata.name}" + ) + if rc != 0 or not output.strip(): + return + for machine_name in output.strip().split(): + run_unchecked(*base_args, "annotate", f"machines.cluster.x-k8s.io/{machine_name}", "-n", cp_ns, f"{hook}-") + + +def wait_for_cluster_deleting(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_cluster_order_phase(name=name, checked=False), + until=lambda v: v == "Deleting", + retries=30, + delay=5, + description=f"{name} ClusterOrder Deleting phase", + ) + + +def wait_for_cluster_grpc_deleting_or_archived(*, grpc: GRPCClient, uuid: str) -> None: + """Succeed if we catch CLUSTER_STATE_DELETING or if the cluster is already archived. + + The DELETING window in the fulfillment-service is extremely short (one + Update + Signal round-trip). Polling for the exact state is racey; accepting + either DELETING or 'already gone' makes the assertion reliable. + """ + + def _done() -> bool: + try: + cluster = grpc.get_cluster(cluster_id=uuid) + state = cluster.get("object", {}).get("status", {}).get("state", "") + return state == "CLUSTER_STATE_DELETING" + except subprocess.CalledProcessError: + return True + + poll_until( + fn=_done, + until=lambda v: v is True, + retries=30, + delay=2, + description=f"{uuid} gRPC DELETING or already archived", + ) + + +def wait_for_cluster_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + # retry_on_error=True: a flaky grpcurl call hitting a momentarily-busy + # route right after heavy cluster-deletion activity shouldn't fail the + # whole test on the first hiccup. + poll_until( + fn=lambda: uuid not in grpc.list_cluster_ids(), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{uuid} removed from gRPC cluster list", + retry_on_error=True, + ) + + +def wait_for_security_group_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_security_group_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"SecurityGroup CR for {uuid}", + ) + + +def wait_for_security_group_ready(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.get_security_group_phase(name=name, checked=False), + until=lambda v: v == "Ready", + retries=60, + delay=5, + description=f"{name} SecurityGroup Ready", + ) + + +def wait_for_security_group_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="securitygroup", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"{name} SecurityGroup deletion", + ) + + +# Tenant helpers + + +def wait_for_tenant_cr(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: k8s.is_present(resource="tenant", name=name), + until=lambda v: v is True, + retries=30, + delay=2, + description=f"Tenant CR {name}", + ) + + +def wait_for_tenant_condition(*, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True") -> None: + def _check() -> str: + if not k8s.is_present(resource="tenant", name=name): + raise AssertionError(f"Tenant {name} disappeared before {condition_type}={expected_status}") + phase: str = k8s.get_tenant_phase(name=name, checked=False) + if phase == "Failed": + cond_status = k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) + if cond_status != expected_status: + raise AssertionError(f"Tenant {name} entered Failed phase before {condition_type}={expected_status}") + return k8s.get_tenant_condition_status(name=name, condition_type=condition_type, checked=False) + + poll_until( + fn=_check, + until=lambda v: v == expected_status, + retries=120, + delay=5, + description=f"Tenant {name} {condition_type}={expected_status}", + ) + + +def wait_for_tenant_deletion(*, k8s: K8sClient, name: str) -> None: + poll_until( + fn=lambda: not k8s.is_present(resource="tenant", name=name), + until=lambda v: v is True, + retries=120, + delay=5, + description=f"Tenant {name} deletion", + ) + + +# CaaS cluster storage helpers + + +def wait_for_cluster_order_condition( + *, k8s: K8sClient, name: str, condition_type: str, expected_status: str = "True" +) -> None: + def _check() -> str: + if not k8s.is_present(resource="clusterorder", name=name): + raise AssertionError(f"ClusterOrder {name} disappeared before {condition_type}={expected_status}") + phase: str = k8s.get_cluster_order_phase(name=name, checked=False) + cond_status = k8s.get_cluster_order_condition_status(name=name, condition_type=condition_type, checked=False) + if phase == "Failed" and cond_status != expected_status: + raise AssertionError(f"ClusterOrder {name} entered Failed phase before {condition_type}={expected_status}") + return cond_status + + poll_until( + fn=_check, + until=lambda v: v == expected_status, + retries=120, + delay=10, + description=f"ClusterOrder {name} {condition_type}={expected_status}", + ) + + +def wait_for_tenant_cluster_storage_entry(*, k8s: K8sClient, tenant_name: str, cluster_name: str) -> dict[str, Any]: + def _check() -> dict[str, Any] | None: + entries = k8s.get_tenant_cluster_storage(name=tenant_name, checked=False) + for entry in entries: + if entry.get("clusterName") == cluster_name and entry.get("ready") is True: + return entry + return None + + return poll_until( + fn=_check, + until=lambda v: v is not None, + retries=60, + delay=10, + description=f"Tenant {tenant_name} clusterStorage entry for {cluster_name}", + ) + + +def wait_for_tenant_cluster_storage_entry_removed(*, k8s: K8sClient, tenant_name: str, cluster_name: str) -> None: + poll_until( + fn=lambda: all( + entry.get("clusterName") != cluster_name + for entry in k8s.get_tenant_cluster_storage(name=tenant_name, checked=False) + ), + until=lambda v: v is True, + retries=60, + delay=10, + description=f"Tenant {tenant_name} clusterStorage entry for {cluster_name} removed", + ) + + +# Storage resource helpers + + +def wait_for_storage_classes_by_tenant(*, k8s: K8sClient, tenant_name: str, min_count: int = 1) -> list[str]: + return poll_until( + fn=lambda: k8s.list_storage_class_names_by_tenant(tenant_name=tenant_name), + until=lambda v: len(v) >= min_count, + retries=120, + delay=5, + description=f"StorageClasses for tenant {tenant_name} (>= {min_count})", + ) + + +def wait_for_storage_classes_removed(*, k8s: K8sClient, tenant_name: str) -> None: + poll_until( + fn=lambda: k8s.count_storage_classes_by_tenant(tenant_name=tenant_name), + until=lambda v: v == 0, + retries=120, + delay=5, + description=f"StorageClasses for tenant {tenant_name} removed", + ) + + +def wait_for_secrets_removed(*, k8s: K8sClient, tenant_name: str, namespace: str) -> None: + poll_until( + fn=lambda: k8s.count_secrets_by_tenant(tenant_name=tenant_name, namespace=namespace), + until=lambda v: v == 0, + retries=120, + delay=5, + description=f"Secrets for tenant {tenant_name} in {namespace} removed", + ) + + +# BareMetalInstance helpers + + +def wait_for_bmi_cr(*, k8s: K8sClient, uuid: str) -> str: + return poll_until( + fn=lambda: k8s.get_baremetal_instance_name(uuid=uuid, checked=False), + until=lambda v: v != "", + retries=30, + delay=2, + description=f"BareMetalInstance CR for {uuid}", + ) + + +def wait_for_bmi_running(*, grpc: GRPCClient, bmi_id: str) -> None: + def _check_state() -> str: + state: str = grpc.get_baremetal_instance_state(bmi_id=bmi_id) + assert "FAILED" not in state, f"BareMetalInstance {bmi_id} entered {state}" + return state + + poll_until( + fn=_check_state, + until=lambda v: v == "BARE_METAL_INSTANCE_STATE_RUNNING", + retries=120, + delay=10, + description=f"{bmi_id} RUNNING", + ) + + +def assert_bmi_lifecycle_on_running( + *, + grpc: GRPCClient, + k8s: K8sClient, + bmi_id: str, + bmh_namespace: str, + power_cycle: bool = True, +) -> tuple[str, str]: + """Assert BMH binding/provisioning on an already-RUNNING BMI. + + Does not create or delete the instance. Inventory exhaust calls this after all + claim BMIs reach RUNNING so provisioning is not blocked by interleaved checks. + + Returns (bmi_cr_name, bmh_name). + """ + assert bmi_id in grpc.list_baremetal_instance_ids() + bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s, uuid=bmi_id) + + external_host_id: str = k8s.get_baremetal_instance_external_host_id(name=bmi_cr_name) + assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}" + bmh_ns, bmh_name = external_host_id.split("/", 1) + assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}" + + wait_for_bmh_provisioned(k8s=k8s, name=bmh_name, bmh_namespace=bmh_ns) + + image_url: str = k8s.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns) + assert image_url != "", f"BMH {bmh_name} has no image URL after provisioning" + + consumer_ref: str = k8s.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns) + assert consumer_ref != "", f"BMH {bmh_name} has no consumerRef after allocation" + + online: str = k8s.get_bmh_online(name=bmh_name, bmh_namespace=bmh_ns) + assert online == "true", f"BMH {bmh_name} should be online after provisioning, got: {online}" + + if power_cycle: + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_HALTED" + ) + poll_until( + fn=lambda: k8s.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "false", + retries=60, + delay=5, + description=f"{bmh_name} powered off", + ) + grpc.update_baremetal_instance_run_strategy( + bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_ALWAYS" + ) + poll_until( + fn=lambda: k8s.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns), + until=lambda v: v == "true", + retries=60, + delay=5, + description=f"{bmh_name} powered on", + ) + + return bmi_cr_name, bmh_name + + +def assert_bmi_does_not_become_running(*, grpc: GRPCClient, bmi_id: str, retries: int = 36, delay: int = 10) -> str: + """Observe that a BareMetalInstance never reaches RUNNING. + + Returns the last observed state. If the instance enters a FAILED state, + returns immediately (inventory / scheduling failure). + """ + last_state = "" + for _ in range(retries): + last_state = grpc.get_baremetal_instance_state(bmi_id=bmi_id) + assert last_state != "BARE_METAL_INSTANCE_STATE_RUNNING", ( + f"BareMetalInstance {bmi_id} unexpectedly reached RUNNING with no free BMH" + ) + if "FAILED" in last_state: + return last_state + time.sleep(delay) + return last_state + + +def wait_for_bmi_running_after_recovery(*, grpc: GRPCClient, bmi_id: str) -> None: + """Wait until a BMI reaches RUNNING, allowing a prior FAILED/pending state. + + Used after inventory is freed so an overflow instance can be scheduled. + Unlike wait_for_bmi_running, does not fail-fast on FAILED. + """ + poll_until( + fn=lambda: grpc.get_baremetal_instance_state(bmi_id=bmi_id), + until=lambda v: v == "BARE_METAL_INSTANCE_STATE_RUNNING", + retries=120, + delay=10, + description=f"{bmi_id} RUNNING after inventory recovery", + ) + + +def wait_for_bmi_deletion(*, k8s: K8sClient, name: str) -> None: + # 2700s (45min), not the old 1200s (20min): the deprovision AAP job this + # blocks on retries with exponential backoff up to a 30-minute ceiling + # (osac-operator's shared pkg/provisioning, BackoffMaxDelay) when it fails + # -- e.g. under the AAP job-pod attach flakiness tracked in OSAC-3499. A + # 20-minute window can time out here while the operator is still correctly + # retrying and would have succeeded; 45 minutes gives it room for a worst-case + # backoff wait plus job execution time, without silently swallowing an + # actually-stuck deletion (still fails, just later). + poll_until( + fn=lambda: not k8s.is_present(resource="baremetalinstance", name=name), + until=lambda v: v is True, + retries=270, + delay=10, + description=f"{name} BareMetalInstance deletion", + ) + + +def wait_for_bmi_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None: + poll_until( + fn=lambda: uuid not in grpc.list_baremetal_instance_ids(), + until=lambda v: v is True, + retries=60, + delay=5, + description=f"{uuid} removed from gRPC BareMetalInstance list", + ) + + +def wait_for_bmh_provisioned(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None: + def _check() -> str: + state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace) + assert state != "error", f"BMH {name} entered error state" + return state + + poll_until( + fn=_check, until=lambda v: v == "provisioned", retries=120, delay=10, description=f"{name} BMH provisioned" + ) + + +def wait_for_bmh_available(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None: + def _check() -> str: + state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace) + assert state != "error", f"BMH {name} entered error state" + return state + + poll_until( + fn=_check, + until=lambda v: v in ("available", "ready"), + retries=120, + delay=10, + description=f"{name} BMH available", + ) diff --git a/tests/core/k8s_client.py b/tests/core/k8s_client.py new file mode 100644 index 0000000000..998758d8c6 --- /dev/null +++ b/tests/core/k8s_client.py @@ -0,0 +1,547 @@ +from __future__ import annotations + +import json +import subprocess +from typing import Any + +from tests.core.runner import run, run_unchecked + + +class K8sClient: + def __init__(self, *, namespace: str, kubeconfig: str | None = None) -> None: + self.namespace: str = namespace + self.kubeconfig: str | None = kubeconfig + + def _base(self) -> list[str]: + args: list[str] = ["kubectl"] + if self.kubeconfig is not None: + args.extend(["--kubeconfig", self.kubeconfig]) + args.extend(["--as", "system:admin"]) + return args + + def _get(self, *args: str, checked: bool = True) -> tuple[str, int]: + if checked: + return run(*self._base(), *args), 0 + return run_unchecked(*self._base(), *args) + + # Generic kubectl operations + + def get_json(self, *, resource: str, name: str) -> dict[str, Any]: + return json.loads(run(*self._base(), "get", resource, name, "-n", self.namespace, "-o", "json")) + + def get_jsonpath(self, *, resource: str, name: str, jsonpath: str) -> str: + return run(*self._base(), "get", resource, name, "-n", self.namespace, "-o", f"jsonpath={jsonpath}") + + def get_by_label(self, *, resource: str, label: str, jsonpath: str) -> str: + return run(*self._base(), "get", resource, "-n", self.namespace, "-l", label, "-o", f"jsonpath={jsonpath}") + + def patch(self, *, resource: str, name: str, patch: str) -> tuple[str, int]: + return run_unchecked(*self._base(), "patch", resource, name, "-n", self.namespace, "--type=merge", "-p", patch) + + def apply(self, *, manifest: str) -> None: + args: list[str] = [*self._base(), "apply", "-f", "-"] + subprocess.run(args, input=manifest, capture_output=True, text=True, check=True) + + def delete(self, *, resource: str, name: str, wait: bool = True) -> None: + args = [*self._base(), "delete", resource, name, "-n", self.namespace] + if not wait: + args.append("--wait=false") + run(*args, timeout=600) + + def is_present(self, *, resource: str, name: str) -> bool: + _, rc = run_unchecked(*self._base(), "get", resource, name, "-n", self.namespace) + return rc == 0 + + def count_by_label_all_namespaces(self, *, resource: str, label: str) -> int: + output: str = run(*self._base(), "get", resource, "-A", "-l", label, "--no-headers") + if not output: + return 0 + return len(output.strip().splitlines()) + + # ComputeInstance queries + + def get_compute_instance_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "computeinstance", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/computeinstance-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_compute_instance_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "computeinstance", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_compute_instance_last_restarted_at(self, *, name: str) -> str: + return self.get_jsonpath(resource="computeinstance", name=name, jsonpath="{.status.lastRestartedAt}") + + def get_compute_instance_latest_job_id(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0]["jobID"] + + def get_compute_instance_latest_job_state(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + + def get_compute_instance_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "computeinstance", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_compute_instance_vm_namespace(self, *, name: str) -> str: + return self.get_jsonpath( + resource="computeinstance", name=name, jsonpath="{.status.virtualMachineReference.namespace}" + ) + + # VirtualNetwork queries + + def get_virtual_network_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "virtualnetwork", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/virtualnetwork-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_virtual_network_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "virtualnetwork", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # Subnet queries + + def get_subnet_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "subnet", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/subnet-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_subnet_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "subnet", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # VirtualMachine/VMI queries (explicit namespace — may be on a different cluster) + + def get_vmi_creation_timestamp(self, *, vmi_namespace: str, compute_instance_name: str) -> str: + return run( + *self._base(), + "get", + "virtualmachineinstance", + "-n", + vmi_namespace, + "-l", + f"osac.openshift.io/computeinstance={compute_instance_name}", + "-o", + "jsonpath={.items[0].metadata.creationTimestamp}", + ) + + def get_vm_printable_status(self, *, name: str, vm_namespace: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "virtualmachine", + name, + "-n", + vm_namespace, + "-o", + "jsonpath={.status.printableStatus}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_vm_run_strategy(self, *, name: str, vm_namespace: str) -> str: + return run( + *self._base(), "get", "virtualmachine", name, "-n", vm_namespace, "-o", "jsonpath={.spec.runStrategy}" + ) + + # ExternalIPPool queries + + def get_external_ip_pool_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "externalippool", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/externalippool-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_external_ip_pool_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "externalippool", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # ExternalIP queries + + def get_external_ip_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "externalip", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/externalip-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_external_ip_state(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "externalip", name, "-n", self.namespace, "-o", "jsonpath={.status.state}", checked=checked + ) + return output if rc == 0 else "" + + # ExternalIPAttachment queries + + def get_external_ip_attachment_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "externalipattachment", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/externalipattachment-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_external_ip_attachment_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "externalipattachment", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # ClusterOrder queries + + def get_cluster_order_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "clusterorder", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/clusterorder-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_cluster_order_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "clusterorder", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_cluster_order_latest_job_id(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0]["jobID"] + + def get_cluster_order_latest_job_state(self, *, name: str, job_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + jobs: list[dict[str, Any]] = [ + j for j in json.loads(output).get("status", {}).get("jobs", []) if j["type"] == job_type + ] + if not jobs: + return "" + return sorted(jobs, key=lambda j: j["timestamp"], reverse=True)[0].get("state", "") + + def get_cluster_order_hosted_cluster_name(self, *, name: str) -> str: + return self.get_jsonpath( + resource="clusterorder", name=name, jsonpath="{.status.clusterReference.hostedClusterName}" + ) + + def get_cluster_order_namespace(self, *, name: str) -> str: + return self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.status.clusterReference.namespace}") + + def get_cluster_order_spec(self, *, name: str) -> dict[str, Any]: + output = self.get_jsonpath(resource="clusterorder", name=name, jsonpath="{.spec}") + return json.loads(output) if output else {} + + def get_cluster_order_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_cluster_order_finalizers(self, *, name: str, checked: bool = True) -> list[str]: + output, rc = self._get("get", "clusterorder", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("metadata", {}).get("finalizers", []) + + # Tenant queries + + def get_tenant_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "tenant", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + def get_tenant_condition_status(self, *, name: str, condition_type: str, checked: bool = True) -> str: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return "" + conditions: list[dict[str, Any]] = json.loads(output).get("status", {}).get("conditions", []) + for cond in conditions: + if cond.get("type") == condition_type: + return cond.get("status", "") + return "" + + def get_tenant_storage_classes(self, *, name: str, checked: bool = True) -> list[dict[str, str]]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("status", {}).get("storageClasses", []) + + def get_tenant_finalizers(self, *, name: str, checked: bool = True) -> list[str]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("metadata", {}).get("finalizers", []) + + def get_tenant_cluster_storage(self, *, name: str, checked: bool = True) -> list[dict[str, Any]]: + output, rc = self._get("get", "tenant", name, "-n", self.namespace, "-o", "json", checked=checked) + if rc != 0: + return [] + return json.loads(output).get("status", {}).get("clusterStorage", []) + + # Cluster-scoped storage resource queries (no -n flag) + + def count_storage_classes_by_tenant(self, *, tenant_name: str) -> int: + output, rc = self._get( + "get", "storageclass", "-l", f"osac.openshift.io/tenant={tenant_name}", "--no-headers", checked=False + ) + if rc != 0 or not output.strip(): + return 0 + return len(output.strip().splitlines()) + + def list_storage_class_names_by_tenant(self, *, tenant_name: str) -> list[str]: + output, rc = self._get( + "get", + "storageclass", + "-l", + f"osac.openshift.io/tenant={tenant_name}", + "-o", + "jsonpath={.items[*].metadata.name}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return output.strip().split() + + def get_storage_class_labels(self, *, name: str) -> dict[str, str]: + output, rc = self._get("get", "storageclass", name, "-o", "json", checked=False) + if rc != 0: + return {} + return json.loads(output).get("metadata", {}).get("labels", {}) + + # Namespaced secret queries with explicit namespace (for cross-namespace lookups) + + def count_secrets_by_tenant(self, *, tenant_name: str, namespace: str) -> int: + output, rc = self._get( + "get", + "secret", + "-n", + namespace, + "-l", + f"osac.openshift.io/tenant={tenant_name}", + "--no-headers", + checked=False, + ) + if rc != 0 or not output.strip(): + return 0 + return len(output.strip().splitlines()) + + # SecurityGroup queries + + def get_security_group_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "securitygroup", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/securitygroup-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_security_group_phase(self, *, name: str, checked: bool = True) -> str: + output, rc = self._get( + "get", "securitygroup", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked + ) + return output if rc == 0 else "" + + # BareMetalInstance queries + + def get_baremetal_instance_name(self, *, uuid: str, checked: bool = True) -> str: + output, rc = self._get( + "get", + "baremetalinstance", + "-n", + self.namespace, + "-l", + f"osac.openshift.io/baremetalinstance-uuid={uuid}", + "-o", + "jsonpath={.items[0].metadata.name}", + checked=checked, + ) + return output if rc == 0 else "" + + def get_baremetal_instance_external_host_id(self, *, name: str) -> str: + return self.get_jsonpath(resource="baremetalinstance", name=name, jsonpath="{.spec.externalHostID}") + + def get_bmi_hardware_nics(self, *, name: str) -> list[str]: + """Return lowercased MAC addresses from BareMetalInstance CR status.hardware.nics.""" + output, rc = self._get( + "get", + "baremetalinstance", + name, + "-n", + self.namespace, + "-o", + "jsonpath={.status.hardware.nics[*].mac}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return [mac.lower() for mac in output.split()] + + def get_baremetal_instance_tenant_ip(self, *, name: str) -> str: + return self.get_jsonpath( + resource="baremetalinstance", name=name, jsonpath="{.status.networkAttachmentStatuses[0].ipAddress}" + ) + + # BareMetalHost queries (explicit namespace — BMHs live in a different namespace) + + def get_bmh_provisioning_state(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", + "baremetalhost", + name, + "-n", + bmh_namespace, + "-o", + "jsonpath={.status.provisioning.state}", + checked=False, + ) + return output if rc == 0 else "" + + def get_bmh_image_url(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.image.url}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_consumer_ref(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.consumerRef.name}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_online(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.online}", checked=False + ) + return output if rc == 0 else "" + + def get_bmh_powered_on(self, *, name: str, bmh_namespace: str) -> str: + output, rc = self._get( + "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.status.poweredOn}", checked=False + ) + return output if rc == 0 else "" + + def count_bmhs_by_provisioning_state(self, *, bmh_namespace: str, states: set[str]) -> int: + """Return how many BareMetalHosts in ``bmh_namespace`` are in any of ``states``.""" + raw: str = run(*self._base(), "get", "baremetalhost", "-n", bmh_namespace, "-o", "json") + items: list[dict[str, Any]] = json.loads(raw).get("items", []) + count = 0 + for item in items: + state: str = item.get("status", {}).get("provisioning", {}).get("state", "") + if state in states: + count += 1 + return count + + def get_bmh_hardware_nics(self, *, name: str, bmh_namespace: str) -> list[str]: + """Return lowercased MAC addresses from BareMetalHost hardware inspection data.""" + output, rc = self._get( + "get", + "baremetalhost", + name, + "-n", + bmh_namespace, + "-o", + "jsonpath={.status.hardware.nics[*].mac}", + checked=False, + ) + if rc != 0 or not output.strip(): + return [] + return [mac.lower() for mac in output.split()] diff --git a/tests/core/keycloak.py b/tests/core/keycloak.py new file mode 100644 index 0000000000..55a3859706 --- /dev/null +++ b/tests/core/keycloak.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json + +from tests.core.runner import poll_until, run_unchecked + + +def get_jwt(*, keycloak_url: str, realm: str, client_id: str, username: str, password: str) -> str: + """Get JWT token from Keycloak with retry logic for startup flakiness.""" + token_url = f"{keycloak_url}/realms/{realm}/protocol/openid-connect/token" + + def try_get_token() -> str | None: + stdout, returncode = run_unchecked( + "curl", + "-sk", + "--fail-with-body", + "--max-time", + "10", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + f"client_id={client_id}", + "-d", + f"username={username}", + "-d", + f"password={password}", + "-d", + "scope=openid organization", + ) + + if returncode != 0: + return None + + try: + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get JWT from Keycloak: {error}") + return token + except json.JSONDecodeError: + # Malformed response - Keycloak might still be starting + return None + + return poll_until( + fn=try_get_token, until=lambda token: token is not None, retries=24, delay=5, description="Keycloak JWT token" + ) diff --git a/tests/core/keycloak_admin.py b/tests/core/keycloak_admin.py new file mode 100644 index 0000000000..9358b22693 --- /dev/null +++ b/tests/core/keycloak_admin.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +import time +from typing import Any +from urllib.parse import urlencode + +from tests.core.runner import run + + +def get_admin_token(*, keycloak_url: str, username: str, password: str) -> str: + """Get an admin access token for Keycloak admin API calls.""" + token_url = f"{keycloak_url}/realms/master/protocol/openid-connect/token" + stdout: str = run( + "curl", + "-sk", + "--fail-with-body", + "-X", + "POST", + token_url, + "-d", + "grant_type=password", + "-d", + "client_id=admin-cli", + "-d", + f"username={username}", + "-d", + f"password={password}", + ) + response: dict[str, str] = json.loads(stdout) + token: str | None = response.get("access_token") + if not token: + error: str = response.get("error_description", response.get("error", "unknown error")) + raise RuntimeError(f"Failed to get admin token from Keycloak: {error}") + return token + + +def keycloak_admin_request( + *, keycloak_url: str, admin_token: str, method: str, path: str, data: Any = None +) -> tuple[int, bytes]: + """ + Make an authenticated request to the Keycloak admin API for the 'osac' realm. + The path is relative to /admin/realms/osac (e.g., "/organizations", "/users/{id}"). + Returns (status_code, response_body). + """ + url = f"{keycloak_url}/admin/realms/osac{path}" + args = [ + "curl", + "-sk", + "-w", + "\n%{http_code}", + "-X", + method, + "-H", + f"Authorization: Bearer {admin_token}", + "-H", + "Content-Type: application/json", + ] + if data is not None: + if isinstance(data, str): + args.extend(["-d", data]) + else: + args.extend(["-d", json.dumps(data)]) + + args.append(url) + + output: str = run(*args) + lines = output.strip().split("\n") + status_code = int(lines[-1]) + body = "\n".join(lines[:-1]).encode("utf-8") + + return status_code, body + + +def wait_for_organization( + *, keycloak_url: str, admin_token: str, org_name: str, timeout_seconds: int = 60 +) -> str: + """ + Wait for an organization to be synced to Keycloak and return its ID. + Polls with exponential backoff until the organization exists or timeout is reached. + """ + start_time = time.time() + interval = 1.0 + max_interval = 10.0 + + while time.time() - start_time < timeout_seconds: + query = urlencode({"exact": "true", "search": org_name}) + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="GET", + path=f"/organizations?{query}", + ) + + if status != 200: + raise RuntimeError(f"Failed to query organizations: status={status} body={body.decode()}") + + orgs: list[dict[str, Any]] = json.loads(body) + if len(orgs) > 0: + org_id: str = orgs[0]["id"] + return org_id + + time.sleep(interval) + interval = min(interval * 2, max_interval) + + raise RuntimeError(f"Organization '{org_name}' not found in Keycloak after {timeout_seconds}s") + + +def get_user_id(*, keycloak_url: str, admin_token: str, username: str) -> str: + """Get a user's ID by username.""" + query = urlencode({"username": username, "exact": "true"}) + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/users?{query}" + ) + + if status != 200: + raise RuntimeError(f"Failed to get user '{username}': status={status} body={body.decode()}") + + users: list[dict[str, Any]] = json.loads(body) + if len(users) == 0: + raise RuntimeError(f"User '{username}' not found in Keycloak") + + return users[0]["id"] + + +def add_user_to_organization( + *, keycloak_url: str, admin_token: str, org_id: str, user_id: str, username: str, org_name: str +) -> None: + """Add a user to a Keycloak organization.""" + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="POST", + path=f"/organizations/{org_id}/members", + data=user_id, + ) + + # 201 Created, 204 No Content, or 409 Conflict (already a member) are all acceptable + # 400 with "Duplicate resource error" is also acceptable (Keycloak returns this when user is already a member) + if status == 400: + error_msg = body.decode().lower() + if "duplicate" not in error_msg: + raise RuntimeError( + f"Failed to add user '{username}' to organization '{org_name}': status={status} body={body.decode()}" + ) + elif status not in (201, 204, 409): + raise RuntimeError( + f"Failed to add user '{username}' to organization '{org_name}': status={status} body={body.decode()}" + ) + + +def ensure_organization_group(*, keycloak_url: str, admin_token: str, org_id: str, org_name: str) -> str: + """ + Ensure a /members group exists in the organization and return its ID. + Creates the group if it doesn't exist. + """ + group_name = "/members" + group_payload = {"name": group_name} + + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="POST", + path=f"/organizations/{org_id}/groups", + data=group_payload, + ) + + # 201 Created or 409 Conflict (already exists) are acceptable + if status == 201: + group_resp: dict[str, Any] = json.loads(body) + return group_resp["id"] + elif status == 409: + # Group already exists, need to fetch it + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, admin_token=admin_token, method="GET", path=f"/organizations/{org_id}/groups" + ) + + if status != 200: + raise RuntimeError( + f"Failed to get groups for organization '{org_name}': status={status} body={body.decode()}" + ) + + groups: list[dict[str, Any]] = json.loads(body) + for g in groups: + if g.get("name") == group_name: + return g["id"] + + raise RuntimeError(f"Failed to find group '{group_name}' in organization '{org_name}'") + else: + raise RuntimeError( + f"Failed to create group '{group_name}' in organization '{org_name}': status={status} body={body.decode()}" + ) + + +def add_user_to_organization_group( + *, keycloak_url: str, admin_token: str, org_id: str, group_id: str, user_id: str, username: str, org_name: str +) -> None: + """Add a user to a group within a Keycloak organization.""" + status, body = keycloak_admin_request( + keycloak_url=keycloak_url, + admin_token=admin_token, + method="PUT", + path=f"/organizations/{org_id}/groups/{group_id}/members/{user_id}", + ) + + # 200 OK, 201 Created, 204 No Content, or 409 Conflict are all acceptable + # 400 with "Duplicate resource error" is also acceptable (Keycloak returns this when user is already a member) + if status == 400: + error_msg = body.decode().lower() + if "duplicate" not in error_msg: + raise RuntimeError( + f"Failed to add user '{username}' to group in organization '{org_name}': " + f"status={status} body={body.decode()}" + ) + elif status not in (200, 201, 204, 409): + raise RuntimeError( + f"Failed to add user '{username}' to group in organization '{org_name}': " + f"status={status} body={body.decode()}" + ) diff --git a/tests/core/metering.py b/tests/core/metering.py new file mode 100644 index 0000000000..c7f398e254 --- /dev/null +++ b/tests/core/metering.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +import logging +import time +import urllib.error +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from tests.core.runner import poll_until + +logger = logging.getLogger(__name__) + + +def _parse_time(s: str) -> datetime: + if s.endswith("Z"): + s = s[:-1] + "+00:00" + return datetime.fromisoformat(s) + + +@dataclass +class ExpectedEvent: + event_type: str + resource_id: str + timeout: int = 60 + + +class MeteringCollector: + """Collects CloudEvents from the metering test adapter HTTP API and verifies expectations. + + The test adapter runs inside the Kubernetes cluster and exposes + ``GET /events?type={event_type}&resource_id={resource_id}&since={RFC3339}`` + which returns a JSON array of CloudEvents. + + Usage in tests:: + + metering.expect("osac.resource.created.v1", resource_id=uuid) + # ... test actions ... + # metering.verify() runs automatically on fixture teardown + """ + + def __init__(self, *, base_url: str) -> None: + self._base_url = base_url.rstrip("/") + self._expectations: list[ExpectedEvent] = [] + self._matched: set[str] = set() + self._verified: dict[tuple[str, str], dict[str, Any]] = {} + self._start_time: str = "" + + def start(self) -> None: + """Record the start time so event queries only return events after this point.""" + self._start_time = datetime.now(UTC).isoformat() + + def stop(self) -> None: + """No-op -- no background resources to clean up.""" + + def expect(self, event_type: str, resource_id: str, *, timeout: int = 60) -> None: + self._expectations.append(ExpectedEvent(event_type=event_type, resource_id=resource_id, timeout=timeout)) + + def verify(self) -> None: + if not self._expectations: + if not self._verified: + logger.warning("verify() called with no expectations — did you forget to call expect()?") + return + for exp in self._expectations: + event = self._poll_for_event(exp) + self._validate_structure(event, exp) + self._verified[(exp.event_type, exp.resource_id)] = event + self._expectations.clear() + + def get_event(self, event_type: str, resource_id: str, *, timeout: int = 60) -> dict[str, Any]: + """Return a verified event, or poll for one if not yet verified.""" + key = (event_type, resource_id) + if key in self._verified: + return self._verified[key] + exp = ExpectedEvent(event_type=event_type, resource_id=resource_id, timeout=timeout) + return self._poll_for_event(exp) + + def assert_no_events(self, event_type: str, resource_id: str, *, since: str, within: int = 60) -> None: + """Assert that no events of the given type appear for the resource within a time window.""" + time.sleep(within) + events = self._fetch_events(event_type, resource_id) + since_dt = datetime.fromisoformat(since) + matching = [ + ev for ev in events + if ev.get("type") == event_type + and (ev.get("resource_id") == resource_id + or ev.get("osacresourceid") == resource_id + or ev.get("data", {}).get("resource_id") == resource_id) + and _parse_time(ev.get("time", "")) >= since_dt + ] + assert not matching, ( + f"Expected no {event_type} events for {resource_id} after {since}, " + f"but found {len(matching)}: {[{'id': e.get('id'), 'time': e.get('time')} for e in matching]}" + ) + + def get_all_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: + """Return all events matching type and resource_id (for N+1 count assertions).""" + return self._fetch_events(event_type, resource_id) + + def _fetch_events(self, event_type: str, resource_id: str) -> list[dict[str, Any]]: + params = urlencode({ + "type": event_type, + "resource_id": resource_id, + "since": self._start_time, + }) + url = f"{self._base_url}/events?{params}" + last_exc: OSError | None = None + for attempt in range(3): + try: + req = Request(url) + with urlopen(req, timeout=10) as resp: # noqa: S310 + return json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, OSError) as exc: + last_exc = exc + logger.warning("Transient HTTP error fetching events (attempt %d/3): %s", attempt + 1, exc) + time.sleep(2) + raise last_exc # type: ignore[misc] + + def _poll_for_event(self, expected: ExpectedEvent) -> dict[str, Any]: + result: list[dict[str, Any]] = [] + + def find() -> bool: + try: + events = self._fetch_events(expected.event_type, expected.resource_id) + except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc: + logger.warning("Transient error fetching metering events, will retry: %s", exc) + return False + for ev in events: + ev_id = ev.get("id", "") + if ev_id and ev_id in self._matched: + continue + if ev.get("type") == expected.event_type and ( + ev.get("resource_id") == expected.resource_id + or ev.get("osacresourceid") == expected.resource_id + or ev.get("data", {}).get("resource_id") == expected.resource_id + ): + if ev_id: + self._matched.add(ev_id) + result.append(ev) + return True + return False + + poll_until( + fn=find, + until=lambda found: found is True, + retries=expected.timeout // 2, + delay=2, + description=f"metering event type={expected.event_type} resource_id={expected.resource_id}", + ) + return result[0] + + @staticmethod + def _validate_structure(event: dict[str, Any], expected: ExpectedEvent) -> None: + assert event.get("specversion") == "1.0", f"Wrong specversion: {event.get('specversion')}" + assert event.get("source") in ("osac-metering", "osac-metering/reconciler"), \ + f"Wrong source: {event.get('source')}" + assert event.get("id"), "Missing event id" + assert event.get("time"), "Missing event time" + assert event.get("osacresourceid") == expected.resource_id, \ + f"Wrong osacresourceid: {event.get('osacresourceid')}" + assert event.get("osacresourcetype") in ("compute_instance", "cluster_order"), \ + f"Wrong osacresourcetype: {event.get('osacresourcetype')}" + assert event.get("osactenant"), "Missing osactenant" + + data = event.get("data", {}) + assert data.get("resource_id") == expected.resource_id, f"Wrong resource_id in data: {data.get('resource_id')}" + assert data.get("resource_type"), "Missing resource_type in data" + assert data.get("tenant_id"), "Missing tenant_id in data" + assert data.get("schema_version") == "v1", f"Wrong schema_version: {data.get('schema_version')}" + assert "project_id" in data, "Missing project_id in data" + assert "billing_dimensions" in data, "Missing billing_dimensions in data" + + if expected.event_type != "osac.resource.heartbeat.v1": + assert data.get("transition_time"), "Missing transition_time in data" + try: + datetime.fromisoformat(data["transition_time"]) + except (ValueError, TypeError) as exc: + raise AssertionError(f"Invalid RFC3339 transition_time: {data.get('transition_time')}") from exc + + transition_types = { + "osac.resource.started.v1", + "osac.resource.suspended.v1", + "osac.resource.resumed.v1", + } + if expected.event_type in transition_types: + assert "previous_state" in data, f"Missing previous_state in {expected.event_type}" + assert "duration_seconds" in data, f"Missing duration_seconds in {expected.event_type}" + + if expected.event_type == "osac.resource.suspended.v1": + valid = ("RUNNING", "STOPPING", "STARTING", "PROGRESSING", "READY") + assert data.get("previous_state") in valid, ( + f"suspended.v1 previous_state should be one of {valid}, " + f"got {data.get('previous_state')!r}" + ) + + if expected.event_type == "osac.resource.resumed.v1": + valid = ("STOPPED", "PAUSED", "FAILED", "DELETE_FAILED") + assert data.get("previous_state") in valid, ( + f"resumed.v1 previous_state should be one of {valid}, " + f"got {data.get('previous_state')!r}" + ) + + resource_type = event.get("osacresourcetype") + if resource_type == "compute_instance": + _validate_vmaas_billing(event) + elif resource_type == "cluster_order": + _validate_caas_billing(event, expected.event_type) + + +def _validate_vmaas_billing(event: dict[str, Any]) -> None: + bd = event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("instance_type"), "Missing or empty instance_type in billing_dimensions" + assert bd.get("image_ref"), "Missing or empty image_ref in billing_dimensions" + assert "boot_disk_size_gib" in bd, "Missing boot_disk_size_gib in billing_dimensions" + assert isinstance( + bd["boot_disk_size_gib"], (int, float) + ), f"boot_disk_size_gib should be numeric, got {type(bd['boot_disk_size_gib']).__name__}" + + +def _validate_caas_billing(event: dict[str, Any], event_type: str) -> None: + bd = event.get("data", {}).get("billing_dimensions", {}) + assert bd.get("cluster_template"), "Missing cluster_template in billing_dimensions" + # created.v1 and deleted.v1 use topLevelDims (cluster_template + + # release_image only, no per-component decomposition) + if event_type in ("osac.resource.created.v1", "osac.resource.deleted.v1"): + assert bd.get("release_image"), "Missing release_image in billing_dimensions" + return + assert bd.get("component") in ("control_plane", "worker"), \ + f"component should be control_plane or worker, got {bd.get('component')!r}" + assert bd.get("node_set"), "Missing node_set in billing_dimensions" + assert bd.get("host_type"), "Missing host_type in billing_dimensions" + assert isinstance(bd.get("node_count"), (int, float)) and bd["node_count"] > 0, \ + f"node_count should be positive number, got {bd.get('node_count')!r}" diff --git a/tests/core/osac_cli.py b/tests/core/osac_cli.py new file mode 100644 index 0000000000..534ad49da3 --- /dev/null +++ b/tests/core/osac_cli.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import re +import shutil +import tempfile +from typing import Any + +from tests.core.runner import run, run_unchecked + + +class OsacCLI: + def __init__( + self, + *, + binary: str, + address: str, + token_script: str, + namespace: str, + default_instance_type: str | None = None, + default_disk_image: str | None = None, + private: bool = False, + ) -> None: + self.binary: str = binary + self.namespace: str = namespace + self._address: str = address + self._token_script: str = token_script + self._private: bool = private + self.default_instance_type: str | None = default_instance_type + self.default_disk_image: str | None = default_disk_image + # Each OsacCLI instance gets its own config directory so that parallel + # xdist workers (or multiple CLI fixtures) don't overwrite each other's + # login credentials via the shared ~/.config/osac/config.json. + self._config_dir: str = tempfile.mkdtemp(prefix="osac-config-") + login_args = ["login", "--address", address, "--insecure", "--token-script", token_script] + if self._private: + login_args.append("--private") + self._run(*login_args) + + def close(self) -> None: + shutil.rmtree(self._config_dir, ignore_errors=True) + + @property + def config_dir(self) -> str: + return self._config_dir + + def _run(self, *args: str, timeout: int = 300) -> str: + return run(self.binary, "--config", self._config_dir, *args, timeout=timeout) + + def _run_unchecked(self, *args: str, timeout: int = 300) -> tuple[str, int]: + return run_unchecked(self.binary, "--config", self._config_dir, *args, timeout=timeout) + + def relogin(self) -> None: + login_args = ["login", "--address", self._address, "--insecure", "--token-script", self._token_script] + if self._private: + login_args.append("--private") + self._run(*login_args) + + @staticmethod + def _parse_uuid(stdout: str) -> str: + match: re.Match[str] | None = re.search(r"'([^']+)'", stdout) + assert match is not None, f"Failed to parse UUID from CLI output: {stdout}" + return match.group(1) + + def create_hub(self, *, hub_id: str, kubeconfig: str) -> None: + self._run("create", "hub", "--id", hub_id, "--kubeconfig", kubeconfig, "--namespace", self.namespace) + + def create_compute_instance( + self, + *, + template: str, + name: str | None = None, + network_attachments: list[dict[str, Any]] | None = None, + boot_disk_size: int = 20, + disk_image: str | None = None, + run_strategy: str = "Always", + user_data_secret_ref: str | None = None, + instance_type: str | None = None, + ) -> str: + args: list[str] = [ + "create", + "computeinstance", + "--template", + template, + "--boot-disk-size", + str(boot_disk_size), + "--run-strategy", + run_strategy, + ] + if name is not None: + args.extend(["--name", name]) + + effective_instance_type = instance_type if instance_type is not None else self.default_instance_type + if effective_instance_type is not None: + args.extend(["--instance-type", effective_instance_type]) + else: + raise ValueError("instance_type or default_instance_type must be set") + + effective_disk_image = disk_image if disk_image is not None else self.default_disk_image + if effective_disk_image is not None: + args.extend(["--disk-image", effective_disk_image]) + else: + raise ValueError("disk_image or default_disk_image must be set") + + # Add network attachments + if network_attachments is not None: + for idx, attachment in enumerate(network_attachments): + subnet = attachment.get("subnet") + if not subnet or not isinstance(subnet, str): + raise ValueError(f"network_attachments[{idx}]: 'subnet' must be a non-empty string, got {subnet!r}") + + security_groups = attachment.get("security_groups", []) + if not isinstance(security_groups, list): + raise ValueError( + f"network_attachments[{idx}]: 'security_groups' must be a list," + f" got {type(security_groups).__name__}" + ) + + if security_groups and not all(isinstance(sg, str) and sg for sg in security_groups): + raise ValueError(f"network_attachments[{idx}]: all security_groups must be non-empty strings") + + # Build network-attachment flag value + # Format: subnet=,security-groups=, + parts = [f"subnet={subnet}"] + if security_groups: + sg_list = ",".join(security_groups) + parts.append(f"security-groups={sg_list}") + + args.extend(["--network-attachment", ",".join(parts)]) + + if user_data_secret_ref is not None: + args.extend(["--user-data", user_data_secret_ref]) + + return self._parse_uuid(self._run(*args)) + + def delete_compute_instance(self, *, uuid: str) -> None: + self._run("delete", "computeinstance", uuid) + + def create_instance_type( + self, + *, + name: str, + cores: int, + memory_gib: int, + description: str = "", + gpu_pci_device_selector: str = "", + gpu_resource_name: str = "", + gpu_count: int = 0, + ) -> str: + args: list[str] = [ + "create", + "instancetype", + "--name", + name, + "--cores", + str(cores), + "--memory-gib", + str(memory_gib), + ] + if description: + args.extend(["--description", description]) + if gpu_pci_device_selector: + args.extend(["--gpu-pci-device-selector", gpu_pci_device_selector]) + if gpu_resource_name: + args.extend(["--gpu-resource-name", gpu_resource_name]) + if gpu_count: + args.extend(["--gpu-count", str(gpu_count)]) + return self._parse_uuid(self._run(*args)) + + def describe_instance_type(self, *, name: str) -> str: + return self._run("describe", "instancetype", name) + + def delete_instance_type(self, *, name: str) -> None: + self._run("delete", "instancetype", name) + + def create_cluster( + self, + *, + template: str, + name: str | None = None, + pull_secret_file: str | None = None, + ssh_public_key_file: str | None = None, + version: str | None = None, + template_parameters: dict[str, str] | None = None, + template_parameter_files: dict[str, str] | None = None, + ) -> str: + args: list[str] = ["create", "cluster", "--template", template] + if name is not None: + args.extend(["--name", name]) + if pull_secret_file is not None: + args.extend(["--pull-secret-file", pull_secret_file]) + if ssh_public_key_file is not None: + args.extend(["--ssh-public-key-file", ssh_public_key_file]) + if version is not None: + args.extend(["--version", version]) + if template_parameters is not None: + for key, value in template_parameters.items(): + args.extend(["-p", f"{key}={value}"]) + if template_parameter_files is not None: + for key, path in template_parameter_files.items(): + args.extend(["-f", f"{key}={path}"]) + + return self._parse_uuid(self._run(*args)) + + def get(self, resource: str, *, output: str | None = None) -> str: + args: list[str] = ["get", resource] + if output is not None: + args.extend(["-o", output]) + return self._run(*args) + + def get_cluster_credential(self, credential: str, *, uuid: str) -> str: + return self._run("get", credential, uuid) + + def get_unchecked(self, resource: str) -> tuple[str, int]: + return self._run_unchecked("get", resource) + + def create_cluster_with_catalog_item(self, *, catalog_item: str, name: str, version: str | None = None) -> str: + args = ["create", "cluster", "--catalog-item", catalog_item, "--name", name] + if version is not None: + args.extend(["--version", version]) + return self._parse_uuid(self._run(*args)) + + def create_compute_instance_with_catalog_item( + self, *, catalog_item: str, name: str | None = None, subnet: str | None = None + ) -> str: + args: list[str] = ["create", "computeinstance", "--catalog-item", catalog_item] + if name is not None: + args.extend(["--name", name]) + if subnet is not None: + args.extend(["--network-attachment", f"subnet={subnet}"]) + return self._parse_uuid(self._run(*args)) + + def scale_cluster(self, *, uuid: str, node_set: str, size: int) -> None: + self._run("scale", "cluster", uuid, "--node-set", node_set, "--size", str(size)) + + def delete_cluster(self, *, uuid: str) -> None: + self._run("delete", "cluster", uuid) + + def create_baremetal_instance( + self, + *, + name: str, + catalog_item: str, + ssh_key: str | None = None, + user_data: str | None = None, + network_attachments: list[str] | None = None, + external_ip_attachment: bool = False, + ) -> str: + args: list[str] = ["create", "baremetalinstance", "--name", name, "--catalog-item", catalog_item] + if ssh_key is not None: + args.extend(["--ssh-key", ssh_key]) + if user_data is not None: + args.extend(["--user-data", user_data]) + if external_ip_attachment: + args.extend(["--external-ip-attachment"]) + for na in network_attachments or []: + args.extend(["--network-attachment", na]) + return self._parse_uuid(self._run(*args)) + + def describe_baremetal_instance(self, *, name: str) -> str: + return self._run("describe", "baremetalinstance", name) + + def delete_baremetal_instance(self, *, uuid: str) -> None: + self._run("delete", "baremetalinstance", uuid) diff --git a/tests/core/runner.py b/tests/core/runner.py new file mode 100644 index 0000000000..1d4f16aa7a --- /dev/null +++ b/tests/core/runner.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import logging +import os +import subprocess +import time +from collections.abc import Callable +from typing import TypeVar + +T = TypeVar("T") + +logger = logging.getLogger(__name__) + +# How often to log a "still waiting" progress line during a poll_until call, +# regardless of its own retry/delay settings -- long polls (e.g. waiting up +# to ~60 minutes for a bare-metal cluster to become Ready) would otherwise +# print nothing until they finish or time out. +_PROGRESS_LOG_INTERVAL_S = 30 + + +def run(*args: str, timeout: int = 300) -> str: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=True) + return result.stdout.strip() + + +def run_unchecked(*args: str, timeout: int = 300) -> tuple[str, int]: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False) + combined = (result.stdout.strip() + "\n" + result.stderr.strip()).strip() + return combined, result.returncode + + +def poll_until( + *, + fn: Callable[[], T], + until: Callable[[T], bool], + retries: int = 60, + delay: int = 5, + description: str, + retry_on_error: bool = False, +) -> T: + value: T | None = None + last_error: subprocess.CalledProcessError | None = None + start = time.monotonic() + last_logged = start + logger.info("Waiting for %s...", description) + for attempt in range(retries): + # retry_on_error is opt-in: most callers' fn() raising CalledProcessError + # indicates a real bug (bad namespace, typo'd resource, auth + # misconfig) and should fail fast, not be swallowed for the whole + # retry budget. Only pollers built on a checked (raising) subprocess + # call where a transient hiccup is expected -- e.g. a flaky grpcurl + # call hitting a momentarily-busy route -- should set this. + if retry_on_error: + try: + value = fn() + except subprocess.CalledProcessError as exc: + last_error = exc + value = None + else: + last_error = None + if until(value): + logger.info("%s — done after %.0fs", description, time.monotonic() - start) + return value + else: + value = fn() + if until(value): + logger.info("%s — done after %.0fs", description, time.monotonic() - start) + return value + now = time.monotonic() + if now - last_logged >= _PROGRESS_LOG_INTERVAL_S: + logger.info( + "Still waiting for %s (attempt %d/%d, %.0fs elapsed, last value: %r%s)", + description, + attempt + 1, + retries, + now - start, + value, + f", last error: {last_error}" if last_error else "", + ) + last_logged = now + time.sleep(delay) + if last_error is not None: + raise TimeoutError( + f"{description} — timeout after {retries * delay}s, last call failed: {last_error}" + ) from last_error + raise TimeoutError(f"{description} — timeout after {retries * delay}s, last value: {value!r}") + + +def env(name: str, default: str | None = None) -> str: + value = os.environ.get(name, default) + if value is None: + msg = f"Required environment variable {name} is not set" + raise RuntimeError(msg) + return value