diff --git a/craft_application/models/spread.py b/craft_application/models/spread.py index 7fc3efaff..1e6640075 100644 --- a/craft_application/models/spread.py +++ b/craft_application/models/spread.py @@ -39,6 +39,7 @@ class CraftSpreadSystem(SpreadBase): """Simplified spread system configuration.""" workers: int | None = None + image: str | None = None class CraftSpreadBackend(SpreadBase): @@ -116,12 +117,15 @@ class SpreadSystem(SpreadBaseModel): username: str | None = None password: str | None = None workers: int | None = None + image: str | None = None @classmethod - def from_craft(cls, simple: CraftSpreadSystem | None) -> Self: + def from_craft( + cls, simple: CraftSpreadSystem | None, image: str | None = None + ) -> Self: """Create a spread system configuration from the simplified version.""" workers = simple.workers if simple else 1 - return cls(workers=workers) + return cls(workers=workers, image=image) class SpreadBackend(SpreadBaseModel): @@ -140,14 +144,22 @@ class SpreadBackend(SpreadBaseModel): restore_each: str | None = None debug_each: str | None = None + # For the openstack backend + endpoint: str | None = None + account: str | None = None + key: str | None = None + location: str | None = None + plan: str | None = None + halt_timeout: str | None = None + @classmethod - def from_craft(cls, simple: CraftSpreadBackend) -> Self: + def from_craft(cls, simple: CraftSpreadBackend, images: dict[str, str]) -> Self: """Create a spread backend configuration from the simplified version.""" return cls( type=simple.type, allocate=simple.allocate, discard=simple.discard, - systems=cls.systems_from_craft(simple.systems), + systems=cls.systems_from_craft(simple.systems, images=images), prepare=simple.prepare, restore=simple.restore, debug=simple.debug, @@ -158,17 +170,27 @@ def from_craft(cls, simple: CraftSpreadBackend) -> Self: @staticmethod def systems_from_craft( - simple: list[str | dict[str, CraftSpreadSystem | None]], + simple: list[str | dict[str, CraftSpreadSystem | None]], images: dict[str, str] ) -> list[str | dict[str, SpreadSystem]]: """Create spread systems from the simplified version.""" systems: list[str | dict[str, SpreadSystem]] = [] for item in simple: + entry: dict[str, SpreadSystem] = {} if isinstance(item, str): - systems.append(item) + image = images.get(item) + if image: + entry[item] = SpreadSystem(workers=1, image=image) + systems.append(entry) + else: + systems.append(item) continue - entry: dict[str, SpreadSystem] = {} + for name, ssys in item.items(): - entry[name] = SpreadSystem.from_craft(ssys) + if ssys: + image = ssys.image if ssys.image else images.get(name) + else: + image = images.get(name) + entry[name] = SpreadSystem.from_craft(ssys, image=image) systems.append(entry) return systems @@ -231,6 +253,7 @@ def from_craft( craft_backend: SpreadBackend, artifact: pathlib.Path, resources: dict[str, pathlib.Path], + images: dict[str, str], ) -> Self: """Create the spread configuration from the simplified version.""" environment = { @@ -249,7 +272,7 @@ def from_craft( return cls( project="craft-test", environment=environment, - backends=cls._backends_from_craft(simple.backends, craft_backend), + backends=cls._backends_from_craft(simple.backends, craft_backend, images), suites=cls._suites_from_craft(simple.suites), exclude=simple.exclude or [".git", ".tox"], path="/root/proj", @@ -269,18 +292,20 @@ def _translate_resource_name(name: str) -> str: @staticmethod def _backends_from_craft( - simple: dict[str, CraftSpreadBackend], craft_backend: SpreadBackend + simple: dict[str, CraftSpreadBackend], + craft_backend: SpreadBackend, + images: dict[str, str], ) -> dict[str, SpreadBackend]: backends: dict[str, SpreadBackend] = {} for name, backend in simple.items(): # Spread assumes the backend name as the type when it's not explicitly declared. if name == "craft" and (not backend.type or backend.type == "craft"): craft_backend.systems = SpreadBackend.systems_from_craft( - backend.systems + backend.systems, images=images ) backends[name] = craft_backend else: - backends[name] = SpreadBackend.from_craft(backend) + backends[name] = SpreadBackend.from_craft(backend, images={}) return backends diff --git a/craft_application/services/testing.py b/craft_application/services/testing.py index 18515b65d..18062e8fe 100644 --- a/craft_application/services/testing.py +++ b/craft_application/services/testing.py @@ -32,6 +32,14 @@ from . import base +_SYSTEM_IMAGES = { + "lp-test": { + "ubuntu-20.04": "ubuntu-focal-daily-amd64", + "ubuntu-22.04": "ubuntu-jammy-daily-amd64", + "ubuntu-24.04": "ubuntu-noble-daily-amd64", + }, +} + class TestingService(base.AppService): """Service class for testing a project.""" @@ -94,8 +102,6 @@ def process_spread_yaml( retcode=os.EX_CONFIG, ) - craft_backend = self._get_backend() - if not pack_state.artifact: raise CraftError( f"No {self._app.artifact_type} files to test.", @@ -107,11 +113,16 @@ def process_spread_yaml( simple = models.CraftSpreadYaml.unmarshal(data) + backend_type = self._get_backend_type() + craft_backend = self._get_backend(backend_type) + images = _SYSTEM_IMAGES.get(backend_type) or {} + spread_yaml = models.SpreadYaml.from_craft( simple, craft_backend=craft_backend, artifact=pack_state.artifact, resources=pack_state.resources or {}, + images=images, ) emit.trace(f"Writing processed spread file to {dest}") @@ -208,8 +219,8 @@ def run_spread( is_interactive = shell or shell_after or debug try: + # Don't pipe output into stream if spread runs in interactive if is_interactive: - # Don't pipe output into stream if spread runs in interactive # mode. This allows spread to run with proper terminal management # until we implement a protocol to pause the emitter and handle # terminal input and output inside an open_stream context. See @@ -234,7 +245,13 @@ def run_spread( ) def _get_backend_type(self) -> str: - return "ci" if os.environ.get("CI") else "lxd-vm" + if os.environ.get("LP_TEST_ACCOUNT"): + return "lp-test" + + if os.environ.get("CI"): + return "ci" + + return "lxd-vm" def _running_on_ci(self) -> bool: return self._get_backend_type() == "ci" @@ -253,21 +270,30 @@ def _get_ci_system(self) -> str: return system - def _get_backend(self) -> models.SpreadBackend: - name = self._get_backend_type() - - return models.SpreadBackend( + def _get_backend(self, name: str) -> models.SpreadBackend: + backend = models.SpreadBackend( type="adhoc", - # Allocate and discard occur on the host. - allocate=f"ADDRESS $(./spread/.extension allocate {name})", - discard=f"./spread/.extension discard {name}", - # Each of these occur within the spread runner. prepare=f'"$PROJECT_PATH"/spread/.extension backend-prepare {name}', restore=f'"$PROJECT_PATH"/spread/.extension backend-restore {name}', prepare_each=f'"$PROJECT_PATH"/spread/.extension backend-prepare-each {name}', restore_each=f'"$PROJECT_PATH"/spread/.extension backend-restore-each {name}', ) + if name == "lp-test": + backend.type = "openstack" + backend.endpoint = "https://lp-test-endpoint:5000/v3" + backend.account = os.getenv("LP_TEST_ACCOUNT") + backend.key = os.getenv("LP_TEST_KEY") + backend.location = "lp-test-project/lp-test-region" + backend.plan = "cpu4-ram8-disk10" + backend.halt_timeout = "4h" + else: + backend.type = "adhoc" + backend.allocate = f"ADDRESS $(./spread/.extension allocate {name})" + backend.discard = f"./spread/.extension discard {name}" + + return backend + def _get_spread_executable(self) -> str: """Get the executable to run for spread. diff --git a/tests/unit/models/test_spread.py b/tests/unit/models/test_spread.py index 84ca0e562..904df13a9 100644 --- a/tests/unit/models/test_spread.py +++ b/tests/unit/models/test_spread.py @@ -38,7 +38,7 @@ ], ) def test_systems_from_craft(systems, expected): - assert model.SpreadBackend.systems_from_craft(systems) == expected + assert model.SpreadBackend.systems_from_craft(systems, {}) == expected _CRAFT_SPREAD = """ @@ -122,6 +122,7 @@ def test_spread_yaml_from_craft_spread(): craft_backend=backend, artifact=pathlib.Path("artifact"), resources={"my-resource": pathlib.Path("resource")}, + images={}, ) assert ( @@ -186,6 +187,107 @@ def test_spread_yaml_from_craft_spread(): ) +def test_spread_yaml_from_lp_test_craft_spread(): + backend = model.SpreadBackend( + type="openstack", + allocate="allocate", + discard="discard", + prepare="prepare", + restore="restore", + prepare_each="prepare_each", + restore_each="restore each", + endpoint="https://lp-test-endpoint:5000/v3", + account="lp-test-account", + key="lp-test-key", + location="lp-test-project/lp-test-region", + plan="cpu2-ram4-disk10", + halt_timeout="1h", + ) + data = util.safe_yaml_load(io.StringIO(_CRAFT_SPREAD)) + craft_spread = model.CraftSpreadYaml.unmarshal(data) + + spread = model.SpreadYaml.from_craft( + craft_spread, + craft_backend=backend, + artifact=pathlib.Path("artifact"), + resources={"my-resource": pathlib.Path("resource")}, + images={"ubuntu-24.04": "jammy-image"}, + ) + + assert ( + spread.marshal() + == model.SpreadYaml( + project="craft-test", + environment={ + "SUDO_USER": "", + "SUDO_UID": "", + "LANG": "C.UTF-8", + "LANGUAGE": "en", + "PROJECT_PATH": "/root/proj", + "CRAFT_ARTIFACT": "$PROJECT_PATH/artifact", + "CRAFT_RESOURCE_MY_RESOURCE": "$PROJECT_PATH/resource", + }, + backends={ + "craft": model.SpreadBackend( + type="openstack", + allocate="allocate", + discard="discard", + systems=[ + { + "ubuntu-24.04": model.SpreadSystem( + workers=1, image="jammy-image" + ) + } + ], + prepare="prepare", + restore="restore", + prepare_each="prepare_each", + restore_each="restore each", + endpoint="https://lp-test-endpoint:5000/v3", + account="lp-test-account", + key="lp-test-key", + location="lp-test-project/lp-test-region", + plan="cpu2-ram4-disk10", + halt_timeout="1h", + ), + "other": model.SpreadBackend( + type="adhoc", + systems=[{"ubuntu-24.04": model.SpreadSystem(workers=1)}], + prepare="echo Preparing backend\n", + restore="echo Restoring backend\n", + debug="echo Debugging backend\n", + prepare_each="echo Preparing-each on backend\n", + restore_each="echo Restoring-each on backend\n", + debug_each="echo Debugging-each on backend\n", + ), + }, + suites={ + "spread/general/": model.SpreadSuite( + summary="General integration tests", + systems=[], + environment={"FOO": "bar"}, + prepare="snap install $CRAFT_ARTIFACT --dangerous\n", + restore="snap remove my-snap --purge\n", + debug="echo Debugging suite\n", + prepare_each="echo Preparing-each on suite\n", + restore_each="echo Restoring-each on suite\n", + debug_each="echo Debugging-each on suite\n", + ) + }, + exclude=[".git"], + path="/root/proj", + kill_timeout="1h", + reroot="..", + prepare="echo Preparing project\n", + restore="echo Restoring project\n", + debug="echo Debugging project\n", + prepare_each="echo Preparing-each on project\n", + restore_each="echo Restoring-each on project\n", + debug_each="echo Debugging-each on project\n", + ).marshal() + ) + + @pytest.mark.parametrize( ("name", "var"), [ diff --git a/tests/unit/services/test_testing.py b/tests/unit/services/test_testing.py index bdd1c5722..c535c36cc 100644 --- a/tests/unit/services/test_testing.py +++ b/tests/unit/services/test_testing.py @@ -17,6 +17,7 @@ import pathlib import stat +import textwrap from collections.abc import Iterable from typing import Any from unittest import mock @@ -208,6 +209,138 @@ def test_process_without_spread_file(new_dir, testing_service): testing_service.process_spread_yaml(new_dir / "wherever", state) +def test_process_spread_file(new_dir, monkeypatch, testing_service): + monkeypatch.delenv("LP_TEST_ACCOUNT", raising=False) + monkeypatch.delenv("CI", raising=False) + + pathlib.Path("spread.yaml").write_text( + textwrap.dedent( + """ + project: fetch-service + backends: + craft: + type: craft + systems: + - ubuntu-24.04: + - ubuntu-22.04 + - ubuntu-20.04 + suites: + tests/general/: + summary: Just a test + """ + ) + ) + state = models.PackState(artifact=pathlib.Path("foo"), resources=None) + testing_service.process_spread_yaml(new_dir / "processed", state) + + processed = pathlib.Path("processed").read_text() + assert processed == textwrap.dedent( + """\ + project: craft-test + environment: + SUDO_USER: '' + SUDO_UID: '' + LANG: C.UTF-8 + LANGUAGE: en + PROJECT_PATH: /root/proj + CRAFT_ARTIFACT: $PROJECT_PATH/foo + backends: + craft: + type: adhoc + allocate: ADDRESS $(./spread/.extension allocate lxd-vm) + discard: ./spread/.extension discard lxd-vm + systems: + - ubuntu-24.04: + workers: 1 + - ubuntu-22.04 + - ubuntu-20.04 + prepare: '"$PROJECT_PATH"/spread/.extension backend-prepare lxd-vm' + restore: '"$PROJECT_PATH"/spread/.extension backend-restore lxd-vm' + prepare-each: '"$PROJECT_PATH"/spread/.extension backend-prepare-each lxd-vm' + restore-each: '"$PROJECT_PATH"/spread/.extension backend-restore-each lxd-vm' + suites: + tests/general/: + summary: Just a test + systems: [] + exclude: + - .git + - .tox + path: /root/proj + reroot: .. + """ + ) + + +def test_process_lp_test_spread_file(new_dir, monkeypatch, testing_service): + monkeypatch.setenv("LP_TEST_ACCOUNT", "lp-test-account") + monkeypatch.setenv("LP_TEST_KEY", "lp-test-key") + pathlib.Path("spread.yaml").write_text( + textwrap.dedent( + """ + project: fetch-service + backends: + craft: + type: craft + systems: + - ubuntu-24.04: + - ubuntu-22.04 + - ubuntu-20.04 + suites: + tests/general/: + summary: Just a test + """ + ) + ) + state = models.PackState(artifact=pathlib.Path("foo"), resources=None) + testing_service.process_spread_yaml(new_dir / "processed", state) + + processed = pathlib.Path("processed").read_text() + assert processed == textwrap.dedent( + """\ + project: craft-test + environment: + SUDO_USER: '' + SUDO_UID: '' + LANG: C.UTF-8 + LANGUAGE: en + PROJECT_PATH: /root/proj + CRAFT_ARTIFACT: $PROJECT_PATH/foo + backends: + craft: + type: openstack + systems: + - ubuntu-24.04: + workers: 1 + image: ubuntu-noble-daily-amd64 + - ubuntu-22.04: + workers: 1 + image: ubuntu-jammy-daily-amd64 + - ubuntu-20.04: + workers: 1 + image: ubuntu-focal-daily-amd64 + prepare: '"$PROJECT_PATH"/spread/.extension backend-prepare lp-test' + restore: '"$PROJECT_PATH"/spread/.extension backend-restore lp-test' + prepare-each: '"$PROJECT_PATH"/spread/.extension backend-prepare-each lp-test' + restore-each: '"$PROJECT_PATH"/spread/.extension backend-restore-each lp-test' + endpoint: https://lp-test-endpoint:5000/v3 + account: lp-test-account + key: lp-test-key + location: lp-test-project/lp-test-region + plan: cpu4-ram8-disk10 + halt-timeout: 4h + suites: + tests/general/: + summary: Just a test + systems: [] + exclude: + - .git + - .tox + path: /root/proj + reroot: .. + """ + ) + + @pytest.mark.parametrize( ("env_var", "value", "testspec"), [("", "", "craft"), ("CI", "1", "craft:id-1.0")],