Generate bootc-image-builder test jobs dynamically and enable all tests [HMS-11073]

.gitlab-ci.yml

@@ -1068,7 +1068,7 @@ fail:
     - "generate-build-config: [rhel-9.9, x86_64]"
 
 
-"bootc-image-builder/test-build-disk [x86_64]":
+"bootc-image-builder/test/test_build_cross.py [x86_64]":
   stage: gen
   extends: .terraform
   variables:
@@ -1082,9 +1082,10 @@ fail:
   script:
     - sudo ./test/scripts/install-dependencies
     - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
-    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_build_cross.py
 
-"bootc-image-builder/test-manifest [x86_64]":
+
+"bootc-image-builder/test/test_build_disk.py [x86_64]":
   stage: gen
   extends: .terraform
   variables:
@@ -1098,9 +1099,10 @@ fail:
   script:
     - sudo ./test/scripts/install-dependencies
     - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
-    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_manifest.py
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_build_disk.py
+
 
-"bootc-image-builder/test-opts-and-progress [x86_64]":
+"bootc-image-builder/test/test_build_iso.py [x86_64]":
   stage: gen
   extends: .terraform
   variables:
@@ -1114,10 +1116,61 @@ fail:
   script:
     - sudo ./test/scripts/install-dependencies
     - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
-    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_opts.py ./bootc-image-builder/test/test_progress.py
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_build_iso.py
 
 
-"bootc-image-builder/test-build-disk [aarch64]":
+"bootc-image-builder/test/test_manifest.py [x86_64]":
+  stage: gen
+  extends: .terraform
+  variables:
+    RUNNER: aws/fedora-44-x86_64
+  rules:
+    # 1. Skip the job if we're on the github merge queue
+    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
+      when: never
+    # 2. Fallback rule for any other branch name
+    - if: '$CI_COMMIT_BRANCH'
+  script:
+    - sudo ./test/scripts/install-dependencies
+    - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_manifest.py
+
+
+"bootc-image-builder/test/test_opts.py [x86_64]":
+  stage: gen
+  extends: .terraform
+  variables:
+    RUNNER: aws/fedora-44-x86_64
+  rules:
+    # 1. Skip the job if we're on the github merge queue
+    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
+      when: never
+    # 2. Fallback rule for any other branch name
+    - if: '$CI_COMMIT_BRANCH'
+  script:
+    - sudo ./test/scripts/install-dependencies
+    - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_opts.py
+
+
+"bootc-image-builder/test/test_build_cross.py [aarch64]":
+  stage: gen
+  extends: .terraform
+  variables:
+    RUNNER: aws/fedora-44-aarch64
+  rules:
+    # 1. Skip the job if we're on the github merge queue
+    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
+      when: never
+    # 2. Fallback rule for any other branch name
+    - if: '$CI_COMMIT_BRANCH'
+  script:
+    - sudo ./test/scripts/install-dependencies
+    - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_build_cross.py
+
+
+"bootc-image-builder/test/test_build_disk.py [aarch64]":
   stage: gen
   extends: .terraform
   variables:
@@ -1131,9 +1184,10 @@ fail:
   script:
     - sudo ./test/scripts/install-dependencies
     - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
-    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_build_disk.py
+
 
-"bootc-image-builder/test-manifest [aarch64]":
+"bootc-image-builder/test/test_manifest.py [aarch64]":
   stage: gen
   extends: .terraform
   variables:
@@ -1147,9 +1201,10 @@ fail:
   script:
     - sudo ./test/scripts/install-dependencies
     - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
-    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_manifest.py
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_manifest.py
+
 
-"bootc-image-builder/test-opts-and-progress [aarch64]":
+"bootc-image-builder/test/test_opts.py [aarch64]":
   stage: gen
   extends: .terraform
   variables:
@@ -1163,7 +1218,7 @@ fail:
   script:
     - sudo ./test/scripts/install-dependencies
     - sudo mkdir -pv /var/tmp/bootc-image-builder-tests
-    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_opts.py ./bootc-image-builder/test/test_progress.py
+    - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests bootc-image-builder/test/test_opts.py
 
 "generate-ostree-build-config: [centos-9, aarch64]":
   stage: ostree-gen

bootc-image-builder/test/test_build_cross.py

@@ -1,16 +1,23 @@
 import platform
 
 import pytest
-from test_build_disk import (  # pylint: disable=unused-import
-    assert_disk_image_boots, build_container_fixture, gpg_conf_fixture,
-    image_type_fixture, registry_conf_fixture, shared_tmpdir_fixture)
+
+# pylint: disable=unused-import
+from test_build_disk import (ImageBuildResult, assert_disk_image_boots,
+                             build_container_fixture, gpg_conf_fixture,
+                             image_type_fixture, registry_conf_fixture,
+                             shared_tmpdir_fixture)
 from testcases import gen_testcases
 
 
-# This testcase is not part of "test_build_disk.py:test_image_boots"
-# because it takes ~30min on the GH runners so moving it into a
-# separate file ensures it is run in parallel on GH.
 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")
 @pytest.mark.parametrize("image_type", gen_testcases("qemu-cross"), indirect=["image_type"])
+@pytest.mark.skip("disabled while rewriting to boot in AWS")
 def test_image_boots_cross(image_type):
     assert_disk_image_boots(image_type)
+
+
+@pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")
+@pytest.mark.parametrize("image_type", gen_testcases("qemu-cross"), indirect=["image_type"])
+def test_image_builds_cross(image_type: ImageBuildResult):
+    assert image_type.img_path.exists()

bootc-image-builder/test/test_build_disk.py

@@ -269,6 +269,8 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_
     container_ref = tc.container_ref
 
     if tc.sign:
+        if not registry_conf:
+            raise RuntimeError("testing signed containers requires configuring a local registry")
         container_ref = get_signed_container_ref(registry_conf.local_registry, tc.container_ref)
 
     # params can be long and the qmp socket (that has a limit of 100ish

bootc-image-builder/test/test_build_iso.py

@@ -19,18 +19,33 @@
     build_container_fixture, make_container)
 from test_build_disk import ImageBuildResult  # pylint: disable=unused-import
 from test_build_disk import (  # pylint: disable=unused-import
-    assert_kernel_args, gpg_conf_fixture, image_type_fixture,
-    registry_conf_fixture, shared_tmpdir_fixture)
+    assert_kernel_args, build_images, gpg_conf_fixture, registry_conf_fixture,
+    shared_tmpdir_fixture)
 from testcases import gen_testcases
 
 ISO_BOOT_TIMEOUT = 1800
 
 
+@pytest.fixture(name="anaconda_iso", scope="session")
+# pylint: disable=too-many-arguments,too-many-positional-arguments
+def anaconda_iso_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf):
+    """
+    Build an anconda-iso image inside the passed build_container and return an ImageBuildResult with the resulting image
+    path and user/password. In the case an image is being built from a local container, the function will build the
+    required local container for the test.
+    """
+    testutil.pull_container(request.param.container_ref, request.param.target_arch)
+
+    with build_images(shared_tmpdir, build_container, request,
+                      force_aws_upload, gpg_conf, registry_conf) as build_results:
+        return build_results[0]
+
+
 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")
-@pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"])
+@pytest.mark.parametrize("anaconda_iso", gen_testcases("anaconda-iso"), indirect=["anaconda_iso"])
 @pytest.mark.skip(reason="kvm boot tests are currently disabled")
-def test_iso_installs(image_type):
-    installer_iso_path = image_type.img_path
+def test_iso_installs(anaconda_iso):
+    installer_iso_path = anaconda_iso.img_path
     test_disk_path = installer_iso_path.with_name("test-disk.img")
     with open(test_disk_path, "w", encoding="utf8") as fp:
         fp.truncate(10_1000_1000_1000)
@@ -41,45 +56,48 @@ def test_iso_installs(image_type):
     # boot test disk and do extremly simple check
     with testlib.vm.QEMU(test_disk_path) as vm:
         vm.start(use_ovmf=True)
-        vm.run("true", user=image_type.username, password=image_type.password)
-        assert_kernel_args(vm, image_type)
-        ret = vm.run(["bootc", "status"], user="root", keyfile=image_type.ssh_keyfile_private_path)
-        assert f"image: {image_type.container_ref}" in ret.stdout
+        vm.run("true", user=anaconda_iso.username, password=anaconda_iso.password)
+        assert_kernel_args(vm, anaconda_iso)
+        ret = vm.run(["bootc", "status"], user="root", keyfile=anaconda_iso.ssh_keyfile_private_path)
+        assert f"image: {anaconda_iso.container_ref}" in ret.stdout
 
 
 def osinfo_for(it: ImageBuildResult, arch: str) -> str:
     base = "Media is an installer for OS"
     if it.container_ref.endswith("/centos-bootc/centos-bootc:stream9"):
         return f"{base} 'CentOS Stream 9 ({arch})'\n"
     if it.container_ref.endswith("/centos-bootc/centos-bootc:stream10"):
-        return f"Media is an installer for OS 'CentOS Stream 10 ({arch})'\n"
+        return f"{base} 'CentOS Stream 10 ({arch})'\n"
     if "/fedora/fedora-bootc:" in it.container_ref:
         ver = it.container_ref.rsplit(":", maxsplit=1)[1]
+        if ver == "44":
+            # This condition should be removed when osinfo is updated to include Fedora 44 detection
+            return f"{base} 'Fedora ({arch})'\n"
         return f"{base} 'Fedora Server {ver} ({arch})'\n"
     raise ValueError(f"unknown osinfo string for '{it.container_ref}'")
 
 
 @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now")
-@pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"])
-def test_iso_os_detection(image_type):
-    installer_iso_path = image_type.img_path
-    arch = image_type.img_arch
+@pytest.mark.parametrize("anaconda_iso", gen_testcases("anaconda-iso"), indirect=["anaconda_iso"])
+def test_iso_os_detection(anaconda_iso):
+    installer_iso_path = anaconda_iso.img_path
+    arch = anaconda_iso.img_arch
     if not arch:
         arch = platform.machine()
     result = subprocess.run([
         "osinfo-detect",
         installer_iso_path,
     ], capture_output=True, text=True, check=True)
     osinfo_output = result.stdout
-    expected_output = f"Media is bootable.\n{osinfo_for(image_type, arch)}"
+    expected_output = f"Media is bootable.\n{osinfo_for(anaconda_iso, arch)}"
     assert osinfo_output == expected_output
 
 
 @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now")
 @pytest.mark.skipif(not testutil.has_executable("unsquashfs"), reason="need unsquashfs")
-@pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"])
-def test_iso_install_img_is_squashfs(tmp_path, image_type):
-    installer_iso_path = image_type.img_path
+@pytest.mark.parametrize("anaconda_iso", gen_testcases("anaconda-iso"), indirect=["anaconda_iso"])
+def test_iso_install_img_is_squashfs(tmp_path, anaconda_iso):
+    installer_iso_path = anaconda_iso.img_path
     with ExitStack() as cm:
         mount_point = tmp_path / "cdrom"
         mount_point.mkdir()

bootc-image-builder/test/test_opts.py

@@ -6,9 +6,9 @@
 import pytest
 
 import testutil
-# pylint: disable=unused-import
-from containerbuild import (build_container_fixture,
-                            build_fake_container_fixture)
+from containerbuild import (  # pylint: disable=unused-import
+    build_container_fixture, build_erroring_container_fixture,
+    build_fake_container_fixture)
 
 
 @pytest.fixture(name="container_storage", scope="session")
@@ -183,3 +183,72 @@ def test_bib_no_outside_container_warning_in_container(tmp_path, container_stora
         container_ref,
     ], check=True, capture_output=True, text=True)
     assert "running outside a container" not in res.stderr
+
+
+def test_progress_debug(tmp_path, build_fake_container):
+    container_ref = "quay.io/centos-bootc/centos-bootc:stream9"
+    testutil.pull_container(container_ref)
+
+    output_path = tmp_path / "output"
+    output_path.mkdir(exist_ok=True)
+
+    cmdline = [
+        *testutil.podman_run_common,
+        build_fake_container,
+        "build",
+        "--progress=debug",
+        container_ref,
+    ]
+    res = subprocess.run(cmdline, capture_output=True, check=True, text=True)
+    assert res.stderr.count("Start progressbar") == 1
+    assert res.stderr.count("Manifest generation step") == 1
+    assert res.stderr.count("Disk image building step") == 1
+    assert res.stderr.count("Build complete") == 1
+    assert res.stderr.count("Stop progressbar") == 1
+    assert res.stdout.strip() == ""
+
+
+def test_progress_term_works_without_tty(tmp_path, build_fake_container):
+    container_ref = "quay.io/centos-bootc/centos-bootc:stream9"
+    testutil.pull_container(container_ref)
+
+    output_path = tmp_path / "output"
+    output_path.mkdir(exist_ok=True)
+
+    cmdline = [
+        *testutil.podman_run_common,
+        # note that "-t" is missing here
+        build_fake_container,
+        "build",
+        # explicitly selecting term progress works even when there is no tty
+        # (i.e. we just need ansi terminal support)
+        "--progress=term",
+        container_ref,
+    ]
+    res = subprocess.run(cmdline, capture_output=True, text=True, check=False)
+    assert res.returncode == 0
+    assert "[|] Manifest generation step" in res.stderr
+
+
+@pytest.mark.skipif(not testutil.can_start_rootful_containers(), reason="require a rootful containers (try: sudo)")
+@pytest.mark.parametrize("progress", ["term", "verbose"])
+def test_progress_error_reporting(tmp_path, build_erroring_container, progress):
+    output_path = tmp_path / "output"
+    output_path.mkdir(exist_ok=True)
+
+    cmdline = [
+        *testutil.podman_run_common,
+        "-v", "/var/lib/containers/storage:/var/lib/containers/storage",
+        # we have a terminal
+        "-t",
+        build_erroring_container,
+        "build",
+        f"--progress={progress}",
+        "quay.io/centos-bootc/centos-bootc:stream9",
+    ]
+    res = subprocess.run(cmdline, capture_output=True, text=True, check=False)
+    assert "osbuild-stage-stdout-output" in res.stdout
+    assert "osbuild-stage-stderr-output" in res.stdout
+    assert "output-from-osbuild-stdout" in res.stdout
+    assert "output-from-osbuild-stderr" in res.stdout
+    assert res.returncode == 1