From b869c8255d4cf309b5e3ab62bcfb38ac945310fc Mon Sep 17 00:00:00 2001 From: Alexander van der Grinten Date: Mon, 27 Apr 2026 21:08:10 +0200 Subject: [PATCH 1/6] base: Take xbstrap-built rootfs properties from bootstrap.yml --- xbstrap/base.py | 57 +++++++++++++++++++++++++++------------------- xbstrap/schema.yml | 8 +++++++ 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/xbstrap/base.py b/xbstrap/base.py index dd02e7d..ca5bc32 100644 --- a/xbstrap/base.py +++ b/xbstrap/base.py @@ -598,6 +598,14 @@ def container_runtime(self): return None return self._site_yml["container"].get("runtime") + @property + def container_build_yml(self): + site_container_yml = self._site_yml.get("container") or {} + if "rootfs" in site_container_yml: + return site_container_yml + root_container_yml = self._root_yml.get("container") or {} + return root_container_yml + @property def site_architectures(self): return self._site_archs @@ -2340,7 +2348,7 @@ def _build_rootfs_layers(cfg, rootfs): def prepare_rootfs(cfg, rootfs): # Inspired by https://codeberg.org/Mintsuki/jinx. - site_container_yml = cfg._site_yml.get("container", dict()) + build_yml = cfg.container_build_yml rootfs_cache = os.path.join(_util.find_cache_dir(), "rootfs_cache") _util.try_mkdir(rootfs_cache) @@ -2375,8 +2383,8 @@ def run_cbuildrt( "environ": environ or {}, }, "mapCurrentUserTo": { - "uid": site_container_yml["uid"], - "gid": site_container_yml["gid"], + "uid": build_yml["uid"], + "gid": build_yml["gid"], }, "bindMounts": bind_mounts or [], "volumes": volumes or [], @@ -2434,8 +2442,8 @@ def run_cbuildrt( suite = container_yml["suite"] snapshot = container_yml["snapshot"] - src_mount = site_container_yml.get("src_mount") - build_mount = site_container_yml.get("build_mount") + src_mount = build_yml.get("src_mount") + build_mount = build_yml.get("build_mount") if src_mount.startswith("/"): src_mount = src_mount[1:] @@ -2662,6 +2670,7 @@ def run_program( runtime = cfg.container_runtime container_yml = cfg._site_yml.get("container", dict()) + build_yml = cfg.container_build_yml use_container = True if containerless: @@ -2695,13 +2704,15 @@ def run_program( if proc.returncode != 0: raise ProgramFailureError() elif runtime == "docker": - if any(prop not in container_yml for prop in ["src_mount", "build_mount", "image"]): + if any(prop not in build_yml for prop in ["src_mount", "build_mount"]) or ( + "image" not in container_yml + ): raise GenericError( "Docker runtime requires src_mount, build_mount and image properties" ) - manifest["source_root"] = container_yml["src_mount"] - manifest["build_root"] = container_yml["build_mount"] + manifest["source_root"] = build_yml["src_mount"] + manifest["build_root"] = build_yml["build_mount"] _util.log_info( "Running {} (tools: {}) in Docker".format(args, [tool.name for tool in pkg_queue]) @@ -2717,9 +2728,9 @@ def run_program( "-i", "--init", "-v", - cfg.source_root + ":" + container_yml["src_mount"], + cfg.source_root + ":" + build_yml["src_mount"], "-v", - cfg.build_root + ":" + container_yml["build_mount"], + cfg.build_root + ":" + build_yml["build_mount"], ] if os.isatty(0): # FD zero = stdin. docker_args += ["-t"] @@ -2737,8 +2748,8 @@ def run_program( if proc.returncode != 0: raise ProgramFailureError() elif runtime == "runc": - manifest["source_root"] = container_yml["src_mount"] - manifest["build_root"] = container_yml["build_mount"] + manifest["source_root"] = build_yml["src_mount"] + manifest["build_root"] = build_yml["build_mount"] _util.log_info( "Running {} (tools: {}) via runc".format(args, [tool.name for tool in pkg_queue]) @@ -2767,13 +2778,13 @@ def run_program( "hostname": container_yml["id"], "mounts": [ { - "destination": container_yml["src_mount"], + "destination": build_yml["src_mount"], "source": cfg.source_root, "options": ["bind"], "type": "none", }, { - "destination": container_yml["build_mount"], + "destination": build_yml["build_mount"], "source": cfg.build_root, "options": ["bind"], "type": "none", @@ -2817,8 +2828,8 @@ def run_program( rootfs = cfg.get_rootfs(rootfs_pkgs) rootfs = {"layers": _build_rootfs_layers(cfg, rootfs)} - manifest["source_root"] = container_yml["src_mount"] - manifest["build_root"] = container_yml["build_mount"] + manifest["source_root"] = build_yml["src_mount"] + manifest["build_root"] = build_yml["build_mount"] _util.log_info( "Running {} (tools: {}) via cbuildrt".format( @@ -2833,28 +2844,26 @@ def run_program( _util.try_mkdir(cfg.sysroot_dir) cbuild_json = { - "user": {"uid": container_yml["uid"], "gid": container_yml["gid"]}, + "user": {"uid": build_yml["uid"], "gid": build_yml["gid"]}, "process": {"args": ["xbstrap", "execute-manifest", "-c", yaml.dump(manifest)]}, "rootfs": rootfs, "isolateNetwork": isolate_network, "bindMounts": [ - {"destination": container_yml["src_mount"], "source": cfg.source_root}, - {"destination": container_yml["build_mount"], "source": cfg.build_root}, + {"destination": build_yml["src_mount"], "source": cfg.source_root}, + {"destination": build_yml["build_mount"], "source": cfg.build_root}, ], } if is_xbstrap_rootfs: cbuild_json["mapCurrentUserTo"] = { - "uid": container_yml["uid"], - "gid": container_yml["gid"], + "uid": build_yml["uid"], + "gid": build_yml["gid"], } if sysroot is not None: if verbosity: _util.log_info(f"Bind mounting {sysroot} as sysroot") cbuild_json["bindMounts"].append( { - "destination": os.path.join( - container_yml["build_mount"], cfg.sysroot_subdir - ), + "destination": os.path.join(build_yml["build_mount"], cfg.sysroot_subdir), "source": sysroot, }, ) diff --git a/xbstrap/schema.yml b/xbstrap/schema.yml index fa1e2dc..883d80f 100644 --- a/xbstrap/schema.yml +++ b/xbstrap/schema.yml @@ -422,3 +422,11 @@ properties: type: array items: type: string + 'uid': + type: integer + 'gid': + type: integer + 'src_mount': + type: string + 'build_mount': + type: string From 3c515ed73c0b69b1876b814c6ccf9d0286d480da Mon Sep 17 00:00:00 2001 From: Alexander van der Grinten Date: Mon, 27 Apr 2026 21:32:35 +0200 Subject: [PATCH 2/6] base: Import initial files via importUpper This is needed because cbuildrt does not create mount points for volumes anymore. --- xbstrap/base.py | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/xbstrap/base.py b/xbstrap/base.py index ca5bc32..b4d0543 100644 --- a/xbstrap/base.py +++ b/xbstrap/base.py @@ -2328,6 +2328,22 @@ def check_if_prepared(self, _, cfg): return ItemState(missing=not os.path.exists(rootfs_marker_path)) +# Build the initial file system that debootstrap runs on. +def _build_rootfs_seed_tar(tar_path): + components = ["var", "var/cache", "var/cache/apt", "var/cache/apt/archives"] + with tarfile.open(tar_path, "w") as tf: + for comp in components: + info = tarfile.TarInfo(name=comp) + info.type = tarfile.DIRTYPE + info.mode = 0o755 + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = 0 + tf.addfile(info) + + # Build the cbuildrt lower layer configuration for a given rootfs. def _build_rootfs_layers(cfg, rootfs): rootfs_cache = os.path.join(_util.find_cache_dir(), "rootfs_cache") @@ -2483,19 +2499,22 @@ def run_cbuildrt( done """ _util.log_info("Building base rootfs") - run_cbuildrt( - args=["sh", "-c", script], - rootfs={ - "layers": [empty_layer_dir], - "withUpper": True, - "extractUpper": base_tar_path, - }, - no_chroot_or_mounts=True, - volumes=[ - {"name": "apt_cache", "destination": "/var/cache/apt/archives"}, - ], - host_environ=host_environ, - ) + with tempfile.NamedTemporaryFile(suffix=".tar") as seed_tar: + _build_rootfs_seed_tar(seed_tar.name) + run_cbuildrt( + args=["sh", "-c", script], + rootfs={ + "layers": [empty_layer_dir], + "withUpper": True, + "extractUpper": base_tar_path, + "importUpper": seed_tar.name, + }, + no_chroot_or_mounts=True, + volumes=[ + {"name": "apt_cache", "destination": "/var/cache/apt/archives"}, + ], + host_environ=host_environ, + ) # Install packages and xbstrap itself. # This creates the -full.tar. From 14070e00e019272ba8b0c3632520edc0873fd2e4 Mon Sep 17 00:00:00 2001 From: Alexander van der Grinten Date: Mon, 27 Apr 2026 22:57:48 +0200 Subject: [PATCH 3/6] base: Unify base + full layers into a single layer --- xbstrap/base.py | 123 +++++++++++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/xbstrap/base.py b/xbstrap/base.py index b4d0543..3ad6c03 100644 --- a/xbstrap/base.py +++ b/xbstrap/base.py @@ -2348,15 +2348,17 @@ def _build_rootfs_seed_tar(tar_path): def _build_rootfs_layers(cfg, rootfs): rootfs_cache = os.path.join(_util.find_cache_dir(), "rootfs_cache") + # OverlayFS requires at least two lower layers when no upper layer is used. + # Hence, always prepend an empty layer. + empty_layer_dir = os.path.join(rootfs_cache, "empty") + _util.try_mkdir(empty_layer_dir) + base_rootfs = cfg.get_rootfs(()) - layers = [ - os.path.join(rootfs_cache, base_rootfs.hash + "-base.tar"), - os.path.join(rootfs_cache, base_rootfs.hash + "-full.tar"), - ] + layers = [empty_layer_dir, os.path.join(rootfs_cache, base_rootfs.hash + ".tar")] for i in range(len(rootfs.packages)): pkg_rootfs = cfg.get_rootfs(rootfs.packages[: i + 1]) - layers.append(os.path.join(rootfs_cache, pkg_rootfs.hash + "-full.tar")) + layers.append(os.path.join(rootfs_cache, pkg_rootfs.hash + ".tar")) return layers @@ -2369,16 +2371,14 @@ def prepare_rootfs(cfg, rootfs): rootfs_cache = os.path.join(_util.find_cache_dir(), "rootfs_cache") _util.try_mkdir(rootfs_cache) - base_tar_path = os.path.join(rootfs_cache, rootfs.hash + "-base.tar") - full_tar_path = os.path.join(rootfs_cache, rootfs.hash + "-full.tar") + out_tar_path = os.path.join(rootfs_cache, rootfs.hash + ".tar") rootfs_marker_path = os.path.join(rootfs_cache, rootfs.hash + ".prepared") - # Remove existing tar files. - for tar_path in (base_tar_path, full_tar_path): - try: - os.remove(tar_path) - except FileNotFoundError: - pass + # Remove existing tar file. + try: + os.remove(out_tar_path) + except FileNotFoundError: + pass def run_cbuildrt( args, @@ -2445,7 +2445,7 @@ def run_cbuildrt( rootfs={ "layers": lower_dirs, "withUpper": True, - "extractUpper": full_tar_path, + "extractUpper": out_tar_path, }, volumes=[ {"name": "apt_cache", "destination": "/var/cache/apt/archives"}, @@ -2480,34 +2480,37 @@ def run_cbuildrt( prepend=[os.path.join(_util.find_home(), "bin"), "/sbin"], ) - # Run debootstrap via cbuildrt with noChroot + noSystemMounts. - # This creates the -base.tar. - script = f""" - set -e + with tempfile.TemporaryDirectory() as scratch: + seed_tar = os.path.join(scratch, "seed.tar") + base_tar = os.path.join(scratch, "base.tar") + _build_rootfs_seed_tar(seed_tar) - target="$(pwd)" + # Run debootstrap via cbuildrt with noChroot + noSystemMounts. + script = f""" + set -e - debootstrap '{suite}' "$target" \ - 'https://snapshot.debian.org/archive/debian/{snapshot}' + target="$(pwd)" - mkdir -p "$target/{src_mount}" - mkdir -p "$target/{build_mount}" + debootstrap '{suite}' "$target" \ + 'https://snapshot.debian.org/archive/debian/{snapshot}' - for dev in tty null zero full random urandom; do - rm -f "$target/dev/$dev" - touch "$target/dev/$dev" - done - """ - _util.log_info("Building base rootfs") - with tempfile.NamedTemporaryFile(suffix=".tar") as seed_tar: - _build_rootfs_seed_tar(seed_tar.name) + mkdir -p "$target/{src_mount}" + mkdir -p "$target/{build_mount}" + + for dev in tty null zero full random urandom; do + rm -f "$target/dev/$dev" + touch "$target/dev/$dev" + done + """ + + _util.log_info("Building base rootfs") run_cbuildrt( args=["sh", "-c", script], rootfs={ "layers": [empty_layer_dir], "withUpper": True, - "extractUpper": base_tar_path, - "importUpper": seed_tar.name, + "extractUpper": base_tar, + "importUpper": seed_tar, }, no_chroot_or_mounts=True, volumes=[ @@ -2516,44 +2519,44 @@ def run_cbuildrt( host_environ=host_environ, ) - # Install packages and xbstrap itself. - # This creates the -full.tar. - base_pkgs = container_yml.get("base_packages", []) - base_pkgs_str = " ".join(shlex.quote(p) for p in base_pkgs) + # Install packages and xbstrap itself. + base_pkgs = container_yml.get("base_packages", []) + base_pkgs_str = " ".join(shlex.quote(p) for p in base_pkgs) - script = f""" - set -e + script = f""" + set -e - printf '%s\\n' 'en_US.UTF-8 UTF-8' > /etc/locale.gen + printf '%s\\n' 'en_US.UTF-8 UTF-8' > /etc/locale.gen - cat > /etc/apt/apt.conf < /etc/apt/apt.conf < Date: Mon, 27 Apr 2026 23:12:46 +0200 Subject: [PATCH 4/6] base: Do not use marker files for rootfs Check for the presence of rootfs tar directly instead. --- xbstrap/base.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/xbstrap/base.py b/xbstrap/base.py index 3ad6c03..d5dc170 100644 --- a/xbstrap/base.py +++ b/xbstrap/base.py @@ -2323,9 +2323,9 @@ def name(self): def check_if_prepared(self, _, cfg): rootfs_cache = os.path.join(_util.find_cache_dir(), "rootfs_cache") - rootfs_marker_path = os.path.join(rootfs_cache, self.hash + ".prepared") + rootfs_tar_path = os.path.join(rootfs_cache, self.hash + ".tar") - return ItemState(missing=not os.path.exists(rootfs_marker_path)) + return ItemState(missing=not os.path.exists(rootfs_tar_path)) # Build the initial file system that debootstrap runs on. @@ -2372,7 +2372,6 @@ def prepare_rootfs(cfg, rootfs): _util.try_mkdir(rootfs_cache) out_tar_path = os.path.join(rootfs_cache, rootfs.hash + ".tar") - rootfs_marker_path = os.path.join(rootfs_cache, rootfs.hash + ".prepared") # Remove existing tar file. try: @@ -2558,9 +2557,6 @@ def run_cbuildrt( environ=container_environ, ) - # Create the marker file - open(rootfs_marker_path, "w").close() - def run_program( cfg, From 9616ef46171f5f20beb38a38c7b6ca5b55e97b78 Mon Sep 17 00:00:00 2001 From: Alexander van der Grinten Date: Tue, 28 Apr 2026 08:45:20 +0200 Subject: [PATCH 5/6] base: Simplify debootstrap stage of prepare-rootfs --- xbstrap/base.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/xbstrap/base.py b/xbstrap/base.py index d5dc170..19d6f86 100644 --- a/xbstrap/base.py +++ b/xbstrap/base.py @@ -2401,6 +2401,7 @@ def run_cbuildrt( "uid": build_yml["uid"], "gid": build_yml["gid"], }, + "provideDev": True, "bindMounts": bind_mounts or [], "volumes": volumes or [], } @@ -2485,26 +2486,14 @@ def run_cbuildrt( _build_rootfs_seed_tar(seed_tar) # Run debootstrap via cbuildrt with noChroot + noSystemMounts. - script = f""" - set -e - - target="$(pwd)" - - debootstrap '{suite}' "$target" \ - 'https://snapshot.debian.org/archive/debian/{snapshot}' - - mkdir -p "$target/{src_mount}" - mkdir -p "$target/{build_mount}" - - for dev in tty null zero full random urandom; do - rm -f "$target/dev/$dev" - touch "$target/dev/$dev" - done - """ - _util.log_info("Building base rootfs") run_cbuildrt( - args=["sh", "-c", script], + args=[ + "debootstrap", + suite, + ".", + f"https://snapshot.debian.org/archive/debian/{snapshot}", + ], rootfs={ "layers": [empty_layer_dir], "withUpper": True, @@ -2540,6 +2529,9 @@ def run_cbuildrt( apt-get install -y python3 python3-pip {base_pkgs_str} python3 -m pip install --progress-bar off --root-user-action ignore --break-system-packages xbstrap + + mkdir -p "$target/{src_mount}" + mkdir -p "$target/{build_mount}" """ _util.log_info("Finalizing rootfs") @@ -2872,6 +2864,7 @@ def run_program( ], } if is_xbstrap_rootfs: + cbuild_json["provideDev"] = True cbuild_json["mapCurrentUserTo"] = { "uid": build_yml["uid"], "gid": build_yml["gid"], From c58fbb27a29f794e8604343fb94ebf9ddd494ae4 Mon Sep 17 00:00:00 2001 From: Alexander van der Grinten Date: Wed, 29 Apr 2026 09:16:40 +0200 Subject: [PATCH 6/6] base: Use zstd compression for rootfs tars --- xbstrap/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xbstrap/base.py b/xbstrap/base.py index 19d6f86..3d39099 100644 --- a/xbstrap/base.py +++ b/xbstrap/base.py @@ -2354,11 +2354,11 @@ def _build_rootfs_layers(cfg, rootfs): _util.try_mkdir(empty_layer_dir) base_rootfs = cfg.get_rootfs(()) - layers = [empty_layer_dir, os.path.join(rootfs_cache, base_rootfs.hash + ".tar")] + layers = [empty_layer_dir, os.path.join(rootfs_cache, base_rootfs.hash + ".tar.zstd")] for i in range(len(rootfs.packages)): pkg_rootfs = cfg.get_rootfs(rootfs.packages[: i + 1]) - layers.append(os.path.join(rootfs_cache, pkg_rootfs.hash + ".tar")) + layers.append(os.path.join(rootfs_cache, pkg_rootfs.hash + ".tar.zstd")) return layers @@ -2371,7 +2371,7 @@ def prepare_rootfs(cfg, rootfs): rootfs_cache = os.path.join(_util.find_cache_dir(), "rootfs_cache") _util.try_mkdir(rootfs_cache) - out_tar_path = os.path.join(rootfs_cache, rootfs.hash + ".tar") + out_tar_path = os.path.join(rootfs_cache, rootfs.hash + ".tar.zstd") # Remove existing tar file. try: