Sysroot support

README.md

@@ -65,6 +65,9 @@ The [examples](/examples) demonstrate how to accomplish typical tasks such as
 - [apt-get install](/examples/debian_snapshot) from Debian repositories.
 - [apt-get install](/examples/ubuntu_snapshot) from Ubuntu repositories.
 
+You can create sysroots for toolchains_llvm on the fly using rules_distroless.
+See [examples/sysroot/README.md](examples/sysroot/README.md) for more details.
+
 We also have `distroless`-specific rules that could be useful:
 
 - [flatten](/examples/flatten): flatten multiple `tar` archives.

apt/BUILD.bazel

@@ -24,6 +24,7 @@ bzl_library(
         "//apt/private:deb_filemap",
         "//apt/private:deb_import",
         "//apt/private:lockfile",
+        "//apt/private:sysroot_repository",
         "//apt/private:translate_dependency_set",
         "//apt/private:util",
         "//apt/private:version_constraint",

apt/extensions.bzl

@@ -6,6 +6,7 @@ load("//apt/private:apt_dep_resolver.bzl", "dependency_resolver")
 load("//apt/private:deb_filemap.bzl", "deb_filemap")
 load("//apt/private:deb_import.bzl", "deb_import")
 load("//apt/private:lockfile.bzl", "lockfile")
+load("//apt/private:sysroot_repository.bzl", "sysroot_repository")
 load("//apt/private:translate_dependency_set.bzl", "translate_dependency_set")
 load("//apt/private:util.bzl", "util")
 load("//apt/private:version_constraint.bzl", "version_constraint")
@@ -329,6 +330,15 @@ def _distroless_extension(mctx):
         False: {},
         True: {},
     }
+    sysroot_repos = []
+
+    for mod in mctx.modules:
+        for sysroot_tag in mod.tags.sysroot:
+            sysroot_repos.append((
+                sysroot_tag.name,
+                sysroot_tag.dependency_set,
+                sysroot_tag.architecture,
+            ))
 
     for mod in mctx.modules:
         for install in mod.tags.install:
@@ -484,6 +494,17 @@ def _distroless_extension(mctx):
             mergedusr = depset_mergedusr,
         )
 
+    # Generate separate sysroot repositories for each architecture
+    for (sysroot_name, depset_name, arch) in sysroot_repos:
+        if depset_name not in dependency_sets:
+            fail("apt.sysroot refers to unknown dependency_set '{}'. Add apt.install with the same dependency_set first.".format(depset_name))
+        sysroot_repository(
+            name = sysroot_name,
+            depset_name = depset_name,
+            lock_content = lock_content,
+            architecture = arch,
+        )
+
     # Generate a repo per package which will be aliased by hub repo.
     for (package_key, package) in glock.packages().items():
         (suite, name, arch, version) = lockfile.parse_package_key(package_key)
@@ -593,6 +614,38 @@ You can use the package like so: `@<REPO>//<PACKAGE>/<ARCH>:<TARGET>`.
 
 E.g. for the previous example, you could use `@bullseye//perl/amd64:data`.
 
+## Creating Unpacked Sysroots
+
+To create unpacked sysroot repositories for use with toolchains like `toolchains_llvm`,
+use `apt.sysroot`. This creates a separate repository for the specified architecture:
+
+```starlark
+apt.install(
+    dependency_set = "my_sysroot",
+    packages = ["libc6", "libstdc++6"],
+    suites = ["noble"],
+)
+
+apt.sysroot(
+    dependency_set = "my_sysroot",
+    architecture = "amd64",
+    name = "my_sysroot_amd64",
+)
+apt.sysroot(
+    dependency_set = "my_sysroot",
+    architecture = "arm64",
+    name = "my_sysroot_arm64",
+)
+```
+
+This creates separate unpacked sysroot repositories:
+- `@my_sysroot_amd64` with unpacked content at `//sysroot`
+- `@my_sysroot_arm64` with unpacked content at `//sysroot`
+
+Each sysroot repository is independent and contains only the unpacked files for
+that specific architecture. The unpacking happens at repository fetch time, making
+the sysroots available to toolchains immediately.
+
 ### Lockfiles
 
 As mentioned, the macro can be used without a lock because the lock will be
@@ -660,6 +713,23 @@ install = tag_class(
     },
 )
 
+sysroot = tag_class(
+    attrs = {
+        "name": attr.string(
+            mandatory = True,
+            doc = "The name of the sysroot repository.",
+        ),
+        "dependency_set": attr.string(
+            mandatory = True,
+            doc = "The dependency set to create a sysroot for.",
+        ),
+        "architecture": attr.string(
+            mandatory = True,
+            doc = "The architecture to unpack the sysroot for.",
+        ),
+    },
+)
+
 lock = tag_class(
     attrs = {
         "into": attr.label(
@@ -674,6 +744,7 @@ apt = module_extension(
     tag_classes = {
         "install": install,
         "sources_list": sources_list,
+        "sysroot": sysroot,
         "lock": lock,
     },
 )

apt/private/BUILD.bazel

@@ -142,3 +142,15 @@ bzl_library(
     srcs = ["util.bzl"],
     visibility = ["//apt:__subpackages__"],
 )
+
+bzl_library(
+    name = "sysroot_repository",
+    srcs = ["sysroot_repository.bzl"],
+    visibility = ["//apt:__subpackages__"],
+    deps = [
+        ":lockfile",
+        ":translate_dependency_set",
+        ":util",
+        "@bazel_lib//lib:repo_utils",
+    ],
+)

apt/private/lockfile.bzl

@@ -58,6 +58,8 @@ def _add_package_dependency(lock, package, dependency, arch = None):
     if k not in lock.packages:
         fail("illegal state: %s is not in the lockfile." % package["Package"])
     sk = _package_key(dependency, arch)
+    if sk not in lock.packages:
+        fail("illegal state: %s is not in the lockfile." % dependency["Package"])
     if sk in lock.packages[k]["depends_on"]:
         return
     lock.packages[k]["depends_on"].append(sk)

apt/private/sysroot_repository.bzl

@@ -0,0 +1,95 @@
+"repository rule for generating unpacked sysroot repositories from a lockfile."
+
+load(":lockfile.bzl", "lockfile")
+load(":translate_dependency_set.bzl", "dependency_set_transitive_package_keys")
+load(":util.bzl", "util")
+
+_SYSROOT_BUILD_TMPL = """\
+"Generated by rules_distroless. DO NOT EDIT."
+
+filegroup(
+    name = "sysroot",
+    srcs = ["."],
+    visibility = ["//visibility:public"],
+)
+"""
+
+def _materialize_deb_data_into_root(rctx, deb_path, ar_dir, unpack_dir):
+    """Unpacks the data.tar.* payload of a .deb archive into unpack_dir.
+
+    This relies exclusively on Bazel's built-in support for extracting
+    the `ar` container format used by .deb files, and the nested `tar.*`
+    payload, via repository_ctx.extract().
+    """
+
+    # .deb files are `ar` archives containing debian-binary, control.tar.*
+    # and data.tar.* members. Extracting unpacks those members as plain,
+    # uncompressed files (the ".deb" extension is auto-detected as ar).
+    rctx.extract(archive = deb_path, output = ar_dir)
+
+    data_member = None
+    for entry in rctx.path(ar_dir).readdir():
+        if entry.basename.startswith("data.tar"):
+            data_member = entry
+            break
+
+    if data_member == None:
+        fail("No data archive found in {}".format(deb_path))
+
+    # The archive type (tar.gz/tar.xz/tar.zst/...) is auto-detected from
+    # the data_member's file extension.
+    rctx.extract(archive = data_member, output = unpack_dir)
+
+def _unpack(rctx, packages, dependency_set, sources, architecture):
+    if architecture not in dependency_set["sets"]:
+        fail("architecture '{}' is not present in dependency set '{}'".format(architecture, rctx.attr.depset_name))
+
+    unpack_dir = "sysroot"
+    work_dir = ".unpack_work_%s" % architecture
+
+    package_keys = dependency_set_transitive_package_keys(packages, dependency_set, [architecture, "all"])
+    for package_key in package_keys:
+        package = packages[package_key]
+        sanitized_key = util.sanitize(package_key)
+        deb_path = "{}/{}.deb".format(work_dir, sanitized_key)
+        ar_dir = "{}/{}.ar".format(work_dir, sanitized_key)
+
+        rctx.download(
+            output = deb_path,
+            sha256 = package["sha256"],
+            url = [
+                uri + "/" + package["filename"]
+                for uri in sources[package["suite"]]["uris"]
+            ],
+        )
+
+        _materialize_deb_data_into_root(rctx, deb_path, ar_dir, unpack_dir)
+
+    rctx.delete(work_dir)
+
+def _sysroot_repository_impl(rctx):
+    lockf = lockfile.from_json(rctx, rctx.attr.lock_content)
+
+    sources = lockf.sources()
+    packages = lockf.packages()
+    dependency_sets = lockf.dependency_sets()
+    dependency_set = dependency_sets[rctx.attr.depset_name]
+
+    _unpack(
+        rctx,
+        packages,
+        dependency_set,
+        sources,
+        rctx.attr.architecture,
+    )
+
+    rctx.file("sysroot/BUILD.bazel", _SYSROOT_BUILD_TMPL)
+
+sysroot_repository = repository_rule(
+    implementation = _sysroot_repository_impl,
+    attrs = {
+        "depset_name": attr.string(doc = "INTERNAL: DO NOT USE"),
+        "lock_content": attr.string(doc = "INTERNAL: DO NOT USE"),
+        "architecture": attr.string(doc = "INTERNAL: Architecture to unpack and expose as //sysroot:sysroot."),
+    },
+)

apt/private/translate_dependency_set.bzl

@@ -133,21 +133,93 @@ alias(
 )
 """
 
-# Computes the `@repo//:data` labels a package's per-architecture BUILD.bazel
-# should depend on for the given architecture.
+# Builds the fully-qualified package key (`<short_key>=<version>`) used to
+# index into the lockfile's `packages` map from a dependency_set's per-
+# architecture `{short_key: version}` entries.
+def _package_key(short_key, version):
+    return short_key + "=" + version
+
+# Returns the `depends_on` keys of `package` whose architecture is present in
+# `allowed_architectures` (a set-like dict of architecture -> True).
 #
 # Avoids mixing architectures: a dependency is only included if it was built
-# for this exact architecture, or is architecture-independent ("all"). Without
+# for an allowed architecture, or is architecture-independent ("all"). Without
 # this scoping, a package's `depends_on` (which spans every architecture the
 # lockfile knows about) would leak foreign-architecture deps into a single
-# architecture's target, causing file duplication that confuses `flatten`.
-def package_deps_for_architecture(packages, package, architecture, mergedusr = False):
+# architecture's target/closure, causing file duplication that confuses
+# `flatten`.
+def _package_dep_keys_for_architectures(packages, package, allowed_architectures):
     return [
-        "@" + util.package_repo_name(dep_key, mergedusr = mergedusr) + "//:data"
+        dep_key
         for dep_key in package["depends_on"]
-        if packages[dep_key]["architecture"] in [architecture, "all"]
+        if packages[dep_key]["architecture"] in allowed_architectures
+    ]
+
+# Computes the `@repo//:data` labels a package's per-architecture BUILD.bazel
+# should depend on for the given architecture.
+def _package_deps_for_architecture(packages, package, architecture, mergedusr = False):
+    allowed_architectures = {architecture: True, "all": True}
+    return [
+        "@" + util.package_repo_name(dep_key, mergedusr = mergedusr) + "//:data"
+        for dep_key in _package_dep_keys_for_architectures(packages, package, allowed_architectures)
     ]
 
+# Validates a dependency_set against the lockfile's `packages` map.
+def verify_dependency_set(packages, dependency_set):
+    package_coords_to_versions = {}
+
+    for (architecture, entries) in dependency_set["sets"].items():
+        for (short_key, version) in entries.items():
+            package_key = _package_key(short_key, version)
+            if package_key not in packages:
+                    fail("illegal state: package %s is not in lockfile" % package_key)
+
+            package_name = packages[package_key]["name"]
+            package_coords = (package_name, architecture)
+            recorded_version = package_coords_to_versions.setdefault(package_coords, version)
+            if recorded_version != version:
+                fail("""Two different source versions detected for package `{name}:{arch}` : {v1} and {v2}.
+This usually means that a distribution ships different versions of this dependency for different architectures.
+
+Please unify the versions manually, or use separate `apt.install` calls (with distinct `dependency_set` names) for each version of the dependency.
+""".format(name = package_name, arch = architecture, v1 = recorded_version, v2 = version))
+
+def dependency_set_transitive_package_keys(packages, dependency_set, architectures):
+    keys = {}
+    pending = []
+    allowed_architectures = {
+        architecture: True
+        for architecture in architectures
+    }
+
+    verify_dependency_set(packages, dependency_set)
+
+    for architecture in architectures:
+        entries = dependency_set["sets"].get(architecture, {})
+        for (short_key, version) in entries.items():
+            pending.append(_package_key(short_key, version))
+
+    for _ in range(len(packages)):
+        if not pending:
+            break
+
+        current = pending
+        pending = []
+        for package_key in current:
+            if package_key in keys:
+                continue
+
+            keys[package_key] = True
+
+            # Keep closure architecture-scoped even when traversing from
+            # architecture = all packages with mixed-arch dependency metadata.
+            pending.extend(_package_dep_keys_for_architectures(packages, packages[package_key], allowed_architectures))
+
+    if pending:
+        fail("dependency traversal for package keys did not converge")
+
+    return sorted(keys.keys())
+
 def _translate_dependency_set_impl(rctx):
     package_template = rctx.read(rctx.attr.package_template)
     lockf = lockfile.from_json(rctx, rctx.attr.lock_content)
@@ -163,33 +235,19 @@ def _translate_dependency_set_impl(rctx):
     # Used to populate the root repo's _PACKAGES table.
     architectures_to_package_names = {}
 
-    # Maps a package coordinates key (e.g. `(<package name>, <architecture>)` to its version.
-    #
-    # It is possible to specify multiple versions of a single dependency for a single architecture in a dependency set.
-    # In those cases, we want to hard-error.
-    package_coords_to_versions = {}
+    verify_dependency_set(packages, dependency_set)
 
     for architecture in dependency_set["sets"].keys():
         architectures_to_package_names[architecture] = []
 
         for (short_key, version) in dependency_set["sets"][architecture].items():
-            package_key = short_key + "=" + version
+            package_key = _package_key(short_key, version)
             repo_name = util.package_repo_name(package_key, mergedusr = rctx.attr.mergedusr)
             package = packages[package_key]
             package_name = package["name"]
 
             architectures_to_package_names[architecture].append(package_name)
 
-            source_version = version
-            package_coords = (package_name, architecture)
-            recorded_version = package_coords_to_versions.setdefault(package_coords, source_version)
-            if recorded_version != source_version:
-                fail("""Two different source versions detected for package `{name}:{arch}` : {v1} and {v2}.
-This usually means that a distribution ships different versions of this dependency for different architectures.
-
-Please unify the versions manually, or use separate `apt.install` calls (with distinct `dependency_set` names) for each version of the dependency.
-""".format(name = package_name, arch = architecture, v1 = recorded_version, v2 = source_version))
-
             # Keyed by package name (not name+version): a package rebuilt with a
             # different binNMU per architecture is a single target whose data,
             # control and deps are selected per architecture.
@@ -208,7 +266,7 @@ Please unify the versions manually, or use separate `apt.install` calls (with di
                     data_targets = '"@%s//:data"' % repo_name,
                     control_targets = '"@%s//:control"' % repo_name,
                     src = '"@%s//:data"' % repo_name,
-                    deps = package_deps_for_architecture(packages, package, architecture, mergedusr = rctx.attr.mergedusr),
+                    deps = _package_deps_for_architecture(packages, package, architecture, mergedusr = rctx.attr.mergedusr),
                     urls = [
                         uri + "/" + package["filename"]
                         for uri in sources[package["suite"]]["uris"]