From 93497a57025be656ee1fe8f7777f53222e1218e3 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 11 Sep 2026 18:44:45 -0400 Subject: [PATCH 1/3] fix: inspect debs with hermetic bsdtar Use tar.bzl's pinned host binary during repository evaluation. Discover the data archive instead of assuming XZ compression, so Zstandard debs work without host tar or zstd. Remove directory exclusions that suppress file discovery with BSD tar, and report inspection/extraction failures. --- MODULE.bazel | 13 +++++++++++-- apt/private/BUILD.bazel | 8 ++++++++ apt/private/deb_archive.bzl | 16 ++++++++++++++++ apt/private/deb_import.bzl | 15 +++++++++++---- 4 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 apt/private/deb_archive.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 5ab4b8cc..9eb7f149 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -12,7 +12,7 @@ bazel_dep(name = "rules_cc", version = "0.2.8") bazel_dep(name = "rules_java", version = "8.8.0") bazel_dep(name = "rules_shell", version = "0.4.1") bazel_dep(name = "gawk", version = "5.3.2.bcr.3") -bazel_dep(name = "tar.bzl", version = "0.6.0") +bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "yq.bzl", version = "0.3.1") yq_toolchains = use_extension("@yq.bzl//yq:extensions.bzl", "yq") @@ -29,7 +29,16 @@ use_repo(bazel_lib_toolchains, "zstd_toolchains") use_repo(bazel_lib_toolchains, "coreutils_toolchains") tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") -use_repo(tar_toolchains, "bsd_tar_toolchains") +use_repo( + tar_toolchains, + "bsd_tar_toolchains", + "bsd_tar_toolchains_darwin_amd64", + "bsd_tar_toolchains_darwin_arm64", + "bsd_tar_toolchains_linux_amd64", + "bsd_tar_toolchains_linux_arm64", + "bsd_tar_toolchains_windows_amd64", + "bsd_tar_toolchains_windows_arm64", +) # Dev dependencies bazel_dep(name = "gazelle", version = "0.34.0", dev_dependency = True, repo_name = "bazel_gazelle") diff --git a/apt/private/BUILD.bazel b/apt/private/BUILD.bazel index bea14893..ed5777b4 100644 --- a/apt/private/BUILD.bazel +++ b/apt/private/BUILD.bazel @@ -55,11 +55,19 @@ bzl_library( deps = ["@bazel_lib//lib:strings"], ) +bzl_library( + name = "deb_archive", + srcs = ["deb_archive.bzl"], + visibility = ["//apt:__subpackages__"], + deps = ["@bazel_lib//lib:repo_utils"], +) + bzl_library( name = "deb_import", srcs = ["deb_import.bzl"], visibility = ["//apt:__subpackages__"], deps = [ + ":deb_archive", ":linker_script", ":pkgconfig", ], diff --git a/apt/private/deb_archive.bzl b/apt/private/deb_archive.bzl new file mode 100644 index 00000000..65f967d7 --- /dev/null +++ b/apt/private/deb_archive.bzl @@ -0,0 +1,16 @@ +"""Inspect Debian data archives with tar.bzl's host toolchain binary.""" + +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") + +# Repository rules run before toolchain resolution. Use the same pinned +# binary tar.bzl makes available for the repository host. +def host_bsdtar(rctx): + platform = repo_utils.platform(rctx) + binary = "tar.exe" if platform.startswith("windows_") else "tar" + return rctx.path(Label("@bsd_tar_toolchains_{}//:{}".format(platform, binary))) + +def data_archive(rctx, directory = "."): + archives = [path for path in rctx.path(directory).readdir() if path.basename.startswith("data.tar")] + if len(archives) != 1: + fail("expected one data archive, found: %s" % archives) + return archives[0] diff --git a/apt/private/deb_import.bzl b/apt/private/deb_import.bzl index d83dca18..7dccd269 100644 --- a/apt/private/deb_import.bzl +++ b/apt/private/deb_import.bzl @@ -1,7 +1,8 @@ "deb_import" -load(":lockfile.bzl", "lockfile") +load(":deb_archive.bzl", "data_archive", "host_bsdtar") load(":linker_script.bzl", "linker_script") +load(":lockfile.bzl", "lockfile") load(":pkgconfig.bzl", "pkgconfig") load(":util.bzl", "util") @@ -183,7 +184,11 @@ def _remap_linkopts(rctx, extract_dir, so_regular_files, self_files, depends_fil return result.linkopts def _discover_contents(rctx, depends_on, depends_file_map, target_name, mergedusr = False): - result = rctx.execute(["tar", "--exclude='./usr/share/**'", "--exclude='./**/'", "-tvf", "data.tar.xz"]) + archive = data_archive(rctx) + tar = host_bsdtar(rctx) + result = rctx.execute([tar, "-tvf", archive]) + if result.return_code: + fail("failed to inspect %s: %s" % (archive, result.stderr)) contents_raw = result.stdout.splitlines() so_files = [] @@ -283,10 +288,12 @@ def _discover_contents(rctx, depends_on, depends_file_map, target_name, mergedus files_to_extract = so_regular_files + pc_files if files_to_extract: rctx.execute(["mkdir", "-p", _EXTRACT_DIR]) - rctx.execute( - ["tar", "-xf", "data.tar.xz", "-C", _EXTRACT_DIR] + + result = rctx.execute( + [tar, "-xf", archive, "-C", _EXTRACT_DIR] + ["./" + f for f in files_to_extract], ) + if result.return_code: + fail("failed to extract %s from %s: %s" % (files_to_extract, archive, result.stderr)) remap_linkopts = _remap_linkopts( rctx, From 2b57e1e482f640645d92386cb721945687865c67 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 11 Sep 2026 18:44:45 -0400 Subject: [PATCH 2/3] fix: retain apt package source identity Keep indexes from different URLs separate even when suite, component, and architecture match. Scope snapshot facts by source URL and retain the selected package's URLs instead of reconstructing them from the suite. Continue reading existing v2 locks and refresh their provenance when packages are resolved. Cover mixed snapshot/rolling sources and package URLs that differ from the suite's URLs. --- apt/extensions.bzl | 43 ++++++++++-------------- apt/private/lockfile.bzl | 12 +++++++ apt/private/translate_dependency_set.bzl | 6 +--- apt/private/util.bzl | 6 ++-- apt/tests/facts_test.bzl | 6 ++-- apt/tests/lockfile_test.bzl | 33 ++++++++++++++++++ 6 files changed, 69 insertions(+), 37 deletions(-) diff --git a/apt/extensions.bzl b/apt/extensions.bzl index 5969d271..8166073b 100644 --- a/apt/extensions.bzl +++ b/apt/extensions.bzl @@ -23,7 +23,7 @@ def _get_auth(mctx, urls): netrc = read_user_netrc(mctx) return use_netrc(netrc, urls, {}) -def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, cached_format = None): +def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, source_id, cached_format = None): """Initiate all format downloads for a given index type with block=False. If cached_format is set, only that extension is attempted — avoiding @@ -63,7 +63,7 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, cached # Without this, the uncompressed variant ("") and a decompressed # .xz/.gz/.bz2 would both write to the same final path. ext_name = ext.lstrip(".") if ext else "raw" - output = "{}/{}/{}/{}{}".format(target_triple, url_idx, ext_name, index_type, ext) + output = "{}/{}/{}/{}/{}{}".format(source_id, target_triple, url_idx, ext_name, index_type, ext) if index_type == "Packages": dist_url = "{}/dists/{}/{}/binary-{}/{}{}".format(url, dist, comp, arch, index_type, ext) else: @@ -98,10 +98,7 @@ def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): if download.success: decompress_r = mctx.execute(cmd + [output]) if decompress_r.return_code == 0: - target_triple = "{}/{}/{}".format(dist, comp, arch) - - # Decompressed file lives in its own ext_name subdirectory - result = ("{}/{}/{}/{}".format(target_triple, url_idx, ext_name, index_type), url, download.integrity, ext) + result = (output.removesuffix(ext) if ext else output, url, download.integrity, ext) continue failed_attempts.append((url + "/.../" + index_type + ext, download, decompress_r)) if result != None: @@ -129,7 +126,7 @@ def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): {} """.format(len(failed_attempts), "\n".join(attempt_messages))) -def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): +def _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats): """Fetch all package indices and contents in parallel, then parse them. Returns the set (as a dict) of fact keys that belong to the current sources, @@ -143,8 +140,8 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): # Deduplicate: multiple dict entries can map to the same logical source # (one entry per URL in the urls list). Only process each unique - # (dist, component, architecture) combination once. - dedup_key = "{}/{}/{}".format(dist, component, architecture) + # (URLs, dist, component, architecture) combination once. + dedup_key = util.index_fact_key(dist, component, architecture, "Packages", urls) if dedup_key in seen: continue seen[dedup_key] = True @@ -158,6 +155,9 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): cnt_fact_key = util.index_fact_key(dist, component, architecture, "Contents", urls) used_keys[pkg_fact_key] = True used_keys[cnt_fact_key] = True + if urls and all([util.is_snapshot_uri(url) for url in urls]): + snapshot_indices[pkg_fact_key] = True + snapshot_indices[cnt_fact_key] = True # Check cached format info to avoid 404 warnings on subsequent runs cached_pkg_format = formats.get(pkg_fact_key) @@ -175,6 +175,7 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): architecture, glock.facts().get(pkg_fact_key, ""), "Packages", + source_id = len(seen), cached_format = cached_pkg_format, ) @@ -188,6 +189,7 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): architecture, glock.facts().get(cnt_fact_key, ""), "Contents", + source_id = len(seen), cached_format = cached_cnt_format, ) @@ -206,7 +208,7 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): for (urls, dist, comp, arch, pkg_tokens, cnt_tokens, pkg_fk, cnt_fk) in pending: mctx.report_progress("resolving Package indices: {}/{} for {}".format(dist, comp, arch)) (output, url, integrity, ext) = _resolve_downloads(mctx, pkg_tokens, "Packages", dist, comp, arch) - if dist in snapshot_suites: + if pkg_fk in snapshot_indices: glock.facts()[pkg_fk] = integrity formats[pkg_fk] = ext @@ -221,7 +223,7 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): if contents_result != None: (output, url, integrity, ext) = contents_result - if dist in snapshot_suites: + if cnt_fk in snapshot_indices: glock.facts()[cnt_fk] = integrity formats[cnt_fk] = ext @@ -278,15 +280,7 @@ def _distroless_extension(mctx): for lock in mod.tags.lock ]) - # First pass over sources_list: classify suites as snapshot or rolling - snapshot_suites = {} - for mod in mctx.modules: - for sl in mod.tags.sources_list: - uris = [uri.removeprefix("mirror+") for uri in sl.uris] - is_snapshot = len(uris) > 0 and all([util.is_snapshot_uri(uri) for uri in uris]) - if is_snapshot: - for suite in sl.suites: - snapshot_suites[suite] = True + snapshot_indices = {} repo = deb_repository.new() resolver = dependency_resolver.new(repo) @@ -317,7 +311,7 @@ def _distroless_extension(mctx): # Fetch all sources_list in parallel and parse them. `used_keys` is the set # of fact keys for the current sources, used below to prune stale facts. - used_keys = _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats) + used_keys = _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats) sources = glock.sources() dependency_sets = glock.dependency_sets() @@ -504,10 +498,7 @@ def _distroless_extension(mctx): deb_import( name = repo_name, target_name = repo_name, - urls = [ - uri + "/" + package["filename"] - for uri in sources[package["suite"]]["uris"] - ], + urls = package["urls"], sha256 = package["sha256"], mergedusr = mergedusr, depends_on = package["depends_on"], @@ -541,7 +532,7 @@ def _distroless_extension(mctx): glock.facts(), formats, used_keys, - snapshot_suites, + snapshot_indices, ) return mctx.extension_metadata( facts = {"indices": cacheable_indices, "formats": cacheable_formats}, diff --git a/apt/private/lockfile.bzl b/apt/private/lockfile.bzl index bdb2af8e..70519b6c 100644 --- a/apt/private/lockfile.bzl +++ b/apt/private/lockfile.bzl @@ -40,6 +40,8 @@ def _package_key(package, arch = None): def _add_package(lock, package, arch = None): k = _package_key(package, arch) if k in lock.packages: + # Refresh provenance from the selected index when reading an older lock. + lock.packages[k]["urls"] = [root + "/" + package["Filename"] for root in package["Roots"]] return lock.packages[k] = { "name": package["Package"], @@ -47,6 +49,7 @@ def _add_package(lock, package, arch = None): "architecture": package["Architecture"], "sha256": package["SHA256"], "filename": package["Filename"], + "urls": [root + "/" + package["Filename"] for root in package["Roots"]], "suite": package["Dist"], "section": package["Section"], "size": int(package["Size"]), @@ -116,6 +119,15 @@ def _from_json(mctx, content): if lock["version"] != 2: fail("lock file version %d is not supported anymore. please upgrade your lock file" % lock["version"]) + # Existing v2 locks only recorded source URLs at suite granularity. + # Preserve their download locations until a selected index refreshes them. + for package in lock.get("packages", {}).values(): + if "urls" not in package: + package["urls"] = [ + root + "/" + package["filename"] + for root in lock["sources"][package["suite"]]["uris"] + ] + lock = struct( version = lock["version"], dependency_sets = lock["dependency_sets"] if "dependency_sets" in lock else dict(), diff --git a/apt/private/translate_dependency_set.bzl b/apt/private/translate_dependency_set.bzl index d0147bf4..cf52ddda 100644 --- a/apt/private/translate_dependency_set.bzl +++ b/apt/private/translate_dependency_set.bzl @@ -152,7 +152,6 @@ def _translate_dependency_set_impl(rctx): package_template = rctx.read(rctx.attr.package_template) lockf = lockfile.from_json(rctx, rctx.attr.lock_content) - sources = lockf.sources() packages = lockf.packages() dependency_sets = lockf.dependency_sets() dependency_set = dependency_sets[rctx.attr.depset_name] @@ -209,10 +208,7 @@ Please unify the versions manually, or use separate `apt.install` calls (with di control_targets = '"@%s//:control"' % repo_name, src = '"@%s//:data"' % repo_name, deps = package_deps_for_architecture(packages, package, architecture, mergedusr = rctx.attr.mergedusr), - urls = [ - uri + "/" + package["filename"] - for uri in sources[package["suite"]]["uris"] - ], + urls = package["urls"], name = package["name"], arch = package["architecture"], sha256 = package["sha256"], diff --git a/apt/private/util.bzl b/apt/private/util.bzl index 901dea16..066a2e23 100644 --- a/apt/private/util.bzl +++ b/apt/private/util.bzl @@ -61,14 +61,14 @@ def _index_fact_key(dist, component, architecture, index_type, urls): url_token = "|".join(sorted_deduplicated_urls) return "{}/{}/{}/{}/{}".format(dist, component, architecture, index_type, url_token) -def _prune_uncacheable_facts(indices, formats, used_keys, snapshot_suites): +def _prune_uncacheable_facts(indices, formats, used_keys, snapshot_indices): """Keep only the facts that can be cached. `used_keys` holds the fact keys produced for this run's sources (see `index_fact_key`). Entries left over from a previous snapshot URL are not in `used_keys`, so they get dropped here instead of accumulating across runs. - `snapshot_suites` holds a list of suites from snapshots, + `snapshot_indices` holds the fact keys from snapshot sources, because we don't want to cache rolling suites. Returns `(cacheable_indices, cacheable_formats)`. @@ -76,7 +76,7 @@ def _prune_uncacheable_facts(indices, formats, used_keys, snapshot_suites): cacheable_indices = { k: v for k, v in indices.items() - if k in used_keys and k.split("/")[0] in snapshot_suites + if k in used_keys and k in snapshot_indices } cacheable_formats = { k: v diff --git a/apt/tests/facts_test.bzl b/apt/tests/facts_test.bzl index a1054f04..c1fe4da9 100644 --- a/apt/tests/facts_test.bzl +++ b/apt/tests/facts_test.bzl @@ -45,16 +45,16 @@ def _prune_facts_test(ctx): old_key = util.index_fact_key("bookworm", "main", "amd64", "Packages", _TEST_SNAPSHOT_1) new_key = util.index_fact_key("bookworm", "main", "amd64", "Packages", _TEST_SNAPSHOT_2) - rolling_key = util.index_fact_key("sid", "main", "amd64", "Packages", ["https://deb.debian.org/debian"]) + rolling_key = util.index_fact_key("bookworm", "main", "amd64", "Packages", ["https://deb.debian.org/debian"]) indices = {old_key: "sha256-OLD", new_key: "sha256-NEW", rolling_key: "sha256-ROLLING"} formats = {old_key: ".xz", new_key: ".xz", rolling_key: ".xz"} # Only the current sources' keys are used this run. used_keys = {new_key: True, rolling_key: True} - snapshot_suites = {"bookworm": True} + snapshot_indices = {new_key: True} - (cacheable_indices, cacheable_formats) = util.prune_uncacheable_facts(indices, formats, used_keys, snapshot_suites) + (cacheable_indices, cacheable_formats) = util.prune_uncacheable_facts(indices, formats, used_keys, snapshot_indices) # The stale previous-URL entry is dropped, and indices from rolling indexes are not cached. asserts.equals(env, {new_key: "sha256-NEW"}, cacheable_indices) diff --git a/apt/tests/lockfile_test.bzl b/apt/tests/lockfile_test.bzl index 384b7108..6b2be40e 100644 --- a/apt/tests/lockfile_test.bzl +++ b/apt/tests/lockfile_test.bzl @@ -63,4 +63,37 @@ def _add_source_merges_architectures_test(ctx): add_source_merges_architectures_test = unittest.make(_add_source_merges_architectures_test) def lockfile_tests(): + package_source_urls_test(name = _TEST_SUITE_PREFIX + "package_source_urls") add_source_merges_architectures_test(name = _TEST_SUITE_PREFIX + "add_source_merges_architectures") + +def _package_source_urls_test(ctx): + env = unittest.begin(ctx) + lock = lockfile.empty(struct()) + package = { + "Architecture": "amd64", + "Dist": "noble", + "Filename": "pool/main/e/example_1_amd64.deb", + "Package": "example", + "Roots": ["https://example.org/ppa"], + "SHA256": "abc", + "Section": "libs", + "Size": "1", + "Version": "1", + } + lock.add_source("noble", ["deb"], ["https://example.org/ubuntu"], ["main"], ["amd64"]) + lock.add_package(package) + key = lockfile.package_key(package) + expected = ["https://example.org/ppa/" + package["Filename"]] + asserts.equals(env, expected, lock.packages()[key]["urls"]) + + # A v2 lock without per-package URLs remains readable. Resolving its + # package again must replace the old suite URL with the selected PPA URL. + legacy = json.decode(lock.as_json()) + legacy["packages"][key].pop("urls") + restored = lockfile.from_json(struct(), json.encode(legacy)) + asserts.equals(env, ["https://example.org/ubuntu/" + package["Filename"]], restored.packages()[key]["urls"]) + restored.add_package(package) + asserts.equals(env, expected, restored.packages()[key]["urls"]) + return unittest.end(env) + +package_source_urls_test = unittest.make(_package_source_urls_test) From cab54912e9184189978bbf465a2676ec060971a5 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 11 Sep 2026 18:44:46 -0400 Subject: [PATCH 3/3] perf: derive file maps from pinned debs Inspect the exact selected Debian archive only when a dependency file map is needed. Avoid downloading and parsing distribution-wide Contents indexes, which third-party apt repositories may not publish and which do not identify the selected package version. Delete the temporary archive after inspection and remove the unused Contents parser. --- apt/extensions.bzl | 108 ++++++----------------------- apt/private/BUILD.bazel | 1 + apt/private/apt_deb_repository.bzl | 22 ------ apt/private/deb_filemap.bzl | 18 +++-- 4 files changed, 36 insertions(+), 113 deletions(-) diff --git a/apt/extensions.bzl b/apt/extensions.bzl index 8166073b..de4cbf92 100644 --- a/apt/extensions.bzl +++ b/apt/extensions.bzl @@ -23,8 +23,8 @@ def _get_auth(mctx, urls): netrc = read_user_netrc(mctx) return use_netrc(netrc, urls, {}) -def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, source_id, cached_format = None): - """Initiate all format downloads for a given index type with block=False. +def _start_downloads(mctx, urls, dist, comp, arch, integrity, source_id, cached_format = None): + """Initiate package index downloads with block=False. If cached_format is set, only that extension is attempted — avoiding 404 warnings for formats the remote doesn't serve. @@ -36,20 +36,12 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, source # --force -> overwrite the output if it exists # --decompress -> decompress # Order of these matter, we want to try the one that is most likely first. - if index_type == "Packages": - extensions = [ - (".xz", ["xz", "--decompress", "--keep", "--force"]), - (".gz", ["gzip", "--decompress", "--keep", "--force"]), - (".bz2", ["bzip2", "--decompress", "--keep", "--force"]), - ("", ["true"]), - ] - else: - extensions = [ - (".gz", ["gzip", "--decompress", "--keep", "--force"]), - (".xz", ["xz", "--decompress", "--keep", "--force"]), - (".bz2", ["bzip2", "--decompress", "--keep", "--force"]), - ("", ["true"]), - ] + extensions = [ + (".xz", ["xz", "--decompress", "--keep", "--force"]), + (".gz", ["gzip", "--decompress", "--keep", "--force"]), + (".bz2", ["bzip2", "--decompress", "--keep", "--force"]), + ("", ["true"]), + ] if cached_format != None: extensions = [(ext, cmd) for (ext, cmd) in extensions if ext == cached_format] @@ -63,11 +55,8 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, source # Without this, the uncompressed variant ("") and a decompressed # .xz/.gz/.bz2 would both write to the same final path. ext_name = ext.lstrip(".") if ext else "raw" - output = "{}/{}/{}/{}/{}{}".format(source_id, target_triple, url_idx, ext_name, index_type, ext) - if index_type == "Packages": - dist_url = "{}/dists/{}/{}/binary-{}/{}{}".format(url, dist, comp, arch, index_type, ext) - else: - dist_url = "{}/dists/{}/{}/Contents-{}{}".format(url, dist, comp, arch, ext) + output = "{}/{}/{}/{}/Packages{}".format(source_id, target_triple, url_idx, ext_name, ext) + dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) auth = {} if url in base_auth: auth = {dist_url: base_auth[url]} @@ -79,18 +68,17 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, source auth = auth, block = False, ) - tokens.append((ext, cmd, url, url_idx, ext_name, output, token)) + tokens.append((ext, cmd, url, output, token)) return tokens -def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): +def _resolve_downloads(mctx, tokens): """Wait on tokens in priority order, decompress the first success. Returns (output_path, url, integrity, ext) on success. - Returns None for optional Contents when all attempts fail. """ failed_attempts = [] result = None - for (ext, cmd, url, url_idx, ext_name, output, token) in tokens: + for (ext, cmd, url, output, token) in tokens: download = token.wait() decompress_r = None if result != None: @@ -100,17 +88,10 @@ def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): if decompress_r.return_code == 0: result = (output.removesuffix(ext) if ext else output, url, download.integrity, ext) continue - failed_attempts.append((url + "/.../" + index_type + ext, download, decompress_r)) + failed_attempts.append((url + "/.../Packages" + ext, download, decompress_r)) if result != None: return result - if index_type == "Contents": - # Contents files are optional; some repositories (e.g. packages.cloud.google.com/apt) - # don't provide them. Print a warning and return None instead of failing. - print("Warning: Could not fetch Contents index for {}/{}/{}. Contents files are optional.".format(dist, comp, arch)) - return None - - # For Packages, fail with details attempt_messages = [] for (failed_url, download, decompress) in failed_attempts: reason = "unknown" @@ -127,7 +108,7 @@ def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): """.format(len(failed_attempts), "\n".join(attempt_messages))) def _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats): - """Fetch all package indices and contents in parallel, then parse them. + """Fetch all package indices in parallel, then parse them. Returns the set (as a dict) of fact keys that belong to the current sources, so the caller can prune stale facts left behind by previous URLs. @@ -152,16 +133,12 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats): urls = [url.rstrip("/") for url in urls] pkg_fact_key = util.index_fact_key(dist, component, architecture, "Packages", urls) - cnt_fact_key = util.index_fact_key(dist, component, architecture, "Contents", urls) used_keys[pkg_fact_key] = True - used_keys[cnt_fact_key] = True if urls and all([util.is_snapshot_uri(url) for url in urls]): snapshot_indices[pkg_fact_key] = True - snapshot_indices[cnt_fact_key] = True # Check cached format info to avoid 404 warnings on subsequent runs cached_pkg_format = formats.get(pkg_fact_key) - cached_cnt_format = formats.get(cnt_fact_key) # Pass 1: Initiate all downloads with block=False # For snapshot suites, integrity hashes from facts enable instant cache hits. @@ -174,40 +151,16 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats): component, architecture, glock.facts().get(pkg_fact_key, ""), - "Packages", source_id = len(seen), cached_format = cached_pkg_format, ) - cnt_tokens = None - if cached_cnt_format != "unavailable": - cnt_tokens = _start_downloads( - mctx, - urls, - dist, - component, - architecture, - glock.facts().get(cnt_fact_key, ""), - "Contents", - source_id = len(seen), - cached_format = cached_cnt_format, - ) - - pending.append(( - urls, - dist, - component, - architecture, - pkg_tokens, - cnt_tokens, - pkg_fact_key, - cnt_fact_key, - )) + pending.append((urls, dist, component, architecture, pkg_tokens, pkg_fact_key)) # Pass 2: Wait, decompress, parse - for (urls, dist, comp, arch, pkg_tokens, cnt_tokens, pkg_fk, cnt_fk) in pending: + for (urls, dist, comp, arch, pkg_tokens, pkg_fk) in pending: mctx.report_progress("resolving Package indices: {}/{} for {}".format(dist, comp, arch)) - (output, url, integrity, ext) = _resolve_downloads(mctx, pkg_tokens, "Packages", dist, comp, arch) + (output, url, integrity, ext) = _resolve_downloads(mctx, pkg_tokens) if pkg_fk in snapshot_indices: glock.facts()[pkg_fk] = integrity formats[pkg_fk] = ext @@ -215,23 +168,6 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats): mctx.report_progress("parsing Package indices: {}/{} for {}".format(dist, comp, arch)) repo.parse_package_index(mctx.read(output), urls, dist) - if cnt_tokens != None: - mctx.report_progress("resolving Contents: {}/{} for {}".format(dist, comp, arch)) - contents_result = _resolve_downloads(mctx, cnt_tokens, "Contents", dist, comp, arch) - else: - contents_result = None - - if contents_result != None: - (output, url, integrity, ext) = contents_result - if cnt_fk in snapshot_indices: - glock.facts()[cnt_fk] = integrity - formats[cnt_fk] = ext - - mctx.report_progress("parsing Contents: {}/{} for {}".format(dist, comp, arch)) - repo.parse_contents(mctx.read(output), arch) - else: - formats[cnt_fk] = "unavailable" - return used_keys def compute_package_repo_modes(packages, roots_by_mode): @@ -421,7 +357,7 @@ def _distroless_extension(mctx): ) # Key every package by the architecture we are resolving for, not by the - # package's own `Architecture` field. + # package's own `Architecture` field. # For arch-specific packages these are the same. # For `Architecture: all` packages this "expands" them into one entry per target architecture, # so each carries its own arch-specific dependency closure instead of a single frozen one shared across arches. @@ -486,7 +422,8 @@ def _distroless_extension(mctx): # Storing these in a file instead of passing filemaps as attributes cuts down lockfile size considerably. deb_filemap( name = util.sanitize(package_key) + "_filemap", - files = json.encode(repo.filemap(name = name, arch = arch) or []), + urls = package["urls"], + sha256 = package["sha256"], ) modes = package_repo_modes.get(package_key, {False: True}) @@ -502,8 +439,7 @@ def _distroless_extension(mctx): sha256 = package["sha256"], mergedusr = mergedusr, depends_on = package["depends_on"], - # Label of each dependency's own filemap, in depends_on order, - # so deb_import can rebuild the {file: dependency} index by lookup. + # Dependency filemaps retain resolver order. dep_filemaps = [ "@" + util.sanitize(dep) + "_filemap//:filemap.json" for dep in package["depends_on"] diff --git a/apt/private/BUILD.bazel b/apt/private/BUILD.bazel index ed5777b4..96cd6780 100644 --- a/apt/private/BUILD.bazel +++ b/apt/private/BUILD.bazel @@ -77,6 +77,7 @@ bzl_library( name = "deb_filemap", srcs = ["deb_filemap.bzl"], visibility = ["//apt:__subpackages__"], + deps = [":deb_archive"], ) bzl_library( diff --git a/apt/private/apt_deb_repository.bzl b/apt/private/apt_deb_repository.bzl index 037dbee6..4c1ef128 100644 --- a/apt/private/apt_deb_repository.bzl +++ b/apt/private/apt_deb_repository.bzl @@ -37,17 +37,6 @@ def _parse_repository(state, contents, roots, dist): last_key = "" pkg = {} -def _parse_contents(state, rcontents, arch): - contents = state.filemap.setdefault(arch, {}) - for line in rcontents.splitlines(): - last_empty_char = line.rfind(" ") - first_empty_char = line.find(" ") - filepath = line[:first_empty_char] - pkgs = line[last_empty_char + 1:].split(",") - for pkg in pkgs: - contents.setdefault(pkg[pkg.find("/") + 1:], []).append(filepath) - state.filemap[arch] = contents - def _add_package(state, package): util.set_dict( state.packages, @@ -111,14 +100,6 @@ def _package(state, name, version, arch, suites = None): return None return package -def _filemap(state, name, arch): - if arch not in state.filemap: - return None - all = state.filemap[arch] - if name not in all: - return None - return state.filemap[arch][name] - def _add_source_if_not_present(state, source): (urls, dist, components, architectures) = source @@ -140,7 +121,6 @@ def _add_source_if_not_present(state, source): def _create(): state = struct( sources = dict(), - filemap = dict(), packages = dict(), virtual_packages = dict(), ) @@ -149,11 +129,9 @@ def _create(): add_source = lambda source: _add_source_if_not_present(state, source), sources = lambda: state.sources, parse_package_index = lambda contents, roots, dist: _parse_repository(state, contents, roots, dist), - parse_contents = lambda rcontents, arch: _parse_contents(state, rcontents, arch), package_versions = lambda **kwargs: _package_versions(state, **kwargs), virtual_packages = lambda **kwargs: _virtual_packages(state, **kwargs), package = lambda **kwargs: _package(state, **kwargs), - filemap = lambda **kwargs: _filemap(state, **kwargs), ) deb_repository = struct( diff --git a/apt/private/deb_filemap.bzl b/apt/private/deb_filemap.bzl index f8f50ae5..cb8f8eed 100644 --- a/apt/private/deb_filemap.bzl +++ b/apt/private/deb_filemap.bzl @@ -5,16 +5,24 @@ Using a file instead of raw JSON-encoded attrs saves significant space in MODULE This is a separate repository to avoid dependency cycle issues, which Debian package graphs allow. """ +load(":deb_archive.bzl", "data_archive", "host_bsdtar") + def _deb_filemap_impl(rctx): - # `files` is a JSON-encoded list of the paths this package installs. - rctx.file("filemap.json", rctx.attr.files) + # Contents indexes describe entire distributions and may be absent from + # third-party repositories. Inspect the exact pinned package on demand. + rctx.download_and_extract(url = rctx.attr.urls, sha256 = rctx.attr.sha256, output = "_archive") + result = rctx.execute([host_bsdtar(rctx), "-tf", data_archive(rctx, "_archive")]) + if result.return_code: + fail("failed to list package files: %s" % result.stderr) + rctx.delete("_archive") + files = [path.removeprefix("./") for path in result.stdout.splitlines() if not path.endswith("/")] + rctx.file("filemap.json", json.encode(files)) rctx.file("BUILD.bazel", 'exports_files(["filemap.json"], visibility = ["//visibility:public"])\n') deb_filemap = repository_rule( implementation = _deb_filemap_impl, attrs = { - "files": attr.string( - doc = "JSON-encoded list of file paths this package installs.", - ), + "urls": attr.string_list(mandatory = True, allow_empty = False), + "sha256": attr.string(mandatory = True), }, )