diff --git a/MODULE.bazel b/MODULE.bazel index 5ab4b8c..9eb7f14 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -12,7 +12,7 @@ bazel_dep(name = "rules_cc", version = "0.2.8") bazel_dep(name = "rules_java", version = "8.8.0") bazel_dep(name = "rules_shell", version = "0.4.1") bazel_dep(name = "gawk", version = "5.3.2.bcr.3") -bazel_dep(name = "tar.bzl", version = "0.6.0") +bazel_dep(name = "tar.bzl", version = "0.10.4") bazel_dep(name = "yq.bzl", version = "0.3.1") yq_toolchains = use_extension("@yq.bzl//yq:extensions.bzl", "yq") @@ -29,7 +29,16 @@ use_repo(bazel_lib_toolchains, "zstd_toolchains") use_repo(bazel_lib_toolchains, "coreutils_toolchains") tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") -use_repo(tar_toolchains, "bsd_tar_toolchains") +use_repo( + tar_toolchains, + "bsd_tar_toolchains", + "bsd_tar_toolchains_darwin_amd64", + "bsd_tar_toolchains_darwin_arm64", + "bsd_tar_toolchains_linux_amd64", + "bsd_tar_toolchains_linux_arm64", + "bsd_tar_toolchains_windows_amd64", + "bsd_tar_toolchains_windows_arm64", +) # Dev dependencies bazel_dep(name = "gazelle", version = "0.34.0", dev_dependency = True, repo_name = "bazel_gazelle") diff --git a/apt/extensions.bzl b/apt/extensions.bzl index 5969d27..3a39f1c 100644 --- a/apt/extensions.bzl +++ b/apt/extensions.bzl @@ -23,8 +23,8 @@ def _get_auth(mctx, urls): netrc = read_user_netrc(mctx) return use_netrc(netrc, urls, {}) -def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, cached_format = None): - """Initiate all format downloads for a given index type with block=False. +def _start_downloads(mctx, urls, dist, comp, arch, integrity, source_id, cached_format = None): + """Initiate package index downloads with block=False. If cached_format is set, only that extension is attempted — avoiding 404 warnings for formats the remote doesn't serve. @@ -36,20 +36,12 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, cached # --force -> overwrite the output if it exists # --decompress -> decompress # Order of these matter, we want to try the one that is most likely first. - if index_type == "Packages": - extensions = [ - (".xz", ["xz", "--decompress", "--keep", "--force"]), - (".gz", ["gzip", "--decompress", "--keep", "--force"]), - (".bz2", ["bzip2", "--decompress", "--keep", "--force"]), - ("", ["true"]), - ] - else: - extensions = [ - (".gz", ["gzip", "--decompress", "--keep", "--force"]), - (".xz", ["xz", "--decompress", "--keep", "--force"]), - (".bz2", ["bzip2", "--decompress", "--keep", "--force"]), - ("", ["true"]), - ] + extensions = [ + (".xz", ["xz", "--decompress", "--keep", "--force"]), + (".gz", ["gzip", "--decompress", "--keep", "--force"]), + (".bz2", ["bzip2", "--decompress", "--keep", "--force"]), + ("", ["true"]), + ] if cached_format != None: extensions = [(ext, cmd) for (ext, cmd) in extensions if ext == cached_format] @@ -63,11 +55,8 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, cached # Without this, the uncompressed variant ("") and a decompressed # .xz/.gz/.bz2 would both write to the same final path. ext_name = ext.lstrip(".") if ext else "raw" - output = "{}/{}/{}/{}{}".format(target_triple, url_idx, ext_name, index_type, ext) - if index_type == "Packages": - dist_url = "{}/dists/{}/{}/binary-{}/{}{}".format(url, dist, comp, arch, index_type, ext) - else: - dist_url = "{}/dists/{}/{}/Contents-{}{}".format(url, dist, comp, arch, ext) + output = "{}/{}/{}/{}/Packages{}".format(source_id, target_triple, url_idx, ext_name, ext) + dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) auth = {} if url in base_auth: auth = {dist_url: base_auth[url]} @@ -79,18 +68,17 @@ def _start_downloads(mctx, urls, dist, comp, arch, integrity, index_type, cached auth = auth, block = False, ) - tokens.append((ext, cmd, url, url_idx, ext_name, output, token)) + tokens.append((ext, cmd, url, output, token)) return tokens -def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): +def _resolve_downloads(mctx, tokens): """Wait on tokens in priority order, decompress the first success. Returns (output_path, url, integrity, ext) on success. - Returns None for optional Contents when all attempts fail. """ failed_attempts = [] result = None - for (ext, cmd, url, url_idx, ext_name, output, token) in tokens: + for (ext, cmd, url, output, token) in tokens: download = token.wait() decompress_r = None if result != None: @@ -98,22 +86,12 @@ def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): if download.success: decompress_r = mctx.execute(cmd + [output]) if decompress_r.return_code == 0: - target_triple = "{}/{}/{}".format(dist, comp, arch) - - # Decompressed file lives in its own ext_name subdirectory - result = ("{}/{}/{}/{}".format(target_triple, url_idx, ext_name, index_type), url, download.integrity, ext) + result = (output.removesuffix(ext) if ext else output, url, download.integrity, ext) continue - failed_attempts.append((url + "/.../" + index_type + ext, download, decompress_r)) + failed_attempts.append((url + "/.../Packages" + ext, download, decompress_r)) if result != None: return result - if index_type == "Contents": - # Contents files are optional; some repositories (e.g. packages.cloud.google.com/apt) - # don't provide them. Print a warning and return None instead of failing. - print("Warning: Could not fetch Contents index for {}/{}/{}. Contents files are optional.".format(dist, comp, arch)) - return None - - # For Packages, fail with details attempt_messages = [] for (failed_url, download, decompress) in failed_attempts: reason = "unknown" @@ -129,8 +107,8 @@ def _resolve_downloads(mctx, tokens, index_type, dist, comp, arch): {} """.format(len(failed_attempts), "\n".join(attempt_messages))) -def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): - """Fetch all package indices and contents in parallel, then parse them. +def _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats): + """Fetch all package indices in parallel, then parse them. Returns the set (as a dict) of fact keys that belong to the current sources, so the caller can prune stale facts left behind by previous URLs. @@ -143,8 +121,8 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): # Deduplicate: multiple dict entries can map to the same logical source # (one entry per URL in the urls list). Only process each unique - # (dist, component, architecture) combination once. - dedup_key = "{}/{}/{}".format(dist, component, architecture) + # (URLs, dist, component, architecture) combination once. + dedup_key = util.index_fact_key(dist, component, architecture, "Packages", urls) if dedup_key in seen: continue seen[dedup_key] = True @@ -155,13 +133,12 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): urls = [url.rstrip("/") for url in urls] pkg_fact_key = util.index_fact_key(dist, component, architecture, "Packages", urls) - cnt_fact_key = util.index_fact_key(dist, component, architecture, "Contents", urls) used_keys[pkg_fact_key] = True - used_keys[cnt_fact_key] = True + if urls and all([util.is_snapshot_uri(url) for url in urls]): + snapshot_indices[pkg_fact_key] = True # Check cached format info to avoid 404 warnings on subsequent runs cached_pkg_format = formats.get(pkg_fact_key) - cached_cnt_format = formats.get(cnt_fact_key) # Pass 1: Initiate all downloads with block=False # For snapshot suites, integrity hashes from facts enable instant cache hits. @@ -174,62 +151,23 @@ def _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats): component, architecture, glock.facts().get(pkg_fact_key, ""), - "Packages", + source_id = len(seen), cached_format = cached_pkg_format, ) - cnt_tokens = None - if cached_cnt_format != "unavailable": - cnt_tokens = _start_downloads( - mctx, - urls, - dist, - component, - architecture, - glock.facts().get(cnt_fact_key, ""), - "Contents", - cached_format = cached_cnt_format, - ) - - pending.append(( - urls, - dist, - component, - architecture, - pkg_tokens, - cnt_tokens, - pkg_fact_key, - cnt_fact_key, - )) + pending.append((urls, dist, component, architecture, pkg_tokens, pkg_fact_key)) # Pass 2: Wait, decompress, parse - for (urls, dist, comp, arch, pkg_tokens, cnt_tokens, pkg_fk, cnt_fk) in pending: + for (urls, dist, comp, arch, pkg_tokens, pkg_fk) in pending: mctx.report_progress("resolving Package indices: {}/{} for {}".format(dist, comp, arch)) - (output, url, integrity, ext) = _resolve_downloads(mctx, pkg_tokens, "Packages", dist, comp, arch) - if dist in snapshot_suites: + (output, url, integrity, ext) = _resolve_downloads(mctx, pkg_tokens) + if pkg_fk in snapshot_indices: glock.facts()[pkg_fk] = integrity formats[pkg_fk] = ext mctx.report_progress("parsing Package indices: {}/{} for {}".format(dist, comp, arch)) repo.parse_package_index(mctx.read(output), urls, dist) - if cnt_tokens != None: - mctx.report_progress("resolving Contents: {}/{} for {}".format(dist, comp, arch)) - contents_result = _resolve_downloads(mctx, cnt_tokens, "Contents", dist, comp, arch) - else: - contents_result = None - - if contents_result != None: - (output, url, integrity, ext) = contents_result - if dist in snapshot_suites: - glock.facts()[cnt_fk] = integrity - formats[cnt_fk] = ext - - mctx.report_progress("parsing Contents: {}/{} for {}".format(dist, comp, arch)) - repo.parse_contents(mctx.read(output), arch) - else: - formats[cnt_fk] = "unavailable" - return used_keys def compute_package_repo_modes(packages, roots_by_mode): @@ -278,15 +216,7 @@ def _distroless_extension(mctx): for lock in mod.tags.lock ]) - # First pass over sources_list: classify suites as snapshot or rolling - snapshot_suites = {} - for mod in mctx.modules: - for sl in mod.tags.sources_list: - uris = [uri.removeprefix("mirror+") for uri in sl.uris] - is_snapshot = len(uris) > 0 and all([util.is_snapshot_uri(uri) for uri in uris]) - if is_snapshot: - for suite in sl.suites: - snapshot_suites[suite] = True + snapshot_indices = {} repo = deb_repository.new() resolver = dependency_resolver.new(repo) @@ -317,7 +247,7 @@ def _distroless_extension(mctx): # Fetch all sources_list in parallel and parse them. `used_keys` is the set # of fact keys for the current sources, used below to prune stale facts. - used_keys = _fetch_and_parse_sources(mctx, repo, glock, snapshot_suites, formats) + used_keys = _fetch_and_parse_sources(mctx, repo, glock, snapshot_indices, formats) sources = glock.sources() dependency_sets = glock.dependency_sets() @@ -427,7 +357,7 @@ def _distroless_extension(mctx): ) # Key every package by the architecture we are resolving for, not by the - # package's own `Architecture` field. + # package's own `Architecture` field. # For arch-specific packages these are the same. # For `Architecture: all` packages this "expands" them into one entry per target architecture, # so each carries its own arch-specific dependency closure instead of a single frozen one shared across arches. @@ -492,7 +422,9 @@ def _distroless_extension(mctx): # Storing these in a file instead of passing filemaps as attributes cuts down lockfile size considerably. deb_filemap( name = util.sanitize(package_key) + "_filemap", - files = json.encode(repo.filemap(name = name, arch = arch) or []), + package_key = package_key, + urls = package["urls"], + sha256 = package["sha256"], ) modes = package_repo_modes.get(package_key, {False: True}) @@ -503,16 +435,11 @@ def _distroless_extension(mctx): for (repo_name, mergedusr) in repo_variants: deb_import( name = repo_name, - target_name = repo_name, - urls = [ - uri + "/" + package["filename"] - for uri in sources[package["suite"]]["uris"] - ], + urls = package["urls"], sha256 = package["sha256"], mergedusr = mergedusr, - depends_on = package["depends_on"], - # Label of each dependency's own filemap, in depends_on order, - # so deb_import can rebuild the {file: dependency} index by lookup. + # Each filemap carries its package key, avoiding a second copy + # of every dependency list in MODULE.bazel.lock. dep_filemaps = [ "@" + util.sanitize(dep) + "_filemap//:filemap.json" for dep in package["depends_on"] @@ -541,7 +468,7 @@ def _distroless_extension(mctx): glock.facts(), formats, used_keys, - snapshot_suites, + snapshot_indices, ) return mctx.extension_metadata( facts = {"indices": cacheable_indices, "formats": cacheable_formats}, diff --git a/apt/private/BUILD.bazel b/apt/private/BUILD.bazel index bea1489..96cd678 100644 --- a/apt/private/BUILD.bazel +++ b/apt/private/BUILD.bazel @@ -55,11 +55,19 @@ bzl_library( deps = ["@bazel_lib//lib:strings"], ) +bzl_library( + name = "deb_archive", + srcs = ["deb_archive.bzl"], + visibility = ["//apt:__subpackages__"], + deps = ["@bazel_lib//lib:repo_utils"], +) + bzl_library( name = "deb_import", srcs = ["deb_import.bzl"], visibility = ["//apt:__subpackages__"], deps = [ + ":deb_archive", ":linker_script", ":pkgconfig", ], @@ -69,6 +77,7 @@ bzl_library( name = "deb_filemap", srcs = ["deb_filemap.bzl"], visibility = ["//apt:__subpackages__"], + deps = [":deb_archive"], ) bzl_library( diff --git a/apt/private/apt_deb_repository.bzl b/apt/private/apt_deb_repository.bzl index 037dbee..4c1ef12 100644 --- a/apt/private/apt_deb_repository.bzl +++ b/apt/private/apt_deb_repository.bzl @@ -37,17 +37,6 @@ def _parse_repository(state, contents, roots, dist): last_key = "" pkg = {} -def _parse_contents(state, rcontents, arch): - contents = state.filemap.setdefault(arch, {}) - for line in rcontents.splitlines(): - last_empty_char = line.rfind(" ") - first_empty_char = line.find(" ") - filepath = line[:first_empty_char] - pkgs = line[last_empty_char + 1:].split(",") - for pkg in pkgs: - contents.setdefault(pkg[pkg.find("/") + 1:], []).append(filepath) - state.filemap[arch] = contents - def _add_package(state, package): util.set_dict( state.packages, @@ -111,14 +100,6 @@ def _package(state, name, version, arch, suites = None): return None return package -def _filemap(state, name, arch): - if arch not in state.filemap: - return None - all = state.filemap[arch] - if name not in all: - return None - return state.filemap[arch][name] - def _add_source_if_not_present(state, source): (urls, dist, components, architectures) = source @@ -140,7 +121,6 @@ def _add_source_if_not_present(state, source): def _create(): state = struct( sources = dict(), - filemap = dict(), packages = dict(), virtual_packages = dict(), ) @@ -149,11 +129,9 @@ def _create(): add_source = lambda source: _add_source_if_not_present(state, source), sources = lambda: state.sources, parse_package_index = lambda contents, roots, dist: _parse_repository(state, contents, roots, dist), - parse_contents = lambda rcontents, arch: _parse_contents(state, rcontents, arch), package_versions = lambda **kwargs: _package_versions(state, **kwargs), virtual_packages = lambda **kwargs: _virtual_packages(state, **kwargs), package = lambda **kwargs: _package(state, **kwargs), - filemap = lambda **kwargs: _filemap(state, **kwargs), ) deb_repository = struct( diff --git a/apt/private/deb_archive.bzl b/apt/private/deb_archive.bzl new file mode 100644 index 0000000..65f967d --- /dev/null +++ b/apt/private/deb_archive.bzl @@ -0,0 +1,16 @@ +"""Inspect Debian data archives with tar.bzl's host toolchain binary.""" + +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") + +# Repository rules run before toolchain resolution. Use the same pinned +# binary tar.bzl makes available for the repository host. +def host_bsdtar(rctx): + platform = repo_utils.platform(rctx) + binary = "tar.exe" if platform.startswith("windows_") else "tar" + return rctx.path(Label("@bsd_tar_toolchains_{}//:{}".format(platform, binary))) + +def data_archive(rctx, directory = "."): + archives = [path for path in rctx.path(directory).readdir() if path.basename.startswith("data.tar")] + if len(archives) != 1: + fail("expected one data archive, found: %s" % archives) + return archives[0] diff --git a/apt/private/deb_filemap.bzl b/apt/private/deb_filemap.bzl index f8f50ae..63792df 100644 --- a/apt/private/deb_filemap.bzl +++ b/apt/private/deb_filemap.bzl @@ -5,16 +5,25 @@ Using a file instead of raw JSON-encoded attrs saves significant space in MODULE This is a separate repository to avoid dependency cycle issues, which Debian package graphs allow. """ +load(":deb_archive.bzl", "data_archive", "host_bsdtar") + def _deb_filemap_impl(rctx): - # `files` is a JSON-encoded list of the paths this package installs. - rctx.file("filemap.json", rctx.attr.files) + # Contents indexes describe entire distributions and may be absent from + # third-party repositories. Inspect the exact pinned package on demand. + rctx.download_and_extract(url = rctx.attr.urls, sha256 = rctx.attr.sha256, output = "_archive") + result = rctx.execute([host_bsdtar(rctx), "-tf", data_archive(rctx, "_archive")]) + if result.return_code: + fail("failed to list package files: %s" % result.stderr) + rctx.delete("_archive") + files = [path.removeprefix("./") for path in result.stdout.splitlines() if not path.endswith("/")] + rctx.file("filemap.json", json.encode({"package_key": rctx.attr.package_key, "files": files})) rctx.file("BUILD.bazel", 'exports_files(["filemap.json"], visibility = ["//visibility:public"])\n') deb_filemap = repository_rule( implementation = _deb_filemap_impl, attrs = { - "files": attr.string( - doc = "JSON-encoded list of file paths this package installs.", - ), + "package_key": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True, allow_empty = False), + "sha256": attr.string(mandatory = True), }, ) diff --git a/apt/private/deb_import.bzl b/apt/private/deb_import.bzl index d83dca1..d3c638c 100644 --- a/apt/private/deb_import.bzl +++ b/apt/private/deb_import.bzl @@ -1,7 +1,8 @@ "deb_import" -load(":lockfile.bzl", "lockfile") +load(":deb_archive.bzl", "data_archive", "host_bsdtar") load(":linker_script.bzl", "linker_script") +load(":lockfile.bzl", "lockfile") load(":pkgconfig.bzl", "pkgconfig") load(":util.bzl", "util") @@ -183,7 +184,11 @@ def _remap_linkopts(rctx, extract_dir, so_regular_files, self_files, depends_fil return result.linkopts def _discover_contents(rctx, depends_on, depends_file_map, target_name, mergedusr = False): - result = rctx.execute(["tar", "--exclude='./usr/share/**'", "--exclude='./**/'", "-tvf", "data.tar.xz"]) + archive = data_archive(rctx) + tar = host_bsdtar(rctx) + result = rctx.execute([tar, "-tvf", archive]) + if result.return_code: + fail("failed to inspect %s: %s" % (archive, result.stderr)) contents_raw = result.stdout.splitlines() so_files = [] @@ -283,10 +288,12 @@ def _discover_contents(rctx, depends_on, depends_file_map, target_name, mergedus files_to_extract = so_regular_files + pc_files if files_to_extract: rctx.execute(["mkdir", "-p", _EXTRACT_DIR]) - rctx.execute( - ["tar", "-xf", "data.tar.xz", "-C", _EXTRACT_DIR] + + result = rctx.execute( + [tar, "-xf", archive, "-C", _EXTRACT_DIR] + ["./" + f for f in files_to_extract], ) + if result.return_code: + fail("failed to extract %s from %s: %s" % (files_to_extract, archive, result.stderr)) remap_linkopts = _remap_linkopts( rctx, @@ -452,17 +459,20 @@ def _deb_import_impl(rctx): # Rebuild the {file: canonical_name(dependency_repo)} index from each dependency's own filemap. # The first dependency that provides a path wins. provided_by = {} - for i in range(len(rctx.attr.dep_filemaps)): - filemap_path = rctx.path(rctx.attr.dep_filemaps[i]) + depends_on = [] + for label in rctx.attr.dep_filemaps: + filemap_path = rctx.path(label) + filemap = json.decode(rctx.read(filemap_path)) + depends_on.append(filemap["package_key"]) dep_repo = filemap_path.dirname.basename.removesuffix("_filemap") - for file in json.decode(rctx.read(filemap_path)): + for file in filemap["files"]: if file not in provided_by: provided_by[file] = dep_repo # TODO: only do this if package is -dev or dependent of a -dev pkg. cc_import_targets, outs, symlinks = _discover_contents( rctx, - rctx.attr.depends_on, + depends_on, provided_by, rctx.attr.package_name.removesuffix("-dev"), mergedusr = rctx.attr.mergedusr, @@ -481,8 +491,8 @@ def _deb_import_impl(rctx): rctx.file("BUILD.bazel", _DEB_IMPORT_BUILD_TMPL.format( mergedusr = rctx.attr.mergedusr, - depends_on = ["@" + util.package_repo_name(dep_key, mergedusr = rctx.attr.mergedusr) + "//:data" for dep_key in rctx.attr.depends_on], - target_name = rctx.attr.target_name, + depends_on = ["@" + util.package_repo_name(dep_key, mergedusr = rctx.attr.mergedusr) + "//:data" for dep_key in depends_on], + target_name = util.get_repo_name(rctx.attr.name), cc_import_targets = cc_import_targets, outs = outs, foreign_symlinks = foreign_symlinks, @@ -494,10 +504,8 @@ deb_import = repository_rule( attrs = { "urls": attr.string_list(mandatory = True, allow_empty = False), "sha256": attr.string(), - "depends_on": attr.string_list(doc = "Names of packages this package depends on"), - "dep_filemaps": attr.label_list(doc = "Each dependency's filemap.json, in depends_on order, used to resolve cross-package symlinks."), + "dep_filemaps": attr.label_list(doc = "Ordered dependency filemaps containing package keys and installed paths."), "mergedusr": attr.bool(), - "target_name": attr.string(), "package_name": attr.string(), }, ) diff --git a/apt/private/lockfile.bzl b/apt/private/lockfile.bzl index bdb2af8..70519b6 100644 --- a/apt/private/lockfile.bzl +++ b/apt/private/lockfile.bzl @@ -40,6 +40,8 @@ def _package_key(package, arch = None): def _add_package(lock, package, arch = None): k = _package_key(package, arch) if k in lock.packages: + # Refresh provenance from the selected index when reading an older lock. + lock.packages[k]["urls"] = [root + "/" + package["Filename"] for root in package["Roots"]] return lock.packages[k] = { "name": package["Package"], @@ -47,6 +49,7 @@ def _add_package(lock, package, arch = None): "architecture": package["Architecture"], "sha256": package["SHA256"], "filename": package["Filename"], + "urls": [root + "/" + package["Filename"] for root in package["Roots"]], "suite": package["Dist"], "section": package["Section"], "size": int(package["Size"]), @@ -116,6 +119,15 @@ def _from_json(mctx, content): if lock["version"] != 2: fail("lock file version %d is not supported anymore. please upgrade your lock file" % lock["version"]) + # Existing v2 locks only recorded source URLs at suite granularity. + # Preserve their download locations until a selected index refreshes them. + for package in lock.get("packages", {}).values(): + if "urls" not in package: + package["urls"] = [ + root + "/" + package["filename"] + for root in lock["sources"][package["suite"]]["uris"] + ] + lock = struct( version = lock["version"], dependency_sets = lock["dependency_sets"] if "dependency_sets" in lock else dict(), diff --git a/apt/private/translate_dependency_set.bzl b/apt/private/translate_dependency_set.bzl index d0147bf..cf52ddd 100644 --- a/apt/private/translate_dependency_set.bzl +++ b/apt/private/translate_dependency_set.bzl @@ -152,7 +152,6 @@ def _translate_dependency_set_impl(rctx): package_template = rctx.read(rctx.attr.package_template) lockf = lockfile.from_json(rctx, rctx.attr.lock_content) - sources = lockf.sources() packages = lockf.packages() dependency_sets = lockf.dependency_sets() dependency_set = dependency_sets[rctx.attr.depset_name] @@ -209,10 +208,7 @@ Please unify the versions manually, or use separate `apt.install` calls (with di control_targets = '"@%s//:control"' % repo_name, src = '"@%s//:data"' % repo_name, deps = package_deps_for_architecture(packages, package, architecture, mergedusr = rctx.attr.mergedusr), - urls = [ - uri + "/" + package["filename"] - for uri in sources[package["suite"]]["uris"] - ], + urls = package["urls"], name = package["name"], arch = package["architecture"], sha256 = package["sha256"], diff --git a/apt/private/util.bzl b/apt/private/util.bzl index 901dea1..066a2e2 100644 --- a/apt/private/util.bzl +++ b/apt/private/util.bzl @@ -61,14 +61,14 @@ def _index_fact_key(dist, component, architecture, index_type, urls): url_token = "|".join(sorted_deduplicated_urls) return "{}/{}/{}/{}/{}".format(dist, component, architecture, index_type, url_token) -def _prune_uncacheable_facts(indices, formats, used_keys, snapshot_suites): +def _prune_uncacheable_facts(indices, formats, used_keys, snapshot_indices): """Keep only the facts that can be cached. `used_keys` holds the fact keys produced for this run's sources (see `index_fact_key`). Entries left over from a previous snapshot URL are not in `used_keys`, so they get dropped here instead of accumulating across runs. - `snapshot_suites` holds a list of suites from snapshots, + `snapshot_indices` holds the fact keys from snapshot sources, because we don't want to cache rolling suites. Returns `(cacheable_indices, cacheable_formats)`. @@ -76,7 +76,7 @@ def _prune_uncacheable_facts(indices, formats, used_keys, snapshot_suites): cacheable_indices = { k: v for k, v in indices.items() - if k in used_keys and k.split("/")[0] in snapshot_suites + if k in used_keys and k in snapshot_indices } cacheable_formats = { k: v diff --git a/apt/tests/facts_test.bzl b/apt/tests/facts_test.bzl index a1054f0..c1fe4da 100644 --- a/apt/tests/facts_test.bzl +++ b/apt/tests/facts_test.bzl @@ -45,16 +45,16 @@ def _prune_facts_test(ctx): old_key = util.index_fact_key("bookworm", "main", "amd64", "Packages", _TEST_SNAPSHOT_1) new_key = util.index_fact_key("bookworm", "main", "amd64", "Packages", _TEST_SNAPSHOT_2) - rolling_key = util.index_fact_key("sid", "main", "amd64", "Packages", ["https://deb.debian.org/debian"]) + rolling_key = util.index_fact_key("bookworm", "main", "amd64", "Packages", ["https://deb.debian.org/debian"]) indices = {old_key: "sha256-OLD", new_key: "sha256-NEW", rolling_key: "sha256-ROLLING"} formats = {old_key: ".xz", new_key: ".xz", rolling_key: ".xz"} # Only the current sources' keys are used this run. used_keys = {new_key: True, rolling_key: True} - snapshot_suites = {"bookworm": True} + snapshot_indices = {new_key: True} - (cacheable_indices, cacheable_formats) = util.prune_uncacheable_facts(indices, formats, used_keys, snapshot_suites) + (cacheable_indices, cacheable_formats) = util.prune_uncacheable_facts(indices, formats, used_keys, snapshot_indices) # The stale previous-URL entry is dropped, and indices from rolling indexes are not cached. asserts.equals(env, {new_key: "sha256-NEW"}, cacheable_indices) diff --git a/apt/tests/lockfile_test.bzl b/apt/tests/lockfile_test.bzl index 384b710..6b2be40 100644 --- a/apt/tests/lockfile_test.bzl +++ b/apt/tests/lockfile_test.bzl @@ -63,4 +63,37 @@ def _add_source_merges_architectures_test(ctx): add_source_merges_architectures_test = unittest.make(_add_source_merges_architectures_test) def lockfile_tests(): + package_source_urls_test(name = _TEST_SUITE_PREFIX + "package_source_urls") add_source_merges_architectures_test(name = _TEST_SUITE_PREFIX + "add_source_merges_architectures") + +def _package_source_urls_test(ctx): + env = unittest.begin(ctx) + lock = lockfile.empty(struct()) + package = { + "Architecture": "amd64", + "Dist": "noble", + "Filename": "pool/main/e/example_1_amd64.deb", + "Package": "example", + "Roots": ["https://example.org/ppa"], + "SHA256": "abc", + "Section": "libs", + "Size": "1", + "Version": "1", + } + lock.add_source("noble", ["deb"], ["https://example.org/ubuntu"], ["main"], ["amd64"]) + lock.add_package(package) + key = lockfile.package_key(package) + expected = ["https://example.org/ppa/" + package["Filename"]] + asserts.equals(env, expected, lock.packages()[key]["urls"]) + + # A v2 lock without per-package URLs remains readable. Resolving its + # package again must replace the old suite URL with the selected PPA URL. + legacy = json.decode(lock.as_json()) + legacy["packages"][key].pop("urls") + restored = lockfile.from_json(struct(), json.encode(legacy)) + asserts.equals(env, ["https://example.org/ubuntu/" + package["Filename"]], restored.packages()[key]["urls"]) + restored.add_package(package) + asserts.equals(env, expected, restored.packages()[key]["urls"]) + return unittest.end(env) + +package_source_urls_test = unittest.make(_package_source_urls_test)