diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5c5f1aa6113..05bedad56a6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,11 @@ updates: - "/python" - "/java/lance-jni" versioning-strategy: lockfile-only + # Matches the 48h floor enforced by ci/check_dependency_age.py, so grouped + # bumps do not land a version that CI will then reject. Cooldown does not + # apply to security updates, which we still want as fast as possible. + cooldown: + default-days: 2 schedule: interval: "weekly" day: "wednesday" diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml index 5835f744d82..80ebff1589f 100644 --- a/.github/workflows/ci-scripts.yml +++ b/.github/workflows/ci-scripts.yml @@ -18,6 +18,8 @@ on: - ci/test_labeler_area.py - ci/check_proto_comments.py - ci/test_check_proto_comments.py + - ci/check_dependency_age.py + - ci/test_check_dependency_age.py - .github/labeler-area.yml - .github/workflows/format-vote-gate.yml - .github/workflows/ci-scripts.yml @@ -39,4 +41,4 @@ jobs: - name: Install test dependencies run: pip install pytest PyYAML - name: Run tests - run: pytest ci/test_format_vote_gate.py ci/test_labeler_area.py ci/test_check_proto_comments.py + run: pytest ci/test_format_vote_gate.py ci/test_labeler_area.py ci/test_check_proto_comments.py ci/test_check_dependency_age.py diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 252422c1b07..0cba62d83da 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -15,9 +15,12 @@ on: - rust-toolchain.toml - Cargo.toml - Cargo.lock + - python/Cargo.lock + - java/lance-jni/Cargo.lock - .cargo/config.toml - clippy.toml - deny.toml + - ci/check_dependency_age.py permissions: contents: read @@ -103,6 +106,10 @@ jobs: with: log-level: warn command: check + # Runs after cargo-deny so it can reuse the index cargo-deny already + # fetched. The script falls back to the network when that misses. + - name: Check dependency age + run: python3 ci/check_dependency_age.py linux-build: runs-on: "ubuntu-24.04-8x" diff --git a/ci/check_dependency_age.py b/ci/check_dependency_age.py new file mode 100644 index 00000000000..d29f518ccc5 --- /dev/null +++ b/ci/check_dependency_age.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Fail if any crates.io dependency in a Cargo.lock is younger than a minimum age. + +A freshly published version is the window in which a compromised crate is most +likely to still be live, so we refuse to ship one until it has had time to be +noticed and yanked. Cargo's own `registry.global-min-publish-age` only constrains +resolution: versions already written into Cargo.lock are grandfathered in and are +never re-checked, so it cannot answer "is what we committed too new?". + +Publish timestamps come from the `pubtime` field of the sparse index. Cargo keeps +the index lines it has downloaded in `$CARGO_HOME/registry/index/*/.cache`, which +lets a full scan run without any network calls; anything not found there is +fetched from index.crates.io, and anything the index has not published yet is +looked up through the crates.io API, which is authoritative and never lags. + +The same floor is enforced on developer machines by Aikido endpoint protection, +which filters the index it serves them. A security fix that has to land sooner +therefore needs an Aikido allowlist entry as well as one in +ci/dependency-age-allowlist.toml; the entry here only unblocks CI. +""" + +import argparse +import json +import os +import sys +import time +import tomllib +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from pathlib import Path + +DEFAULT_LOCKFILES = ("Cargo.lock", "python/Cargo.lock", "java/lance-jni/Cargo.lock") +DEFAULT_ALLOWLIST = "ci/dependency-age-allowlist.toml" +DEFAULT_MIN_AGE_HOURS = 48 +CRATES_IO_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" +INDEX_URL = "https://index.crates.io" +API_URL = "https://crates.io/api/v1/crates" +USER_AGENT = "lance-ci-dependency-age (https://github.com/lance-format/lance)" +FETCH_THREADS = 16 +# The API is rate limited and is only ever asked about the few versions the +# index CDN has not caught up with, so it gets a much smaller pool. +API_THREADS = 4 +FETCH_TIMEOUT_SECONDS = 30 +FETCH_ATTEMPTS = 4 + +# The cache file is a cargo internal: a format byte, an index-format u32, an +# `etag: ...` line, then a NUL-separated run of alternating version and index +# JSON. Only the JSON matters here, so rather than validating the framing we +# pick out the chunks that parse, and let the caller fall back to HTTP if a +# cargo release ever changes the layout underneath us. +CACHE_JSON_PREFIX = b"{" + + +def index_path(name): + """Return the sparse-index path prefix cargo uses for a crate name.""" + name = name.lower() + if len(name) <= 2: + return f"{len(name)}/{name}" + if len(name) == 3: + return f"3/{name[0]}/{name}" + return f"{name[:2]}/{name[2:4]}/{name}" + + +def parse_index_entries(chunks): + """Map version -> pubtime (or None) from an iterable of index JSON chunks.""" + entries = {} + for chunk in chunks: + chunk = chunk.strip() + if not chunk.startswith(CACHE_JSON_PREFIX): + continue + entry = json.loads(chunk) + entries[entry["vers"]] = entry.get("pubtime") + return entries + + +def parse_cache_file(data): + try: + return parse_index_entries(data.split(b"\x00")) + except (json.JSONDecodeError, KeyError, UnicodeDecodeError): + return {} + + +def parse_index_response(data): + return parse_index_entries(data.splitlines()) + + +def parse_api_response(data): + """Map version -> publish time from a crates.io `/versions` response.""" + versions = json.loads(data)["versions"] + return {version["num"]: version.get("created_at") for version in versions} + + +def crates_io_packages(lock_text): + """Return (name, version) for every crates.io package in a lockfile.""" + lock = tomllib.loads(lock_text) + return [ + (package["name"], package["version"]) + for package in lock.get("package", []) + if package.get("source") == CRATES_IO_SOURCE + ] + + +def load_allowlist(path): + """Map (crate, version) -> reason for the exemptions in an allowlist file.""" + if not Path(path).is_file(): + return {} + entries = tomllib.loads(Path(path).read_text()).get("allow", []) + allowed = {} + for entry in entries: + missing = {"crate", "version", "reason"} - entry.keys() + if missing: + raise SystemExit( + f"{path}: allow entry {entry} is missing {sorted(missing)}" + ) + allowed[(entry["crate"], entry["version"])] = entry["reason"] + return allowed + + +def cache_dirs(cargo_home): + return sorted(Path(cargo_home).glob("registry/index/*/.cache")) + + +def pubtimes_from_cache(names, cargo_home): + dirs = cache_dirs(cargo_home) + found = {} + for name in names: + for directory in dirs: + path = directory / index_path(name) + if not path.is_file(): + continue + entries = parse_cache_file(path.read_bytes()) + if entries: + found[name] = entries + break + return found + + +def http_get(url): + """Return the body, or None for a 404. Retries transient failures.""" + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + for attempt in range(FETCH_ATTEMPTS): + try: + with urllib.request.urlopen( + request, timeout=FETCH_TIMEOUT_SECONDS + ) as response: + return response.read() + except urllib.error.HTTPError as error: + if error.code == 404: + return None + if attempt == FETCH_ATTEMPTS - 1: + raise + except OSError: + if attempt == FETCH_ATTEMPTS - 1: + raise + time.sleep(2**attempt) + + +def fetch_index_pubtimes(name): + body = http_get(f"{INDEX_URL}/{index_path(name)}") + return name, {} if body is None else parse_index_response(body) + + +def fetch_api_pubtimes(name): + body = http_get(f"{API_URL}/{name}/versions") + return name, {} if body is None else parse_api_response(body) + + +def pubtimes_from_index(names): + return fetch_all(fetch_index_pubtimes, names, FETCH_THREADS) + + +def pubtimes_from_api(names): + return fetch_all(fetch_api_pubtimes, names, API_THREADS) + + +def fetch_all(fetch, names, threads): + if not names: + return {} + with ThreadPoolExecutor(threads) as pool: + return dict(pool.map(fetch, names)) + + +def resolve_pubtimes(packages, cargo_home): + """Sort packages into dated, undated, and not-in-the-index. + + Returns ({(name, version): pubtime}, [undated], [absent]). + """ + known = pubtimes_from_cache({name for name, _ in packages}, cargo_home) + for lookup in (pubtimes_from_index, pubtimes_from_api): + stale = sorted( + {name for name, version in packages if version not in known.get(name, {})} + ) + for name, entries in lookup(stale).items(): + known[name] = {**known.get(name, {}), **entries} + + dated, undated, absent = {}, [], [] + for name, version in packages: + entries = known.get(name, {}) + if version not in entries: + absent.append((name, version)) + elif entries[version] is None: + undated.append((name, version)) + else: + dated[(name, version)] = entries[version] + return dated, undated, absent + + +def too_new(dated, cutoff): + """Return (name, version, published) for versions published after the cutoff.""" + violations = [] + for (name, version), pubtime in dated.items(): + published = datetime.fromisoformat(pubtime.replace("Z", "+00:00")) + if published > cutoff: + violations.append((name, version, published)) + return violations + + +def check_lockfile(path, cutoff, cargo_home, allowed): + """Print this lockfile's findings and return whether it passed.""" + packages = crates_io_packages(Path(path).read_text()) + dated, undated, absent = resolve_pubtimes(packages, cargo_home) + + if undated: + # The index carries no pubtime for a few dozen versions predating the + # field. Those are years old, so treating them as passing is safe. + print(f"{path}: {len(undated)} of {len(packages)} versions predate `pubtime`") + + violations = [ + violation + for violation in too_new(dated, cutoff) + if (violation[0], violation[1]) not in allowed + ] + absent = [package for package in absent if package not in allowed] + + for name, version, published in sorted(violations): + short_by = (published - cutoff).total_seconds() / 3600 + print( + f"{path}: {name} {version} is too new — published {published.isoformat()}, " + f"{short_by:.1f}h short of the minimum age" + ) + for name, version in sorted(absent): + print(f"{path}: {name} {version} is not in the index, so it cannot be dated") + + return not violations and not absent + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("lockfiles", nargs="*", default=list(DEFAULT_LOCKFILES)) + parser.add_argument("--min-age-hours", type=int, default=DEFAULT_MIN_AGE_HOURS) + parser.add_argument("--allowlist", default=DEFAULT_ALLOWLIST) + parser.add_argument( + "--cargo-home", default=os.environ.get("CARGO_HOME", Path.home() / ".cargo") + ) + args = parser.parse_args(argv) + + cutoff = datetime.now(timezone.utc) - timedelta(hours=args.min_age_hours) + allowed = load_allowlist(args.allowlist) + for (name, version), reason in sorted(allowed.items()): + print(f"{args.allowlist} exempts {name} {version}: {reason}") + + passed = [ + check_lockfile(path, cutoff, args.cargo_home, allowed) + for path in args.lockfiles + ] + if all(passed): + return 0 + + print( + f"\nEvery crates.io dependency must be at least {args.min_age_hours}h old. " + "Wait for these versions to age, or pin the previous version. Dependabot's " + "cooldown should normally keep this from firing.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/dependency-age-allowlist.toml b/ci/dependency-age-allowlist.toml new file mode 100644 index 00000000000..60866509a4e --- /dev/null +++ b/ci/dependency-age-allowlist.toml @@ -0,0 +1,15 @@ +# Versions exempt from the minimum publish age enforced by +# ci/check_dependency_age.py. +# +# Only add an entry once the same version has been allowlisted in Aikido +# endpoint protection. Aikido filters the crates.io index it serves to +# developer machines, so an entry here on its own unblocks CI while leaving +# every developer unable to build. +# +# Entries become inert once the version passes the minimum age, so pruning +# them is housekeeping rather than urgent. + +# [[allow]] +# crate = "some-crate" +# version = "1.2.3" +# reason = "RUSTSEC-2026-0000, allowlisted in Aikido 2026-09-16" diff --git a/ci/test_check_dependency_age.py b/ci/test_check_dependency_age.py new file mode 100644 index 00000000000..d9b29e6b460 --- /dev/null +++ b/ci/test_check_dependency_age.py @@ -0,0 +1,208 @@ +"""Unit tests for the crates.io dependency age check. + +Run with: pytest ci/test_check_dependency_age.py +""" + +import json +from datetime import datetime, timedelta, timezone + +import pytest + +from check_dependency_age import ( + check_lockfile, + crates_io_packages, + index_path, + load_allowlist, + parse_api_response, + parse_cache_file, + parse_index_response, + too_new, +) + +NOW = datetime(2026, 9, 16, 12, 0, tzinfo=timezone.utc) +CUTOFF = NOW - timedelta(hours=48) + + +def cache_blob(entries): + """Build a cargo index cache file the way cargo lays one out.""" + out = b"\x03\x02\x00\x00\x00" + b'etag: "abc"' + b"\x00" + for version, pubtime in entries.items(): + entry = {"name": "demo", "vers": version, "deps": [], "yanked": False} + if pubtime is not None: + entry["pubtime"] = pubtime + out += version.encode() + b"\x00" + json.dumps(entry).encode() + b"\x00" + return out + + +def lockfile(packages): + blocks = ["version = 4\n"] + for name, version, source in packages: + block = f'[[package]]\nname = "{name}"\nversion = "{version}"\n' + if source is not None: + block += f'source = "{source}"\n' + blocks.append(block) + return "\n".join(blocks) + + +CRATES_IO = "registry+https://github.com/rust-lang/crates.io-index" + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + pytest.param("a", "1/a", id="one_char"), + pytest.param("ab", "2/ab", id="two_chars"), + pytest.param("abc", "3/a/abc", id="three_chars"), + pytest.param("serde", "se/rd/serde", id="long"), + pytest.param("Inflector", "in/fl/inflector", id="uppercase_is_folded"), + ], +) +def test_index_path(name, expected): + assert index_path(name) == expected + + +def test_parse_cache_file_reads_pubtimes(): + blob = cache_blob({"1.0.0": "2020-01-01T00:00:00Z", "1.1.0": None}) + assert parse_cache_file(blob) == { + "1.0.0": "2020-01-01T00:00:00Z", + "1.1.0": None, + } + + +def test_parse_cache_file_tolerates_an_unknown_layout(): + """A cargo change to the cache format must degrade to the HTTP fallback.""" + assert parse_cache_file(b"\x09\x99{not json at all}\x00") == {} + + +def test_parse_index_response_reads_newline_delimited_json(): + body = b'{"vers":"1.0.0","pubtime":"2020-01-01T00:00:00Z"}\n{"vers":"2.0.0"}\n' + assert parse_index_response(body) == { + "1.0.0": "2020-01-01T00:00:00Z", + "2.0.0": None, + } + + +def test_parse_api_response_reads_created_at(): + body = b'{"versions":[{"num":"1.0.0","created_at":"2020-01-01T00:00:00.123456Z"}]}' + assert parse_api_response(body) == {"1.0.0": "2020-01-01T00:00:00.123456Z"} + + +def test_crates_io_packages_ignores_path_and_git_packages(): + text = lockfile( + [ + ("serde", "1.0.0", CRATES_IO), + ("lance", "0.1.0", None), + ("forked", "0.2.0", "git+https://example.com/forked#abc123"), + ] + ) + assert crates_io_packages(text) == [("serde", "1.0.0")] + + +@pytest.mark.parametrize( + ("pubtime", "expected"), + [ + pytest.param("2026-09-10T00:00:00Z", [], id="old"), + pytest.param("2026-09-14T11:59:00Z", [], id="just_past_cutoff"), + pytest.param("2026-09-14T12:01:00Z", [("serde", "1.0.0")], id="just_too_new"), + pytest.param("2026-09-16T11:00:00Z", [("serde", "1.0.0")], id="an_hour_old"), + ], +) +def test_too_new_uses_the_cutoff_as_the_boundary(pubtime, expected): + violations = too_new({("serde", "1.0.0"): pubtime}, CUTOFF) + assert [(name, version) for name, version, _ in violations] == expected + + +def cargo_home_with(tmp_path, crates): + home = tmp_path / "cargo" + for name, entries in crates.items(): + path = ( + home / "registry/index/index.crates.io-1949cf8c/.cache" / index_path(name) + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(cache_blob(entries)) + return home + + +def test_load_allowlist_is_empty_when_the_file_is_absent(tmp_path): + assert load_allowlist(tmp_path / "nope.toml") == {} + + +def test_load_allowlist_reads_entries(tmp_path): + path = tmp_path / "allow.toml" + path.write_text( + '[[allow]]\ncrate = "serde"\nversion = "1.0.0"\nreason = "RUSTSEC-0"\n' + ) + assert load_allowlist(path) == {("serde", "1.0.0"): "RUSTSEC-0"} + + +def test_load_allowlist_rejects_an_entry_without_a_reason(tmp_path): + """An exemption with no rationale is not reviewable, so refuse to run.""" + path = tmp_path / "allow.toml" + path.write_text('[[allow]]\ncrate = "serde"\nversion = "1.0.0"\n') + with pytest.raises(SystemExit, match="reason"): + load_allowlist(path) + + +def test_check_lockfile_passes_when_every_version_is_old_enough(tmp_path, capsys): + lock = tmp_path / "Cargo.lock" + lock.write_text(lockfile([("serde", "1.0.0", CRATES_IO)])) + home = cargo_home_with(tmp_path, {"serde": {"1.0.0": "2026-09-01T00:00:00Z"}}) + + assert check_lockfile(lock, CUTOFF, home, {}) is True + assert capsys.readouterr().out == "" + + +def test_check_lockfile_reports_a_too_new_version(tmp_path, capsys): + lock = tmp_path / "Cargo.lock" + lock.write_text(lockfile([("serde", "1.0.0", CRATES_IO)])) + home = cargo_home_with(tmp_path, {"serde": {"1.0.0": "2026-09-16T00:00:00Z"}}) + + assert check_lockfile(lock, CUTOFF, home, {}) is False + assert "serde 1.0.0 is too new" in capsys.readouterr().out + + +def test_check_lockfile_accepts_versions_predating_pubtime(tmp_path, capsys): + lock = tmp_path / "Cargo.lock" + lock.write_text(lockfile([("serde", "1.0.0", CRATES_IO)])) + home = cargo_home_with(tmp_path, {"serde": {"1.0.0": None}}) + + assert check_lockfile(lock, CUTOFF, home, {}) is True + assert "1 of 1 versions predate `pubtime`" in capsys.readouterr().out + + +def test_check_lockfile_fails_when_the_index_does_not_list_the_version( + tmp_path, capsys, monkeypatch +): + """An undatable version is a hole in the check, so it must not pass silently.""" + monkeypatch.setattr("check_dependency_age.pubtimes_from_index", lambda names: {}) + monkeypatch.setattr("check_dependency_age.pubtimes_from_api", lambda names: {}) + lock = tmp_path / "Cargo.lock" + lock.write_text(lockfile([("serde", "9.9.9", CRATES_IO)])) + home = cargo_home_with(tmp_path, {"serde": {"1.0.0": "2026-09-01T00:00:00Z"}}) + + assert check_lockfile(lock, CUTOFF, home, {}) is False + assert "serde 9.9.9 is not in the index" in capsys.readouterr().out + + +def test_check_lockfile_honours_an_allowlisted_version(tmp_path, capsys): + lock = tmp_path / "Cargo.lock" + lock.write_text(lockfile([("serde", "1.0.0", CRATES_IO)])) + home = cargo_home_with(tmp_path, {"serde": {"1.0.0": "2026-09-16T00:00:00Z"}}) + allowed = {("serde", "1.0.0"): "RUSTSEC-0"} + + assert check_lockfile(lock, CUTOFF, home, allowed) is True + assert "too new" not in capsys.readouterr().out + + +def test_check_lockfile_allowlist_also_covers_an_undatable_version( + tmp_path, capsys, monkeypatch +): + """Aikido hides allowlisted-but-young versions from the index it serves.""" + monkeypatch.setattr("check_dependency_age.pubtimes_from_index", lambda names: {}) + monkeypatch.setattr("check_dependency_age.pubtimes_from_api", lambda names: {}) + lock = tmp_path / "Cargo.lock" + lock.write_text(lockfile([("serde", "9.9.9", CRATES_IO)])) + home = cargo_home_with(tmp_path, {"serde": {"1.0.0": "2026-09-01T00:00:00Z"}}) + + assert check_lockfile(lock, CUTOFF, home, {("serde", "9.9.9"): "RUSTSEC-0"}) is True + assert capsys.readouterr().out == ""