From a47fcd89ab4afbbc9e97750f746ebe59851a26c6 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:11:01 +0000 Subject: [PATCH] build(tools): serve the FastQC archive from this repository's releases On 2026-09-22 CI went red on a commit that touched nothing near FastQC: www.bioinformatics.babraham.ac.uk served an expired TLS certificate. The gate fix in #51 made that legible and retryable, but it could not remove the dependency, because FastQC publishes no GitHub release assets -- all six of its GitHub releases carry none -- so the pinned URL could only ever point at that one host. A third party's certificate renewal could stop our builds, and did. So the archive is vendored: same bytes, served from this repository's own releases (tagged vendored/fastqc-v0.12.1, marked pre-release because it is a build input rather than a version of this software). CI now installs it over a connection to GitHub, the same host that already serves vsearch and swarm, and the FastQC host leaves the critical path for building and reproducing the pipeline. The provenance is stated rather than glossed. The capture used `curl --insecure`, because upstream's certificate was already invalid when the file was taken. Integrity does not rest on that transfer: the sha256 in the pin is UNCHANGED from the original upstream pin, established while upstream was healthy, and the captured file verifies against it. Nothing changed except who serves the file -- and that is checkable, because the vendored download was verified byte-for-byte against a fresh upstream copy. Guarded, because the failure mode is silent: a URL that quietly points back at a third party still downloads and still verifies. The new testset fails by name if the pin returns to its old host or stops being a release of this repository (mutation-tested: repointing it at babraham.ac.uk fails two assertions by name). The vendored checksum is asserted too, so a future bump has to be a decision rather than a drift. docs/compliance/vendored-archives.md records the rule -- the checksum is the integrity claim and never the transport, never re-checksum to make a download succeed, the licence travels with the archive, and repointing back at a third party is a decision -- with the entry for this archive and the procedure for adding the next one. Refs #30, #51 --- config/defaults/tool_versions.yml | 21 ++++++++-- docs/compliance/vendored-archives.md | 61 ++++++++++++++++++++++++++++ test/unit/test_install_pins.jl | 27 ++++++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 docs/compliance/vendored-archives.md diff --git a/config/defaults/tool_versions.yml b/config/defaults/tool_versions.yml index e3596557..7b9a79f6 100644 --- a/config/defaults/tool_versions.yml +++ b/config/defaults/tool_versions.yml @@ -66,14 +66,29 @@ tools: version: "1.33" source: pypi - # Babraham publish no release API, so the URL is constructed from the version - # and cannot be discovered. Bump both together. + # Babraham publish no release API, and no GitHub release assets either -- all six + # of their GitHub releases carry none -- so upstream the URL is constructed from + # the version and cannot be discovered. Because of that, and only because of that, + # the archive is VENDORED: see docs/compliance/vendored-archives.md. + # + # MEASURED 2026-09-22: CI went red on a commit that touched nothing near FastQC + # because www.bioinformatics.babraham.ac.uk served an expired TLS certificate. + # With no upstream release asset to fall back on, the pin could only point at that + # one host, so a third party's certificate renewal was able to stop our builds. + # + # The vendored asset is the SAME BYTE-FOR-BYTE ARTEFACT: the sha256 below is + # unchanged from the upstream pin, and it was established while upstream was + # healthy. Integrity comes from that checksum -- the retrieval itself used + # `curl --insecure` (recorded in the release notes) because upstream's certificate + # was already invalid when the file was captured. Nothing changed but who serves it. + # + # Bump the version, the URL and the checksum together. fastqc: version: "0.12.1" source: archive archives: any: - url: "https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v0.12.1.zip" + url: "https://github.com/hyperpolymath/MetaManifold-WebUI/releases/download/vendored/fastqc-v0.12.1/fastqc_v0.12.1.zip" sha256: "5f4dba8780231a25a6b8e11ab2c238601920c9704caa5458d9de559575d58aa7" # No precompiled cd-hit binary is published for Linux, so the installer prefers diff --git a/docs/compliance/vendored-archives.md b/docs/compliance/vendored-archives.md new file mode 100644 index 00000000..33bdfa80 --- /dev/null +++ b/docs/compliance/vendored-archives.md @@ -0,0 +1,61 @@ + +# Vendored archives + +Third-party build inputs that are served from **this repository's own releases** instead +of from the party that publishes them. Vendoring is a decision, not a convenience: every +entry here states why the upstream location cannot be relied on, and every archive is +byte-identical to the upstream release it came from. + +## The rule + +1. **The checksum is the integrity claim, never the transport.** A vendored archive is + accepted only against the SHA-256 that was already pinned while upstream was healthy. + If a capture had to be made over a broken connection, that is recorded below rather + than glossed. +2. **Never re-checksum to make a download succeed.** If the bytes differ, the vendoring + failed — investigate, do not "fix" the pin. +3. **Licence travels with the archive.** The upstream licence stays inside the archive, + unmodified, and is named below. +4. **Repointing back at a third party is a decision.** `test/unit/test_install_pins.jl` + fails by name if a vendored URL quietly returns to its original host. + +## Entries + +### `fastqc_v0.12.1.zip` — FastQC 0.12.1 + +| | | +| --- | --- | +| Release | [`vendored/fastqc-v0.12.1`](https://github.com/hyperpolymath/MetaManifold-WebUI/releases/tag/vendored%2Ffastqc-v0.12.1) (marked *pre-release* — it is a build input, not a version of this software) | +| Upstream | `https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v0.12.1.zip` | +| SHA-256 | `5f4dba8780231a25a6b8e11ab2c238601920c9704caa5458d9de559575d58aa7` | +| Licence | GNU GPL v3 (the archive carries `FastQC/LICENSE.txt`), redistributed unmodified | +| Captured | 2026-09-22 | +| Pinned in | `config/defaults/tool_versions.yml` (`tools.fastqc.archives.any`) | + +**Why.** On 2026-09-22 CI went red on a commit that touched nothing near FastQC: the +upstream host served an **expired TLS certificate**. There was no way out through the pin +file, because FastQC publishes **no GitHub release assets** — all six of its GitHub +releases carry none — so the URL could only ever point at that single host. A third +party's certificate renewal could therefore stop our builds, and did. + +**Provenance.** The archive was captured once from the upstream URL above. The capture +used `curl --insecure`, because upstream's certificate was already invalid at that +moment; this is stated plainly because it is the one detail a reader should not have to +infer. Integrity does not rest on that transfer: the SHA-256 above is **unchanged from +the original upstream pin**, which was established while upstream was healthy, and the +captured file verifies against it. These are the bytes CI has always installed; only the +serving host changed. + +**Effect.** CI installs FastQC from this repository's releases, over a connection to +GitHub — the same host that already serves `vsearch` and `swarm` — so the FastQC host is +no longer on the critical path for building or reproducing the pipeline. + +## Adding an entry + +Capture the archive, verify it against the existing pinned checksum (or, for a new tool, +establish a checksum through a healthy source and record how), attach it to a release +tagged `vendored/-v` and marked as a pre-release, repoint the URL in +`config/defaults/tool_versions.yml` keeping the checksum unchanged, add an entry here +with the same fields, and extend the guard in `test/unit/test_install_pins.jl`. diff --git a/test/unit/test_install_pins.jl b/test/unit/test_install_pins.jl index 9d99e107..3b54afc8 100644 --- a/test/unit/test_install_pins.jl +++ b/test/unit/test_install_pins.jl @@ -93,6 +93,33 @@ end end end + @testset "the FastQC archive is vendored, and stays vendored" begin + # Guarded as a decision rather than described as a fact, because the failure + # mode is silent: a URL that quietly points back at a third party still + # downloads, still verifies against the checksum, and gives no sign that CI is + # once again dependent on someone else's certificate renewal. + # + # MEASURED 2026-09-22: CI went red on a commit that touched nothing near FastQC + # because www.bioinformatics.babraham.ac.uk served an expired TLS certificate. + # FastQC publishes no release assets upstream -- all six of its GitHub releases + # carry none -- so the pin had nowhere else to point. The archive is now served + # from this repository's own releases (see docs/compliance/vendored-archives.md): + # the same bytes, kept against the checksum that was already pinned, so only the + # serving host changed. + # + # Repointing this at a third-party host again is a decision somebody should make + # deliberately, so it fails here by name instead of drifting back in silence. + fastqc_url = pins["tools"]["fastqc"]["archives"]["any"]["url"] + @test startswith(fastqc_url, + "https://github.com/hyperpolymath/MetaManifold-WebUI/releases/download/") + @test !occursin("babraham.ac.uk", fastqc_url) + + # The vendored copy is only defensible while it is provably the upstream + # artefact. This is that checksum, unchanged from the original upstream pin. + @test pins["tools"]["fastqc"]["archives"]["any"]["sha256"] == + "5f4dba8780231a25a6b8e11ab2c238601920c9704caa5458d9de559575d58aa7" + end + @testset "install.jl holds no versions of its own" begin src = read(INSTALL_JL, String)