From 706ca9e95d01a0f82ba02e3d108de73a1c802b62 Mon Sep 17 00:00:00 2001 From: skjnldsv Date: Thu, 8 Oct 2026 15:03:40 +0200 Subject: [PATCH] ci: squash gh-pages history on every release Each deploy commits a full set of rebuilt PDF and ePub files, so gh-pages grows by ~18 MiB a day and is now 3.5 GiB of the repository's 3.7 GB. On each final release tag, rebuild gh-pages as one snapshot commit per release day, keep the deploys made since the last release, and store out-of-support version folders only once. The tip tree is unchanged and the rewrite is deterministic, so re-running it is a no-op. Assisted-by: ClaudeCode Signed-off-by: skjnldsv --- .github/workflows/squash-gh-pages.yml | 136 ++++++++++++++++++++ build/squash-gh-pages.sh | 141 ++++++++++++++++++++ build/tests/test-squash-gh-pages.sh | 178 ++++++++++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 .github/workflows/squash-gh-pages.yml create mode 100755 build/squash-gh-pages.sh create mode 100755 build/tests/test-squash-gh-pages.sh diff --git a/.github/workflows/squash-gh-pages.yml b/.github/workflows/squash-gh-pages.yml new file mode 100644 index 00000000000..f45dba2396c --- /dev/null +++ b/.github/workflows/squash-gh-pages.yml @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# Squash the gh-pages history on every release. +# +# Every deploy commits a full set of rebuilt PDF and ePub files, so gh-pages +# grows by ~18 MiB a day. build/squash-gh-pages.sh rebuilds the branch as one +# snapshot commit per release day, keeps the deploys made since the last +# release, and stores out of support version folders only once. The published +# tree is unchanged; see the script for details. +# +# Required setup (repository admin): +# GH_PAGES_DEPLOY_KEY – private half of a deploy key with write access, added +# as a bypass actor to the "Pages" ruleset, which +# otherwise blocks force pushes to gh-pages. +name: "Squash gh-pages history" + +on: + push: + tags: + - 'v*' + pull_request: + paths: + - '.github/workflows/squash-gh-pages.yml' + - 'build/squash-gh-pages.sh' + - 'build/tests/**' + workflow_dispatch: + inputs: + dry_run: + description: "Compute the rewrite without pushing it" + type: boolean + default: true + +permissions: + contents: read + +concurrency: + group: squash-gh-pages + cancel-in-progress: false + +jobs: + test: + name: Test squash script + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run unit tests + run: bash build/tests/test-squash-gh-pages.sh + + squash: + name: Squash gh-pages + needs: test + # Final releases only: rc, beta and alpha tags do not get a snapshot. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && !contains(github.ref_name, 'rc') && + !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha')) + runs-on: ubuntu-latest + + permissions: + contents: read + + steps: + # Full history: the release dates are read from the tags. + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + + - name: Setup PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2 + with: + php-version: '8.3' + + - name: Detect lowest supported version + id: versions + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mapfile -t nums < <(git ls-remote --heads origin "heads/stable[0-9][0-9]" \ + | sed -n 's?.*refs/heads/stable\([0-9]\{2\}\)$?\1?p' | sort -n) + eval "$(php build/detect-versions.php "${nums[@]}")" + echo "Folders below server/${lowest_stable} are frozen" + echo "lowest_stable=${lowest_stable}" >> "$GITHUB_OUTPUT" + + - name: Fetch gh-pages history + run: git fetch --no-tags origin +refs/heads/gh-pages:refs/heads/gh-pages + + - name: Rewrite gh-pages + id: rewrite + run: | + echo "old=$(git rev-parse gh-pages)" >> "$GITHUB_OUTPUT" + build/squash-gh-pages.sh gh-pages "${{ steps.versions.outputs.lowest_stable }}" \ + refs/heads/gh-pages-squashed | tee -a "$GITHUB_OUTPUT" + + # Deploy PRs are based on the old history: merging one after the rewrite + # would bring every squashed commit back. The next nightly deploy opens a + # fresh one on top of the new history. + - name: Close open deploy pull requests + if: steps.rewrite.outputs.changed == 'true' && inputs.dry_run != true + env: + GH_TOKEN: ${{ secrets.COMMAND_BOT_PAT }} + run: | + gh pr list --repo "${{ github.repository }}" --base gh-pages --state open \ + --json number,headRefName \ + --jq '.[] | select(.headRefName | startswith("automated/deploy/")) | .number' \ + | while read -r number; do + gh pr close "${number}" --repo "${{ github.repository }}" --delete-branch \ + --comment "Closed because gh-pages was squashed. The next scheduled deploy reopens it on the new history." + done + + # The lease makes the push fail if anything landed on gh-pages since the + # fetch, rather than dropping it. Re-run the workflow in that case. + - name: Push rewritten gh-pages + if: steps.rewrite.outputs.changed == 'true' && inputs.dry_run != true + env: + DEPLOY_KEY: ${{ secrets.GH_PAGES_DEPLOY_KEY }} + run: | + key="${RUNNER_TEMP}/gh-pages-deploy-key" + hosts="${RUNNER_TEMP}/github-known-hosts" + install -m 600 /dev/null "${key}" + printf '%s\n' "${DEPLOY_KEY}" > "${key}" + # GitHub's published host keys, fetched over TLS rather than trusted on first use. + curl -fsSL https://api.github.com/meta | jq -r '.ssh_keys[] | "github.com " + .' > "${hosts}" + GIT_SSH_COMMAND="ssh -i ${key} -o IdentitiesOnly=yes -o UserKnownHostsFile=${hosts} -o StrictHostKeyChecking=yes" git push \ + --force-with-lease="refs/heads/gh-pages:${{ steps.rewrite.outputs.old }}" \ + "git@github.com:${{ github.repository }}.git" \ + refs/heads/gh-pages-squashed:refs/heads/gh-pages + rm -f "${key}" diff --git a/build/squash-gh-pages.sh b/build/squash-gh-pages.sh new file mode 100755 index 00000000000..7928b9d7286 --- /dev/null +++ b/build/squash-gh-pages.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# Rewrite the gh-pages history to keep it from growing without bounds. +# +# Every deploy commits a full set of rebuilt PDF and ePub files, so the branch +# grows by tens of MiB per deploy even when the docs barely change. The built +# output has no value as history, so this script rebuilds the branch as: +# +# - one snapshot commit per release day (final vX.Y.Z tags only), holding the +# tree of the last deploy made before the end of that day (UTC), +# - every deploy made after the last release, unchanged, +# - version folders below (out of support, no more releases) +# replaced by their current content in every commit, so they are stored once. +# +# The tip tree is never modified, so the published site is identical. The +# rewrite is deterministic: running it again on its own output yields the same +# commits, which makes it safe to run on every release tag. +# +# Usage: squash-gh-pages.sh +# +# Prints changed=true|false and tip= for $GITHUB_OUTPUT, and a commit +# count summary on stderr. + +set -euo pipefail + +# Group final release tags by UTC day. +# stdin: " " lines +# stdout: " [,...]" lines, oldest day first +release_days() { + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+ ' \ + | awk '{ print int($2 / 86400) * 86400 + 86399, $1 }' \ + | LC_ALL=C sort -k1,1n -k2,2 \ + | awk '$1 == day { tags = tags "," $2; next } + day { print day, tags } + { day = $1; tags = $2 } + END { if (day) print day, tags }' +} + +# Pick the commits to keep. +# $1: file of " " lines, oldest first +# $2: file of release_days output +# stdout: " " for a snapshot, " -" for an unreleased commit +plan() { + awk 'NR == FNR { sha[NR] = $1; ts[NR] = $2; n = NR; next } + { + last = 0 + for (i = 1; i <= n; i++) if (ts[i] <= $1) last = i + if (!last) next + if (last in tags) tags[last] = tags[last] "," $2 + else { order[++m] = last; tags[last] = $2 } + } + END { + for (k = 1; k <= m; k++) print sha[order[k]], tags[order[k]] + for (i = (m ? order[m] + 1 : 1); i <= n; i++) print sha[i], "-" + }' "$1" "$2" +} + +# ls-tree lines of server/ folders below $2 in tree-ish $1 (or the others with $3 = keep) +version_entries() { + git ls-tree "$1" 2>/dev/null | awk -F '\t' -v below="$2" -v keep="${3:-}" ' + { frozen = $2 ~ /^[0-9]+$/ && $2 + 0 < below } + frozen != (keep == "keep")' +} + +# Tree of commit $1 with every frozen server/ folder replaced by its tip content. +frozen_tree() { + local server + server=$( { version_entries "$1:server" "$FROZEN_BELOW" keep; printf '%s' "$FROZEN_ENTRIES"; } \ + | awk NF | git mktree) + { + git ls-tree "$1^{tree}" | awk -F '\t' '$2 != "server"' + if [ "$server" != "$(git mktree " + author=$(sed -n 's/^author //p' <<<"$header") + committer=$(sed -n 's/^committer //p' <<<"$header") + author_email=${author##*<} + committer_email=${committer##*<} + GIT_AUTHOR_NAME=${author% <*} GIT_AUTHOR_EMAIL=${author_email%%>*} GIT_AUTHOR_DATE=${author##*> } \ + GIT_COMMITTER_NAME=${committer% <*} GIT_COMMITTER_EMAIL=${committer_email%%>*} GIT_COMMITTER_DATE=${committer##*> } \ + git commit-tree "$2" ${3:+-p "$3"} +} + +# Rebuild branch $1 with folders below $2 frozen, print the new tip (the branch is not moved). +rewrite() { + local branch=$1 tmp parent='' sha tags tree + FROZEN_BELOW=$2 + FROZEN_ENTRIES=$(version_entries "$branch:server" "$FROZEN_BELOW") + tmp=$(mktemp -d) + + git log --first-parent --reverse --format='%H %ct' "$branch" >"$tmp/commits" + git for-each-ref --format='%(refname:short) %(creatordate:unix)' refs/tags \ + | { release_days || true; } >"$tmp/days" + + while read -r sha tags; do + tree=$(frozen_tree "$sha") + if [ "$tags" = - ]; then + parent=$(git cat-file commit "$sha" | sed '1,/^$/d' | commit_like "$sha" "$tree" "$parent") + else + parent=$(printf 'chore: documentation snapshot for %s\n\nSquashed gh-pages history up to this release.\n' \ + "${tags//,/, }" | commit_like "$sha" "$tree" "$parent") + fi + done < <(plan "$tmp/commits" "$tmp/days") + + rm -rf "$tmp" + echo "$parent" +} + +main() { + if [ $# -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 + fi + local branch=$1 tip + tip=$(rewrite "$branch" "$2") + + if [ "$(git rev-parse "$tip^{tree}")" != "$(git rev-parse "$branch^{tree}")" ]; then + echo "Rewritten tip tree differs from the current branch, aborting" >&2 + exit 1 + fi + git update-ref "$3" "$tip" + + echo "$branch: $(git rev-list --count --first-parent "$branch") commits -> $(git rev-list --count "$tip") commits" >&2 + if [ "$tip" = "$(git rev-parse "$branch")" ]; then echo changed=false; else echo changed=true; fi + echo "tip=$tip" +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + main "$@" +fi diff --git a/build/tests/test-squash-gh-pages.sh b/build/tests/test-squash-gh-pages.sh new file mode 100755 index 00000000000..9c0d8dc5374 --- /dev/null +++ b/build/tests/test-squash-gh-pages.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# Tests for build/squash-gh-pages.sh. Run: bash build/tests/test-squash-gh-pages.sh + +set -euo pipefail + +# shellcheck source-path=SCRIPTDIR source=../squash-gh-pages.sh +source "$(dirname "${BASH_SOURCE[0]}")/../squash-gh-pages.sh" + +failures=0 + +# Unix timestamp of at UTC (default noon) +ts() { + date -u -d "$1 ${2:-12}:00:00" +%s +} + +assert_eq() { + if [ "$2" = "$3" ]; then + echo "ok - $1" + else + echo "not ok - $1" + printf ' expected: %s\n actual: %s\n' "$2" "$3" + failures=$((failures + 1)) + fi +} + +test_release_days_keeps_final_releases_grouped_by_utc_day() { + local actual + actual=$(printf '%s\n' \ + "v34.0.4 $(ts 2026-09-10 18)" \ + "v33.0.9 $(ts 2026-09-10 9)" \ + "v35.0.0rc1 $(ts 2026-09-11)" \ + "v35.0.0beta2 $(ts 2026-09-12)" \ + "v35.0.0 $(ts 2026-09-15)" \ + "not-a-release $(ts 2026-09-16)" | release_days) + assert_eq "${FUNCNAME[0]}" \ + "$(($(ts 2026-09-11 0) - 1)) v33.0.9,v34.0.4 +$(($(ts 2026-09-16 0) - 1)) v35.0.0" "$actual" +} + +# Plan for commits a..d (2026-04-02, 03, 05, 07) and the given " " lines. +plan_for() { + local tmp + tmp=$(mktemp -d) + printf '%s\n' "a $(ts 2026-04-02)" "b $(ts 2026-04-03)" "c $(ts 2026-04-05)" "d $(ts 2026-04-07)" >"$tmp/commits" + { printf '%s\n' "$@" | release_days || true; } >"$tmp/days" + plan "$tmp/commits" "$tmp/days" | tr '\n' ' ' + rm -rf "$tmp" +} + +test_plan_one_snapshot_per_release_then_unreleased() { + assert_eq "${FUNCNAME[0]}" "b v1.0.0 c v1.0.1 d - " \ + "$(plan_for "v1.0.0 $(ts 2026-04-03 20)" "v1.0.1 $(ts 2026-04-05 20)")" +} + +test_plan_releases_without_deploy_in_between_share_a_snapshot() { + assert_eq "${FUNCNAME[0]}" "b v1.0.0,v2.0.0 c - d - " \ + "$(plan_for "v1.0.0 $(ts 2026-04-03)" "v2.0.0 $(ts 2026-04-04)")" +} + +test_plan_release_before_first_deploy_is_ignored() { + assert_eq "${FUNCNAME[0]}" "a v1.0.0 b - c - d - " \ + "$(plan_for "v0.9.0 $(ts 2026-03-01)" "v1.0.0 $(ts 2026-04-02)")" +} + +test_plan_no_release_keeps_everything() { + assert_eq "${FUNCNAME[0]}" "a - b - c - d - " "$(plan_for)" +} + +# End to end on a throwaway repository shaped like gh-pages + +repo_write() { + mkdir -p "$(dirname "$1")" + printf '%s' "$2" >"$1" +} + +repo_env() { + local date + date="$(ts "$1" "${2:-12}") +0000" + export GIT_AUTHOR_NAME=bot GIT_AUTHOR_EMAIL=bot@example.com GIT_AUTHOR_DATE="$date" + export GIT_COMMITTER_NAME=bot GIT_COMMITTER_EMAIL=bot@example.com GIT_COMMITTER_DATE="$date" +} + +repo_commit() { + (repo_env "$1" && git add -A && git commit -q --allow-empty -m "$2") +} + +# Lightweight tag on its own commit, like the release tags pointing at master +repo_tag() { + (repo_env "$2" "${3:-12}" && git tag "$1" "$(echo release | git commit-tree "$(git mktree