diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cdf16725ac..7cbfef381d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,6 +93,15 @@ jobs: - name: Check the branch out uses: actions/checkout@v4 + with: + # Full history, because `make ci` runs build/generate_page_moves.py, which + # reads git rename records to date each entry in the published redirect map. + # checkout defaults to fetch-depth: 1, and in a shallow clone that scan finds + # nothing and reports zero moves without failing -- so the map would ship with + # no dates at all while a local full-clone build shows hundreds. Only this job + # needs it; the versioned matrix builds run bare `hugo` and never generate the + # map. See DOC-6951. + fetch-depth: 0 - name: Install dependencies run: make deps diff --git a/.gitignore b/.gitignore index 0c7268678d..d67cd86400 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /resources/ /content/tmp/ /data/examples.json +/data/page-moves.json /data/languages.json /data/repos.json /data/tool_types.json diff --git a/Makefile b/Makefile index 1d0c8f538c..261ee90def 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,14 @@ components: components_local: @python3 build/make.py --stack ./data/components_local/index.json -hugo: +# Move dates live only in git history, which Hugo cannot read, so they are written +# into data/ before the build. Generated rather than committed, like the other +# derived files in data/, so a snapshot of history cannot go stale. +page_moves: + @echo "Recording page move dates..." + @python3 build/generate_page_moves.py + +hugo: page_moves @hugo $(HUGO_DEBUG) $(HUGO_BUILD) # json_transform requires hugo to have populated public/ with index.json files @@ -45,7 +52,7 @@ ndjson: redirect_tombstones @echo "Compressing NDJSON feed..." @gzip -kf public/docs.ndjson -serve_hugo: +serve_hugo: page_moves @hugo serve # Passive post-build report of unusually large rendered pages (warn-only). diff --git a/build/generate_page_moves.py b/build/generate_page_moves.py new file mode 100644 index 0000000000..d9694a9c2f --- /dev/null +++ b/build/generate_page_moves.py @@ -0,0 +1,97 @@ +"""Record when each page move happened, for the published redirect map. + +The map in ``layouts/index.redirects.json`` is rendered from ``.Aliases``, which is +everything Hugo knows. It cannot say *when* a page moved, because frontmatter does +not record that -- only git does. This writes those dates into ``data/page-moves.json`` +before the build, so the template can attach a ``moved_on`` to each redirect it +publishes. + +Why a consumer wants it: a date separates "this redirect is years old, I have surely +seen it" from "this appeared last week, my index is stale". Without one, every entry +in a thousand-line map looks equally new. + +Generated at build time and gitignored, matching ``data/examples.json`` and the other +derived data files in this repo. That keeps it from going stale, which a committed +snapshot of git history would do immediately. + +Deliberately does **not** record deleted pages, though the redirect map would be a +natural home for them. git cannot reliably distinguish a deletion from a move it +failed to detect: of 195 apparent deletions in this repo's history, 83 have a +same-named page somewhere else today, so they almost certainly moved by a +delete-plus-add that fell below git's rename similarity threshold. Publishing those as +deleted would tell a consumer to discard a citation that still resolves, which is +worse than saying nothing at all. The remaining 112 are not safe either, since a page +can be renamed *and* relocated in one go, which no name-matching heuristic can see. +See DOC-6951. +""" + +import json +import logging +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_missing_aliases import ( # noqa: E402 + DEFAULT_THRESHOLD, find_moves, git, norm, +) + +logger = logging.getLogger("generate_page_moves") + +OUTPUT = os.path.join("data", "page-moves.json") + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + + # A shallow clone has no rename records, so the scan below finds nothing and + # reports zero moves perfectly happily -- which is how this shipped with no dates + # at all while local full-clone builds showed hundreds. Say so loudly rather than + # writing an empty file that looks like a corpus with no history. + shallow = git("rev-parse", "--is-shallow-repository").strip() == "true" + if shallow: + logger.warning("::warning::generate_page_moves: this is a shallow clone, so " + "no move dates can be read. The redirect map will publish " + "none. Check out with fetch-depth: 0.") + + # Deliberately not classified. classify() decides whether each move is already + # aliased, shadowed, contested or a draft target, which costs a scan of every + # published URL and of the frontmatter of every file declaring an alias -- and + # dating needs only old_url and date. This runs on every build, so the work has to + # be work that is used. + moves = find_moves(None, DEFAULT_THRESHOLD) + + # One record per redirect, keyed the way the map keys its entries so the template + # can look a date up directly. Where a page moved more than once the earliest + # date wins, because that is when the old URL stopped resolving -- which is what + # a consumer holding a stale citation actually cares about. + dates: dict[str, str] = {} + for move in sorted(moves, key=lambda m: m.date): + key = "/" + norm(move.old_url) + dates.setdefault(key, move.date) + + try: + head = git("rev-parse", "--short", "HEAD").strip() + except Exception: # noqa: BLE001 - a missing commit must not fail the build + head = "" + + payload = { + "generated_from": head, + # Recorded so a consumer of this file, or anyone reading a build log, can tell + # "this history has no moves" from "this clone could not see the history". + "shallow_clone": shallow, + "count": len(dates), + "moved_on": dates, + } + + os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) + with open(OUTPUT, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=1, sort_keys=True) + handle.write("\n") + + logger.info("generate_page_moves: wrote %d move date(s) to %s.", len(dates), OUTPUT) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/content/ai-agent-resources.md b/content/ai-agent-resources.md index 96541d6558..39579ef7b7 100644 --- a/content/ai-agent-resources.md +++ b/content/ai-agent-resources.md @@ -125,7 +125,7 @@ the page it resolves to: "ambiguous_count": 27, "shadowed_count": 10, "redirects": [ - {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/"} + {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/", "moved_on": "2026-05-11"} ], "ambiguous": [ {"from": "/develop/use/pipelining", "candidates": ["https://redis.io/docs/latest/develop/using-commands/", "https://redis.io/docs/latest/develop/using-commands/pipelining/"]} @@ -143,8 +143,14 @@ Four things worth knowing before you rely on it: record of its own at `/index.json`, so you can resolve one URL without fetching the whole map. - **It is an alias map, not a move log.** Many entries are vanity or legacy paths that - were never a page's location, and there is no date, because the source data does not - record when a page moved. + were never a page's location. Where we can date a move from the repository history, + the entry carries a `moved_on`; entries without one are not undated moves, they are + aliases that were never a page's location in the first place. +- **A URL missing from the map is not necessarily gone.** We publish redirects we can + establish, not a complete account of every URL that ever existed. In particular we + deliberately do not publish a list of deleted pages: the repository history cannot + reliably tell a deletion from a move it failed to detect, and telling you a page was + deleted when it merely moved would be worse than telling you nothing. - `ambiguous` holds the keys that more than one page claims, with every candidate listed. We publish them separately rather than picking one, because the site itself resolves those arbitrarily — so any single answer we gave you would sometimes diff --git a/layouts/index.redirects.json b/layouts/index.redirects.json index c4da604c74..fe488362c9 100644 --- a/layouts/index.redirects.json +++ b/layouts/index.redirects.json @@ -26,8 +26,8 @@ is an even split -- so normalizing here saves every consumer doing it. `to` is absolute, matching the `url` field every per-page record already publishes. - This is an *alias* map, not a move log. Many entries are vanity or legacy paths - that were never a page's location, and it carries no date, because frontmatter - does not record when a page moved. + that were never a page's location, so only the ones git can date carry a + `moved_on`. Only pages Hugo publishes appear, so drafts contribute nothing -- which matches the site, since Hugo emits no stub for a draft's aliases either. @@ -74,6 +74,13 @@ {{- $seen.SetInMap "map" $from ($targets | append $to | uniq) -}} {{- end -}} {{- end -}} +{{- /* Move dates, written into data/ before the build by + build/generate_page_moves.py, because git records when a page moved and + frontmatter does not. Absent when the site is built with bare `hugo` rather + than `make`, so every lookup tolerates a missing file: a date is extra + information, never a precondition. Only moves have one -- a vanity or legacy + alias was never a page's location, so there is no date to give. */ -}} +{{- $movedOn := index (index site.Data "page-moves" | default dict) "moved_on" | default dict -}} {{- $redirects := slice -}} {{- $ambiguous := slice -}} {{- $shadowed := slice -}} @@ -81,7 +88,11 @@ {{- if index ($occupied.Get "url" | default dict) $from -}} {{- $shadowed = $shadowed | append (dict "from" $from "declared" (sort $targets)) -}} {{- else if eq (len $targets) 1 -}} - {{- $redirects = $redirects | append (dict "from" $from "to" (index $targets 0)) -}} + {{- $entry := dict "from" $from "to" (index $targets 0) -}} + {{- with index $movedOn $from -}} + {{- $entry = merge $entry (dict "moved_on" .) -}} + {{- end -}} + {{- $redirects = $redirects | append $entry -}} {{- else -}} {{- $ambiguous = $ambiguous | append (dict "from" $from "candidates" (sort $targets)) -}} {{- end -}}