DOC-6951 Date each redirect from git history, and say why deletions are not published

.github/workflows/main.yml

@@ -93,6 +93,15 @@ jobs:
 
     - name: Check the branch out
       uses: actions/checkout@v4
+      with:
+        # Full history, because `make ci` runs build/generate_page_moves.py, which
+        # reads git rename records to date each entry in the published redirect map.
+        # checkout defaults to fetch-depth: 1, and in a shallow clone that scan finds
+        # nothing and reports zero moves without failing -- so the map would ship with
+        # no dates at all while a local full-clone build shows hundreds. Only this job
+        # needs it; the versioned matrix builds run bare `hugo` and never generate the
+        # map. See DOC-6951.
+        fetch-depth: 0
 
     - name: Install dependencies
       run: make deps

.gitignore

@@ -2,6 +2,7 @@
 /resources/
 /content/tmp/
 /data/examples.json
+/data/page-moves.json
 /data/languages.json
 /data/repos.json
 /data/tool_types.json

Makefile

@@ -22,7 +22,14 @@ components:
 components_local:
 	@python3 build/make.py --stack ./data/components_local/index.json
 
-hugo:
+# Move dates live only in git history, which Hugo cannot read, so they are written
+# into data/ before the build. Generated rather than committed, like the other
+# derived files in data/, so a snapshot of history cannot go stale.
+page_moves:
+	@echo "Recording page move dates..."
+	@python3 build/generate_page_moves.py
+
+hugo: page_moves
 	@hugo $(HUGO_DEBUG) $(HUGO_BUILD)
 
 # json_transform requires hugo to have populated public/ with index.json files
@@ -45,7 +52,7 @@ ndjson: redirect_tombstones
 	@echo "Compressing NDJSON feed..."
 	@gzip -kf public/docs.ndjson
 
-serve_hugo:
+serve_hugo: page_moves
 	@hugo serve
 
 # Passive post-build report of unusually large rendered pages (warn-only).

build/generate_page_moves.py

@@ -0,0 +1,97 @@
+"""Record when each page move happened, for the published redirect map.
+
+The map in ``layouts/index.redirects.json`` is rendered from ``.Aliases``, which is
+everything Hugo knows. It cannot say *when* a page moved, because frontmatter does
+not record that -- only git does. This writes those dates into ``data/page-moves.json``
+before the build, so the template can attach a ``moved_on`` to each redirect it
+publishes.
+
+Why a consumer wants it: a date separates "this redirect is years old, I have surely
+seen it" from "this appeared last week, my index is stale". Without one, every entry
+in a thousand-line map looks equally new.
+
+Generated at build time and gitignored, matching ``data/examples.json`` and the other
+derived data files in this repo. That keeps it from going stale, which a committed
+snapshot of git history would do immediately.
+
+Deliberately does **not** record deleted pages, though the redirect map would be a
+natural home for them. git cannot reliably distinguish a deletion from a move it
+failed to detect: of 195 apparent deletions in this repo's history, 83 have a
+same-named page somewhere else today, so they almost certainly moved by a
+delete-plus-add that fell below git's rename similarity threshold. Publishing those as
+deleted would tell a consumer to discard a citation that still resolves, which is
+worse than saying nothing at all. The remaining 112 are not safe either, since a page
+can be renamed *and* relocated in one go, which no name-matching heuristic can see.
+See DOC-6951.
+"""
+
+import json
+import logging
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from check_missing_aliases import (  # noqa: E402
+    DEFAULT_THRESHOLD, find_moves, git, norm,
+)
+
+logger = logging.getLogger("generate_page_moves")
+
+OUTPUT = os.path.join("data", "page-moves.json")
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+    # A shallow clone has no rename records, so the scan below finds nothing and
+    # reports zero moves perfectly happily -- which is how this shipped with no dates
+    # at all while local full-clone builds showed hundreds. Say so loudly rather than
+    # writing an empty file that looks like a corpus with no history.
+    shallow = git("rev-parse", "--is-shallow-repository").strip() == "true"
+    if shallow:
+        logger.warning("::warning::generate_page_moves: this is a shallow clone, so "
+                       "no move dates can be read. The redirect map will publish "
+                       "none. Check out with fetch-depth: 0.")
+
+    # Deliberately not classified. classify() decides whether each move is already
+    # aliased, shadowed, contested or a draft target, which costs a scan of every
+    # published URL and of the frontmatter of every file declaring an alias -- and
+    # dating needs only old_url and date. This runs on every build, so the work has to
+    # be work that is used.
+    moves = find_moves(None, DEFAULT_THRESHOLD)
+
+    # One record per redirect, keyed the way the map keys its entries so the template
+    # can look a date up directly. Where a page moved more than once the earliest
+    # date wins, because that is when the old URL stopped resolving -- which is what
+    # a consumer holding a stale citation actually cares about.
+    dates: dict[str, str] = {}
+    for move in sorted(moves, key=lambda m: m.date):
+        key = "/" + norm(move.old_url)
+        dates.setdefault(key, move.date)
+
+    try:
+        head = git("rev-parse", "--short", "HEAD").strip()
+    except Exception:  # noqa: BLE001 - a missing commit must not fail the build
+        head = ""
+
+    payload = {
+        "generated_from": head,
+        # Recorded so a consumer of this file, or anyone reading a build log, can tell
+        # "this history has no moves" from "this clone could not see the history".
+        "shallow_clone": shallow,
+        "count": len(dates),
+        "moved_on": dates,
+    }
+
+    os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
+    with open(OUTPUT, "w", encoding="utf-8") as handle:
+        json.dump(payload, handle, indent=1, sort_keys=True)
+        handle.write("\n")
+
+    logger.info("generate_page_moves: wrote %d move date(s) to %s.", len(dates), OUTPUT)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

content/ai-agent-resources.md

@@ -125,7 +125,7 @@ the page it resolves to:
   "ambiguous_count": 27,
   "shadowed_count": 10,
   "redirects": [
-    {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/"}
+    {"from": "/develop/ai/langcache", "to": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/", "moved_on": "2026-05-11"}
   ],
   "ambiguous": [
     {"from": "/develop/use/pipelining", "candidates": ["https://redis.io/docs/latest/develop/using-commands/", "https://redis.io/docs/latest/develop/using-commands/pipelining/"]}
@@ -143,8 +143,14 @@ Four things worth knowing before you rely on it:
   record of its own at `<from>/index.json`, so you can resolve one URL without
   fetching the whole map.
 - **It is an alias map, not a move log.** Many entries are vanity or legacy paths that
-  were never a page's location, and there is no date, because the source data does not
-  record when a page moved.
+  were never a page's location. Where we can date a move from the repository history,
+  the entry carries a `moved_on`; entries without one are not undated moves, they are
+  aliases that were never a page's location in the first place.
+- **A URL missing from the map is not necessarily gone.** We publish redirects we can
+  establish, not a complete account of every URL that ever existed. In particular we
+  deliberately do not publish a list of deleted pages: the repository history cannot
+  reliably tell a deletion from a move it failed to detect, and telling you a page was
+  deleted when it merely moved would be worse than telling you nothing.
 - `ambiguous` holds the keys that more than one page claims, with every candidate
   listed. We publish them separately rather than picking one, because the site itself
   resolves those arbitrarily — so any single answer we gave you would sometimes

layouts/index.redirects.json

@@ -26,8 +26,8 @@
     is an even split -- so normalizing here saves every consumer doing it. `to` is
     absolute, matching the `url` field every per-page record already publishes.
   - This is an *alias* map, not a move log. Many entries are vanity or legacy paths
-    that were never a page's location, and it carries no date, because frontmatter
-    does not record when a page moved.
+    that were never a page's location, so only the ones git can date carry a
+    `moved_on`.
 
   Only pages Hugo publishes appear, so drafts contribute nothing -- which matches
   the site, since Hugo emits no stub for a draft's aliases either.
@@ -74,14 +74,25 @@
     {{- $seen.SetInMap "map" $from ($targets | append $to | uniq) -}}
   {{- end -}}
 {{- end -}}
+{{- /* Move dates, written into data/ before the build by
+       build/generate_page_moves.py, because git records when a page moved and
+       frontmatter does not. Absent when the site is built with bare `hugo` rather
+       than `make`, so every lookup tolerates a missing file: a date is extra
+       information, never a precondition. Only moves have one -- a vanity or legacy
+       alias was never a page's location, so there is no date to give. */ -}}
+{{- $movedOn := index (index site.Data "page-moves" | default dict) "moved_on" | default dict -}}
 {{- $redirects := slice -}}
 {{- $ambiguous := slice -}}
 {{- $shadowed := slice -}}
 {{- range $from, $targets := ($seen.Get "map" | default dict) -}}
   {{- if index ($occupied.Get "url" | default dict) $from -}}
     {{- $shadowed = $shadowed | append (dict "from" $from "declared" (sort $targets)) -}}
   {{- else if eq (len $targets) 1 -}}
-    {{- $redirects = $redirects | append (dict "from" $from "to" (index $targets 0)) -}}
+    {{- $entry := dict "from" $from "to" (index $targets 0) -}}
+    {{- with index $movedOn $from -}}
+      {{- $entry = merge $entry (dict "moved_on" .) -}}
+    {{- end -}}
+    {{- $redirects = $redirects | append $entry -}}
   {{- else -}}
     {{- $ambiguous = $ambiguous | append (dict "from" $from "candidates" (sort $targets)) -}}
   {{- end -}}