diff --git a/build/transform_json_sections.ts b/build/transform_json_sections.ts index e4f736a872..38721b71aa 100644 --- a/build/transform_json_sections.ts +++ b/build/transform_json_sections.ts @@ -84,6 +84,33 @@ function assignRole(title: string, index: number): string { return 'content'; } +/** + * Hugo's explicit heading anchor syntax, e.g. "## Change the socket path {#after-setup}". + * + * Deliberately strict. Plenty of headings end in a brace that is NOT an anchor -- + * Python signatures ending "connection_kwargs={}", dict defaults such as + * "{'extra': 'ignore'}", and relref shortcodes -- and a loose trailing-brace match + * would silently eat them. Requires "{#", then anchor-shaped characters only, then + * "}" at end of the heading. + */ +const EXPLICIT_ANCHOR = /\s*\{#([A-Za-z0-9][A-Za-z0-9_.:-]*)\}\s*$/; + +/** + * Split a heading into its display title and its id. + * + * Hugo takes the explicit anchor as the rendered heading's id verbatim and does not + * show it, so the title must have it removed and the id must not be slugified -- + * slugifying would produce a different string from the anchor on the page and break + * url#section-id deep links. + */ +function parseHeading(raw: string): { title: string; id: string } { + const match = raw.match(EXPLICIT_ANCHOR); + if (match) { + return { title: raw.replace(EXPLICIT_ANCHOR, '').trim(), id: match[1] }; + } + return { title: raw, id: slugify(raw) }; +} + function slugify(text: string): string { return text .toLowerCase() @@ -96,6 +123,30 @@ const FILTERED_SECTION_IDS = new Set([ 'code-examples-legend', ]); +/** + * Make a section id unique within its page. + * + * The id is the join key for examples[].section_id and the target of + * url#section-id deep links, so collisions are not cosmetic: examples from two + * sections merge under one id, examples[].id itself stops being unique, and a + * consumer keying sections by id silently drops one of them. Headings that + * differ only in punctuation collide readily, because slugify() discards it -- + * "The special `$` ID" and "The special `+` ID" both reduce to the-special-id. + * + * Disambiguated with Hugo's own -1, -2 suffix convention. + */ +function makeUniqueId(base: string, used: Set): string { + if (!used.has(base)) { + used.add(base); + return base; + } + let n = 1; + while (used.has(`${base}-${n}`)) n++; + const id = `${base}-${n}`; + used.add(id); + return id; +} + /** * Extract fenced code blocks from text. * Returns the code blocks and the text with code blocks removed. @@ -115,15 +166,20 @@ function extractCodeBlocks(text: string, sectionId: string): { const language = lang || 'plaintext'; const trimmedCode = code.trim(); - if (trimmedCode) { - examples.push({ - id: `${sectionId}-ex${exampleIndex++}`, - language, - code: trimmedCode, - section_id: sectionId, - }); + // An empty code block yields no example, so a placeholder here would be + // unresolvable: the consumer sees [code example] with nothing in examples[] + // to substitute back in. Drop it instead. + if (!trimmedCode) { + return ''; } + examples.push({ + id: `${sectionId}-ex${exampleIndex++}`, + language, + code: trimmedCode, + section_id: sectionId, + }); + // Replace code block with placeholder to preserve structure return '[code example]'; }); @@ -134,6 +190,8 @@ function extractCodeBlocks(text: string, sectionId: string): { function splitContentIntoSections(content: string): { sections: Section[]; examples: CodeExample[] } { const rawSections: Section[] = []; const allExamples: CodeExample[] = []; + // Tracks ids already handed out on this page, so they stay unique + const usedIds = new Set(); // First, find all code block ranges to avoid matching headings inside them const codeBlockRanges: { start: number; end: number }[] = []; @@ -174,10 +232,11 @@ function splitContentIntoSections(content: string): { sections: Section[]; examp if (matches.length === 0) { const text = content.trim(); if (text) { - const { examples, textWithoutCode } = extractCodeBlocks(text, 'content'); + const contentId = makeUniqueId('content', usedIds); + const { examples, textWithoutCode } = extractCodeBlocks(text, contentId); allExamples.push(...examples); rawSections.push({ - id: 'content', + id: contentId, title: 'Content', role: 'content', text: textWithoutCode, @@ -186,13 +245,24 @@ function splitContentIntoSections(content: string): { sections: Section[]; examp return { sections: rawSections, examples: allExamples }; } + // Reserve the ids of the real headings BEFORE naming the synthetic intro section. + // A real "## Overview" heading has to keep the plain `overview` id, because that is + // the anchor Hugo renders for it and the id the metadata tableOfContents uses. If + // the intro took `overview` first, the heading would be pushed to `overview-1` and a + // #overview deep link would land on the intro instead of the heading. + const headings = matches.map(m => { + const parsed = parseHeading(m.title); + return { ...parsed, id: makeUniqueId(parsed.id, usedIds) }; + }); + // Extract text before first heading as intro/overview const introText = content.slice(0, matches[0].index).trim(); if (introText) { - const { examples, textWithoutCode } = extractCodeBlocks(introText, 'overview'); + const overviewId = makeUniqueId('overview', usedIds); + const { examples, textWithoutCode } = extractCodeBlocks(introText, overviewId); allExamples.push(...examples); rawSections.push({ - id: 'overview', + id: overviewId, title: 'Overview', role: 'overview', text: textWithoutCode, @@ -210,8 +280,8 @@ function splitContentIntoSections(content: string): { sections: Section[]; examp const headingEnd = newlinePos === -1 ? content.length : newlinePos + 1; const sectionText = content.slice(headingEnd, nextIndex).trim(); - const id = slugify(current.title); - const role = assignRole(current.title, rawSections.length); + const { title, id } = headings[i]; + const role = assignRole(title, rawSections.length); // Extract code blocks from section text const { examples, textWithoutCode } = extractCodeBlocks(sectionText, id); @@ -219,7 +289,7 @@ function splitContentIntoSections(content: string): { sections: Section[]; examp rawSections.push({ id, - title: current.title, + title, role, text: textWithoutCode, }); @@ -259,10 +329,20 @@ function computeContentHash( return createHash('sha256').update(content, 'utf-8').digest('hex'); } -function transformJsonFile(filePath: string, dryRun: boolean): boolean { +type TransformResult = 'transformed' | 'skipped' | 'error'; + +function transformJsonFile(filePath: string, dryRun: boolean): TransformResult { try { const fileContent = readFileSync(filePath, 'utf-8'); - const data: PageJsonInput = JSON.parse(fileContent); + const data: PageJsonInput & { page_type?: PageType } = JSON.parse(fileContent); + + // This script consumes `content` and does not write it back, so it is not + // safe to run twice over the same output: a second pass finds no content and + // would rewrite every content page as an empty index page. A file that has a + // page_type but no content has already been transformed, so leave it alone. + if (data.content === undefined && data.page_type !== undefined) { + return 'skipped'; + } // Remove content field from output const { content: rawContent, ...rest } = data; @@ -313,10 +393,10 @@ function transformJsonFile(filePath: string, dryRun: boolean): boolean { writeFileSync(filePath, JSON.stringify(newData, null, 2) + '\n'); } - return true; + return 'transformed'; } catch (err) { console.error(`Error processing ${filePath}:`, err); - return false; + return 'error'; } } @@ -345,14 +425,15 @@ function main() { let processed = 0; let transformed = 0; + let skipped = 0; if (singlePath) { // Process single file const fullPath = singlePath.startsWith('/') ? singlePath : join(process.cwd(), singlePath); console.log(`Processing single file: ${fullPath}`); - if (transformJsonFile(fullPath, dryRun)) { - transformed++; - } + const result = transformJsonFile(fullPath, dryRun); + if (result === 'transformed') transformed++; + if (result === 'skipped') skipped++; processed++; } else { // Process all JSON files @@ -360,9 +441,9 @@ function main() { for (const filePath of walkJsonFiles(publicDir)) { processed++; - if (transformJsonFile(filePath, dryRun)) { - transformed++; - } + const result = transformJsonFile(filePath, dryRun); + if (result === 'transformed') transformed++; + if (result === 'skipped') skipped++; // Progress indicator if (processed % 500 === 0) { @@ -372,6 +453,9 @@ function main() { } console.log(`\nDone! Processed ${processed} files, transformed ${transformed}.`); + if (skipped > 0) { + console.log(`Skipped ${skipped} already-transformed files. Re-run 'hugo' first if you meant to rebuild them.`); + } if (dryRun) { console.log('(Dry run - no files were modified)'); } diff --git a/content/ai-agent-resources.md b/content/ai-agent-resources.md index 8eb6fc43b9..74765c041f 100644 --- a/content/ai-agent-resources.md +++ b/content/ai-agent-resources.md @@ -28,7 +28,7 @@ A single file containing all documentation pages in [NDJSON](https://github.com/ | NDJSON | [docs.ndjson](https://redis.io/docs/latest/docs.ndjson) | ~30 MB | | Gzipped | [docs.ndjson.gz](https://redis.io/docs/latest/docs.ndjson.gz) | ~5 MB | -Both files contain ~4,100 documents. +Both files contain one record per documentation page, currently more than 2,600. ### Per-page JSON diff --git a/content/integrate/redis-data-integration/installation/upgrade.md b/content/integrate/redis-data-integration/installation/upgrade.md index e4052e327b..da873958c0 100644 --- a/content/integrate/redis-data-integration/installation/upgrade.md +++ b/content/integrate/redis-data-integration/installation/upgrade.md @@ -103,6 +103,7 @@ After upgrading, manually set a unique cluster ID for one of the installations ( ```bash sudo nano /etc/rdi/rdi-sys-config.yaml + ``` {{< /warning >}} ## Upgrading a Kubernetes installation diff --git a/layouts/_default/section.md b/layouts/_default/section.md index fa9608dc5d..800ebf5e4b 100644 --- a/layouts/_default/section.md +++ b/layouts/_default/section.md @@ -25,4 +25,9 @@ {{- /* Process content with shared partial (shortcode expansion, HTML unescaping, etc.) */ -}} {{- $content := partial "process-markdown-content.html" (dict "RawContent" .RawContent "Site" .Site "Page" .) -}} -{{ $content }} +{{- /* The leading newlines are emitted explicitly rather than left as literal blank + lines in this template, because the trim markers above would swallow them: the + content would then start on the same line as the metadata block's closing fence. + A closing fence may be followed only by whitespace, so it would stop closing + anything and the rest of the page would be read as part of the code block. */ -}} +{{ printf "\n\n%s" $content }} diff --git a/layouts/_default/single.md b/layouts/_default/single.md index fa9608dc5d..800ebf5e4b 100644 --- a/layouts/_default/single.md +++ b/layouts/_default/single.md @@ -25,4 +25,9 @@ {{- /* Process content with shared partial (shortcode expansion, HTML unescaping, etc.) */ -}} {{- $content := partial "process-markdown-content.html" (dict "RawContent" .RawContent "Site" .Site "Page" .) -}} -{{ $content }} +{{- /* The leading newlines are emitted explicitly rather than left as literal blank + lines in this template, because the trim markers above would swallow them: the + content would then start on the same line as the metadata block's closing fence. + A closing fence may be followed only by whitespace, so it would stop closing + anything and the rest of the page would be read as part of the code block. */ -}} +{{ printf "\n\n%s" $content }} diff --git a/layouts/partials/markdown-table-children.html b/layouts/partials/markdown-table-children.html new file mode 100644 index 0000000000..f2078d5afb --- /dev/null +++ b/layouts/partials/markdown-table-children.html @@ -0,0 +1,134 @@ +{{- /* + Expand table-children shortcodes for Markdown/JSON output. + + The HTML shortcode (layouts/shortcodes/table-children.html) renders an HTML + listing a page's children, taking each column from a child page + param. AI-facing output needs the same data as a Markdown table. + + Without this partial the shortcode is stripped by process-markdown-content.html, + and the release-notes index pages -- whose entire body is the table -- lose all + their content. The version columns (min-version-db, min-version-rs) appear + nowhere else in the JSON feed, so they were being dropped outright. + + Input: dict with: + - "RawContent": The Markdown content to process (already HTML-unescaped + by the caller, so shortcodes appear as {{< table-children ... >}}) + - "Page": The current Hugo page (for .Pages and .GetPage) + + Output: Markdown content with table-children shortcodes expanded +*/ -}} + +{{- $content := .RawContent -}} +{{- $page := .Page -}} + +{{- /* Match non-greedily up to the first ">}}". Attribute values legitimately + contain ">" -- columnNames uses
to wrap header text -- so a [^>]* + class stops at that ">" and never matches the shortcode at all. */ -}} +{{- $pattern := `(?s)\{\{<\s*table-children.*?>\}\}` -}} + +{{- range $match := (findRE $pattern $content) -}} + + {{- /* Pull the shortcode's named parameters out of the matched text */ -}} + {{- $names := "" -}} + {{- $sources := "" -}} + {{- $links := "" -}} + {{- $limitTags := "" -}} + {{- $from := "" -}} + + {{- with (findRESubmatch `columnNames="([^"]*)"` $match 1) -}} + {{- $names = index (index . 0) 1 -}} + {{- end -}} + {{- with (findRESubmatch `columnSources="([^"]*)"` $match 1) -}} + {{- $sources = index (index . 0) 1 -}} + {{- end -}} + {{- with (findRESubmatch `enableLinks="([^"]*)"` $match 1) -}} + {{- $links = index (index . 0) 1 -}} + {{- end -}} + {{- with (findRESubmatch `limitTags="([^"]*)"` $match 1) -}} + {{- $limitTags = index (index . 0) 1 -}} + {{- end -}} + {{- with (findRESubmatch `from="([^"]*)"` $match 1) -}} + {{- $from = index (index . 0) 1 -}} + {{- end -}} + + {{- /* Same child selection as the HTML shortcode, so both outputs agree on + contents and ordering */ -}} + {{- $children := $page.Pages -}} + {{- if $from -}} + {{- /* Start empty when from= is given: if the target does not resolve, falling + back to this page's own children would publish a plausible-looking table + of the wrong data into the feed. No rows is the safe answer. */ -}} + {{- $children = slice -}} + {{- with $page.GetPage $from -}} + {{- $children = .Pages -}} + {{- end -}} + {{- end -}} + + {{- $replacement := "" -}} + + {{- if and $names $sources -}} + {{- $sourceList := split $sources "," -}} + {{- $linkList := split $links "," -}} + + {{- /* Header: strip the presentational markup the HTML table relies on + (  spacers and
line breaks) and collapse the whitespace */ -}} + {{- $headerCells := slice -}} + {{- range $name := (split $names ",") -}} + {{- $clean := $name | replaceRE ` ?` " " | replaceRE `` " " -}} + {{- $clean = trim (replaceRE `\s+` " " $clean) " " -}} + {{- $headerCells = $headerCells | append $clean -}} + {{- end -}} + + {{- $rows := slice -}} + {{- range $child := $children -}} + {{- $addRow := true -}} + {{- if $limitTags -}} + {{- $childTags := $child.Param "tags" -}} + {{- if not (in $childTags $limitTags) -}} + {{- $addRow = false -}} + {{- end -}} + {{- end -}} + + {{- if $addRow -}} + {{- $cells := slice -}} + {{- range $source := $sourceList -}} + {{- $value := printf "%v" ($child.Param $source | default "") -}} + {{- /* Keep each row on one line and don't let a value break out of + its table cell */ -}} + {{- $value = trim (replaceRE `\s+` " " $value) " " -}} + {{- $value = replace $value "|" `\|` -}} + {{- if in $linkList $source -}} + {{- if $value -}} + {{- $value = printf "[%s](%s)" $value $child.Permalink -}} + {{- end -}} + {{- end -}} + {{- $cells = $cells | append $value -}} + {{- end -}} + {{- $rows = $rows | append (printf "| %s |" (delimit $cells " | ")) -}} + {{- end -}} + {{- end -}} + + {{- $divider := slice -}} + {{- range $headerCells -}} + {{- $divider = $divider | append "---" -}} + {{- end -}} + {{- /* Leading newline is load-bearing: on pages whose entire body is this + shortcode, the table would otherwise start on the same line as the + closing ``` of the Markdown metadata block, which stops it being a + valid closing fence and swallows the table into the code block. */ -}} + {{- /* The header is emitted even with no rows, matching the HTML shortcode, + which always renders thead. Otherwise a table that filters down to + nothing would remove the whole body of a page whose only content is + this shortcode. */ -}} + {{- $replacement = printf "\n| %s |\n| %s |\n" + (delimit $headerCells " | ") + (delimit $divider " | ") -}} + {{- if $rows -}} + {{- $replacement = printf "%s%s\n" $replacement (delimit $rows "\n") -}} + {{- end -}} + {{- end -}} + + {{- $content = replace $content $match $replacement 1 -}} +{{- end -}} + +{{- return $content -}} diff --git a/layouts/partials/process-markdown-content.html b/layouts/partials/process-markdown-content.html index b5ccccab03..5e47e1eef7 100644 --- a/layouts/partials/process-markdown-content.html +++ b/layouts/partials/process-markdown-content.html @@ -75,8 +75,26 @@ {{- $content = $content | replaceRE "'" "'" -}} {{- $content = $content | replaceRE "+" "+" -}} +{{- /* Expand table-children shortcodes into Markdown tables. Runs after the unescape + above (it matches literal {{< ... >}}) and before the strip below, which would + otherwise discard the table -- taking the whole body of every release-notes + index page with it. */ -}} +{{- $content = partial "markdown-table-children.html" (dict "RawContent" $content "Page" .Page) -}} +{{- /* Unescape again, as Hugo re-escapes partial output. Must cover the same entities + as the block above, including +, or a "+" in a table cell (a version column + such as "6.0+", say) reaches the feed as a literal entity. */ -}} +{{- $content = $content | replaceRE """ "\"" -}} +{{- $content = $content | replaceRE """ "\"" -}} +{{- $content = $content | replaceRE "'" "'" -}} +{{- $content = $content | replaceRE "<" "<" -}} +{{- $content = $content | replaceRE ">" ">" -}} +{{- $content = $content | replaceRE "&" "&" -}} +{{- $content = $content | replaceRE "+" "+" -}} + {{- /* Remove remaining shortcodes AFTER unescape (content now has literal < and >) */ -}} -{{- $content = $content | replaceRE `\{\{<\s*/?[^>]*>\}\}` "" -}} +{{- /* Match non-greedily to the first ">}}": a [^>]* class is defeated by a ">" inside + an attribute value (e.g. columnNames="...
...") and leaks the raw shortcode. */ -}} +{{- $content = $content | replaceRE `(?s)\{\{<\s*/?.*?>\}\}` "" -}} {{- $content = $content | replaceRE `\{\{%\s*/?[^%]*%\}\}` "" -}} {{- return $content -}} diff --git a/layouts/partials/toc-from-markdown.html b/layouts/partials/toc-from-markdown.html index c272e73b80..f59a6e6be5 100644 --- a/layouts/partials/toc-from-markdown.html +++ b/layouts/partials/toc-from-markdown.html @@ -14,6 +14,17 @@ {{- $sections := slice -}} +{{- /* Ids already used on this page, so duplicates can be suffixed rather than + repeated. Keeps the TOC in step with sections[] in the JSON output, which + does the same in build/transform_json_sections.ts. */ -}} +{{- $usedIds := slice -}} + +{{- /* Drop fenced code blocks before looking for headings, so a "## comment" inside a + code sample does not become a navigation entry. Uses the same simple pairing rule + as splitContentIntoSections in build/transform_json_sections.ts, so both agree on + what counts as code and their ids stay in step. */ -}} +{{- $content = $content | replaceRE "(?s)```.*?```" "" -}} + {{- /* Find all ## and ### headers in the raw markdown */ -}} {{- /* Pattern matches lines starting with ## or ### followed by space and title */ -}} {{- $headerPattern := `(?m)^(#{2,3}) +(.+)$` -}} @@ -45,6 +56,19 @@ {{- /* Trim whitespace */ -}} {{- $title = $title | strings.TrimSpace -}} + {{- /* Hugo's explicit heading anchor, "## Title {#custom-id}". Hugo uses the + anchor as the rendered heading id and does not display it, so it has to come + out of the title and be used verbatim as the id. Matched strictly, because a + loose trailing-brace match would eat Python signatures ending in an empty + dict and dict defaults such as {'extra': 'ignore'}. */ -}} + {{- $explicitId := "" -}} + {{- with (findRESubmatch `\s*\{#([A-Za-z0-9][A-Za-z0-9_.:-]*)\}\s*$` $title 1) -}} + {{- $explicitId = index (index . 0) 1 -}} + {{- end -}} + {{- if $explicitId -}} + {{- $title = $title | replaceRE `\s*\{#[A-Za-z0-9][A-Za-z0-9_.:-]*\}\s*$` "" | strings.TrimSpace -}} + {{- end -}} + {{- /* Generate ID (slug) from title */ -}} {{- $id := $title | lower -}} {{- /* Replace spaces with hyphens */ -}} @@ -56,6 +80,27 @@ {{- /* Trim leading/trailing hyphens */ -}} {{- $id = $id | replaceRE `^-+|-+$` "" -}} + {{- /* An explicit anchor overrides the slug entirely */ -}} + {{- if $explicitId -}} + {{- $id = $explicitId -}} + {{- end -}} + + {{- /* Disambiguate a repeated id with Hugo's own -1, -2 suffix convention. Bounded by + the heading count rather than an arbitrary number, since a page can never need + more suffixes than it has headings -- an arbitrary cap would silently leave + duplicates on a page that exceeded it. */ -}} + {{- if in $usedIds $id -}} + {{- $base := $id -}} + {{- range seq 1 (len $headerMatches) -}} + {{- if in $usedIds $id -}} + {{- $id = printf "%s-%d" $base . -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- if $id -}} + {{- $usedIds = $usedIds | append $id -}} + {{- end -}} + {{- /* Add to sections array */ -}} {{- if and $id $title -}} {{- if $isH3 -}}