From eeca382ce8a17cca30a023d2d54828e9ca21532c Mon Sep 17 00:00:00 2001 From: Nigel Brown Date: Mon, 29 Jun 2026 17:53:40 +0530 Subject: [PATCH 1/7] sample search implementation Added Algolia site verification and DocSearch integration. Signed-off-by: Nigel Brown --- templates/default.html | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/templates/default.html b/templates/default.html index 10efa26d..e1189b10 100644 --- a/templates/default.html +++ b/templates/default.html @@ -1,4 +1,7 @@ -{%- block head -%}{%- endblock -%} +{%- block head -%} + + +{%- endblock -%} {% include "includes/head.html" %} + From ea5ef0683dc1341d901e01babd241584380fd1f5 Mon Sep 17 00:00:00 2001 From: Nigel Brown Date: Mon, 29 Jun 2026 18:08:18 +0530 Subject: [PATCH 2/7] Remove unused stylesheet and meta tags from default.html Removed unused stylesheet and meta verification from head block. Signed-off-by: Nigel Brown --- templates/default.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/default.html b/templates/default.html index e1189b10..b051cba3 100644 --- a/templates/default.html +++ b/templates/default.html @@ -1,6 +1,4 @@ {%- block head -%} - - {%- endblock -%} {% include "includes/head.html" %} From 24ece4f09a23953837e24505854cecbb4f6e3791 Mon Sep 17 00:00:00 2001 From: Nigel Brown Date: Mon, 29 Jun 2026 18:10:44 +0530 Subject: [PATCH 3/7] Add Algolia site verification meta tag and stylesheet Signed-off-by: Nigel Brown --- templates/includes/head.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/includes/head.html b/templates/includes/head.html index 68a4ccb4..9696a207 100644 --- a/templates/includes/head.html +++ b/templates/includes/head.html @@ -7,6 +7,7 @@ + {% if page and page.extra and page.extra.custom_meta%} {{ page.extra.custom_meta | safe }} {% endif %} @@ -55,6 +56,7 @@ {%- endif -%}"> + @@ -74,6 +76,7 @@ 'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-MFFCB7SR'); + {% if page and page.extra and page.extra.head_extra %}{{ page.extra.head_extra }}{% endif %} - \ No newline at end of file + From 5e2c858c9f3f6452eaf9c6ad863ffa3025f11623 Mon Sep 17 00:00:00 2001 From: nigel Date: Wed, 9 Sep 2026 12:40:38 -0500 Subject: [PATCH 4/7] removed algolia for fuse.js and addressed some coderabbit feedback Signed-off-by: nigel --- .github/workflows/zola-deploy.yml | 44 ++++ .gitignore | 2 +- README.md | 44 ++++ build/build-search-index.mjs | 281 ++++++++++++++++++++++++++ config.toml | 4 +- package-lock.json | 322 ++++++++++++++++++++++++++++++ package.json | 13 ++ sass/_search.scss | 114 +++++++++++ sass/css/styles.scss | 1 + static/assets/js/search.js | 219 ++++++++++++++++++++ templates/default.html | 32 +-- templates/includes/head.html | 2 - 12 files changed, 1061 insertions(+), 17 deletions(-) create mode 100644 build/build-search-index.mjs create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 sass/_search.scss create mode 100644 static/assets/js/search.js diff --git a/.github/workflows/zola-deploy.yml b/.github/workflows/zola-deploy.yml index 9e39ec3d..5094a775 100644 --- a/.github/workflows/zola-deploy.yml +++ b/.github/workflows/zola-deploy.yml @@ -70,6 +70,50 @@ jobs: BUILD_ONLY: true BUILD_THEMES: false + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + # Runs after `zola build`, before the artifact upload, so the index + # ships with the pages it describes and reflects this exact commit. + # The zola-deploy-action builds `website/public` as root inside Docker, + # so reclaim ownership before the Node step writes search-index.json. + - name: Build search index + run: | + cd website + sudo chown -R "$(id -u):$(id -g)" public + npm ci + npm run build:search-index + + # Fail the deploy if the index is missing, malformed, or too small. + # MIN_RECORDS is a floor below the real count (~780). + - name: Verify search index + env: + MIN_RECORDS: "500" + run: | + cd website + INDEX=public/search-index.json + if [ ! -s "$INDEX" ]; then + echo "::error::$INDEX is missing or empty" + exit 1 + fi + COUNT=$(node -e "const a=require('./public/search-index.json'); if(!Array.isArray(a)){console.error('not an array');process.exit(1)} console.log(a.length)") + echo "search-index.json contains $COUNT records" + if [ "$COUNT" -lt "$MIN_RECORDS" ]; then + echo "::error::search index has $COUNT records, below the expected minimum of $MIN_RECORDS" + exit 1 + fi + # Spot-check that render-time docs content (not just blog stubs) made + # it in: at least one /topics/ and one /commands/ page must be present. + node -e " + const a=require('./public/search-index.json'); + const has=(p)=>a.some(r=>typeof r.url==='string'&&r.url.startsWith(p)&&(r.body||'').length>0); + const topics=has('/topics/'), commands=has('/commands/'); + console.log('topics indexed:', topics, '| commands indexed:', commands); + if(!topics||!commands){console.error('::error::expected indexed /topics/ and /commands/ pages with content');process.exit(1)} + " + - name: Upload artifact if: github.event_name != 'pull_request' uses: actions/upload-pages-artifact@v3 diff --git a/.gitignore b/.gitignore index e9aeccba..44b77583 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .DS_Store public -package-lock.json +node_modules build-bloom-command-json build-command-docs build-command-json diff --git a/README.md b/README.md index 8b0c4824..ab98d905 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,50 @@ Point your browser at `http://127.0.0.1:1111/commands/` and you should see the f All files created in this process are ignored by git. Commit your changes to your local copy of `valkey-io/valkey-doc` for description changes and `valkey-io/valkey` for command JSON changes (if you have any). +## Search + +Site search is powered by [fuse.js](https://www.fusejs.io/) running entirely in the browser against a prebuilt index (`search-index.json`). + +The index is not Zola's native search index. Most of this site's documentation (topics, the command reference, and the clients page) is injected at template-render time from the sibling repos described above, so it never appears in the Markdown page body that Zola's `build_search_index` reads. Instead, `build/build-search-index.mjs` walks the rendered HTML in `public/` after a build and extracts the visible page content, capturing everything the site actually renders. + +### Building the index locally + +The indexer needs [Node.js](https://nodejs.org/) (18 or newer). Install dependencies once: + +```shell +npm install +``` + +Because `zola build` and `zola serve` both wipe `public/`, the index must be generated after each build. The simplest way is the convenience script, which runs `zola build` and then the indexer: + +```shell +npm run build +``` + +To regenerate only the index against an existing `public/` (for example, after a `zola serve` rebuild), run: + +```shell +npm run build:search-index +``` + +The generated `public/search-index.json` is ignored by git; it is always produced fresh at build time. + +To search topics, the command reference, and the clients page locally, first follow [Building additional content](#building-additional-content) so those pages exist to be indexed. Otherwise only the blog, author, download, event, and static pages are searchable. + +### Previewing complete results without the sibling repos + +If you don't have the sibling repos checked out, you can build an index from a running site (production or a local `zola serve`) via its sitemap: + +```shell +node build/build-search-index.mjs --crawl https://valkey.io +``` + +This is a local convenience for previewing complete results and is not used by the deploy pipeline. + +### Automation + +The deploy workflow (`.github/workflows/zola-deploy.yml`) regenerates the index on every deploy, after `zola build` and before the site is published, so it always reflects the commit being deployed. + ## License This project is licensed under the BSD-3-Clause License. diff --git a/build/build-search-index.mjs b/build/build-search-index.mjs new file mode 100644 index 00000000..6600445b --- /dev/null +++ b/build/build-search-index.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node +/** + * Generates public/search-index.json (records: { url, title, body }) for + * fuse.js by extracting text from rendered HTML. Zola's native index only sees + * Markdown bodies, which are empty stubs here (topics/commands/clients are + * injected at render time), so it can't index the docs. + * + * node build/build-search-index.mjs parse public/ (CI + local) + * node build/build-search-index.mjs --crawl fetch a live site via sitemap.xml + * + * Run after `zola build`. Crawl mode is a local insight tool, not used by CI. + */ + +import { readFileSync, writeFileSync, readdirSync, statSync, mkdirSync } from "node:fs"; +import { join, relative, sep, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import * as cheerio from "cheerio"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dirname, ".."); +const PUBLIC_DIR = join(ROOT, "public"); +const OUTPUT_FILE = join(PUBLIC_DIR, "search-index.json"); + +// Most specific content container first; .body is the shared fallback. +const CONTENT_SELECTORS = ["main", ".main-inner", ".event-single", ".body"]; + +// Shared chrome stripped before text extraction so it doesn't pollute the index. +const STRIP_SELECTORS = [ + "script", + "style", + "noscript", + "iframe", + ".header", + ".footer", + ".banner", + ".site-search", + "nav", + ".left-aside", + ".right-aside", + ".edit_box", +]; + +// Pages that are indexes/redirects or otherwise not worth surfacing directly. +const EXCLUDE_URL_PATTERNS = [ + /^\/404\/?$/, + /^\/authors\/?$/, // author index; individual author pages are still indexed +]; + +const MAX_BODY_CHARS = 8000; + +// Crawl-mode politeness. +const CRAWL_CONCURRENCY = 6; +const CRAWL_DELAY_MS = 50; + +function normalizeWhitespace(text) { + return text.replace(/\s+/g, " ").trim(); +} + +function extractTitle($, url) { + const h1 = normalizeWhitespace($("h1").first().text()); + if (h1) return h1; + + let t = normalizeWhitespace($("title").first().text()); + if (t) { + // Drop the "Valkey ·" / "Valkey Documentation ·" prefix, keep the specific part. + const parts = t.split("\u00b7"); + if (parts.length > 1) { + t = parts.slice(1).join("\u00b7").trim(); + } + if (t) return t; + } + return url; +} + +function extractBody($) { + let $container = null; + for (const selector of CONTENT_SELECTORS) { + const found = $(selector).first(); + if (found.length) { + $container = found; + break; + } + } + if (!$container) { + const body = $("body").first(); + $container = body.length ? body : $.root(); + } + + for (const selector of STRIP_SELECTORS) { + $container.find(selector).remove(); + } + + const text = normalizeWhitespace($container.text()); + return text.length > MAX_BODY_CHARS ? text.slice(0, MAX_BODY_CHARS) : text; +} + +// Returns a record, or null for redirect pages (Zola aliases / external-url +// events emit ), which have no useful content. +function recordFromHtml(html, url) { + const $ = cheerio.load(html); + + const refresh = $('meta[http-equiv]').filter( + (i, el) => ($(el).attr("http-equiv") || "").toLowerCase() === "refresh" + ); + if (refresh.length) return null; + + const title = extractTitle($, url); + const body = extractBody($); + if (!body && title === url) return null; + return { url, title, body }; +} + +function isExcluded(url) { + return EXCLUDE_URL_PATTERNS.some((re) => re.test(url)); +} + +function walkHtmlFiles(dir) { + const out = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + out.push(...walkHtmlFiles(full)); + } else if (entry.endsWith(".html")) { + out.push(full); + } + } + return out; +} + +// e.g. public/topics/sentinel/index.html -> /topics/sentinel/ +function fileToUrl(file) { + let rel = relative(PUBLIC_DIR, file).split(sep).join("/"); + if (rel === "index.html") { + rel = ""; + } else if (rel.endsWith("/index.html")) { + rel = rel.slice(0, -"index.html".length); + } else if (rel.endsWith(".html")) { + rel = rel.slice(0, -".html".length) + "/"; + } + return "/" + rel; +} + +function buildFromPublic() { + let files; + try { + files = walkHtmlFiles(PUBLIC_DIR); + } catch (err) { + console.error( + `Could not read ${PUBLIC_DIR}. Run \`zola build\` first.\n${err.message}` + ); + process.exit(1); + } + + const seen = new Set(); + const records = []; + + for (const file of files) { + const url = fileToUrl(file); + if (isExcluded(url) || seen.has(url)) continue; + const html = readFileSync(file, "utf8"); + const record = recordFromHtml(html, url); + if (!record) continue; + seen.add(url); + records.push(record); + } + return records; +} + +function toRelativeUrl(absUrl, baseUrl) { + try { + const u = new URL(absUrl); + let path = u.pathname; + if (!path.endsWith("/") && !path.includes(".")) path += "/"; + return path; + } catch { + return absUrl.startsWith("/") ? absUrl : "/" + absUrl; + } +} + +async function fetchText(url) { + const res = await fetch(url, { + headers: { "user-agent": "valkey-search-indexer/1.0" }, + redirect: "follow", + }); + if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); + return res.text(); +} + +async function getSitemapUrls(baseUrl) { + const sitemapUrl = new URL("sitemap.xml", baseUrl).toString(); + const xml = await fetchText(sitemapUrl); + const locs = [...xml.matchAll(/\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1]); + if (!locs.length) { + throw new Error(`No entries found in ${sitemapUrl}`); + } + return locs; +} + +async function mapWithConcurrency(items, limit, worker) { + const results = []; + let index = 0; + async function run() { + while (index < items.length) { + const current = index++; + results[current] = await worker(items[current], current); + if (CRAWL_DELAY_MS) await new Promise((r) => setTimeout(r, CRAWL_DELAY_MS)); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run)); + return results; +} + +async function buildFromCrawl(baseUrl) { + console.log(`Crawling ${baseUrl} via sitemap.xml ...`); + const locs = await getSitemapUrls(baseUrl); + console.log(`Found ${locs.length} URLs in sitemap.`); + + const seen = new Set(); + const targets = []; + for (const loc of locs) { + const url = toRelativeUrl(loc, baseUrl); + if (isExcluded(url) || seen.has(url)) continue; + seen.add(url); + targets.push({ abs: loc, url }); + } + + let failures = 0; + const settled = await mapWithConcurrency( + targets, + CRAWL_CONCURRENCY, + async ({ abs, url }) => { + try { + const html = await fetchText(abs); + return recordFromHtml(html, url); + } catch (err) { + failures++; + console.warn(` skip ${url}: ${err.message}`); + return null; + } + } + ); + + if (failures) console.warn(`Crawl completed with ${failures} failed page(s).`); + return settled.filter(Boolean); +} + +function parseArgs(argv) { + const args = { crawl: null }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--crawl") { + args.crawl = argv[i + 1]; + i++; + if (!args.crawl || args.crawl.startsWith("--")) { + console.error("--crawl requires a base URL, e.g. --crawl https://valkey.io"); + process.exit(1); + } + } + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const records = args.crawl + ? await buildFromCrawl(args.crawl) + : buildFromPublic(); + + records.sort((a, b) => a.url.localeCompare(b.url)); + + mkdirSync(PUBLIC_DIR, { recursive: true }); + writeFileSync(OUTPUT_FILE, JSON.stringify(records), "utf8"); + + console.log(`Wrote ${records.length} records to ${relative(ROOT, OUTPUT_FILE)}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/config.toml b/config.toml index 0d6cc47f..52e90024 100644 --- a/config.toml +++ b/config.toml @@ -3,8 +3,8 @@ title = "Valkey" compile_sass = true -build_search_index = true - +# Search uses fuse.js over a post-build index (build/build-search-index.mjs), +# not Zola's native index, which only sees the empty Markdown stubs here. generate_feeds = true taxonomies = [ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..fcb8efef --- /dev/null +++ b/package-lock.json @@ -0,0 +1,322 @@ +{ + "name": "valkey-io-website", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "valkey-io-website", + "version": "1.0.0", + "devDependencies": { + "cheerio": "1.0.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true, + "license": "ISC" + }, + "node_modules/cheerio": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^6.19.5", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=18.17" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "dev": true, + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/undici": { + "version": "6.28.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "dev": true, + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..66b8427d --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "valkey-io-website", + "version": "1.0.0", + "private": true, + "description": "Build tooling for the valkey.io website (Zola site).", + "scripts": { + "build:search-index": "node build/build-search-index.mjs", + "build": "zola build && npm run build:search-index" + }, + "devDependencies": { + "cheerio": "1.0.0" + } +} diff --git a/sass/_search.scss b/sass/_search.scss new file mode 100644 index 00000000..9d2d0407 --- /dev/null +++ b/sass/_search.scss @@ -0,0 +1,114 @@ +// +// Site search (fuse.js) — search box in the main nav and results dropdown. +// + +.site-search { + position: relative; + // Don't stretch to the nav's full height, else the dropdown detaches from the input. + display: inline-flex; + align-self: center; + align-items: center; + + &__input { + width: 160px; + padding: 6px 12px; + border: 1px solid $line; + border-radius: 16px; + background-color: $background-lightest; + color: $text; + font-size: 14px; + line-height: 1.4; + outline: none; + transition: width 0.2s ease, border-color 0.2s ease; + + &::placeholder { + color: $text-light; + } + + &:focus { + width: 220px; + border-color: $attention; + } + } + + &__input, + &__results { + box-sizing: border-box; + } + + &__results { + position: absolute; + top: calc(100% + 6px); + right: 0; + z-index: 1000; + width: 420px; + max-width: 90vw; + max-height: 70vh; + margin: 0; + padding: 4px 0; + overflow-x: hidden; + overflow-y: auto; + list-style: none; + background-color: $background-lightest; + border: 1px solid $line; + border-radius: 8px; + box-shadow: 0 8px 24px rgba(0, 42, 58, 0.15); + } + + &__result { + width: 100%; + margin: 0; + padding: 0; + + &.is-active, + &:hover { + background-color: $highlight-lightest; + } + } + + &__link { + display: block; + width: 100%; + box-sizing: border-box; + padding: 10px 18px; + text-decoration: none; + color: $text; + + &:hover { + text-decoration: none; + } + } + + // Override `.header nav a` (nowrap + zero padding) so results wrap and don't hug the edge. + .header nav & &__link { + white-space: normal; + padding: 10px 18px; + } + + &__title { + display: block; + font-weight: 600; + font-size: 14px; + color: $text-link-alternate; + overflow-wrap: anywhere; + } + + &__snippet { + margin-top: 2px; + font-size: 12px; + line-height: 1.4; + color: $text-light; + overflow-wrap: anywhere; + // Clamp long snippets to two lines. + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; + } + + &__empty { + padding: 12px 14px; + font-size: 13px; + color: $text-light; + } +} diff --git a/sass/css/styles.scss b/sass/css/styles.scss index 8f7c1389..30868fec 100755 --- a/sass/css/styles.scss +++ b/sass/css/styles.scss @@ -8,6 +8,7 @@ @import '../pygments'; @import '../valkey'; @import '../email-form'; +@import '../search'; /* Styling for Markdown Tables */ table { diff --git a/static/assets/js/search.js b/static/assets/js/search.js new file mode 100644 index 00000000..09cf0b73 --- /dev/null +++ b/static/assets/js/search.js @@ -0,0 +1,219 @@ +/** + * Client-side search powered by fuse.js over the index produced by + * build/build-search-index.mjs (records: { url, title, body }). + * The index is fetched lazily on first interaction. + */ +(function () { + "use strict"; + + var SEARCH_INDEX_URL = "/search-index.json"; + var MAX_RESULTS = 8; + var DEBOUNCE_MS = 150; + + var input = document.getElementById("search-input"); + var resultsList = document.getElementById("search-results"); + + if (!input || !resultsList || typeof Fuse === "undefined") { + return; + } + + var fuse = null; + var indexPromise = null; + var activeIndex = -1; + var currentResults = []; + + var fuseOptions = { + includeScore: true, + ignoreLocation: true, + threshold: 0.4, + minMatchCharLength: 2, + keys: [ + { name: "title", weight: 0.7 }, + { name: "body", weight: 0.3 }, + ], + }; + + function loadIndex() { + if (indexPromise) { + return indexPromise; + } + + indexPromise = fetch(SEARCH_INDEX_URL) + .then(function (response) { + if (!response.ok) { + throw new Error("Failed to load search index: " + response.status); + } + return response.json(); + }) + .then(function (records) { + fuse = new Fuse(records, fuseOptions); + return fuse; + }) + .catch(function (error) { + indexPromise = null; // allow retry on a later interaction + console.error(error); + throw error; + }); + + return indexPromise; + } + + function makeSnippet(body, maxLength) { + if (!body) { + return ""; + } + var text = body.replace(/\s+/g, " ").trim(); + if (text.length <= maxLength) { + return text; + } + return text.slice(0, maxLength).trimEnd() + "\u2026"; + } + + function clearResults() { + resultsList.innerHTML = ""; + resultsList.hidden = true; + activeIndex = -1; + currentResults = []; + input.setAttribute("aria-expanded", "false"); + input.removeAttribute("aria-activedescendant"); + } + + function renderResults(results) { + currentResults = results; + activeIndex = -1; + resultsList.innerHTML = ""; + input.removeAttribute("aria-activedescendant"); + + if (!results.length) { + var empty = document.createElement("li"); + empty.className = "site-search__empty"; + empty.setAttribute("role", "option"); + empty.textContent = "No results found"; + resultsList.appendChild(empty); + resultsList.hidden = false; + input.setAttribute("aria-expanded", "true"); + return; + } + + results.forEach(function (result, i) { + var item = result.item; + var li = document.createElement("li"); + li.className = "site-search__result"; + li.setAttribute("role", "option"); + li.id = "search-result-" + i; + + var link = document.createElement("a"); + link.href = item.url; + link.className = "site-search__link"; + + var title = document.createElement("span"); + title.className = "site-search__title"; + title.textContent = item.title || item.url; + link.appendChild(title); + + var snippet = makeSnippet(item.body, 90); + if (snippet) { + var desc = document.createElement("span"); + desc.className = "site-search__snippet"; + desc.textContent = snippet; + link.appendChild(desc); + } + + li.appendChild(link); + resultsList.appendChild(li); + }); + + resultsList.hidden = false; + input.setAttribute("aria-expanded", "true"); + } + + function runSearch(query) { + var trimmed = query.trim(); + if (trimmed.length < 2) { + clearResults(); + return; + } + + loadIndex() + .then(function (index) { + // Guard against a stale async response after the box was cleared. + if (input.value.trim() !== trimmed) { + return; + } + var results = index.search(trimmed).slice(0, MAX_RESULTS); + renderResults(results); + }) + .catch(function () { + clearResults(); + }); + } + + function setActive(nextIndex) { + var items = resultsList.querySelectorAll(".site-search__result"); + if (!items.length) { + return; + } + + if (activeIndex > -1 && items[activeIndex]) { + items[activeIndex].classList.remove("is-active"); + } + + activeIndex = ((nextIndex % items.length) + items.length) % items.length; + var active = items[activeIndex]; + active.classList.add("is-active"); + active.scrollIntoView({ block: "nearest" }); + input.setAttribute("aria-activedescendant", active.id); + } + + var debounceTimer = null; + input.addEventListener("input", function () { + var value = input.value; + window.clearTimeout(debounceTimer); + debounceTimer = window.setTimeout(function () { + runSearch(value); + }, DEBOUNCE_MS); + }); + + input.addEventListener("keydown", function (event) { + var items = resultsList.querySelectorAll(".site-search__result"); + + switch (event.key) { + case "ArrowDown": + if (items.length) { + event.preventDefault(); + setActive(activeIndex + 1); + } + break; + case "ArrowUp": + if (items.length) { + event.preventDefault(); + setActive(activeIndex - 1); + } + break; + case "Enter": + if (activeIndex > -1 && currentResults[activeIndex]) { + event.preventDefault(); + window.location.href = currentResults[activeIndex].item.url; + } + break; + case "Escape": + clearResults(); + input.blur(); + break; + default: + break; + } + }); + + // Close the dropdown when focus leaves the search widget. + document.addEventListener("click", function (event) { + if (!event.target.closest(".site-search")) { + clearResults(); + } + }); + + // Warm the index on first focus so the first query feels instant. + input.addEventListener("focus", function () { + loadIndex().catch(function () {}); + }); +})(); diff --git a/templates/default.html b/templates/default.html index b051cba3..9a4f9357 100644 --- a/templates/default.html +++ b/templates/default.html @@ -1,5 +1,6 @@ + {%- block head -%} -{%- endblock -%} +{%- endblock -%} {% include "includes/head.html" %} - + + + diff --git a/templates/includes/head.html b/templates/includes/head.html index 9696a207..575d9572 100644 --- a/templates/includes/head.html +++ b/templates/includes/head.html @@ -7,7 +7,6 @@ - {% if page and page.extra and page.extra.custom_meta%} {{ page.extra.custom_meta | safe }} {% endif %} @@ -56,7 +55,6 @@ {%- endif -%}"> - From 540156968167c519243ec69a329026133ea0704f Mon Sep 17 00:00:00 2001 From: nigel Date: Wed, 9 Sep 2026 13:32:21 -0500 Subject: [PATCH 5/7] tweaking the function for creating the search index to include more records and relevant hits Signed-off-by: nigel --- .github/workflows/zola-deploy.yml | 7 +- README.md | 2 + build/build-search-index.mjs | 161 ++++++++++++++++++++++++++---- static/assets/js/search.js | 69 +++++++++++-- 4 files changed, 211 insertions(+), 28 deletions(-) diff --git a/.github/workflows/zola-deploy.yml b/.github/workflows/zola-deploy.yml index 5094a775..affb36cd 100644 --- a/.github/workflows/zola-deploy.yml +++ b/.github/workflows/zola-deploy.yml @@ -87,10 +87,13 @@ jobs: npm run build:search-index # Fail the deploy if the index is missing, malformed, or too small. - # MIN_RECORDS is a floor below the real count (~780). + # Long pages are split into per-section records, so the real count is + # ~2,100+. MIN_RECORDS is a floor with headroom for content churn that is + # still high enough to catch a regression back to one-record-per-page + # (which would collapse the count to roughly 800). - name: Verify search index env: - MIN_RECORDS: "500" + MIN_RECORDS: "1500" run: | cd website INDEX=public/search-index.json diff --git a/README.md b/README.md index ab98d905..9e99f28c 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,8 @@ Site search is powered by [fuse.js](https://www.fusejs.io/) running entirely in The index is not Zola's native search index. Most of this site's documentation (topics, the command reference, and the clients page) is injected at template-render time from the sibling repos described above, so it never appears in the Markdown page body that Zola's `build_search_index` reads. Instead, `build/build-search-index.mjs` walks the rendered HTML in `public/` after a build and extracts the visible page content, capturing everything the site actually renders. +Long pages are indexed as one record per top-level (`h2`) section rather than a single whole-page record, so deep content stays searchable and a result can link straight to the matching section via its heading anchor. Content before the first `h2`, and pages with no `h2`, produce a single page-level record. All records for one page share the same `title` (the page title); the section heading is stored separately and weighted well below the title, so splitting a page into sections does not let a thin section out-rank, or dilute, a page-name match. Because a long page contributes several records that share a base url, the client (`static/assets/js/search.js`) also caps how many sections from the same page appear in the results list. + ### Building the index locally The indexer needs [Node.js](https://nodejs.org/) (18 or newer). Install dependencies once: diff --git a/build/build-search-index.mjs b/build/build-search-index.mjs index 6600445b..5af2c127 100644 --- a/build/build-search-index.mjs +++ b/build/build-search-index.mjs @@ -1,10 +1,19 @@ #!/usr/bin/env node /** - * Generates public/search-index.json (records: { url, title, body }) for - * fuse.js by extracting text from rendered HTML. Zola's native index only sees + * Generates public/search-index.json (records: { url, title, heading, body }) + * for fuse.js by extracting text from rendered HTML. `heading` is the section + * heading for section records and "" for page-level records. Zola's native index only sees * Markdown bodies, which are empty stubs here (topics/commands/clients are * injected at render time), so it can't index the docs. * + * Long pages are split into one record per top-level (h2) section rather than a + * single whole-page record. This keeps deep content searchable instead of being + * dropped by a whole-page character cap, and lets results deep-link to the + * matching section via its heading anchor (e.g. /topics/sentinel/#sentinel-api). + * Content before the first h2, and pages with no h2, become a single page-level + * record as before. Nested h3/h4 text is folded into its parent h2 section so + * the record count stays modest. + * * node build/build-search-index.mjs parse public/ (CI + local) * node build/build-search-index.mjs --crawl fetch a live site via sitemap.xml * @@ -46,7 +55,11 @@ const EXCLUDE_URL_PATTERNS = [ /^\/authors\/?$/, // author index; individual author pages are still indexed ]; -const MAX_BODY_CHARS = 8000; +// Safety valve for a single section's body. Sections rarely approach this; +// it only guards against a pathologically large h2 block (e.g. a generated API +// reference with no subheadings). Page-level records for pages without any h2 +// are also bounded by this. +const MAX_SECTION_CHARS = 12000; // Crawl-mode politeness. const CRAWL_CONCURRENCY = 6; @@ -72,7 +85,9 @@ function extractTitle($, url) { return url; } -function extractBody($) { +// Resolves the primary content container and strips shared chrome from it, +// returning the cheerio node ready for text extraction (or null if empty). +function getContentContainer($) { let $container = null; for (const selector of CONTENT_SELECTORS) { const found = $(selector).first(); @@ -89,25 +104,133 @@ function extractBody($) { for (const selector of STRIP_SELECTORS) { $container.find(selector).remove(); } + return $container; +} + +function capBody(text) { + return text.length > MAX_SECTION_CHARS ? text.slice(0, MAX_SECTION_CHARS) : text; +} + +// Splits a page's content container into search records, one per top-level +// (h2) section, plus a page-level record for the content preceding the first +// h2. h3/h4 headings and their text are folded into the enclosing h2 section +// rather than becoming their own records, to keep the record count modest. +// +// Section records deep-link to the heading anchor when the h2 carries an id +// (Zola emits ids on all headings), so a hit lands on the relevant section. +// +// Pages with no h2 collapse to a single page-level record, matching the +// previous whole-page behavior. +// +// Ranking note: every record for a page carries the SAME `title` (the page +// title), so a query naming the page scores identically whether it matches the +// page-level record or a section record. The section heading goes in a separate +// low-weight `heading` field (for a light relevance boost and for display), and +// its text is also folded into `body`. Keeping the heading out of the +// high-weight `title` field is deliberate: putting it there both diluted page +// title matches (longer field = weaker fuzzy score) and let section-heading +// words win spurious high-weight matches, pushing better page results down. +function recordsFromContainer($, $container, pageUrl, pageTitle) { + const records = []; + + // A section accumulates the text of its heading and everything up to the + // next h2. `null` heading means the page-level lead section (before any h2). + let current = { heading: null, id: null, parts: [] }; + const sections = [current]; + + // Walk the rendered content in document order. cheerio's contents() over the + // container's descendants would double-count nested text, so we walk only the + // top-level flow children and rely on .text() to gather nested content, while + // treating an h2 as a section boundary. Since headings are flat siblings of + // the flow content in this site's markup, iterating children of the heading's + // parent captures the true order. + const flowRoot = $container.find("h2").first().length + ? $($container.find("h2").first().get(0)).parent() + : $container; + + flowRoot.children().each((_, el) => { + const tag = (el.tagName || "").toLowerCase(); + const $el = $(el); + if (tag === "h2") { + current = { + heading: normalizeWhitespace($el.text()), + id: $el.attr("id") || null, + parts: [], + }; + sections.push(current); + // Include the heading text itself in the searchable body. + current.parts.push(current.heading); + } else { + const text = normalizeWhitespace($el.text()); + if (text) current.parts.push(text); + } + }); + + // A section only becomes its own record if its h2 has an id to deep-link to. + // Anchorless sections (h2 with no id) can't be linked individually, so their + // text is folded into the page-level lead record instead of producing several + // records that share the bare page url. + const leadParts = []; + const sectionRecords = []; + for (const section of sections) { + const text = normalizeWhitespace(section.parts.join(" ")); + if (!text) continue; + + if (section.heading === null || !section.id) { + leadParts.push(text); + } else { + sectionRecords.push({ + url: `${pageUrl}#${section.id}`, + title: pageTitle, + heading: section.heading, + body: capBody(text), + }); + } + } - const text = normalizeWhitespace($container.text()); - return text.length > MAX_BODY_CHARS ? text.slice(0, MAX_BODY_CHARS) : text; + const leadBody = capBody(normalizeWhitespace(leadParts.join(" "))); + if (leadBody) { + records.push({ url: pageUrl, title: pageTitle, heading: "", body: leadBody }); + } + records.push(...sectionRecords); + + // Guarantee at least one record for a page that has content but produced + // none above (e.g. text only inside unexpected wrappers): fall back to the + // whole-container text as a page-level record. + if (!records.length) { + const whole = capBody(normalizeWhitespace($container.text())); + if (whole) + records.push({ url: pageUrl, title: pageTitle, heading: "", body: whole }); + } + + return records; } -// Returns a record, or null for redirect pages (Zola aliases / external-url -// events emit ), which have no useful content. -function recordFromHtml(html, url) { +// Returns an array of records for a page (possibly empty). Redirect pages +// (Zola aliases / external-url events emit ) have no +// useful content and yield no records. +function recordsFromHtml(html, url) { const $ = cheerio.load(html); const refresh = $('meta[http-equiv]').filter( (i, el) => ($(el).attr("http-equiv") || "").toLowerCase() === "refresh" ); - if (refresh.length) return null; + if (refresh.length) return []; const title = extractTitle($, url); - const body = extractBody($); - if (!body && title === url) return null; - return { url, title, body }; + const $container = getContentContainer($); + const records = recordsFromContainer($, $container, url, title); + + // Drop a lone page-level record that has neither body nor a real title + // (matches the previous "no body and title === url" skip). + if ( + records.length === 1 && + !records[0].body && + records[0].title === url + ) { + return []; + } + return records; } function isExcluded(url) { @@ -159,10 +282,10 @@ function buildFromPublic() { const url = fileToUrl(file); if (isExcluded(url) || seen.has(url)) continue; const html = readFileSync(file, "utf8"); - const record = recordFromHtml(html, url); - if (!record) continue; + const pageRecords = recordsFromHtml(html, url); + if (!pageRecords.length) continue; seen.add(url); - records.push(record); + records.push(...pageRecords); } return records; } @@ -232,17 +355,17 @@ async function buildFromCrawl(baseUrl) { async ({ abs, url }) => { try { const html = await fetchText(abs); - return recordFromHtml(html, url); + return recordsFromHtml(html, url); } catch (err) { failures++; console.warn(` skip ${url}: ${err.message}`); - return null; + return []; } } ); if (failures) console.warn(`Crawl completed with ${failures} failed page(s).`); - return settled.filter(Boolean); + return settled.flat(); } function parseArgs(argv) { diff --git a/static/assets/js/search.js b/static/assets/js/search.js index 09cf0b73..ef03d250 100644 --- a/static/assets/js/search.js +++ b/static/assets/js/search.js @@ -1,13 +1,24 @@ /** * Client-side search powered by fuse.js over the index produced by - * build/build-search-index.mjs (records: { url, title, body }). + * build/build-search-index.mjs (records: { url, title, heading, body }). * The index is fetched lazily on first interaction. + * + * Long pages are indexed as multiple section records whose urls share a base + * path and differ only by "#anchor". All records for one page share the same + * `title` (the page title); the section heading lives in `heading`. To keep + * ranking fair, `heading` is weighted well below `title` so section records + * neither dilute nor out-compete the page-name match, and results are capped + * per base page (see MAX_PER_PAGE) so one page's sections cannot fill the whole + * dropdown. */ (function () { "use strict"; var SEARCH_INDEX_URL = "/search-index.json"; var MAX_RESULTS = 8; + // Cap on how many section results from the same base page (url without its + // "#anchor") may appear, so a broad query still surfaces multiple pages. + var MAX_PER_PAGE = 2; var DEBOUNCE_MS = 150; var input = document.getElementById("search-input"); @@ -28,8 +39,9 @@ threshold: 0.4, minMatchCharLength: 2, keys: [ - { name: "title", weight: 0.7 }, - { name: "body", weight: 0.3 }, + { name: "title", weight: 0.6 }, + { name: "heading", weight: 0.15 }, + { name: "body", weight: 0.25 }, ], }; @@ -58,11 +70,46 @@ return indexPromise; } - function makeSnippet(body, maxLength) { + // The base page of a record url is everything before the "#anchor". + function basePage(url) { + if (!url) { + return url; + } + var hash = url.indexOf("#"); + return hash === -1 ? url : url.slice(0, hash); + } + + // fuse returns results best-first; keep at most `limit` per base page so a + // single long page's sections can't crowd out other pages. Order preserved. + function capPerPage(results, limit) { + var counts = Object.create(null); + var kept = []; + for (var i = 0; i < results.length; i++) { + var page = basePage(results[i].item && results[i].item.url); + var seen = counts[page] || 0; + if (seen >= limit) { + continue; + } + counts[page] = seen + 1; + kept.push(results[i]); + } + return kept; + } + + // Section bodies begin with the heading text (the indexer prepends it so the + // heading is searchable in `body` too). The heading is already shown in the + // result title, so strip that leading copy to avoid a redundant snippet. + function makeSnippet(body, heading, maxLength) { if (!body) { return ""; } var text = body.replace(/\s+/g, " ").trim(); + if (heading) { + var h = heading.replace(/\s+/g, " ").trim(); + if (h && text.slice(0, h.length) === h) { + text = text.slice(h.length).trim(); + } + } if (text.length <= maxLength) { return text; } @@ -108,10 +155,15 @@ var title = document.createElement("span"); title.className = "site-search__title"; - title.textContent = item.title || item.url; + // Show the section heading (when present) after the page title so users + // can tell which section a result points to, without it affecting rank. + title.textContent = + item.title && item.heading + ? item.title + " \u203a " + item.heading + : item.title || item.url; link.appendChild(title); - var snippet = makeSnippet(item.body, 90); + var snippet = makeSnippet(item.body, item.heading, 90); if (snippet) { var desc = document.createElement("span"); desc.className = "site-search__snippet"; @@ -140,7 +192,10 @@ if (input.value.trim() !== trimmed) { return; } - var results = index.search(trimmed).slice(0, MAX_RESULTS); + var results = capPerPage(index.search(trimmed), MAX_PER_PAGE).slice( + 0, + MAX_RESULTS + ); renderResults(results); }) .catch(function () { From 05948873e139063b80a8b93311d194077dc83162 Mon Sep 17 00:00:00 2001 From: nigel Date: Wed, 9 Sep 2026 14:56:25 -0500 Subject: [PATCH 6/7] fixing bugs that coderabbit found Signed-off-by: nigel --- build/build-search-index.mjs | 62 ++++++++++++++++++++++-------------- sass/_search.scss | 23 +++++++++++++ static/assets/js/search.js | 38 ++++++++++++++++------ templates/default.html | 1 + 4 files changed, 91 insertions(+), 33 deletions(-) diff --git a/build/build-search-index.mjs b/build/build-search-index.mjs index 5af2c127..d0f67121 100644 --- a/build/build-search-index.mjs +++ b/build/build-search-index.mjs @@ -138,33 +138,47 @@ function recordsFromContainer($, $container, pageUrl, pageTitle) { let current = { heading: null, id: null, parts: [] }; const sections = [current]; - // Walk the rendered content in document order. cheerio's contents() over the - // container's descendants would double-count nested text, so we walk only the - // top-level flow children and rely on .text() to gather nested content, while - // treating an h2 as a section boundary. Since headings are flat siblings of - // the flow content in this site's markup, iterating children of the heading's - // parent captures the true order. - const flowRoot = $container.find("h2").first().length - ? $($container.find("h2").first().get(0)).parent() - : $container; - - flowRoot.children().each((_, el) => { - const tag = (el.tagName || "").toLowerCase(); + // Walk the container's descendants in document order, treating every h2 as a + // section boundary regardless of how deeply it is nested. We accumulate text + // from text nodes only (never an element's aggregate .text()), so a wrapping + // element does not double-count the text of its children. This covers lead + // content, content after a nested-heading block, and sibling blocks such as a + // feature-comparison table that live outside the first h2's parent -- an + // earlier version iterated only the first h2's parent and silently dropped + // all of that. + const root = $container.get(0); + const startSection = (el) => { const $el = $(el); - if (tag === "h2") { - current = { - heading: normalizeWhitespace($el.text()), - id: $el.attr("id") || null, - parts: [], - }; - sections.push(current); - // Include the heading text itself in the searchable body. - current.parts.push(current.heading); - } else { - const text = normalizeWhitespace($el.text()); + current = { + heading: normalizeWhitespace($el.text()), + id: $el.attr("id") || null, + parts: [], + }; + sections.push(current); + // Include the heading text itself in the searchable body. + if (current.heading) current.parts.push(current.heading); + }; + + const visit = (node) => { + if (!node) return; + if (node.type === "text") { + const text = normalizeWhitespace(node.data || ""); if (text) current.parts.push(text); + return; } - }); + if (node.type !== "tag") return; + const tag = (node.tagName || "").toLowerCase(); + if (tag === "h2") { + // Start a new section; the heading's own text is captured via .text() + // in startSection, so we do not descend into it again. + startSection(node); + return; + } + const children = node.children || []; + for (const child of children) visit(child); + }; + + for (const child of root.children || []) visit(child); // A section only becomes its own record if its h2 has an id to deep-link to. // Anchorless sections (h2 with no id) can't be linked individually, so their diff --git a/sass/_search.scss b/sass/_search.scss index 9d2d0407..048a0d80 100644 --- a/sass/_search.scss +++ b/sass/_search.scss @@ -111,4 +111,27 @@ font-size: 13px; color: $text-light; } + + // Status live region ("No results found"). Sits where the dropdown would and + // is only visible when it carries a message; an empty status shows nothing. + &__status { + position: absolute; + top: calc(100% + 6px); + right: 0; + z-index: 1000; + box-sizing: border-box; + width: 420px; + max-width: 90vw; + padding: 12px 14px; + font-size: 13px; + color: $text-light; + background-color: $background-lightest; + border: 1px solid $line; + border-radius: 8px; + box-shadow: 0 8px 24px rgba(0, 42, 58, 0.15); + + &:empty { + display: none; + } + } } diff --git a/static/assets/js/search.js b/static/assets/js/search.js index ef03d250..9e37ca6e 100644 --- a/static/assets/js/search.js +++ b/static/assets/js/search.js @@ -23,6 +23,10 @@ var input = document.getElementById("search-input"); var resultsList = document.getElementById("search-results"); + // Live region for status messages (e.g. "No results found"). Kept outside the + // listbox because a listbox must only own option/group children; a message + // placed inside it would be exposed as a non-navigable option. + var statusRegion = document.getElementById("search-status"); if (!input || !resultsList || typeof Fuse === "undefined") { return; @@ -32,6 +36,15 @@ var indexPromise = null; var activeIndex = -1; var currentResults = []; + // Bumped whenever the dropdown is dismissed so an in-flight search that + // resolves later can tell it was superseded and skip rendering. + var searchGeneration = 0; + + function setStatus(message) { + if (statusRegion) { + statusRegion.textContent = message || ""; + } + } var fuseOptions = { includeScore: true, @@ -121,6 +134,8 @@ resultsList.hidden = true; activeIndex = -1; currentResults = []; + searchGeneration++; + setStatus(""); input.setAttribute("aria-expanded", "false"); input.removeAttribute("aria-activedescendant"); } @@ -132,16 +147,16 @@ input.removeAttribute("aria-activedescendant"); if (!results.length) { - var empty = document.createElement("li"); - empty.className = "site-search__empty"; - empty.setAttribute("role", "option"); - empty.textContent = "No results found"; - resultsList.appendChild(empty); - resultsList.hidden = false; - input.setAttribute("aria-expanded", "true"); + // Keep the listbox empty and hidden; announce via the status region so no + // non-navigable "option" is exposed inside the listbox. + resultsList.hidden = true; + setStatus("No results found"); + input.setAttribute("aria-expanded", "false"); return; } + setStatus(""); + results.forEach(function (result, i) { var item = result.item; var li = document.createElement("li"); @@ -186,10 +201,15 @@ return; } + // Capture the current generation so a resolve after a dismissal (Escape or + // outside-click) is ignored. clearResults does not change input.value, so + // the value check alone would let a dismissed search re-open the dropdown. + var generation = searchGeneration; loadIndex() .then(function (index) { - // Guard against a stale async response after the box was cleared. - if (input.value.trim() !== trimmed) { + // Guard against a stale async response after the box was cleared or the + // query changed. + if (generation !== searchGeneration || input.value.trim() !== trimmed) { return; } var results = capPerPage(index.search(trimmed), MAX_PER_PAGE).slice( diff --git a/templates/default.html b/templates/default.html index 9a4f9357..232e5cd6 100644 --- a/templates/default.html +++ b/templates/default.html @@ -82,6 +82,7 @@ aria-expanded="false" /> +
From 49cabd3357fc9f0ced7b1c4e7afb238555aa1270 Mon Sep 17 00:00:00 2001 From: nigel Date: Fri, 11 Sep 2026 11:19:22 -0500 Subject: [PATCH 7/7] adding instructions for building locally and a message to show errors in search index Signed-off-by: nigel --- README.md | 21 +++++++-------------- static/assets/js/search.js | 11 +++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9e99f28c..2c1e6718 100644 --- a/README.md +++ b/README.md @@ -101,27 +101,20 @@ The index is not Zola's native search index. Most of this site's documentation ( Long pages are indexed as one record per top-level (`h2`) section rather than a single whole-page record, so deep content stays searchable and a result can link straight to the matching section via its heading anchor. Content before the first `h2`, and pages with no `h2`, produce a single page-level record. All records for one page share the same `title` (the page title); the section heading is stored separately and weighted well below the title, so splitting a page into sections does not let a thin section out-rank, or dilute, a page-name match. Because a long page contributes several records that share a base url, the client (`static/assets/js/search.js`) also caps how many sections from the same page appear in the results list. -### Building the index locally +### Testing search locally -The indexer needs [Node.js](https://nodejs.org/) (18 or newer). Install dependencies once: +Search does not work under `zola serve`. The dev server builds the site into memory and does not run the post-build indexer, so `search-index.json` is never generated or served and every query returns nothing. This is expected: `zola serve` is fine for editing content with live reload, but it cannot serve search. -```shell -npm install -``` - -Because `zola build` and `zola serve` both wipe `public/`, the index must be generated after each build. The simplest way is the convenience script, which runs `zola build` and then the indexer: +To test search, build the site to disk (which also generates the index) and serve the `public/` directory with any static file server. The indexer needs [Node.js](https://nodejs.org/) (18 or newer); install dependencies once with `npm install`, then: ```shell -npm run build +npm run build # zola build + generate search-index.json in public/ +python3 -m http.server -d public 8080 # or any static server for public/ ``` -To regenerate only the index against an existing `public/` (for example, after a `zola serve` rebuild), run: - -```shell -npm run build:search-index -``` +Open `http://localhost:8080/` and search will work. Any static server works; the only requirement is that it serves the `public/` directory produced by `npm run build`, including `search-index.json`. -The generated `public/search-index.json` is ignored by git; it is always produced fresh at build time. +The generated `public/search-index.json` is ignored by git; it is always produced fresh at build time. Because both `zola build` and `zola serve` wipe `public/`, re-run `npm run build` after any rebuild to refresh the index. To regenerate only the index against a `public/` that already exists on disk (for example after a plain `zola build`), run `npm run build:search-index`. To search topics, the command reference, and the clients page locally, first follow [Building additional content](#building-additional-content) so those pages exist to be indexed. Otherwise only the blog, author, download, event, and static pages are searchable. diff --git a/static/assets/js/search.js b/static/assets/js/search.js index 9e37ca6e..72248cdf 100644 --- a/static/assets/js/search.js +++ b/static/assets/js/search.js @@ -66,6 +66,17 @@ indexPromise = fetch(SEARCH_INDEX_URL) .then(function (response) { if (!response.ok) { + if (response.status === 404) { + // Most common local cause: running under `zola serve`, which does + // not generate or serve the post-build index. Point developers at + // the fix instead of failing silently. + throw new Error( + SEARCH_INDEX_URL + + " was not found (HTTP 404). Search will not work under `zola serve`, " + + "which does not build the index. Run `npm run build` and serve the " + + "`public/` directory statically. See the Search section of README.md." + ); + } throw new Error("Failed to load search index: " + response.status); } return response.json();