diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..e6cf269823 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,108 @@ +# Builds the mdBook docs site from the Markdown in this repo and publishes it +# to the existing `pages` branch under /book/. +# +# Publishing is safe to run unconditionally because mdBook emits relative asset +# paths, so the book works from a subpath. It is written into a subdirectory of +# the branch Pages already serves, which leaves index/privacy/support/terms +# untouched at the root and needs no change to the Pages source or DNS. +# +# https://block.github.io/buzz/ legal pages, unchanged +# https://block.github.io/buzz/book/ this book + +name: Docs + +on: + push: + branches: [main] + paths: &docs_paths + - '**.md' + - 'docs/book/**' + - '.github/workflows/docs.yml' + pull_request: + paths: *docs_paths + workflow_dispatch: + +concurrency: + group: docs-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + MDBOOK_VERSION: '0.5.4' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Install mdBook + run: | + url="https://github.com/rust-lang/mdBook/releases/download/v${MDBOOK_VERSION}/mdbook-v${MDBOOK_VERSION}-x86_64-unknown-linux-gnu.tar.gz" + mkdir -p "$HOME/.local/bin" + curl -fsSL "$url" | tar -xz -C "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + # --check fails the build if any document matches no SECTIONS rule, so a + # new doc has to be placed deliberately rather than landing in Unsorted + # unnoticed. Links already broken in the source are reported on stderr + # without failing — those are pre-existing and fixing them belongs in the + # document, not here. + - name: Assemble book + run: python3 docs/book/build.py --check + + - name: Build + run: mdbook build docs/book + + - uses: actions/upload-artifact@v4 + with: + name: book + path: docs/book/dist + retention-days: 7 + + publish: + name: Publish + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install mdBook + run: | + url="https://github.com/rust-lang/mdBook/releases/download/v${MDBOOK_VERSION}/mdbook-v${MDBOOK_VERSION}-x86_64-unknown-linux-gnu.tar.gz" + mkdir -p "$HOME/.local/bin" + curl -fsSL "$url" | tar -xz -C "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Build + run: | + python3 docs/book/build.py --check + mdbook build docs/book + + - name: Publish to pages branch + run: | + set -euo pipefail + git fetch origin pages + git worktree add /tmp/pages origin/pages + rm -rf /tmp/pages/book + cp -R docs/book/dist /tmp/pages/book + cd /tmp/pages + git add -A book + if git diff --cached --quiet; then + echo "book unchanged; nothing to publish" + exit 0 + fi + git -c user.name='github-actions[bot]' \ + -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -m "docs: publish book from ${GITHUB_SHA:0:8}" + git push origin HEAD:pages diff --git a/docs/book/.gitignore b/docs/book/.gitignore new file mode 100644 index 0000000000..f86b12c9e9 --- /dev/null +++ b/docs/book/.gitignore @@ -0,0 +1,2 @@ +src/ +dist/ diff --git a/docs/book/README.md b/docs/book/README.md new file mode 100644 index 0000000000..3b7a84743d --- /dev/null +++ b/docs/book/README.md @@ -0,0 +1,53 @@ +# Docs book + +Renders the Markdown already tracked in this repo as a browsable book, using +[mdBook](https://rust-lang.github.io/mdBook/). The Markdown stays the source of +truth — this directory holds only the config and the script that assembles it. + +## Local preview + +```bash +cargo install mdbook # or: brew install mdbook +python3 docs/book/build.py +mdbook serve docs/book +``` + +## How pages are chosen + +Pages are **discovered, not listed**. Every tracked `.md` file is published +unless it matches `EXCLUDE` in `build.py`, so a new document appears in the +book by existing. The inverse — an explicit include list — fails silently: a +file nobody remembered to add looks exactly like a file that was never +written. + +`EXCLUDE` covers machine-facing Markdown: agent instructions, prompt +fragments, persona definitions, skill manifests, issue templates and test +fixtures. + +Placement comes from `SECTIONS`, an ordered list of path patterns, first match +wins. Anything matching no rule is still published, under **Unsorted**, and +named on stdout. `--check` turns that into a build failure, which is what CI +runs — a new document has to be placed deliberately rather than drifting into +a catch-all. + +Section ordering is derived from `SECTIONS` itself, so adding a rule is the +only step; there is no second list to keep in sync. + +## Why files keep their repo paths + +Documents are copied into `src/` at their original paths. Relative links +between them (`[NIP-AE](NIP-AE.md)`, `[architecture](../../ARCHITECTURE.md)`) +therefore resolve unchanged, because the distance between any two files is +preserved, and mdBook rewrites `.md` targets to `.html` at build time. + +Three cases still need handling, in `resolve_links()`: + +- mdBook renders `README.md` as `index.html`, so links *to* a README are + rewritten to `index.md`. +- Links to excluded files are repointed at GitHub, where they do exist. +- Anything else is already broken in the repo. Those are reported on stderr + and left alone — the fix belongs in the document. + +That last report currently lists 36 links, all of them bare-number references +to the upstream nostr-protocol NIPs repo (`01.md`, `44.md`, and similar). They +do not resolve on GitHub either. diff --git a/docs/book/book.toml b/docs/book/book.toml new file mode 100644 index 0000000000..56342fa953 --- /dev/null +++ b/docs/book/book.toml @@ -0,0 +1,26 @@ +[book] +title = "Buzz" +description = "A workspace where humans and agents build together, on a relay you own." +authors = ["Block, Inc."] +src = "src" +language = "en" + +[build] +build-dir = "dist" +# Some documents intentionally show malformed or illustrative snippets; broken +# internal links are still worth failing on, and CI runs `--check` for those. +create-missing = false + +[output.html] +default-theme = "navy" +preferred-dark-theme = "navy" +git-repository-url = "https://github.com/block/buzz" +edit-url-template = "https://github.com/block/buzz/edit/main/{path}" + +[output.html.search] +enable = true +limit-results = 30 + +[output.html.fold] +enable = true +level = 1 diff --git a/docs/book/build.py b/docs/book/build.py new file mode 100755 index 0000000000..8282de0dca --- /dev/null +++ b/docs/book/build.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Assemble the mdBook site from the Markdown already tracked in this repo. + +Design notes +------------ +Pages are *discovered*, not listed. Every tracked `.md` file is published +unless it matches EXCLUDE, so a new document appears in the book by simply +existing. The inverse — an explicit include list — fails silently: a file +nobody remembered to add is indistinguishable from a file that was never +written. + +Discovered files are copied into `src/` at their original repo paths. That is +deliberate: relative links between documents (`[NIP-AE](NIP-AE.md)`, +`[architecture](../../ARCHITECTURE.md)`) keep working untouched, because the +distance between any two files is preserved. mdBook rewrites `.md` targets to +`.html` at build time, so no link rewriting happens here. + +Anything that matches no SECTION rule is still published, under "Unsorted", +and named on stdout. New docs are therefore loud rather than lost. + +Usage: python3 docs/book/build.py [--out DIR] [--check] +""" + +from __future__ import annotations + +import argparse +import pathlib +import re +import shutil +import subprocess +import sys + +REPO = pathlib.Path(__file__).resolve().parents[2] + +# Machine-facing Markdown: agent instructions, prompt fragments, persona +# definitions, skill manifests, templates and test fixtures. These are part of +# how the project is built, not documentation about it. +EXCLUDE = [ + r"^\.github/", + r"^\.(agents|claude|codex|goose)/", + r"^AGENTS\.md$", + r"^CLAUDE\.md$", + r"/tests/fixtures/", + r"^benchmarks/.*/personas/", + r"\.persona\.md$", + r"/SKILL\.md$", + r"^crates/buzz-acp/src/base_prompt\.md$", + r"^desktop/src-tauri/src/managed_agents/.*\.md$", + r"^desktop/src/features/agents/AGENTS\.md$", + r"^desktop/src-tauri/resources/", + r"^mobile/ios/", + r"^docs/book/", +] + +# Ordered; first match wins. (section title, path pattern) +SECTIONS: list[tuple[str, str]] = [ + ("Architecture", r"^(ARCHITECTURE|NOSTR)\.md$"), + ("Architecture", r"^docs/(multi-tenant-|bridge-|git-on-object-storage)"), + ("Architecture", r"^perf/"), + ("Protocol (NIPs)", r"NIP-[A-Z]{2}\.md$"), + ("Agents", r"^VISION_(AGENT|REMOTE_AGENTS)\.md$"), + ("Agents", r"^crates/buzz-(acp|agent|cli)/README\.md$"), + ("Agents", r"^docs/(MCP_DRIVEN_HOOKS|buzz-shared-compute-dev|welcome-kickoff)"), + ("Self-Hosting", r"^deploy/"), + ("Self-Hosting", r"^docs/(push-gateway|admin/|linux-rendering)"), + ("Self-Hosting", r"^scripts/cutover/"), + ("Vision", r"^VISION"), + ("Formal Specs", r"^docs/formal/"), + ("Apps", r"^(desktop|mobile)/"), + ("Examples", r"^(examples|benchmarks)/"), + ("Crates", r"^crates/"), + ("Project", r"^(CONTRIBUTING|TESTING|RELEASING|SECURITY|CHANGELOG|CODE_OF_CONDUCT|GOVERNANCE)\.md$"), + ("Project", r"^bin/"), +] + + +# Section order follows first appearance in SECTIONS, so adding a rule is the +# only step needed — there is no second list to keep in sync. "Unsorted" is +# always last. +SECTION_ORDER = list(dict.fromkeys(title for title, _ in SECTIONS)) + ["Unsorted"] + +FENCE = re.compile(r"^\s*(```|~~~)") +SETEXT = re.compile(r"^(.+)\n[=-]{3,}\s*$", re.MULTILINE) + + +GITHUB_BLOB = "https://github.com/block/buzz/blob/main/" +MD_LINK = re.compile(r"(\]\()([^)\s]+?\.md)((?:#[^)\s]*)?)(\))") + + +def resolve_links(out: pathlib.Path, published: set[str]) -> list[tuple[str, str]]: + """Repoint links that the copy step would otherwise leave dangling. + + Paths are preserved on copy, so most relative links already resolve. Three + cases still need handling: + + * mdBook renders `README.md` as `index.html`, so links *to* a README + must be rewritten or they 404. + * Links to files excluded from the book (agent instructions, prompts) + are sent to GitHub, where they do exist. + * Anything else is already broken in the repo. Those are reported rather + than rewritten — the fix belongs in the source document. + """ + broken: list[tuple[str, str]] = [] + + for page in sorted(out.rglob("*.md")): + if page.name == "SUMMARY.md": + continue + rel = page.relative_to(out).as_posix() + here = pathlib.PurePosixPath(rel).parent + lines, in_fence, changed = [], False, False + + for line in page.read_text(encoding="utf-8").split("\n"): + if FENCE.match(line): + in_fence = not in_fence + lines.append(line) + continue + if in_fence: + lines.append(line) + continue + + def fix(match: re.Match[str]) -> str: + nonlocal changed + prefix, target, anchor, close = match.groups() + if re.match(r"^(https?:|mailto:|/)", target): + return match.group(0) + resolved = str((here / target).as_posix()) + while "/../" in resolved or resolved.startswith("../"): + resolved = re.sub(r"[^/]+/\.\./", "", resolved, count=1) + if resolved.startswith("../"): + break + if resolved in published: + if pathlib.PurePosixPath(resolved).name == "README.md": + changed = True + return f"{prefix}{target[:-len('README.md')]}index.md{anchor}{close}" + return match.group(0) + if (REPO / resolved).is_file(): + changed = True + return f"{prefix}{GITHUB_BLOB}{resolved}{anchor}{close}" + broken.append((rel, target)) + return match.group(0) + + lines.append(MD_LINK.sub(fix, line)) + + if changed: + page.write_text("\n".join(lines), encoding="utf-8") + + return broken + + +def tracked_markdown() -> list[str]: + out = subprocess.run( + ["git", "ls-files", "*.md"], cwd=REPO, capture_output=True, text=True, check=True + ) + return sorted(out.stdout.split()) + + +def title_of(path: pathlib.Path, rel: str) -> str: + """First ATX or setext heading, else a name derived from the path.""" + text = path.read_text(encoding="utf-8", errors="replace") + atx = re.search(r"^#\s+(.+)$", text, re.MULTILINE) + setext = SETEXT.search(text) + if atx and (not setext or atx.start() < setext.start()): + return atx.group(1).strip() + if setext: + return setext.group(1).strip() + stem = pathlib.Path(rel).stem + if stem.upper() in {"README", "NOTE"}: + parent = pathlib.Path(rel).parent.name + return parent or stem + return stem.replace("_", " ").replace("-", " ").title() + + +def section_of(rel: str) -> str: + for title, pattern in SECTIONS: + if re.search(pattern, rel): + return title + return "Unsorted" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", default=str(pathlib.Path(__file__).parent / "src")) + parser.add_argument( + "--check", action="store_true", + help="fail if any document lands in Unsorted", + ) + args = parser.parse_args() + + out = pathlib.Path(args.out) + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + + excluded = re.compile("|".join(EXCLUDE)) + grouped: dict[str, list[tuple[str, str]]] = {} + published: set[str] = set() + intro: str | None = None + + for rel in tracked_markdown(): + if excluded.search(rel): + continue + source = REPO / rel + destination = out / rel + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + published.add(rel) + if rel == "README.md": + intro = rel + continue + grouped.setdefault(section_of(rel), []).append((title_of(source, rel), rel)) + + broken = resolve_links(out, published) + + lines = ["# Summary", ""] + if intro: + lines += [f"[Introduction]({intro})", ""] + for section in SECTION_ORDER: + entries = grouped.get(section) + if not entries: + continue + lines.append(f"# {section}") + lines.append("") + for title, rel in sorted(entries): + lines.append(f"- [{title}]({rel})") + lines.append("") + (out / "SUMMARY.md").write_text("\n".join(lines), encoding="utf-8") + + orphaned = set(grouped) - set(SECTION_ORDER) + if orphaned: + raise AssertionError(f"sections with no slot in SECTION_ORDER: {sorted(orphaned)}") + + written = sum(len(grouped[s]) for s in SECTION_ORDER if s in grouped) + total = written + (1 if intro else 0) + print(f"published {total} pages") + for section in SECTION_ORDER: + if grouped.get(section): + print(f" {section}: {len(grouped[section])}") + + if broken: + print(f"\n{len(broken)} link(s) already broken in the source docs:", file=sys.stderr) + for page_rel, target in sorted(set(broken)): + print(f" {page_rel} -> {target}", file=sys.stderr) + + unsorted = grouped.get("Unsorted", []) + if unsorted: + print("\nUnsorted — add a SECTIONS rule or an EXCLUDE entry:", file=sys.stderr) + for _, rel in sorted(unsorted, key=lambda e: e[1]): + print(f" {rel}", file=sys.stderr) + if args.check: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())