diff --git a/packages/registry/src/build.ts b/packages/registry/src/build.ts index ddefdf4..092c3f1 100644 --- a/packages/registry/src/build.ts +++ b/packages/registry/src/build.ts @@ -24,6 +24,7 @@ import { type UnversionedDefinition, type VersionedDefinition, } from "./definition.js"; +import { excludeFiles } from "./glob.js"; import { downloadAndExtractZip } from "./zip.js"; export interface RegistryBuildResult extends BuildResult { @@ -82,6 +83,7 @@ export async function buildFromDefinition( entry.source.url, constructTag(entry.tag_pattern, version), entry.source.docs_path, + entry.source.exclude_paths, entry.source.lang, outputPath, definition, @@ -172,10 +174,16 @@ export async function buildUnversioned( stdio: ["pipe", "pipe", "pipe"], }).trim(); - const files = readLocalDocsFiles(tempDir, { - path: source.docs_path, - lang: source.lang, - }); + // Filter before the emptiness check, so an over-broad exclude_paths fails + // loudly here instead of publishing an empty package. + const files = excludeFiles( + readLocalDocsFiles(tempDir, { + path: source.docs_path, + lang: source.lang, + }), + source.exclude_paths, + source.docs_path, + ); if (files.length === 0) { throw new Error( @@ -208,6 +216,7 @@ function buildFromGit( url: string, tag: string, docsPath: string | undefined, + excludePaths: string[] | undefined, lang: string, outputPath: string, definition: VersionedDefinition, @@ -216,7 +225,13 @@ function buildFromGit( const { tempDir, cleanup } = cloneRepository(url, tag); try { - const files = readLocalDocsFiles(tempDir, { path: docsPath, lang }); + // Filter before the emptiness check, so an over-broad exclude_paths fails + // loudly here instead of publishing an empty package. + const files = excludeFiles( + readLocalDocsFiles(tempDir, { path: docsPath, lang }), + excludePaths, + docsPath, + ); if (files.length === 0) { throw new Error(`No documentation files found in ${url} at tag ${tag}`); diff --git a/packages/registry/src/definition.ts b/packages/registry/src/definition.ts index 370f1d5..9381e23 100644 --- a/packages/registry/src/definition.ts +++ b/packages/registry/src/definition.ts @@ -35,6 +35,7 @@ const GitSourceSchema = z.object({ */ ref: z.string().optional(), docs_path: z.string().optional(), + exclude_paths: z.array(z.string()).optional(), // glob patterns to exclude lang: z.string().default("en"), }); diff --git a/packages/registry/src/glob.test.ts b/packages/registry/src/glob.test.ts new file mode 100644 index 0000000..9cb4cfe --- /dev/null +++ b/packages/registry/src/glob.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { excludeFiles } from "./glob.js"; + +const files = (...paths: string[]) => paths.map((path) => ({ path })); + +describe("excludeFiles", () => { + it("returns the input untouched when no patterns are given", () => { + const input = files("a.md", "b.md"); + expect(excludeFiles(input, undefined)).toBe(input); + expect(excludeFiles(input, [])).toBe(input); + }); + + it("matches ** across segments and * within one", () => { + const input = files( + "tutorials/scripting/c_sharp/basics.md", + "tutorials/scripting/gdscript/basics.md", + "index.md", + "nested/index.md", + ); + + expect( + excludeFiles(input, ["tutorials/scripting/c_sharp/**"]).map( + (f) => f.path, + ), + ).toEqual([ + "tutorials/scripting/gdscript/basics.md", + "index.md", + "nested/index.md", + ]); + + // A single * must not cross a separator, so "nested/index.md" survives. + expect(excludeFiles(input, ["*.md"]).map((f) => f.path)).toEqual([ + "tutorials/scripting/c_sharp/basics.md", + "tutorials/scripting/gdscript/basics.md", + "nested/index.md", + ]); + }); + + it("matches relative to docs_path, as zip sources do", () => { + // Git file paths keep the docs_path prefix; the pattern must not have to. + const input = files("docs/api/internal/secret.md", "docs/api/public.md"); + + expect( + excludeFiles(input, ["internal/**"], "docs/api").map((f) => f.path), + ).toEqual(["docs/api/public.md"]); + + // Without the docs_path the same pattern matches nothing, since the + // stored path still carries the prefix. + expect(excludeFiles(input, ["internal/**"]).map((f) => f.path)).toEqual( + input.map((f) => f.path), + ); + }); + + it("can exclude everything, leaving the caller to reject an empty build", () => { + // The builders check for zero files after filtering; an over-broad pattern + // must therefore be able to empty the list rather than silently no-op. + expect(excludeFiles(files("a.md", "b.md"), ["**"])).toHaveLength(0); + }); + + it("treats glob metacharacters in a pattern literally where unsupported", () => { + const input = files("a+b.md", "axb.md"); + expect(excludeFiles(input, ["a+b.md"]).map((f) => f.path)).toEqual([ + "axb.md", + ]); + }); +}); diff --git a/packages/registry/src/glob.ts b/packages/registry/src/glob.ts new file mode 100644 index 0000000..f1fabf8 --- /dev/null +++ b/packages/registry/src/glob.ts @@ -0,0 +1,42 @@ +/** + * Glob matching for `exclude_paths`, shared by the zip and git source builders. + */ + +/** + * Compile a simple glob pattern to a RegExp. + * Supports * (any chars except /) and ** (any chars including /). + */ +export function compileGlob(pattern: string): RegExp { + const escaped = pattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*\*/g, "\0") + .replace(/\*/g, "[^/]*") + .replace(/\0/g, ".*"); + return new RegExp(`^${escaped}$`); +} + +/** + * Drop files matching any exclude pattern. + * + * Patterns are matched against the path relative to `docsPath`, so a definition + * reads the same whether its source is a zip or a git clone: the zip builder + * strips the prefix while extracting, and git file paths still carry it. + */ +export function excludeFiles( + files: T[], + excludePaths: string[] | undefined, + docsPath?: string, +): T[] { + if (!excludePaths?.length) return files; + + const patterns = excludePaths.map(compileGlob); + const prefix = docsPath ? `${docsPath.replace(/\/+$/, "")}/` : ""; + + return files.filter((file) => { + const relative = + prefix && file.path.startsWith(prefix) + ? file.path.slice(prefix.length) + : file.path; + return !patterns.some((re) => re.test(relative)); + }); +} diff --git a/packages/registry/src/zip.ts b/packages/registry/src/zip.ts index 35ae5e0..c8ad7e9 100644 --- a/packages/registry/src/zip.ts +++ b/packages/registry/src/zip.ts @@ -7,6 +7,7 @@ */ import { inflateRawSync } from "node:zlib"; +import { compileGlob } from "./glob.js"; const DOCUMENTATION_EXTENSIONS = [ ".md", @@ -99,19 +100,6 @@ export async function downloadAndExtractZip( return files; } -/** - * Compile a simple glob pattern to a RegExp. - * Supports * (any chars except /) and ** (any chars including /). - */ -function compileGlob(pattern: string): RegExp { - const escaped = pattern - .replace(/[.+^${}()|[\]\\]/g, "\\$&") - .replace(/\*\*/g, "\0") - .replace(/\*/g, "[^/]*") - .replace(/\0/g, ".*"); - return new RegExp(`^${escaped}$`); -} - /** Get lowercase file extension including the dot. */ function getExtension(path: string): string { const lastDot = path.lastIndexOf("."); diff --git a/registry/README.md b/registry/README.md index 79eec78..1beac55 100644 --- a/registry/README.md +++ b/registry/README.md @@ -92,6 +92,29 @@ versions: > > Outside those four directories, use **unversioned** or **versioned-by-zip**. +## Excluding parts of a source + +`docs_path` narrows a source to one directory. When the directory you need also holds +material that belongs to a different package, `exclude_paths` prunes it: + +```yaml +source: + type: git + url: https://github.com/godotengine/godot-docs + ref: stable + exclude_paths: + - "tutorials/scripting/c_sharp/**" +``` + +Patterns are glob-style — `*` matches within a path segment, `**` across segments — and +are matched against the path **relative to `docs_path`** when one is set, or to the +repository root when it is not. Excluding everything is an error rather than an empty +package. + +Reach for this when a wider `docs_path` would drag in a sibling language or framework +that then outranks the docs you actually want; prefer narrowing `docs_path` when the +content you want is already isolated in its own directory. + ## Supported documentation formats Markdown (`.md`, `.mdx`), HTML, AsciiDoc (`.adoc`) and reStructuredText (`.rst`).