From 49e4a9789ec4c6a0110c75564a6d04212e3c2ead Mon Sep 17 00:00:00 2001 From: Conner Ruhl Date: Wed, 19 Aug 2026 15:58:36 -0700 Subject: [PATCH] Index a TypeScript file's own documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A leading `/** … */` block documents the file when no declaration follows it to own it: the `@fileoverview` and `@module` convention, and any design note written above the imports. Association is by adjacency, and an `import`, a bare `export {}`, or a re-export carries no symbol to attach such a block to, so the walk dropped it. A file written that way contributed only signatures to the index, and the one piece of prose saying what it is for could not be retrieved at all. It now emits as a `file` chunk keyed `ts/file`, which is for TypeScript and JavaScript what the `package` chunk is for Go. A shebang and a directive prologue are skipped when looking for the block, in either order, and a block that does document a declaration is untouched. The Go pin moves to 1.26.7 because two standard library advisories landed against 1.26.5 after the last release, both fixed in 1.26.6, and the vulnerability scan reaches them through the model download path. Signed-off-by: Conner Ruhl Signed-off-by: Conner Ruhl --- CHANGELOG.md | 15 +++++++ README.md | 4 +- pkg/chunk/typescript.go | 78 ++++++++++++++++++++++++++++++++++- pkg/chunk/typescript_test.go | 80 ++++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2556e92..644cb24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,21 @@ so this is a note about cost rather than an instruction — `semantic index ## [Unreleased] +### Fixed + +- **[reindex]** **A TypeScript or JavaScript file's own documentation is now + indexed.** A leading `/** … */` block is a file's documentation when no + declaration follows it to own it — the `@fileoverview` and `@module` + convention, and any design note written above the imports. The walk dropped + it, because association is by adjacency and an `import`, a bare + `export {}`, or a re-export carries no symbol to attach it to. Such a file + therefore contributed only signatures to the index, and the one piece of + prose saying what the file is for was absent from search. It now emits as a + `file` chunk keyed `ts/file`, which is for TypeScript what the `package` + chunk is for Go. A shebang and a directive prologue (`"use client"`) are + skipped when looking for the block, in either order, and a doc block that + does document a declaration is untouched. + ## [0.1.3] — 2026-08-03 A better default embedding model, a registry so it is no longer the only one, diff --git a/README.md b/README.md index b0a2c92..c9a684d 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ a body is implementation, and embedding it dilutes what the symbol is for. | Markdown | `.md` `.markdown` | Heading tree | | Go | `.go` | package · type · func · method · documented const/var | | Python | `.py` `.pyi` | module · class · method · func · documented constant | -| TypeScript | `.ts` `.mts` `.cts` `.tsx` | func · class · interface · type · enum · const | -| JavaScript | `.js` `.mjs` `.cjs` `.jsx` | func · class · CommonJS export | +| TypeScript | `.ts` `.mts` `.cts` `.tsx` | file doc · func · class · interface · type · enum · const | +| JavaScript | `.js` `.mjs` `.cjs` `.jsx` | file doc · func · class · CommonJS export | | Java | `.java` | class · interface · enum · record · method | | C# | `.cs` | namespace · class · interface · struct · record · method · property | | Rust | `.rs` | struct · enum · trait · mod · func · impl method | diff --git a/pkg/chunk/typescript.go b/pkg/chunk/typescript.go index 2f880e9..3f1ec49 100644 --- a/pkg/chunk/typescript.go +++ b/pkg/chunk/typescript.go @@ -27,11 +27,15 @@ import ( // Variant constants for the TypeScript chunk kinds without a Go analogue. // Functions, methods, and const/var reuse the func/method/value variants from // gosource.go, and a type alias reuses VariantType — they mean the same thing -// across languages, so search filters and displays stay uniform. +// across languages, so search filters and displays stay uniform. VariantFile +// is the file's own documentation, which VariantPackage is for Go; it is not +// VariantModule, which names a declared module or namespace in the languages +// that have one. const ( VariantClass = "class" VariantInterface = "interface" VariantEnum = "enum" + VariantFile = "file" ) // tsValueSigMax caps how much of a plain const/var initializer is inlined into @@ -91,6 +95,7 @@ func tsSource(content string, lang *sitter.Language) []Chunk { defer tree.Close() w := &tsWalker{source: source, seen: map[string]bool{}} + w.fileDoc(tree.RootNode()) w.walk(tree.RootNode(), "") return w.out } @@ -142,6 +147,77 @@ func (w *tsWalker) walk(container *sitter.Node, prefix string) { } } +// fileDoc emits a leading JSDoc block that documents no declaration as the +// file's own chunk, the way GoSource emits a package doc. +// +// A file whose first statement is an import, or whose only export is a +// re-export, leaves its leading block with nothing below it to own. The walk +// then drops that block, which is how a file's own prose — the `@module` and +// `@fileoverview` convention, and every design note written above the imports +// — stayed out of the index. It is the highest-signal prose in such a file: +// the symbols below it carry signatures, and only this says what the file is +// for. +func (w *tsWalker) fileDoc(root *sitter.Node) { + start := afterPrologue(root) + first := root.NamedChild(start) + if first == nil || first.Kind() != "comment" { + return + } + doc := jsDoc(first, w.source) + if doc == "" || documentsNext(root.NamedChild(start+1)) { + return + } + crumb := VariantFile + w.out = append(w.out, Chunk{ + Key: "ts/" + VariantFile, + Heading: crumb, + Variant: VariantFile, + Text: crumb + "\n\n" + doc, + Line: nodeLine(first), + }) +} + +// afterPrologue returns the index of the first child past a shebang and any +// directive prologue, so `"use client"` above the doc block does not hide it. +func afterPrologue(root *sitter.Node) uint { + index := uint(0) + for ; index < root.NamedChildCount(); index++ { + if !prologue(root.NamedChild(index)) { + break + } + } + return index +} + +// prologue reports whether n is a shebang or a bare string statement, the two +// things a file may carry above its own documentation. +func prologue(n *sitter.Node) bool { + if n.Kind() == "hash_bang_line" { + return true + } + if n.Kind() != "expression_statement" || n.NamedChildCount() != 1 { + return false + } + return n.NamedChild(0).Kind() == "string" +} + +// documentsNext reports whether the node after a leading comment is a +// declaration the walk hands that comment to. An import, a re-export carrying +// no declaration, a directive, a second comment block, and the end of the file +// all leave the comment unowned, which makes it documentation about the file. +func documentsNext(next *sitter.Node) bool { + if next == nil { + return false + } + switch next.Kind() { + case "comment", "import_statement": + return false + case "export_statement": + return next.ChildByFieldName("declaration") != nil + } + return !prologue(next) +} + // declare dispatches one declaration to the emitter for its kind. func (w *tsWalker) declare(decl *sitter.Node, prefix, doc string) { switch decl.Kind() { diff --git a/pkg/chunk/typescript_test.go b/pkg/chunk/typescript_test.go index c2446bd..8f02fda 100644 --- a/pkg/chunk/typescript_test.go +++ b/pkg/chunk/typescript_test.go @@ -243,3 +243,83 @@ module.exports = { sub, inline: (a, b) => a / b };` t.Errorf("shorthand re-export should not duplicate sub, got %q", c.Key) } } + +// sampleFileDocTS is a file whose leading block documents the file rather than +// a symbol: an import follows it, so no declaration owns it. +const sampleFileDocTS = `/** + * The widget store is the one place a widget's bytes are written. + * + * A write is idempotent, so a retry after a timeout costs a lookup rather + * than a duplicate. + */ +import { open } from "node:fs/promises"; + +/** put writes one widget. */ +export function put(id: string): void {} +` + +func TestTypeScript_FileDocBeforeImports(t *testing.T) { + t.Parallel() + got := TypeScript(sampleFileDocTS) + c := find(got, "ts/file") + if c.Key == "" { + t.Fatalf("no file chunk emitted (keys: %v)", keysOf(got)) + } + if c.Variant != VariantFile { + t.Errorf("variant = %q, want %q", c.Variant, VariantFile) + } + if !strings.HasPrefix(c.Text, c.Heading) { + t.Errorf("text should lead with the breadcrumb, got %q", c.Text) + } + for _, want := range []string{"one place a widget's bytes are written", "A write is idempotent"} { + if !strings.Contains(c.Text, want) { + t.Errorf("file text missing %q, got %q", want, c.Text) + } + } + if put := find(got, "ts/func/put"); put.Key == "" { + t.Errorf("the declaration after the module doc is still chunked (keys: %v)", keysOf(got)) + } +} + +func TestTypeScript_FileDocWithoutDeclarations(t *testing.T) { + t.Parallel() + cases := []struct { + name string + source string + }{ + {"re-export only", "/** A widget is a thing. */\nexport {};\n"}, + {"nothing at all", "/** A widget is a thing. */\n"}, + {"a second block below", "/** A widget is a thing. */\n\n/** put writes one. */\nexport function put(): void {}\n"}, + {"a directive above", "\"use client\";\n\n/** A widget is a thing. */\nimport { open } from \"node:fs/promises\";\n"}, + {"a directive below", "/** A widget is a thing. */\n\"use client\";\n"}, + {"a shebang above", "#!/usr/bin/env node\n/** A widget is a thing. */\nimport { open } from \"node:fs/promises\";\n"}, + {"a star export below", "/** A widget is a thing. */\nexport * from \"./widget\";\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if c := find(TypeScript(tc.source), "ts/file"); c.Key == "" { + t.Errorf("no file chunk emitted for %s", tc.name) + } + }) + } +} + +func TestTypeScript_DocumentedDeclarationKeepsItsDoc(t *testing.T) { + t.Parallel() + got := TypeScript("/** put writes one widget. */\nexport function put(): void {}\n") + if c := find(got, "ts/file"); c.Key != "" { + t.Errorf("a doc block owned by a declaration became a file chunk: %q", c.Text) + } + if c := find(got, "ts/func/put"); !strings.Contains(c.Text, "writes one widget") { + t.Errorf("declaration lost its doc, got %q", c.Text) + } +} + +func TestTypeScript_LineCommentIsNotAModuleDoc(t *testing.T) { + t.Parallel() + got := TypeScript("// internal notes, not documentation\nimport { open } from \"node:fs/promises\";\n") + if c := find(got, "ts/file"); c.Key != "" { + t.Errorf("a line comment became a file chunk: %q", c.Text) + } +}