diff --git a/CHANGELOG.md b/CHANGELOG.md index 2556e92..644cb24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,21 @@ so this is a note about cost rather than an instruction — `semantic index ## [Unreleased] +### Fixed + +- **[reindex]** **A TypeScript or JavaScript file's own documentation is now + indexed.** A leading `/** … */` block is a file's documentation when no + declaration follows it to own it — the `@fileoverview` and `@module` + convention, and any design note written above the imports. The walk dropped + it, because association is by adjacency and an `import`, a bare + `export {}`, or a re-export carries no symbol to attach it to. Such a file + therefore contributed only signatures to the index, and the one piece of + prose saying what the file is for was absent from search. It now emits as a + `file` chunk keyed `ts/file`, which is for TypeScript what the `package` + chunk is for Go. A shebang and a directive prologue (`"use client"`) are + skipped when looking for the block, in either order, and a doc block that + does document a declaration is untouched. + ## [0.1.3] — 2026-08-03 A better default embedding model, a registry so it is no longer the only one, diff --git a/README.md b/README.md index b0a2c92..c9a684d 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ a body is implementation, and embedding it dilutes what the symbol is for. | Markdown | `.md` `.markdown` | Heading tree | | Go | `.go` | package · type · func · method · documented const/var | | Python | `.py` `.pyi` | module · class · method · func · documented constant | -| TypeScript | `.ts` `.mts` `.cts` `.tsx` | func · class · interface · type · enum · const | -| JavaScript | `.js` `.mjs` `.cjs` `.jsx` | func · class · CommonJS export | +| TypeScript | `.ts` `.mts` `.cts` `.tsx` | file doc · func · class · interface · type · enum · const | +| JavaScript | `.js` `.mjs` `.cjs` `.jsx` | file doc · func · class · CommonJS export | | Java | `.java` | class · interface · enum · record · method | | C# | `.cs` | namespace · class · interface · struct · record · method · property | | Rust | `.rs` | struct · enum · trait · mod · func · impl method | diff --git a/pkg/chunk/typescript.go b/pkg/chunk/typescript.go index 2f880e9..3f1ec49 100644 --- a/pkg/chunk/typescript.go +++ b/pkg/chunk/typescript.go @@ -27,11 +27,15 @@ import ( // Variant constants for the TypeScript chunk kinds without a Go analogue. // Functions, methods, and const/var reuse the func/method/value variants from // gosource.go, and a type alias reuses VariantType — they mean the same thing -// across languages, so search filters and displays stay uniform. +// across languages, so search filters and displays stay uniform. VariantFile +// is the file's own documentation, which VariantPackage is for Go; it is not +// VariantModule, which names a declared module or namespace in the languages +// that have one. const ( VariantClass = "class" VariantInterface = "interface" VariantEnum = "enum" + VariantFile = "file" ) // tsValueSigMax caps how much of a plain const/var initializer is inlined into @@ -91,6 +95,7 @@ func tsSource(content string, lang *sitter.Language) []Chunk { defer tree.Close() w := &tsWalker{source: source, seen: map[string]bool{}} + w.fileDoc(tree.RootNode()) w.walk(tree.RootNode(), "") return w.out } @@ -142,6 +147,77 @@ func (w *tsWalker) walk(container *sitter.Node, prefix string) { } } +// fileDoc emits a leading JSDoc block that documents no declaration as the +// file's own chunk, the way GoSource emits a package doc. +// +// A file whose first statement is an import, or whose only export is a +// re-export, leaves its leading block with nothing below it to own. The walk +// then drops that block, which is how a file's own prose — the `@module` and +// `@fileoverview` convention, and every design note written above the imports +// — stayed out of the index. It is the highest-signal prose in such a file: +// the symbols below it carry signatures, and only this says what the file is +// for. +func (w *tsWalker) fileDoc(root *sitter.Node) { + start := afterPrologue(root) + first := root.NamedChild(start) + if first == nil || first.Kind() != "comment" { + return + } + doc := jsDoc(first, w.source) + if doc == "" || documentsNext(root.NamedChild(start+1)) { + return + } + crumb := VariantFile + w.out = append(w.out, Chunk{ + Key: "ts/" + VariantFile, + Heading: crumb, + Variant: VariantFile, + Text: crumb + "\n\n" + doc, + Line: nodeLine(first), + }) +} + +// afterPrologue returns the index of the first child past a shebang and any +// directive prologue, so `"use client"` above the doc block does not hide it. +func afterPrologue(root *sitter.Node) uint { + index := uint(0) + for ; index < root.NamedChildCount(); index++ { + if !prologue(root.NamedChild(index)) { + break + } + } + return index +} + +// prologue reports whether n is a shebang or a bare string statement, the two +// things a file may carry above its own documentation. +func prologue(n *sitter.Node) bool { + if n.Kind() == "hash_bang_line" { + return true + } + if n.Kind() != "expression_statement" || n.NamedChildCount() != 1 { + return false + } + return n.NamedChild(0).Kind() == "string" +} + +// documentsNext reports whether the node after a leading comment is a +// declaration the walk hands that comment to. An import, a re-export carrying +// no declaration, a directive, a second comment block, and the end of the file +// all leave the comment unowned, which makes it documentation about the file. +func documentsNext(next *sitter.Node) bool { + if next == nil { + return false + } + switch next.Kind() { + case "comment", "import_statement": + return false + case "export_statement": + return next.ChildByFieldName("declaration") != nil + } + return !prologue(next) +} + // declare dispatches one declaration to the emitter for its kind. func (w *tsWalker) declare(decl *sitter.Node, prefix, doc string) { switch decl.Kind() { diff --git a/pkg/chunk/typescript_test.go b/pkg/chunk/typescript_test.go index c2446bd..8f02fda 100644 --- a/pkg/chunk/typescript_test.go +++ b/pkg/chunk/typescript_test.go @@ -243,3 +243,83 @@ module.exports = { sub, inline: (a, b) => a / b };` t.Errorf("shorthand re-export should not duplicate sub, got %q", c.Key) } } + +// sampleFileDocTS is a file whose leading block documents the file rather than +// a symbol: an import follows it, so no declaration owns it. +const sampleFileDocTS = `/** + * The widget store is the one place a widget's bytes are written. + * + * A write is idempotent, so a retry after a timeout costs a lookup rather + * than a duplicate. + */ +import { open } from "node:fs/promises"; + +/** put writes one widget. */ +export function put(id: string): void {} +` + +func TestTypeScript_FileDocBeforeImports(t *testing.T) { + t.Parallel() + got := TypeScript(sampleFileDocTS) + c := find(got, "ts/file") + if c.Key == "" { + t.Fatalf("no file chunk emitted (keys: %v)", keysOf(got)) + } + if c.Variant != VariantFile { + t.Errorf("variant = %q, want %q", c.Variant, VariantFile) + } + if !strings.HasPrefix(c.Text, c.Heading) { + t.Errorf("text should lead with the breadcrumb, got %q", c.Text) + } + for _, want := range []string{"one place a widget's bytes are written", "A write is idempotent"} { + if !strings.Contains(c.Text, want) { + t.Errorf("file text missing %q, got %q", want, c.Text) + } + } + if put := find(got, "ts/func/put"); put.Key == "" { + t.Errorf("the declaration after the module doc is still chunked (keys: %v)", keysOf(got)) + } +} + +func TestTypeScript_FileDocWithoutDeclarations(t *testing.T) { + t.Parallel() + cases := []struct { + name string + source string + }{ + {"re-export only", "/** A widget is a thing. */\nexport {};\n"}, + {"nothing at all", "/** A widget is a thing. */\n"}, + {"a second block below", "/** A widget is a thing. */\n\n/** put writes one. */\nexport function put(): void {}\n"}, + {"a directive above", "\"use client\";\n\n/** A widget is a thing. */\nimport { open } from \"node:fs/promises\";\n"}, + {"a directive below", "/** A widget is a thing. */\n\"use client\";\n"}, + {"a shebang above", "#!/usr/bin/env node\n/** A widget is a thing. */\nimport { open } from \"node:fs/promises\";\n"}, + {"a star export below", "/** A widget is a thing. */\nexport * from \"./widget\";\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if c := find(TypeScript(tc.source), "ts/file"); c.Key == "" { + t.Errorf("no file chunk emitted for %s", tc.name) + } + }) + } +} + +func TestTypeScript_DocumentedDeclarationKeepsItsDoc(t *testing.T) { + t.Parallel() + got := TypeScript("/** put writes one widget. */\nexport function put(): void {}\n") + if c := find(got, "ts/file"); c.Key != "" { + t.Errorf("a doc block owned by a declaration became a file chunk: %q", c.Text) + } + if c := find(got, "ts/func/put"); !strings.Contains(c.Text, "writes one widget") { + t.Errorf("declaration lost its doc, got %q", c.Text) + } +} + +func TestTypeScript_LineCommentIsNotAModuleDoc(t *testing.T) { + t.Parallel() + got := TypeScript("// internal notes, not documentation\nimport { open } from \"node:fs/promises\";\n") + if c := find(got, "ts/file"); c.Key != "" { + t.Errorf("a line comment became a file chunk: %q", c.Text) + } +}