diff --git a/internal/manager/biz/knowledge/chunk_test.go b/internal/manager/biz/knowledge/chunk_test.go index 5c63e2914..b9ef76982 100644 --- a/internal/manager/biz/knowledge/chunk_test.go +++ b/internal/manager/biz/knowledge/chunk_test.go @@ -2,11 +2,64 @@ package knowledge import ( "context" + "reflect" "strings" "testing" "unicode/utf8" ) +func TestSplitKnowledgeContent_SelectsByExtension(t *testing.T) { + markdown := "# Platform\n\n## Network\n\nDNS troubleshooting" + plain := "# Platform\n\n## Network\n\nDNS troubleshooting" + tests := []struct { + name string + filename string + content string + want []string + contains string + }{ + { + name: "markdown", + filename: "runbook.md", + content: markdown, + contains: "# Platform\n## Network", + }, + { + name: "markdown alternate extension", + filename: "runbook.MARKDOWN", + content: markdown, + contains: "# Platform\n## Network", + }, + { + name: "plain text", + filename: "runbook.txt", + content: plain, + want: []string{plain}, + }, + { + name: "reStructuredText", + filename: "runbook.rst", + content: plain, + want: []string{plain}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := splitKnowledgeContent(context.Background(), tt.filename, tt.content) + if err != nil { + t.Fatalf("split knowledge content: %v", err) + } + if tt.want != nil && !reflect.DeepEqual(got, tt.want) { + t.Fatalf("chunks = %#v, want %#v", got, tt.want) + } + if tt.contains != "" && !strings.Contains(strings.Join(got, "\n"), tt.contains) { + t.Fatalf("chunks = %#v, missing heading hierarchy %q", got, tt.contains) + } + }) + } +} + // TestSplitForChunks_ShortDoc — a doc under chunkChars stays as one piece. // Regression for the pre-chunking behaviour: a 1k-char concept page must // produce a single chunk so its qdrant point id stays equal to diff --git a/internal/manager/biz/knowledge/scan_test.go b/internal/manager/biz/knowledge/scan_test.go new file mode 100644 index 000000000..cd273763e --- /dev/null +++ b/internal/manager/biz/knowledge/scan_test.go @@ -0,0 +1,40 @@ +package knowledge + +import ( + "os" + "path/filepath" + "testing" +) + +func TestScanRepoFiles_IncludesMarkdownExtension(t *testing.T) { + root := t.TempDir() + files := map[string]string{ + "guide.md": "# Markdown\n\ncontent", + "guide.markdown": "# Alternate\n\ncontent", + "guide.txt": "plain text", + "guide.rst": "plain rst", + "guide.yaml": "ignored: true", + } + for name, content := range files { + if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + got, err := scanRepoFiles(root) + if err != nil { + t.Fatalf("scanRepoFiles: %v", err) + } + seen := make(map[string]bool, len(got)) + for _, file := range got { + seen[file.URL] = true + } + for _, name := range []string{"guide.md", "guide.markdown", "guide.txt", "guide.rst"} { + if !seen[name] { + t.Errorf("scanRepoFiles omitted %q", name) + } + } + if seen["guide.yaml"] { + t.Error("scanRepoFiles indexed unsupported YAML") + } +} diff --git a/internal/manager/biz/knowledge/usecase.go b/internal/manager/biz/knowledge/usecase.go index 765aeca22..13ef14880 100644 --- a/internal/manager/biz/knowledge/usecase.go +++ b/internal/manager/biz/knowledge/usecase.go @@ -5,8 +5,8 @@ // (or POST /v1/knowledge/docs). We embed and upsert into qdrant. // 2. Repo sync: user registers a git URL; Sync() shells `git clone // --no-single-branch` (or full-history fetch on existing clones) into -// /var/lib/ongrid/repos/, walks the tree for .md / .txt / -// .rst / .yaml / .yml / .toml / .json files, embeds each, replaces +// /var/lib/ongrid/repos/, walks the tree for .md / .markdown / +// .txt / .rst files, embeds each, replaces // the qdrant point set for that repo. // // The repo registrations themselves live in MySQL (knowledge_repos — @@ -282,18 +282,9 @@ func (u *Usecase) UploadDoc(ctx context.Context, in UploadDocInput) (*model.Doc, // docs, and a vault re-sync never touches these (its delete is scoped to // source_type=vault). Returns the logical doc (head chunk id). func (u *Usecase) ingestUpload(ctx context.Context, d model.Doc) (*model.Doc, error) { - parts := make([]string, 0) - - ext := strings.ToLower(filepath.Ext(d.URL)) - switch ext { - case ".md", ".markdown", ".docx", ".pdf": - chunks, err := splitMarkdown(ctx, d.Content) - if err != nil { - return nil, fmt.Errorf("knowledge: split markdown: %w", err) - } - parts = chunks - default: - parts = splitForChunks(d.Content) + parts, err := splitKnowledgeContent(ctx, d.URL, d.Content) + if err != nil { + return nil, fmt.Errorf("knowledge: split %q: %w", d.URL, err) } if len(parts) == 0 { return nil, fmt.Errorf("%w: empty file", errs.ErrInvalid) @@ -1093,7 +1084,10 @@ func (u *Usecase) Sync(ctx context.Context, id uint64) (*model.Repository, error now := time.Now().UTC() chunks := make([]chunkRef, 0, len(files)) for i := range files { - parts := splitForChunks(files[i].Content) + parts, err := splitKnowledgeContent(ctx, files[i].URL, files[i].Content) + if err != nil { + return u.recordSyncFailure(ctx, repo, fmt.Errorf("split file %q: %w", files[i].URL, err)) + } for j, p := range parts { // Chunk 0 prepends the title so the embedding picks up the // "what is this doc" signal — same as the pre-chunking @@ -1228,7 +1222,10 @@ func (u *Usecase) SyncBuiltinVault(ctx context.Context) (int, string, error) { } chunks := make([]chunkRef, 0, len(files)) for i := range files { - parts := splitForChunks(files[i].Content) + parts, err := splitKnowledgeContent(ctx, files[i].URL, files[i].Content) + if err != nil { + return 0, "", fmt.Errorf("knowledge: split vault file %q: %w", files[i].URL, err) + } for j, p := range parts { body := p if j == 0 { @@ -1725,9 +1722,10 @@ func isTransientGitErr(combinedOutput string) bool { // future "code repo" mode lands (HLD TBD), it'll widen the allow-list // behind an explicit kind=code flag. var indexableExts = map[string]bool{ - ".md": true, - ".txt": true, - ".rst": true, + ".md": true, + ".markdown": true, + ".txt": true, + ".rst": true, } // skipDirNames are directories that scanRepoFiles drops without @@ -2064,6 +2062,20 @@ func docID(key string) uint64 { return binary.BigEndian.Uint64(sum[:8]) } +// splitKnowledgeContent keeps document chunking consistent across upload, +// repository, and built-in vault ingestion. DOCX/PDF content has already been +// converted to Markdown by the upload extractor, so it uses the same structured +// splitter as native Markdown files. Plain text and reStructuredText retain the +// fixed-window behavior because they do not have the same heading contract. +func splitKnowledgeContent(ctx context.Context, filename, content string) ([]string, error) { + switch strings.ToLower(filepath.Ext(filename)) { + case ".md", ".markdown", ".docx", ".pdf": + return splitMarkdown(ctx, content) + default: + return splitForChunks(content), nil + } +} + // splitForChunks splits a body into chunkChars-sized overlapping pieces. // Returns one chunk for short docs, multiple for long ones. Operates on // runes (not bytes) so CJK is handled correctly. Hard-caps at