From 2a0fe3c57de60979888aacffb37b59eedae5d1e3 Mon Sep 17 00:00:00 2001 From: Akeem Jenkins Date: Fri, 26 Jun 2026 23:01:23 -0600 Subject: [PATCH 1/2] fix: index command reports empty output for generated index files --- internal/bundle/bundle.go | 14 +++++++++ internal/bundle/bundle_test.go | 55 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go index 73f0b67..3bedcb9 100644 --- a/internal/bundle/bundle.go +++ b/internal/bundle/bundle.go @@ -2,6 +2,7 @@ package bundle import ( + "errors" "fmt" "io/fs" "os" @@ -66,6 +67,19 @@ func Load(root string) (*Bundle, error) { c, perr := concept.Parse(path, relPath) if perr == nil { b.Reserved = append(b.Reserved, c) + } else if errors.Is(perr, concept.ErrNoFrontmatter) { + // Generated index.md files have no frontmatter. Still load + // them as a Concept with empty Frontmatter and the raw file + // content as Body so callers can discover them via Reserved. + raw, rerr := os.ReadFile(path) + if rerr != nil { + return fmt.Errorf("read reserved %s: %w", relPath, rerr) + } + b.Reserved = append(b.Reserved, &concept.Concept{ + ID: concept.ConceptID(relPath), + Path: path, + Body: string(raw), + }) } return nil } diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go index 2e53902..10f2623 100644 --- a/internal/bundle/bundle_test.go +++ b/internal/bundle/bundle_test.go @@ -1,7 +1,11 @@ package bundle import ( + "os" + "path/filepath" "testing" + + "github.com/okfcli/okf/internal/concept" ) func TestLoad_ValidBundle(t *testing.T) { @@ -64,3 +68,54 @@ func TestLoad_FrontmatterParsed(t *testing.T) { t.Errorf("tags len = %d, want 3", len(c.Frontmatter.Tags)) } } + +// TestLoad_ReservedIndexNoFrontmatter verifies that a reserved index.md file +// without YAML frontmatter (as generated by `okf index`) is still loaded into +// Reserved with a correct ID, Path, and raw Body content. +func TestLoad_ReservedIndexNoFrontmatter(t *testing.T) { + tmp := t.TempDir() + + // Write a concept file with frontmatter so the bundle is non-empty. + conceptDir := filepath.Join(tmp, "tables") + if err := os.MkdirAll(conceptDir, 0o755); err != nil { + t.Fatalf("mkdir tables: %v", err) + } + conceptContent := "---\ntype: BigQuery Table\ntitle: Events\n---\n\nEvent data.\n" + if err := os.WriteFile(filepath.Join(conceptDir, "events_.md"), []byte(conceptContent), 0o644); err != nil { + t.Fatalf("write concept: %v", err) + } + + // Write an index.md WITHOUT frontmatter (as `okf index` generates). + indexPath := filepath.Join(conceptDir, "index.md") + indexContent := "# Index\n\n- [events_](events_.md)\n" + if err := os.WriteFile(indexPath, []byte(indexContent), 0o644); err != nil { + t.Fatalf("write index: %v", err) + } + + b, err := Load(tmp) + if err != nil { + t.Fatalf("Load failed: %v", err) + } + + var found *concept.Concept + for _, r := range b.Reserved { + if r.ID == "tables/index" { + found = r + break + } + } + if found == nil { + t.Fatalf("expected Reserved to contain tables/index, got %d entries", len(b.Reserved)) + } + + // Path should be the absolute path to index.md. + absIndex, _ := filepath.Abs(indexPath) + if found.Path != absIndex { + t.Errorf("Path = %q, want %q", found.Path, absIndex) + } + + // Body should contain the raw file content. + if found.Body != indexContent { + t.Errorf("Body = %q, want %q", found.Body, indexContent) + } +} From 43b6c4a1fc22fc857e18f0936718061f20dd09a2 Mon Sep 17 00:00:00 2001 From: Akeem Jenkins Date: Wed, 8 Jul 2026 10:47:42 -0600 Subject: [PATCH 2/2] fix(bundle): read reserved files once via concept.ParseReserved Resolve the gosec G122 CI-lint blocker: reserved files without frontmatter were read a second time via a literal os.ReadFile inside the filepath.WalkDir callback. Move the no-frontmatter fallback into the concept package as ParseReserved, which reads the file exactly once and returns a *Concept with empty Frontmatter and the raw content as Body. No literal os.ReadFile remains in the walk callback. Also stop silently swallowing malformed-frontmatter errors on reserved files: ParseReserved only rescues ErrNoFrontmatter and surfaces every other parse error, so a reserved file with broken frontmatter no longer vanishes without a diagnostic. Add an end-to-end regression test that generates index.md files via index.Generate and asserts they appear in b.Reserved (the slice runIndex scans), covering the user-visible empty-output symptom. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/bundle/bundle.go | 24 +++++---------- internal/bundle/bundle_test.go | 56 ++++++++++++++++++++++++++++++++++ internal/concept/concept.go | 25 +++++++++++++++ 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go index 3bedcb9..52ac3dc 100644 --- a/internal/bundle/bundle.go +++ b/internal/bundle/bundle.go @@ -2,7 +2,6 @@ package bundle import ( - "errors" "fmt" "io/fs" "os" @@ -63,24 +62,15 @@ func Load(root string) (*Bundle, error) { relPath = filepath.ToSlash(relPath) // Reserved filenames are loaded separately; they may lack frontmatter. + // ParseReserved reads the file once and tolerates a missing frontmatter + // block (e.g. generated index.md), but still surfaces malformed + // frontmatter and other errors rather than silently dropping the file. if concept.ReservedNames[strings.ToLower(d.Name())] { - c, perr := concept.Parse(path, relPath) - if perr == nil { - b.Reserved = append(b.Reserved, c) - } else if errors.Is(perr, concept.ErrNoFrontmatter) { - // Generated index.md files have no frontmatter. Still load - // them as a Concept with empty Frontmatter and the raw file - // content as Body so callers can discover them via Reserved. - raw, rerr := os.ReadFile(path) - if rerr != nil { - return fmt.Errorf("read reserved %s: %w", relPath, rerr) - } - b.Reserved = append(b.Reserved, &concept.Concept{ - ID: concept.ConceptID(relPath), - Path: path, - Body: string(raw), - }) + c, perr := concept.ParseReserved(path, relPath) + if perr != nil { + return fmt.Errorf("parse reserved %s: %w", relPath, perr) } + b.Reserved = append(b.Reserved, c) return nil } diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go index 10f2623..20382e3 100644 --- a/internal/bundle/bundle_test.go +++ b/internal/bundle/bundle_test.go @@ -3,9 +3,11 @@ package bundle import ( "os" "path/filepath" + "strings" "testing" "github.com/okfcli/okf/internal/concept" + "github.com/okfcli/okf/internal/index" ) func TestLoad_ValidBundle(t *testing.T) { @@ -119,3 +121,57 @@ func TestLoad_ReservedIndexNoFrontmatter(t *testing.T) { t.Errorf("Body = %q, want %q", found.Body, indexContent) } } + +// TestLoad_GeneratedIndexIsDiscoverable is an end-to-end regression test for +// the user-visible symptom: after `okf index` generates index.md files (which +// have no frontmatter), reloading the bundle must expose them in b.Reserved so +// runIndex reports a non-empty indexes_written. Previously these files were +// silently dropped and the index command reported zero indexes. +func TestLoad_GeneratedIndexIsDiscoverable(t *testing.T) { + tmp := t.TempDir() + + conceptDir := filepath.Join(tmp, "tables") + if err := os.MkdirAll(conceptDir, 0o755); err != nil { + t.Fatalf("mkdir tables: %v", err) + } + conceptContent := "---\ntype: BigQuery Table\ntitle: Events\n---\n\nEvent data.\n" + if err := os.WriteFile(filepath.Join(conceptDir, "events_.md"), []byte(conceptContent), 0o644); err != nil { + t.Fatalf("write concept: %v", err) + } + + // Generate index.md files the same way `okf index` does. + if err := index.Generate(tmp); err != nil { + t.Fatalf("index.Generate: %v", err) + } + + b, err := Load(tmp) + if err != nil { + t.Fatalf("Load failed: %v", err) + } + + // Mirror runIndex's collection logic. + var indexFiles []string + for _, r := range b.Reserved { + if r.ID == "index" || strings.HasSuffix(r.ID, "/index") { + indexFiles = append(indexFiles, r.Path) + } + } + if len(indexFiles) == 0 { + t.Fatalf("expected at least one generated index in Reserved, got %d reserved entries", len(b.Reserved)) + } + + // The tables/ subdirectory index must be among them and carry its body. + var tablesIndex *concept.Concept + for _, r := range b.Reserved { + if r.ID == "tables/index" { + tablesIndex = r + break + } + } + if tablesIndex == nil { + t.Fatal("expected Reserved to contain tables/index") + } + if tablesIndex.Body == "" { + t.Error("generated index Body is empty, want raw index content") + } +} diff --git a/internal/concept/concept.go b/internal/concept/concept.go index 6e376da..085eca8 100644 --- a/internal/concept/concept.go +++ b/internal/concept/concept.go @@ -76,6 +76,31 @@ func ParseBytes(raw []byte, relPath, absPath string) (*Concept, error) { }, nil } +// ParseReserved reads and parses a reserved document (index.md, log.md) from +// path. Unlike Parse, a missing frontmatter block is not an error: such files +// (e.g. those generated by `okf index`) are returned as a Concept with empty +// Frontmatter and the raw file content as Body, so callers can still discover +// them. The file is read exactly once. Malformed-but-present frontmatter still +// returns an error rather than being silently dropped. +func ParseReserved(path, relPath string) (*Concept, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + c, err := ParseBytes(raw, relPath, path) + if err != nil { + if errors.Is(err, ErrNoFrontmatter) { + return &Concept{ + ID: conceptID(relPath), + Path: path, + Body: string(raw), + }, nil + } + return nil, err + } + return c, nil +} + // ConceptID converts a relative file path to a concept ID by stripping the .md // suffix and normalizing separators to forward slashes. // e.g. "tables/users.md" -> "tables/users"