From 811b08c8739b91c79cd97bcffb52a027d41b0aab Mon Sep 17 00:00:00 2001 From: Dead-Bytes <143434285+Dead-Bytes@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:46:08 +0530 Subject: [PATCH 1/2] feat: branch indexing --- packages/ingest-core/src/pipeline/paths.ts | 2 + .../src/pipeline/github-index-source.ts | 4 +- .../src/pipeline/pull-source-resolver.ts | 2 + .../ingest-github/src/pipeline/run-local.ts | 4 +- packages/mcp/src/repoFs.ts | 3 + packages/mongo/src/index.ts | 1 + packages/mongo/src/knowledge.ts | 70 ++++++++++++++++++- packages/path-migration/src/move.ts | 9 ++- packages/types/src/index.ts | 2 + packages/types/src/knowledge.ts | 24 ++++++- packages/types/src/path-layout.ts | 46 +++++++++--- 11 files changed, 153 insertions(+), 14 deletions(-) diff --git a/packages/ingest-core/src/pipeline/paths.ts b/packages/ingest-core/src/pipeline/paths.ts index 6c0fa18..7534bb4 100644 --- a/packages/ingest-core/src/pipeline/paths.ts +++ b/packages/ingest-core/src/pipeline/paths.ts @@ -117,6 +117,7 @@ async function repoLocationFor(knowledgeId: string, commitHash?: string): Promis provider: "local", orgId, knowledgeId, + branch: kDoc.info.branch ?? "main", commitHash: commitHash ?? kDoc.source.sourcePath, // best-effort fallback when no commit }; } @@ -142,6 +143,7 @@ async function repoLocationFor(knowledgeId: string, commitHash?: string): Promis knowledgeId, owner: parsed.owner, repo: parsed.repo, + branch: kDoc.info.branch ?? "main", commitHash: effectiveCommit, }; } diff --git a/packages/ingest-github/src/pipeline/github-index-source.ts b/packages/ingest-github/src/pipeline/github-index-source.ts index 5bf99a0..1a54d79 100644 --- a/packages/ingest-github/src/pipeline/github-index-source.ts +++ b/packages/ingest-github/src/pipeline/github-index-source.ts @@ -47,7 +47,7 @@ export const resolveGithubIndexSource: IndexSourceResolver = async (input) => { source = factoryResult.source; commitHash = factoryResult.commitHash; archiveSink = factoryResult.archiveSink; - location = { provider: "github", orgId, knowledgeId, owner, repo, commitHash }; + location = { provider: "github", orgId, knowledgeId, owner, repo, branch, commitHash }; // The factory has already produced the source tree; meta-output dirs still // need to exist before the strategy writes scan-manifest.json. Idempotent. await ensureCommitDirs(location); @@ -65,7 +65,7 @@ export const resolveGithubIndexSource: IndexSourceResolver = async (input) => { `could not resolve HEAD commit hash for ${owner}/${repo}@${branch} before clone`, ); } - location = { provider: "github", orgId, knowledgeId, owner, repo, commitHash: resolvedSha }; + location = { provider: "github", orgId, knowledgeId, owner, repo, branch, commitHash: resolvedSha }; await ensureCommitDirs(location); const repoDir = pathsFor(location).repositoryDir; const cloneOpts: { repoUrl: string; branch: string; destinationDir: string; gitToken?: string } = { diff --git a/packages/ingest-github/src/pipeline/pull-source-resolver.ts b/packages/ingest-github/src/pipeline/pull-source-resolver.ts index c9d41fd..acebaa2 100644 --- a/packages/ingest-github/src/pipeline/pull-source-resolver.ts +++ b/packages/ingest-github/src/pipeline/pull-source-resolver.ts @@ -52,6 +52,7 @@ export async function resolvePullSource(input: ResolvePullSourceInput): Promise< knowledgeId, owner, repo, + branch, commitHash: factoryResult.targetCommit, }; logger.info( @@ -99,6 +100,7 @@ export async function resolvePullSource(input: ResolvePullSourceInput): Promise< knowledgeId, owner, repo, + branch, commitHash: resolvedTarget, }; await ensureCommitDirs(location); diff --git a/packages/ingest-github/src/pipeline/run-local.ts b/packages/ingest-github/src/pipeline/run-local.ts index 5b39230..5073626 100644 --- a/packages/ingest-github/src/pipeline/run-local.ts +++ b/packages/ingest-github/src/pipeline/run-local.ts @@ -35,7 +35,9 @@ export async function runLocal( // repository/ dir); only meta-output lives under the kube-v2 tree. const commitHash = `local-${startedAt}`; const orgId = resolveOrgId(payload); - const location: RepoLocation = { provider: "local", orgId, knowledgeId, commitHash }; + // Local sources have no git branch; use a stable synthetic segment so the + // branch-scoped layout slot is populated. + const location: RepoLocation = { provider: "local", orgId, knowledgeId, branch: "main", commitHash }; await ensureCommitDirs(location); const metaPaths = pathsFor(location); diff --git a/packages/mcp/src/repoFs.ts b/packages/mcp/src/repoFs.ts index 408ba0f..48b39ec 100644 --- a/packages/mcp/src/repoFs.ts +++ b/packages/mcp/src/repoFs.ts @@ -66,6 +66,9 @@ export async function resolveCloneDir(knowledgeId: string): Promise { knowledgeId, owner: parsed.owner, repo: parsed.repo, + // Read-side resolves the branch the KB was indexed under (info.branch is set + // at index time). Multi-branch read selection is a separate MCP-side effort. + branch: kDoc.info.branch ?? "main", commitHash: commitId, }; return repositoryDirFor(getBytebellHome(), loc); diff --git a/packages/mongo/src/index.ts b/packages/mongo/src/index.ts index 19bb5b0..8ceb14c 100644 --- a/packages/mongo/src/index.ts +++ b/packages/mongo/src/index.ts @@ -8,6 +8,7 @@ export { setKnowledgeCommit, setKnowledgeState, setKnowledgeBranch, + setBranchHead, updateKnowledgeProgress, upsertKnowledge, listKnowledge, diff --git a/packages/mongo/src/knowledge.ts b/packages/mongo/src/knowledge.ts index 89aba56..be73f72 100644 --- a/packages/mongo/src/knowledge.ts +++ b/packages/mongo/src/knowledge.ts @@ -1,4 +1,4 @@ -import type { KnowledgeDoc, KnowledgeState } from "@bb/types"; +import { branchIdFor, type KnowledgeDoc, type KnowledgeState } from "@bb/types"; import { KnowledgeNotFoundError } from "@bb/errors"; import { _getDb } from "./client.ts"; import { Collections } from "./collections.ts"; @@ -89,6 +89,74 @@ export async function setKnowledgeBranch(knowledgeId: string, branch: string): P } } +/** + * Records that `branch` is indexed at `commitHash` on this knowledge, under the + * branch-per-KB model (`source.branches[]`). Upserts the branch element: + * advances its `headCommit`, appends `commitHash` to the branch's own deduped + * `commitHashes` history, and (optionally) sets the branch's processing + * `state`. Seeds `source.defaultBranch` on the first branch recorded. Idempotent + * per (branch, commit). Throws `KnowledgeNotFoundError` if the doc is missing. + * + * Callers: the branch-add trigger (state = PROCESSING at enqueue) and the + * worker on completion (state = PROCESSED + token usage). + */ +export async function setBranchHead( + knowledgeId: string, + branch: string, + commitHash: string, + opts: { state?: KnowledgeState; inputTokens?: string; outputTokens?: string; costUsd?: string } = {}, +): Promise { + const db = _getDb(); + const commitEntry = { + hash: commitHash, + inputTokens: opts.inputTokens ?? "", + outputTokens: opts.outputTokens ?? "", + costUsd: opts.costUsd ?? "0", + }; + + // 1. Create the branch element if it doesn't exist yet (first commit seeds its history). + const inserted = await db.collection(Collections.Knowledge).updateOne( + { knowledgeId, "source.branches.name": { $ne: branch } }, + { + $addToSet: { + "source.branches": { + name: branch, + branchId: branchIdFor(branch), + headCommit: commitHash, + commitHashes: [commitEntry], + ...(opts.state !== undefined ? { state: opts.state } : {}), + }, + }, + $set: { updatedAt: new Date() }, + }, + ); + + // 2. Otherwise advance the existing branch's head + append its commit. + if (inserted.modifiedCount === 0) { + const set: Record = { "source.branches.$.headCommit": commitHash, updatedAt: new Date() }; + if (opts.state !== undefined) { + set["source.branches.$.state"] = opts.state; + } + const updated = await db + .collection(Collections.Knowledge) + .updateOne( + { knowledgeId, "source.branches.name": branch }, + { $set: set, $addToSet: { "source.branches.$.commitHashes": commitEntry } }, + ); + if (updated.matchedCount === 0) { + throw new KnowledgeNotFoundError(knowledgeId); + } + } + + // 3. Seed the default branch on the first branch recorded. + await db + .collection(Collections.Knowledge) + .updateOne( + { knowledgeId, "source.defaultBranch": { $exists: false } }, + { $set: { "source.defaultBranch": branch } }, + ); +} + export async function updateKnowledgeProgress( knowledgeId: string, processedFiles: number, diff --git a/packages/path-migration/src/move.ts b/packages/path-migration/src/move.ts index bbd2ae9..b7ad26e 100644 --- a/packages/path-migration/src/move.ts +++ b/packages/path-migration/src/move.ts @@ -123,7 +123,13 @@ export async function migrateOne(input: MigrateOneInput): Promise { if (doc.source.kind === "local") { const syntheticCommit = `migrated-${doc.updatedAt.getTime()}`; - const newLoc: RepoLocation = { provider: "local", orgId, knowledgeId, commitHash: syntheticCommit }; + const newLoc: RepoLocation = { + provider: "local", + orgId, + knowledgeId, + branch: doc.info.branch ?? "main", + commitHash: syntheticCommit, + }; await moveMetaIfPresent(ctx, legacyMetaRoot, newLoc); return; } @@ -149,6 +155,7 @@ export async function migrateOne(input: MigrateOneInput): Promise { knowledgeId, owner: parsed.owner, repo: parsed.repo, + branch: doc.info.branch ?? "main", commitHash: commitId, }; await moveCloneIfPresent(ctx, legacyCloneDir, newLoc); diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 8aacbc4..eba7fe1 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -22,6 +22,7 @@ export type { LocalKnowledgeSource, KnowledgeListEntry, CommitHashRecord, + BranchRecord, TokenUsage, UsageGuard, } from "./knowledge.ts"; @@ -54,6 +55,7 @@ export type { EnrichmentFailure, EnrichmentFailureReason } from "./knowledge.ts" export { IngestionStrategyType } from "./config.ts"; export { orgsRootFor, + branchIdFor, commitBaseDirFor, repositoryDirFor, metaOutputRootFor, diff --git a/packages/types/src/knowledge.ts b/packages/types/src/knowledge.ts index b1d0621..6188cfc 100644 --- a/packages/types/src/knowledge.ts +++ b/packages/types/src/knowledge.ts @@ -31,12 +31,34 @@ export interface CommitHashRecord { costUsd: string; } +/** + * One indexed branch of a knowledge base. Branch support lets a single + * `knowledgeId` hold many branches side by side (see + * `code_history/004-branch-support.md`). `branchId` is `branchIdFor(name)` + * (sha256) — the same value used as the on-disk `` path segment and + * the branch-scoped graph key. `commitHashes` is this branch's own indexed + * commit history (per-branch, deduped, oldest → newest); `headCommit` is its + * current head; `state` is this branch's own processing state (a branch can be + * PROCESSING while a sibling is PROCESSED). + */ +export interface BranchRecord { + name: string; + branchId: string; + headCommit?: string; + state?: KnowledgeState; + commitHashes?: (string | CommitHashRecord)[]; +} + export interface GithubKnowledgeSource { kind: "github"; - /** Current head pointer — the most recently indexed commit. */ + /** Current head pointer — the most recently indexed commit (of the default branch). */ commitId?: string; /** Every commit this knowledge has been indexed at, oldest → newest. Pull appends to this list. */ commitHashes?: (string | CommitHashRecord)[]; + /** The branch treated as "current" for default reads. Seeded on first branch indexed. */ + defaultBranch?: string; + /** Per-branch indexed state. One `knowledgeId` may hold many branches. */ + branches?: BranchRecord[]; } export interface LocalKnowledgeSource { diff --git a/packages/types/src/path-layout.ts b/packages/types/src/path-layout.ts index 7e1973c..2015725 100644 --- a/packages/types/src/path-layout.ts +++ b/packages/types/src/path-layout.ts @@ -1,16 +1,23 @@ +import crypto from "node:crypto"; import path from "node:path"; // ───────────────────────────────────────────────────────────────────────────── // Pure-typed on-disk path resolver shared across packages that need to read // or write knowledge artifacts. No I/O, no FS calls — every helper returns -// strings derived from the inputs. Callers compose with their own -// `getBytebellHome()` (the package boundary that holds the home-dir state). +// strings derived from the inputs (a sha256 of a branch name is such a +// derivation; `node:crypto` is a runtime builtin, not a package dependency). +// Callers compose with their own `getBytebellHome()` (the package boundary +// that holds the home-dir state). // -// Layout (per knowledge + provider + commit): -// `//////repository/` -// `//////meta-output/` +// Layout (per knowledge + provider + branch + commit): +// `///////repository/` +// `///////meta-output/` // For local sources the `/` segments collapse: -// `/local///repository/` +// `/local////repository/` +// +// `` is `branchIdFor(loc.branch)` — a sha256 of the branch name, so +// one repo knowledge can hold many indexed branches side by side, each a +// self-contained subtree (see `branchIdFor` for why it is hashed, not raw). // // `` is the per-tenant base directory: // • OSS standalone: `~/.bytebell/` (single-tenant; no org segment) @@ -32,12 +39,14 @@ export type RepoLocation = knowledgeId: string; owner: string; repo: string; + branch: string; commitHash: string; } | { provider: "local"; orgId: string; knowledgeId: string; + branch: string; commitHash: string; }; @@ -70,11 +79,32 @@ export function orgsRootFor(home: string): string { return path.join(home, "orgs"); } +const BRANCH_BACKSLASH_RE = /\\/gu; + +/** + * Deterministic, filesystem-safe id for a git branch name — the `` + * path segment and the `branchId` key on `:Branch`/`:FileVersion`/other + * branch-scoped graph nodes. A single 64-hex SHA-256 component sidesteps every + * branch-name hazard at once: embedded slashes (`feat/x`), length caps, and + * case-insensitive filesystems (`Feature` vs `feature` hash differently, so no + * collision). The human-readable name is kept on Mongo `knowledge.info.branch` + * and the `:Branch` node, never on disk. Mirrors the hashing of + * `@bb/ingest-core`'s `metaId`; kept here in the kernel so every tier can + * derive a branch id without importing upward (same rationale as the + * duplicated `parseGithubOwnerRepo`). Backslashes normalise to `/` so a name + * hashes identically regardless of platform; nothing else is normalised + * (branch names are case-sensitive). + */ +export function branchIdFor(branch: string): string { + return crypto.createHash("sha256").update(branch.replace(BRANCH_BACKSLASH_RE, "/")).digest("hex"); +} + export function commitBaseDirFor(home: string, loc: RepoLocation): string { + const branchId = branchIdFor(loc.branch); if (loc.provider === "github") { - return path.join(home, "github", loc.knowledgeId, loc.owner, loc.repo, loc.commitHash); + return path.join(home, "github", loc.knowledgeId, loc.owner, loc.repo, branchId, loc.commitHash); } - return path.join(home, "local", loc.knowledgeId, loc.commitHash); + return path.join(home, "local", loc.knowledgeId, branchId, loc.commitHash); } export function repositoryDirFor(home: string, loc: RepoLocation): string { From b8db1430ce701eb393cfee97d572134963ff0747 Mon Sep 17 00:00:00 2001 From: Dead-Bytes <143434285+Dead-Bytes@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:36:26 +0530 Subject: [PATCH 2/2] feat: bootstrap made clear for the new graph writes --- packages/ingest-github/src/bootstrap.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/ingest-github/src/bootstrap.ts b/packages/ingest-github/src/bootstrap.ts index 639b0e5..375e2d8 100644 --- a/packages/ingest-github/src/bootstrap.ts +++ b/packages/ingest-github/src/bootstrap.ts @@ -21,6 +21,16 @@ import "@bb/neo4j"; export interface BootstrapRuntimeOptions { config: unknown; loggerFactory: LoggerFactory; + /** + * Skip the OSS knowledge/concept-graph index bootstrap (`ensureKnowledgeIndexes` + * / `ensureConceptGraphIndexes`). A composition root sets this when it owns the + * Neo4j graph schema itself — e.g. the enterprise ingestion engine, whose + * `ensureIrGraphSchema()` is the sole (branch-scoped) authority for the `:File` + * / `:FileVersion` / … constraints; the OSS branchless `file_unique` here would + * otherwise reject a second branch's file. Connections are still established. + * Default `false` (OSS standalone keeps creating its own indexes). + */ + skipGraphIndexes?: boolean; } export async function bootstrapRuntime(opts: BootstrapRuntimeOptions): Promise { @@ -35,6 +45,9 @@ export async function bootstrapRuntime(opts: BootstrapRuntimeOptions): Promise