From 40044facf684e7c66c9897d248d42351a608acae Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Sun, 12 Jul 2026 12:29:19 +0200 Subject: [PATCH 1/2] feat: OKF bundle export and git delivery (M1+M2) Add @okf/core (portable Open Knowledge Format primitives) and @flank/okf-export, projecting published dossiers/battlecards into an OKF bundle served at GET /api/okf (M1). Add M2 git delivery: GitHubBundlePublisher, bundle-diff delivery planning, append-only okf_delivery table, and the okf-delivery Inngest cron. Fail-closed on unverified citations. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/web/app/api/inngest/route.ts | 22 +- apps/web/app/api/okf/route.ts | 44 ++++ apps/web/package.json | 1 + packages/core/src/entities.ts | 31 +++ packages/core/src/store.ts | 11 + packages/db/drizzle/0007_okf_deliveries.sql | 27 ++ packages/db/src/drizzle-mappers.ts | 18 ++ packages/db/src/drizzle-store.ts | 51 ++++ packages/db/src/schema.test.ts | 4 + packages/db/src/schema.ts | 30 +++ packages/okf-export/package.json | 23 ++ packages/okf-export/src/deliver.test.ts | 103 ++++++++ packages/okf-export/src/deliver.ts | 74 ++++++ packages/okf-export/src/index.ts | 21 ++ packages/okf-export/src/load.test.ts | 95 +++++++ packages/okf-export/src/load.ts | 34 +++ packages/okf-export/src/log.ts | Bin 0 -> 1614 bytes packages/okf-export/src/manifest.test.ts | 98 +++++++ packages/okf-export/src/manifest.ts | 67 +++++ packages/okf-export/src/project.test.ts | 203 +++++++++++++++ packages/okf-export/src/project.ts | 125 +++++++++ packages/okf-export/src/publisher.ts | 49 ++++ packages/okf-export/src/sections.ts | 129 ++++++++++ packages/okf-export/src/tar.test.ts | 70 +++++ packages/okf-export/src/tar.ts | 61 +++++ packages/okf-export/src/types.ts | 30 +++ packages/okf-export/tsconfig.json | 7 + packages/okf/package.json | 19 ++ packages/okf/src/bundle.test.ts | 177 +++++++++++++ packages/okf/src/bundle.ts | 34 +++ packages/okf/src/concept.ts | 47 ++++ packages/okf/src/frontmatter.test.ts | 93 +++++++ packages/okf/src/frontmatter.ts | 151 +++++++++++ packages/okf/src/index-gen.ts | 55 ++++ packages/okf/src/index.ts | 18 ++ packages/okf/src/links.test.ts | 68 +++++ packages/okf/src/links.ts | 86 +++++++ packages/okf/src/log-gen.ts | 29 +++ packages/okf/src/validate.test.ts | 94 +++++++ packages/okf/src/validate.ts | 110 ++++++++ packages/okf/tsconfig.json | 4 + packages/pipeline/package.json | 2 + .../pipeline/src/github-publisher.test.ts | 173 +++++++++++++ packages/pipeline/src/github-publisher.ts | 242 ++++++++++++++++++ packages/pipeline/src/index.ts | 3 + packages/pipeline/src/inngest.ts | 43 ++++ packages/pipeline/src/memory-store.ts | 29 +++ packages/pipeline/src/okf-config.test.ts | 65 +++++ packages/pipeline/src/okf-config.ts | 41 +++ packages/pipeline/src/okf-delivery.test.ts | 222 ++++++++++++++++ packages/pipeline/src/okf-delivery.ts | 147 +++++++++++ packages/pipeline/src/store-contract.ts | 63 +++++ pnpm-lock.yaml | 40 +++ 53 files changed, 3481 insertions(+), 2 deletions(-) create mode 100644 apps/web/app/api/okf/route.ts create mode 100644 packages/db/drizzle/0007_okf_deliveries.sql create mode 100644 packages/okf-export/package.json create mode 100644 packages/okf-export/src/deliver.test.ts create mode 100644 packages/okf-export/src/deliver.ts create mode 100644 packages/okf-export/src/index.ts create mode 100644 packages/okf-export/src/load.test.ts create mode 100644 packages/okf-export/src/load.ts create mode 100644 packages/okf-export/src/log.ts create mode 100644 packages/okf-export/src/manifest.test.ts create mode 100644 packages/okf-export/src/manifest.ts create mode 100644 packages/okf-export/src/project.test.ts create mode 100644 packages/okf-export/src/project.ts create mode 100644 packages/okf-export/src/publisher.ts create mode 100644 packages/okf-export/src/sections.ts create mode 100644 packages/okf-export/src/tar.test.ts create mode 100644 packages/okf-export/src/tar.ts create mode 100644 packages/okf-export/src/types.ts create mode 100644 packages/okf-export/tsconfig.json create mode 100644 packages/okf/package.json create mode 100644 packages/okf/src/bundle.test.ts create mode 100644 packages/okf/src/bundle.ts create mode 100644 packages/okf/src/concept.ts create mode 100644 packages/okf/src/frontmatter.test.ts create mode 100644 packages/okf/src/frontmatter.ts create mode 100644 packages/okf/src/index-gen.ts create mode 100644 packages/okf/src/index.ts create mode 100644 packages/okf/src/links.test.ts create mode 100644 packages/okf/src/links.ts create mode 100644 packages/okf/src/log-gen.ts create mode 100644 packages/okf/src/validate.test.ts create mode 100644 packages/okf/src/validate.ts create mode 100644 packages/okf/tsconfig.json create mode 100644 packages/pipeline/src/github-publisher.test.ts create mode 100644 packages/pipeline/src/github-publisher.ts create mode 100644 packages/pipeline/src/okf-config.test.ts create mode 100644 packages/pipeline/src/okf-config.ts create mode 100644 packages/pipeline/src/okf-delivery.test.ts create mode 100644 packages/pipeline/src/okf-delivery.ts diff --git a/apps/web/app/api/inngest/route.ts b/apps/web/app/api/inngest/route.ts index f496cbc..d056060 100644 --- a/apps/web/app/api/inngest/route.ts +++ b/apps/web/app/api/inngest/route.ts @@ -1,10 +1,18 @@ -import { createNotifier, createSynthesisClient, createTriageClient } from '@flank/pipeline'; +import { + createNotifier, + createOkfPublisher, + createSynthesisClient, + createTriageClient, + parseOkfTargets, +} from '@flank/pipeline'; import { createDeliverySweepFunction, createNightlySynthesisFunction, + createOkfDeliveryFunction, createScheduledTickFunction, inngest, type DeliveryRuntime, + type OkfDeliveryRuntime, type SchedulerRuntime, type SynthesisRuntime, } from '@flank/pipeline/inngest'; @@ -31,11 +39,21 @@ const buildDeliveryRuntime = async (): Promise => ({ notifier: createNotifier(process.env), }); +const buildOkfDeliveryRuntime = async (): Promise => ({ + store: store(), + // Real GitHub publisher when FLANK_OKF_GITHUB_TOKEN is set; targets come from FLANK_OKF_TARGETS + // (JSON). With neither configured the sweep has no targets and no-ops. + publisher: createOkfPublisher(process.env), + targets: parseOkfTargets(process.env.FLANK_OKF_TARGETS), + baseUrl: process.env.FLANK_APP_ORIGIN ?? 'https://app.flank.example', +}); + const scheduledTick = createScheduledTickFunction(buildSchedulerRuntime); const nightlySynthesis = createNightlySynthesisFunction(buildSynthesisRuntime); const deliverySweep = createDeliverySweepFunction(buildDeliveryRuntime); +const okfDelivery = createOkfDeliveryFunction(buildOkfDeliveryRuntime); export const { GET, POST, PUT } = serve({ client: inngest, - functions: [scheduledTick, nightlySynthesis, deliverySweep], + functions: [scheduledTick, nightlySynthesis, deliverySweep, okfDelivery], }); diff --git a/apps/web/app/api/okf/route.ts b/apps/web/app/api/okf/route.ts new file mode 100644 index 0000000..79a0aef --- /dev/null +++ b/apps/web/app/api/okf/route.ts @@ -0,0 +1,44 @@ +import { bundleToTar, loadWorkspaceExport, projectWorkspaceBundle } from '@flank/okf-export'; +import { NextResponse, type NextRequest } from 'next/server'; +import { resolveActiveWorkspace } from '../../../lib/auth/session'; +import { getStore } from '../../../lib/store'; + +/** + * GET /api/okf — the workspace's competitive knowledge as an OKF bundle. + * Default response is a deterministic ustar tarball; `?format=json` returns + * `{ files, findings }` for programmatic consumers. + * + * Fail closed (projection-only rule): any `error` finding — an unverified or + * unresolvable citation, a structural bundle defect — returns 409 with the + * findings instead of a bundle. Unproven provenance never ships. + */ +export async function GET(request: NextRequest): Promise { + const active = await resolveActiveWorkspace(); + const workspaceName = + active.memberships.find((m) => m.workspace.id === active.workspaceId)?.workspace.name ?? + active.workspaceId; + + const input = await loadWorkspaceExport( + getStore(), + { id: active.workspaceId, name: workspaceName }, + request.nextUrl.origin, + ); + const { files, findings } = projectWorkspaceBundle(input); + + if (findings.some((finding) => finding.severity === 'error')) { + return NextResponse.json({ error: 'bundle failed the publish gate', findings }, { status: 409 }); + } + + if (request.nextUrl.searchParams.get('format') === 'json') { + return NextResponse.json({ files: Object.fromEntries(files), findings }); + } + + return new NextResponse(Buffer.from(bundleToTar(files)), { + headers: { + 'Content-Type': 'application/x-tar', + 'Content-Disposition': `attachment; filename="flank-okf-${active.workspaceId}.tar"`, + // The bundle reflects live published state; never cache across sessions. + 'Cache-Control': 'private, no-store', + }, + }); +} diff --git a/apps/web/package.json b/apps/web/package.json index b39fce7..13696d9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -14,6 +14,7 @@ "dependencies": { "@flank/core": "workspace:*", "@flank/db": "workspace:*", + "@flank/okf-export": "workspace:*", "@flank/pipeline": "workspace:*", "inngest": "^4.7.0", "next": "^16.2.9", diff --git a/packages/core/src/entities.ts b/packages/core/src/entities.ts index 9f682f0..8427b6b 100644 --- a/packages/core/src/entities.ts +++ b/packages/core/src/entities.ts @@ -279,6 +279,37 @@ export interface Alert { readonly deliveredAt: Date | null; } +// --- OKF bundle delivery (M2) --- + +/** + * The outcome of one attempt to push a workspace's OKF bundle to its designated git repo. + * `published` shipped a commit; `failed` is a recorded attempt that errored (publish I/O, or a + * gate that refused the bundle). A run that finds no changes ships nothing and records no row — + * silence isn't a delivery. Append-only (Invariant 5). + */ +export const BUNDLE_DELIVERY_STATUSES = ['published', 'failed'] as const; +export type BundleDeliveryStatus = (typeof BUNDLE_DELIVERY_STATUSES)[number]; + +export interface BundleDelivery { + readonly id: string; + readonly workspaceId: string; + readonly status: BundleDeliveryStatus; + readonly commitSha: string | null; + /** The git ref the commit landed on (e.g. `refs/heads/flank-okf`); null on a failed attempt. */ + readonly branchRef: string | null; + readonly pullRequestUrl: string | null; + /** + * path → sha256 content hash of the bundle actually delivered — the baseline the next run diffs + * against to decide what changed. Empty for a failed attempt (nothing shipped). + */ + readonly manifest: Readonly>; + readonly filesAdded: number; + readonly filesModified: number; + readonly filesRemoved: number; + readonly error: string | null; + readonly createdAt: Date; +} + /** Boundary validation for a channel config arriving from settings/UI — never trust raw input. */ export const AlertChannelConfigSchema = z .object({ diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index cfab730..bbe4b16 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -5,6 +5,7 @@ import type { AppUser, BattlecardSection, BattlecardSectionKind, + BundleDelivery, Claim, Competitor, CoverageRun, @@ -349,6 +350,16 @@ export interface FlankStore { /** A workspace's delivery log, newest first (request-safe, for the /authed/alerts view). */ listAlertsForWorkspace(workspaceId: string): Promise; + // --- OKF bundle delivery (M2) --- + + /** Append-only record of one git-push attempt (Invariant 5). Workspace must exist. */ + insertBundleDelivery(delivery: BundleDelivery): Promise; + /** + * The most recent successfully-published delivery for a workspace, or null if none. Its manifest + * is the baseline the next delivery run diffs against; failed attempts never become the baseline. + */ + latestPublishedBundleDelivery(workspaceId: string): Promise; + /** * Run `fn` as a single atomic unit of work. The handle passed to `fn` is a {@link FlankStore} * bound to the transaction; if `fn` throws, every write performed through that handle is rolled diff --git a/packages/db/drizzle/0007_okf_deliveries.sql b/packages/db/drizzle/0007_okf_deliveries.sql new file mode 100644 index 0000000..7c8b0ca --- /dev/null +++ b/packages/db/drizzle/0007_okf_deliveries.sql @@ -0,0 +1,27 @@ +-- M2 OKF bundle delivery. Append-only record of each git-push attempt: `published` shipped a commit, +-- `failed` errored (publish I/O or a refused bundle). `manifest` (path -> sha256) is the baseline the +-- next run diffs against. Append-only in depth like the history tables (0002): the reject-mutation +-- trigger blocks any UPDATE/DELETE against raw SQL, not just the app layer. + +CREATE TYPE "bundle_delivery_status" AS ENUM('published', 'failed');--> statement-breakpoint + +CREATE TABLE "okf_delivery" ( + "id" text PRIMARY KEY NOT NULL, + "workspace_id" text NOT NULL, + "status" "bundle_delivery_status" NOT NULL, + "commit_sha" text, + "branch_ref" text, + "pull_request_url" text, + "manifest" jsonb DEFAULT '{}'::jsonb NOT NULL, + "files_added" integer DEFAULT 0 NOT NULL, + "files_modified" integer DEFAULT 0 NOT NULL, + "files_removed" integer DEFAULT 0 NOT NULL, + "error" text, + "created_at" timestamp with time zone NOT NULL +);--> statement-breakpoint + +ALTER TABLE "okf_delivery" ADD CONSTRAINT "okf_delivery_workspace_id_workspace_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspace"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "okf_delivery_workspace_status_idx" ON "okf_delivery" USING btree ("workspace_id","status");--> statement-breakpoint + +-- Append-only in depth: reuse flank_reject_mutation() (defined in 0002) to block UPDATE and DELETE. +CREATE TRIGGER okf_delivery_append_only BEFORE UPDATE OR DELETE ON "okf_delivery" FOR EACH ROW EXECUTE FUNCTION flank_reject_mutation(); diff --git a/packages/db/src/drizzle-mappers.ts b/packages/db/src/drizzle-mappers.ts index d4c8155..1c1a6a9 100644 --- a/packages/db/src/drizzle-mappers.ts +++ b/packages/db/src/drizzle-mappers.ts @@ -3,6 +3,7 @@ import type { AlertChannelConfig, AppUser, BattlecardSection, + BundleDelivery, Claim, Competitor, CoverageRun, @@ -24,6 +25,7 @@ import type { deltas, dossierSections, memberships, + okfDeliveries, snapshots, sources, workspaces, @@ -193,6 +195,22 @@ export const toAlert = (row: typeof alerts.$inferSelect): Alert => deliveredAt: row.deliveredAt, }); +export const toBundleDelivery = (row: typeof okfDeliveries.$inferSelect): BundleDelivery => + Object.freeze({ + id: row.id, + workspaceId: row.workspaceId, + status: row.status, + commitSha: row.commitSha, + branchRef: row.branchRef, + pullRequestUrl: row.pullRequestUrl, + manifest: row.manifest, + filesAdded: row.filesAdded, + filesModified: row.filesModified, + filesRemoved: row.filesRemoved, + error: row.error, + createdAt: row.createdAt, + }); + const PG_UNIQUE_VIOLATION = '23505'; /** diff --git a/packages/db/src/drizzle-store.ts b/packages/db/src/drizzle-store.ts index a3afcf5..a4aeeff 100644 --- a/packages/db/src/drizzle-store.ts +++ b/packages/db/src/drizzle-store.ts @@ -10,6 +10,7 @@ import { type AppUser, type BattlecardSection, type BattlecardSectionKind, + type BundleDelivery, type Claim, type Competitor, type CoverageRun, @@ -42,6 +43,7 @@ import { deltas, dossierSections, memberships, + okfDeliveries, snapshots, sources, workspaces, @@ -51,6 +53,7 @@ import { isUniqueViolation, toAppUser, toBattlecardSection, + toBundleDelivery, toClaim, toCompetitor, toCoverageRun, @@ -788,6 +791,54 @@ export class DrizzleFlankStore implements FlankStore { return alertStore.listAlertsForWorkspace(this.db, workspaceId); } + // --- OKF bundle delivery (M2) --- + + async insertBundleDelivery(delivery: BundleDelivery): Promise { + const parent = await this.db + .select({ id: workspaces.id }) + .from(workspaces) + .where(eq(workspaces.id, delivery.workspaceId)) + .limit(1); + if (parent[0] === undefined) { + throw new UnknownEntityError(`workspace ${delivery.workspaceId} does not exist`); + } + return this.insertOne( + () => + this.db + .insert(okfDeliveries) + .values({ + id: delivery.id, + workspaceId: delivery.workspaceId, + status: delivery.status, + commitSha: delivery.commitSha, + branchRef: delivery.branchRef, + pullRequestUrl: delivery.pullRequestUrl, + manifest: delivery.manifest, + filesAdded: delivery.filesAdded, + filesModified: delivery.filesModified, + filesRemoved: delivery.filesRemoved, + error: delivery.error, + createdAt: delivery.createdAt, + }) + .returning(), + toBundleDelivery, + 'okf_delivery', + ); + } + + async latestPublishedBundleDelivery(workspaceId: string): Promise { + const rows = await this.db + .select() + .from(okfDeliveries) + .where( + and(eq(okfDeliveries.workspaceId, workspaceId), eq(okfDeliveries.status, 'published')), + ) + // Deterministic "latest": createdAt desc, id desc — the memory store's byCreatedThenId mirror. + .orderBy(desc(okfDeliveries.createdAt), desc(okfDeliveries.id)) + .limit(1); + return rows[0] === undefined ? null : toBundleDelivery(rows[0]); + } + async withTransaction(fn: (tx: FlankStore) => Promise): Promise { return this.db.transaction((tx) => // `tx` is a transaction-bound handle exposing the same query builder used throughout this diff --git a/packages/db/src/schema.test.ts b/packages/db/src/schema.test.ts index 0d1cc34..eb632af 100644 --- a/packages/db/src/schema.test.ts +++ b/packages/db/src/schema.test.ts @@ -11,6 +11,7 @@ import { deltaStateEnum, deltas, dossierSections, + okfDeliveries, snapshots, sourceTypeEnum, sources, @@ -32,6 +33,7 @@ describe('drizzle schema (no live database required)', () => { battlecardSections, alerts, coverageRuns, + okfDeliveries, ]; // Act @@ -49,6 +51,7 @@ describe('drizzle schema (no live database required)', () => { 'battlecard_section', 'alert', 'coverage_run', + 'okf_delivery', ]); }); @@ -70,6 +73,7 @@ describe('drizzle schema (no live database required)', () => { 'claim', 'dossier_section', 'battlecard_section', + 'okf_delivery', ]); }); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 8de66b8..a1dec3f 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -1,5 +1,6 @@ import { BATTLECARD_SECTION_KINDS, + BUNDLE_DELIVERY_STATUSES, DELTA_STATES, DOSSIER_SECTION_KINDS, LEGAL_STATUSES, @@ -35,6 +36,7 @@ export const alertStatusEnum = pgEnum('alert_status', ['queued', 'delivered', 'f export const sectionKindEnum = pgEnum('dossier_section_kind', DOSSIER_SECTION_KINDS); export const battlecardKindEnum = pgEnum('battlecard_section_kind', BATTLECARD_SECTION_KINDS); export const membershipRoleEnum = pgEnum('membership_role', MEMBERSHIP_ROLES); +export const bundleDeliveryStatusEnum = pgEnum('bundle_delivery_status', BUNDLE_DELIVERY_STATUSES); export const workspaces = pgTable('workspace', { id: text('id').primaryKey(), @@ -295,6 +297,33 @@ export const memberships = pgTable( ], ); +/** + * Append-only record of each OKF bundle git-push (M2). One row per published or failed attempt; + * `manifest` (path → sha256) is the baseline the next run diffs against. Append-only (Invariant 5); + * workspace-scoped (Invariant 8). + */ +export const okfDeliveries = pgTable( + 'okf_delivery', + { + id: text('id').primaryKey(), + workspaceId: text('workspace_id') + .notNull() + .references(() => workspaces.id), + status: bundleDeliveryStatusEnum('status').notNull(), + commitSha: text('commit_sha'), + branchRef: text('branch_ref'), + pullRequestUrl: text('pull_request_url'), + manifest: jsonb('manifest').$type>>().notNull().default({}), + filesAdded: integer('files_added').notNull().default(0), + filesModified: integer('files_modified').notNull().default(0), + filesRemoved: integer('files_removed').notNull().default(0), + error: text('error'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull(), + }, + // The next-run baseline lookup: newest published delivery per workspace. + (table) => [index('okf_delivery_workspace_status_idx').on(table.workspaceId, table.status)], +); + /** Tables whose rows must never be UPDATEd or DELETEd (Invariant 5). */ export const APPEND_ONLY_TABLES = Object.freeze([ snapshots, @@ -302,4 +331,5 @@ export const APPEND_ONLY_TABLES = Object.freeze([ claims, dossierSections, battlecardSections, + okfDeliveries, ] as const); diff --git a/packages/okf-export/package.json b/packages/okf-export/package.json new file mode 100644 index 0000000..c6c7f42 --- /dev/null +++ b/packages/okf-export/package.json @@ -0,0 +1,23 @@ +{ + "name": "@flank/okf-export", + "version": "0.1.0", + "private": true, + "description": "OKF bundle projection (M1): maps a workspace's published dossier/battlecard sections and verified claims into a portable OKF bundle. Pure over the FlankStore read interface; no direct I/O.", + "type": "module", + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@flank/core": "workspace:*", + "@okf/core": "workspace:*" + }, + "devDependencies": { + "@types/node": "^26.0.0", + "typescript": "^6.0.3", + "vitest": "^4.1.9" + } +} diff --git a/packages/okf-export/src/deliver.test.ts b/packages/okf-export/src/deliver.test.ts new file mode 100644 index 0000000..82476ce --- /dev/null +++ b/packages/okf-export/src/deliver.test.ts @@ -0,0 +1,103 @@ +import type { LintFinding } from '@okf/core'; +import { describe, expect, it } from 'vitest'; +import { planDelivery } from './deliver'; +import { manifestOf } from './manifest'; +import type { GitBundleTarget } from './publisher'; + +const TARGET: GitBundleTarget = { + workspaceId: 'ws-a', + workspaceName: 'Acme', + provider: 'github', + repo: 'acme/intel', + branch: 'flank-okf', + baseBranch: 'main', + subdir: 'intel/acme', +}; + +const filesOf = (entries: Record): Map => + new Map(Object.entries(entries)); + +describe('planDelivery', () => { + it('blocks when any finding is an error, pushing nothing', () => { + // Arrange + const findings: LintFinding[] = [ + { check: 'UNVERIFIED_CITATION', severity: 'error', path: 'a.md', message: 'nope' }, + { check: 'ORPHAN', severity: 'warning', path: 'b.md', message: 'meh' }, + ]; + + // Act + const plan = planDelivery({ + target: TARGET, + files: filesOf({ 'index.md': 'x' }), + findings, + previousManifest: new Map(), + }); + + // Assert + expect(plan.kind).toBe('blocked'); + if (plan.kind === 'blocked') { + expect(plan.blockingFindings).toHaveLength(1); + expect(plan.blockingFindings[0].check).toBe('UNVERIFIED_CITATION'); + } + }); + + it('reports unchanged when the bundle matches the previous manifest', () => { + // Arrange + const files = filesOf({ 'index.md': 'x', 'competitors/g/index.md': 'y' }); + + // Act + const plan = planDelivery({ + target: TARGET, + files, + findings: [], + previousManifest: manifestOf(files), + }); + + // Assert + expect(plan.kind).toBe('unchanged'); + }); + + it('builds a subdir-prefixed publish request with deletions and a summary commit message', () => { + // Arrange — previous had an extra file (now removed) and a changed one. + const previous = manifestOf( + filesOf({ 'index.md': 'old', 'competitors/gone/index.md': 'bye' }), + ); + const files = filesOf({ 'index.md': 'new', 'competitors/globex/index.md': 'hi' }); + + // Act + const plan = planDelivery({ target: TARGET, files, findings: [], previousManifest: previous }); + + // Assert + expect(plan.kind).toBe('publish'); + if (plan.kind !== 'publish') return; + // Every file path is prefixed with the target subdir. + expect([...plan.request.files.keys()].sort()).toEqual([ + 'intel/acme/competitors/globex/index.md', + 'intel/acme/index.md', + ]); + // Removed path is prefixed and queued for deletion. + expect(plan.request.deletions).toEqual(['intel/acme/competitors/gone/index.md']); + expect(plan.request.commitMessage).toBe( + 'chore(okf): update Acme competitive bundle (+1 ~1 -1)', + ); + expect(plan.diff.added).toEqual(['competitors/globex/index.md']); + expect(plan.diff.modified).toEqual(['index.md']); + expect(plan.diff.removed).toEqual(['competitors/gone/index.md']); + }); + + it('does not prefix when the subdir is empty (repo root delivery)', () => { + // Act + const plan = planDelivery({ + target: { ...TARGET, subdir: '' }, + files: filesOf({ 'index.md': 'x' }), + findings: [], + previousManifest: new Map(), + }); + + // Assert + expect(plan.kind).toBe('publish'); + if (plan.kind === 'publish') { + expect([...plan.request.files.keys()]).toEqual(['index.md']); + } + }); +}); diff --git a/packages/okf-export/src/deliver.ts b/packages/okf-export/src/deliver.ts new file mode 100644 index 0000000..3eca8b1 --- /dev/null +++ b/packages/okf-export/src/deliver.ts @@ -0,0 +1,74 @@ +import type { LintFinding } from '@okf/core'; +import { + diffManifests, + isEmptyDiff, + manifestOf, + type BundleDiff, + type BundleManifest, +} from './manifest'; +import type { BundlePublishRequest, GitBundleTarget } from './publisher'; + +export interface PlanDeliveryInput { + readonly target: GitBundleTarget; + /** The projected bundle (path → content) from {@link projectWorkspaceBundle}. */ + readonly files: ReadonlyMap; + /** Findings from the projection; any `error` severity blocks the delivery (projection-only rule). */ + readonly findings: readonly LintFinding[]; + /** Manifest of the last published delivery, or an empty manifest for a first delivery. */ + readonly previousManifest: BundleManifest; +} + +/** + * The delivery decision for one workspace, fully determined by the inputs (pure): + * - `blocked` — an error finding refused the bundle; nothing is pushed, a failed record is written. + * - `unchanged` — the bundle is byte-identical to the last delivery; nothing is pushed or recorded. + * - `publish` — the bundle changed; `request` is ready for the {@link BundlePublisher}. + */ +export type PlannedDelivery = + | { readonly kind: 'blocked'; readonly blockingFindings: readonly LintFinding[] } + | { readonly kind: 'unchanged'; readonly manifest: BundleManifest; readonly diff: BundleDiff } + | { + readonly kind: 'publish'; + readonly request: BundlePublishRequest; + readonly manifest: BundleManifest; + readonly diff: BundleDiff; + }; + +const underSubdir = (subdir: string, path: string): string => + subdir === '' ? path : `${subdir}/${path}`; + +const commitMessage = (target: GitBundleTarget, diff: BundleDiff): string => + `chore(okf): update ${target.workspaceName} competitive bundle ` + + `(+${diff.added.length} ~${diff.modified.length} -${diff.removed.length})`; + +/** Decide what to do with a projected bundle for one target. Pure — no I/O, deterministic. */ +export const planDelivery = (input: PlanDeliveryInput): PlannedDelivery => { + const blockingFindings = input.findings.filter((finding) => finding.severity === 'error'); + if (blockingFindings.length > 0) { + return { kind: 'blocked', blockingFindings }; + } + + const manifest = manifestOf(input.files); + const diff = diffManifests(input.previousManifest, manifest); + if (isEmptyDiff(diff)) { + return { kind: 'unchanged', manifest, diff }; + } + + const { subdir } = input.target; + const files = new Map(); + for (const [path, content] of input.files) files.set(underSubdir(subdir, path), content); + const deletions = diff.removed.map((path) => underSubdir(subdir, path)); + + return { + kind: 'publish', + manifest, + diff, + request: { + target: input.target, + files, + deletions, + commitMessage: commitMessage(input.target, diff), + diff, + }, + }; +}; diff --git a/packages/okf-export/src/index.ts b/packages/okf-export/src/index.ts new file mode 100644 index 0000000..dffc7d3 --- /dev/null +++ b/packages/okf-export/src/index.ts @@ -0,0 +1,21 @@ +export { projectWorkspaceBundle } from './project'; +export { loadWorkspaceExport } from './load'; +export type { ExportReadStore } from './load'; +export { bundleToTar } from './tar'; +export type { CompetitorExport, WorkspaceBundle, WorkspaceExportInput } from './types'; +export { + diffManifests, + isEmptyDiff, + manifestFromRecord, + manifestOf, + manifestToRecord, +} from './manifest'; +export type { BundleDiff, BundleManifest } from './manifest'; +export { planDelivery } from './deliver'; +export type { PlanDeliveryInput, PlannedDelivery } from './deliver'; +export type { + BundlePublishRequest, + BundlePublishResult, + BundlePublisher, + GitBundleTarget, +} from './publisher'; diff --git a/packages/okf-export/src/load.test.ts b/packages/okf-export/src/load.test.ts new file mode 100644 index 0000000..53aaf93 --- /dev/null +++ b/packages/okf-export/src/load.test.ts @@ -0,0 +1,95 @@ +import type { BattlecardSection, Claim, Competitor, DossierSection } from '@flank/core'; +import { describe, expect, it } from 'vitest'; +import { loadWorkspaceExport, type ExportReadStore } from './load'; + +const AT = new Date('2026-06-08T06:00:00Z'); + +const COMP: Competitor = { + id: 'comp-a', + workspaceId: 'ws-a', + name: 'Globex', + primaryDomain: 'globex.com', +}; + +const dossier: DossierSection = { + id: 'ds-1', + competitorId: COMP.id, + kind: 'pricing', + version: 1, + contentMd: '# Pricing', + claimIds: ['c-1', 'c-2'], + model: null, + batchId: null, + supersedesId: null, + createdAt: AT, +}; + +const battlecard: BattlecardSection = { + id: 'bc-1', + competitorId: COMP.id, + kind: 'why_we_win', + version: 1, + contentMd: '# Why we win', + claimIds: ['c-2', 'c-3'], + supersedesId: null, + createdAt: AT, +}; + +describe('loadWorkspaceExport', () => { + it('loads competitors with their chains and resolves cited claim ids once, deduplicated', async () => { + // Arrange + const claimRequests: string[][] = []; + const store: ExportReadStore = { + listCompetitors: async () => [COMP], + listDossierSections: async () => [dossier], + listBattlecardSections: async () => [battlecard], + getClaimsByIds: async (_workspaceId, claimIds) => { + claimRequests.push([...claimIds]); + return claimIds.map( + (id): Claim => ({ + id, + deltaId: 'd-1', + snapshotId: 'snap-1', + quoteText: 'q', + charStart: 0, + charEnd: 1, + sourceUrl: 'https://globex.com', + capturedAt: AT, + verifiedAt: AT, + }), + ); + }, + }; + + // Act + const input = await loadWorkspaceExport(store, { id: 'ws-a', name: 'Acme' }, 'https://app.test'); + + // Assert + expect(input.workspace.id).toBe('ws-a'); + expect(input.competitors).toHaveLength(1); + expect(input.competitors[0].claims.map((c) => c.id)).toEqual(['c-1', 'c-2', 'c-3']); + // c-2 is cited by both sections but requested once. + expect(claimRequests).toEqual([['c-1', 'c-2', 'c-3']]); + }); + + it('skips the claim lookup entirely when no section cites anything', async () => { + // Arrange + let claimCalls = 0; + const store: ExportReadStore = { + listCompetitors: async () => [COMP], + listDossierSections: async () => [{ ...dossier, claimIds: [] }], + listBattlecardSections: async () => [], + getClaimsByIds: async () => { + claimCalls += 1; + return []; + }, + }; + + // Act + const input = await loadWorkspaceExport(store, { id: 'ws-a', name: 'Acme' }, 'https://app.test'); + + // Assert + expect(input.competitors[0].claims).toEqual([]); + expect(claimCalls).toBe(0); + }); +}); diff --git a/packages/okf-export/src/load.ts b/packages/okf-export/src/load.ts new file mode 100644 index 0000000..30666c2 --- /dev/null +++ b/packages/okf-export/src/load.ts @@ -0,0 +1,34 @@ +import type { FlankStore } from '@flank/core'; +import type { CompetitorExport, WorkspaceExportInput } from './types'; + +/** The four workspace-scoped reads the export needs — nothing else leaks in. */ +export type ExportReadStore = Pick< + FlankStore, + 'listCompetitors' | 'listDossierSections' | 'listBattlecardSections' | 'getClaimsByIds' +>; + +/** + * Load a workspace's export input through the store's request-safe reads + * (workspace-scoped, fail-closed — Invariant 8 is the store's job, not ours). + * Claims are resolved once per competitor across all of its section heads. + */ +export const loadWorkspaceExport = async ( + store: ExportReadStore, + workspace: { readonly id: string; readonly name: string }, + baseUrl: string, +): Promise => { + const competitors = await store.listCompetitors(workspace.id); + const exports: CompetitorExport[] = []; + for (const competitor of competitors) { + const [dossierSections, battlecardSections] = await Promise.all([ + store.listDossierSections(workspace.id, competitor.id), + store.listBattlecardSections(workspace.id, competitor.id), + ]); + const claimIds = [ + ...new Set([...dossierSections, ...battlecardSections].flatMap((s) => s.claimIds)), + ]; + const claims = claimIds.length > 0 ? await store.getClaimsByIds(workspace.id, claimIds) : []; + exports.push({ competitor, dossierSections, battlecardSections, claims }); + } + return { workspace, baseUrl, competitors: exports }; +}; diff --git a/packages/okf-export/src/log.ts b/packages/okf-export/src/log.ts new file mode 100644 index 0000000000000000000000000000000000000000..53b873f4b50b8f7c6704f4f86432e098ce958a4c GIT binary patch literal 1614 zcma)6L2lbH5bW8n*aAT;1vU*IdU5KcMVdp<0y+3-7!WP3Z8j1qkaSu@Akatjg?&kv z5-FugQ&fOqisH`B&g@cDwbB;ssYYnvMpZRhVU@o9-tIftX;s0P&HM>7z8r}zpn*L~ z)ZF41wV!nHH%jJM+fOPF8}#W`T77~D^FclAX1UTxK%?2k#Rx7SE7hJ=Mb>4q2LxW% zD2u6* zNI^;>py!t|xYq@0EMQA1)lugyoD~%s%c~l`33~tz^s+4xtib73BY&9CVbD0OwfY)} zN}reZlm|3W3WX@}jN%xf|WVWH z*m7%2%(*V&9+^wx4nZbuSw&vKFv>)E{KFV~hRP~lCke$-K}i!WC#DnF@k*3LdK^N= zz)dLFrRijVe97y7P=~0$0k_gT6DPD!WEP5Vn=;lND>uiQIsgkauna^Y7RO>gU%?!% zA%o<}>BqAsr0I`MVeZllNC-%FLJtj;(j;}dh6Z5u(Hs)r7v0nMFzMO3$Z~25i^WIF zth!s>uiA!4dNqmUv8uQx*|W1|FqTPPp?eoRh+JNg(7pLXFk(y_uc$>|B3fvMp*)v%t~#>O|Dv~^ zN|JCI!){nO3*_KEZ2R>!Oy^Y3JZ(`l+M||Fuj?ejwR6|dbr!BUy3RW1Y4AH!=$Z&q M`R_wMmAf4M1}?Z38UO$Q literal 0 HcmV?d00001 diff --git a/packages/okf-export/src/manifest.test.ts b/packages/okf-export/src/manifest.test.ts new file mode 100644 index 0000000..541030a --- /dev/null +++ b/packages/okf-export/src/manifest.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest'; +import { + diffManifests, + isEmptyDiff, + manifestFromRecord, + manifestOf, + manifestToRecord, +} from './manifest'; + +describe('manifestOf', () => { + it('hashes each file and is stable for identical content', () => { + // Arrange + const files = new Map([ + ['index.md', 'one'], + ['a.md', 'two'], + ]); + + // Act + const first = manifestOf(files); + const second = manifestOf(new Map([...files])); + + // Assert + expect(first.get('index.md')).toMatch(/^[0-9a-f]{64}$/); + expect(manifestToRecord(first)).toEqual(manifestToRecord(second)); + }); +}); + +describe('manifestToRecord / manifestFromRecord', () => { + it('round-trips and sorts keys deterministically', () => { + // Arrange + const manifest = manifestOf( + new Map([ + ['z.md', 'z'], + ['a.md', 'a'], + ]), + ); + + // Act + const record = manifestToRecord(manifest); + + // Assert — keys sorted, restore is faithful. + expect(Object.keys(record)).toEqual(['a.md', 'z.md']); + expect(manifestToRecord(manifestFromRecord(record))).toEqual(record); + }); +}); + +describe('diffManifests', () => { + it('classifies added, modified, removed, and unchanged paths, each sorted', () => { + // Arrange + const previous = manifestOf( + new Map([ + ['keep.md', 'same'], + ['change.md', 'old'], + ['gone.md', 'bye'], + ]), + ); + const next = manifestOf( + new Map([ + ['keep.md', 'same'], + ['change.md', 'new'], + ['fresh.md', 'hi'], + ]), + ); + + // Act + const diff = diffManifests(previous, next); + + // Assert + expect(diff).toEqual({ + added: ['fresh.md'], + modified: ['change.md'], + removed: ['gone.md'], + unchanged: ['keep.md'], + }); + expect(isEmptyDiff(diff)).toBe(false); + }); + + it('reports an empty diff when nothing changed', () => { + // Arrange + const manifest = manifestOf(new Map([['index.md', 'x']])); + + // Act + const diff = diffManifests(manifest, manifestOf(new Map([['index.md', 'x']]))); + + // Assert + expect(isEmptyDiff(diff)).toBe(true); + expect(diff.unchanged).toEqual(['index.md']); + }); + + it('treats an empty previous manifest as an all-added first delivery', () => { + // Act + const diff = diffManifests(new Map(), manifestOf(new Map([['index.md', 'x']]))); + + // Assert + expect(diff.added).toEqual(['index.md']); + expect(isEmptyDiff(diff)).toBe(false); + }); +}); diff --git a/packages/okf-export/src/manifest.ts b/packages/okf-export/src/manifest.ts new file mode 100644 index 0000000..71e55f8 --- /dev/null +++ b/packages/okf-export/src/manifest.ts @@ -0,0 +1,67 @@ +import { createHash } from 'node:crypto'; + +/** + * A bundle manifest: bundle-relative path → sha256 of its file content. The manifest is what a + * delivery persists and what the next run diffs against — comparing hashes, never re-reading the + * remote repo, so "what changed" is deterministic and offline-computable. + */ +export type BundleManifest = ReadonlyMap; + +const sha256 = (content: string): string => createHash('sha256').update(content, 'utf8').digest('hex'); + +/** Manifest of a rendered bundle (path → content). */ +export const manifestOf = (files: ReadonlyMap): BundleManifest => { + const manifest = new Map(); + for (const [path, content] of files) manifest.set(path, sha256(content)); + return manifest; +}; + +/** A plain object form for persistence (jsonb), path-sorted so stored manifests are stable. */ +export const manifestToRecord = (manifest: BundleManifest): Readonly> => + Object.fromEntries([...manifest.entries()].sort(([a], [b]) => (a < b ? -1 : 1))); + +/** Restore a manifest from its persisted record form. */ +export const manifestFromRecord = (record: Readonly>): BundleManifest => + new Map(Object.entries(record)); + +export interface BundleDiff { + /** Paths present only in `next`. */ + readonly added: readonly string[]; + /** Paths in both whose content hash changed. */ + readonly modified: readonly string[]; + /** Paths present only in `previous`. */ + readonly removed: readonly string[]; + /** Paths in both with an unchanged hash. */ + readonly unchanged: readonly string[]; +} + +/** True when the diff represents no change at all — the signal to skip a delivery. */ +export const isEmptyDiff = (diff: BundleDiff): boolean => + diff.added.length === 0 && diff.modified.length === 0 && diff.removed.length === 0; + +/** + * Diff two manifests into added/modified/removed/unchanged path lists (each sorted). This is the + * "what changed since the last delivery" the Slack alert links to and the delivery record counts. + */ +export const diffManifests = (previous: BundleManifest, next: BundleManifest): BundleDiff => { + const added: string[] = []; + const modified: string[] = []; + const unchanged: string[] = []; + for (const [path, hash] of next) { + const before = previous.get(path); + if (before === undefined) added.push(path); + else if (before === hash) unchanged.push(path); + else modified.push(path); + } + const removed: string[] = []; + for (const path of previous.keys()) { + if (!next.has(path)) removed.push(path); + } + const sort = (paths: string[]): string[] => paths.sort((a, b) => (a < b ? -1 : 1)); + return { + added: sort(added), + modified: sort(modified), + removed: sort(removed), + unchanged: sort(unchanged), + }; +}; diff --git a/packages/okf-export/src/project.test.ts b/packages/okf-export/src/project.test.ts new file mode 100644 index 0000000..7b96d51 --- /dev/null +++ b/packages/okf-export/src/project.test.ts @@ -0,0 +1,203 @@ +import type { BattlecardSection, Claim, DossierSection } from '@flank/core'; +import { describe, expect, it } from 'vitest'; +import { projectWorkspaceBundle } from './project'; +import type { CompetitorExport, WorkspaceExportInput } from './types'; + +const AT_V1 = new Date('2026-06-01T06:00:00Z'); +const AT_V2 = new Date('2026-06-08T06:00:00Z'); +const VERIFIED_AT = new Date('2026-06-08T07:00:00Z'); + +const COMP = { + id: 'comp-a', + workspaceId: 'ws-a', + name: 'Globex', + primaryDomain: 'globex.com', +} as const; + +const claim = (id: string, verified: boolean): Claim => ({ + id, + deltaId: 'd-1', + snapshotId: 'snap-1', + quoteText: 'Analyst $59 per month', + charStart: 0, + charEnd: 21, + sourceUrl: 'https://globex.com/pricing', + capturedAt: AT_V2, + verifiedAt: verified ? VERIFIED_AT : null, +}); + +const dossier = ( + id: string, + version: number, + over: Partial = {}, +): DossierSection => ({ + id, + competitorId: COMP.id, + kind: 'pricing', + version, + contentMd: `# Pricing v${version}\n\n- Analyst $59/mo`, + claimIds: ['c-1'], + model: 'claude-sonnet-4-6', + batchId: 'batch-1', + supersedesId: version > 1 ? `ds-${version - 1}` : null, + createdAt: version > 1 ? AT_V2 : AT_V1, + ...over, +}); + +const battlecard = (id: string, version: number): BattlecardSection => ({ + id, + competitorId: COMP.id, + kind: 'pricing_counter', + version, + contentMd: `# Counter v${version}`, + claimIds: ['c-1'], + supersedesId: null, + createdAt: AT_V1, + ...{}, +}); + +const exportOf = (over: Partial = {}): CompetitorExport => ({ + competitor: COMP, + dossierSections: [dossier('ds-1', 1), dossier('ds-2', 2)], + battlecardSections: [battlecard('bc-1', 1)], + claims: [claim('c-1', true)], + ...over, +}); + +const inputOf = (competitors: readonly CompetitorExport[]): WorkspaceExportInput => ({ + workspace: { id: 'ws-a', name: 'Acme' }, + baseUrl: 'https://app.flank.test', + competitors, +}); + +describe('projectWorkspaceBundle', () => { + it('renders the section head as a golden concept doc with verified sources only', () => { + // Act + const { files, findings } = projectWorkspaceBundle(inputOf([exportOf()])); + + // Assert — byte-exact head doc; v1 renders no doc of its own. + expect(files.get('competitors/globex/dossier/pricing.md')).toBe( + [ + '---', + 'type: Dossier Section', + 'title: Globex — Pricing', + 'resource: https://app.flank.test/authed/c/comp-a', + 'timestamp: 2026-06-08T06:00:00.000Z', + 'kind: pricing', + 'model: claude-sonnet-4-6', + "version: '2'", + '---', + '', + '# Pricing v2', + '', + '- Analyst $59/mo', + '', + '## Sources', + '', + '- "Analyst $59 per month" — [globex.com](https://globex.com/pricing), captured 2026-06-08T06:00:00.000Z, verified 2026-06-08T07:00:00.000Z', + '', + '[← Globex](../index.md)', + '', + ].join('\n'), + ); + expect(findings).toEqual([]); + }); + + it('bundles exactly the expected files, path-sorted', () => { + // Act + const { files } = projectWorkspaceBundle(inputOf([exportOf()])); + + // Assert + expect([...files.keys()]).toEqual([ + 'competitors/globex/battlecard/pricing-counter.md', + 'competitors/globex/dossier/pricing.md', + 'competitors/globex/index.md', + 'index.md', + 'log.md', + ]); + }); + + it('writes every section version into log.md in chronological order', () => { + // Act + const { files } = projectWorkspaceBundle(inputOf([exportOf()])); + const log = files.get('log.md') ?? ''; + + // Assert — v1, battlecard v1 (same instant, stable tie-break), then v2. + const positions = [ + 'Globex: battlecard/pricing_counter v1 published (first version).', + 'Globex: dossier/pricing v1 published (first version).', + 'Globex: dossier/pricing v2 published (supersedes v1).', + ].map((line) => log.indexOf(line)); + expect(positions.every((p) => p >= 0)).toBe(true); + expect([...positions].sort((a, b) => a - b)).toEqual(positions); + }); + + it('is byte-deterministic under shuffled input order', () => { + // Arrange — same rows, reversed array orders. + const shuffled = exportOf({ + dossierSections: [dossier('ds-2', 2), dossier('ds-1', 1)], + }); + + // Act + const first = projectWorkspaceBundle(inputOf([exportOf()])); + const second = projectWorkspaceBundle(inputOf([shuffled])); + + // Assert + expect([...first.files.entries()]).toEqual([...second.files.entries()]); + }); + + it('gates unverified and unresolvable citations out of the bundle as error findings', () => { + // Arrange — head cites one unverified and one missing claim. + const gated = exportOf({ + dossierSections: [dossier('ds-1', 1, { claimIds: ['c-unverified', 'c-gone'] })], + battlecardSections: [], + claims: [claim('c-unverified', false)], + }); + + // Act + const { files, findings } = projectWorkspaceBundle(inputOf([gated])); + + // Assert — no Sources block rendered, both violations reported as errors. + expect(files.get('competitors/globex/dossier/pricing.md')).not.toContain('## Sources'); + expect(findings).toEqual([ + expect.objectContaining({ + check: 'UNVERIFIED_CITATION', + severity: 'error', + path: 'competitors/globex/dossier/pricing.md', + message: expect.stringContaining('c-unverified'), + }), + expect.objectContaining({ + check: 'UNVERIFIED_CITATION', + severity: 'error', + message: expect.stringContaining('c-gone'), + }), + ]); + }); + + it('keeps two same-named competitors apart with id-suffixed slugs', () => { + // Arrange + const twinA = exportOf(); + const twinB = exportOf({ + competitor: { ...COMP, id: 'comp-b', primaryDomain: 'other.example' }, + dossierSections: [dossier('ds-b', 1, { competitorId: 'comp-b' })], + battlecardSections: [], + }); + + // Act + const { files, findings } = projectWorkspaceBundle(inputOf([twinA, twinB])); + + // Assert + expect(files.has('competitors/globex-comp-a/index.md')).toBe(true); + expect(files.has('competitors/globex-comp-b/index.md')).toBe(true); + expect(findings.filter((f) => f.severity === 'error')).toEqual([]); + }); + + it('produces a minimal valid bundle for an empty workspace', () => { + // Act + const { files, findings } = projectWorkspaceBundle(inputOf([])); + + // Assert + expect([...files.keys()]).toEqual(['index.md', 'log.md']); + expect(findings).toEqual([]); + }); +}); diff --git a/packages/okf-export/src/project.ts b/packages/okf-export/src/project.ts new file mode 100644 index 0000000..c13ad8e --- /dev/null +++ b/packages/okf-export/src/project.ts @@ -0,0 +1,125 @@ +import type { Claim, Competitor } from '@flank/core'; +import { + buildBundle, + generateIndex, + slugify, + validateBundle, + type ConceptDoc, + type IndexEntry, + type LintFinding, +} from '@okf/core'; +import { versionLog } from './log'; +import { + BATTLECARD_KIND_TITLES, + DOSSIER_KIND_TITLES, + headsByKind, + sectionDoc, +} from './sections'; +import type { CompetitorExport, WorkspaceBundle, WorkspaceExportInput } from './types'; + +/** Stable, collision-free slug per competitor (name slug; id-suffixed on clash). */ +const competitorSlugs = ( + competitors: readonly CompetitorExport[], +): ReadonlyMap => { + const byName = new Map(); + for (const { competitor } of competitors) { + const slug = slugify(competitor.name); + byName.set(slug, (byName.get(slug) ?? 0) + 1); + } + const slugs = new Map(); + for (const { competitor } of competitors) { + const slug = slugify(competitor.name); + slugs.set( + competitor.id, + (byName.get(slug) ?? 0) > 1 ? `${slug}-${slugify(competitor.id)}` : slug, + ); + } + return slugs; +}; + +const competitorIndexDoc = ( + competitor: Competitor, + slug: string, + baseUrl: string, + sectionDocs: readonly ConceptDoc[], +): ConceptDoc => { + const group = (family: 'dossier' | 'battlecard'): string => + sectionDocs + .filter((doc) => doc.path.includes(`/${family}/`)) + .map((doc) => `- [${doc.frontmatter.title ?? doc.path}](${doc.path.split('/').slice(2).join('/')})`) + .join('\n'); + + const dossier = group('dossier'); + const battlecard = group('battlecard'); + const body = [ + ...(dossier === '' ? [] : [`## Dossier\n\n${dossier}`]), + ...(battlecard === '' ? [] : [`## Battlecard\n\n${battlecard}`]), + ].join('\n\n'); + + return { + path: `competitors/${slug}/index.md`, + frontmatter: { + type: 'Competitor', + title: competitor.name, + description: `Competitive dossier for ${competitor.name} (${competitor.primaryDomain}).`, + resource: `${baseUrl}/authed/c/${competitor.id}`, + extra: { domain: competitor.primaryDomain }, + }, + body, + }; +}; + +/** + * Project a workspace's published competitive knowledge into an OKF bundle. + * Pure and byte-deterministic: same input rows → identical bytes. Sections + * render at their chain heads only; claims render only when verified — every + * violation surfaces as a finding, and any `error` finding means the caller + * must not ship the bundle (projection-only rule). + */ +export const projectWorkspaceBundle = (input: WorkspaceExportInput): WorkspaceBundle => { + const slugs = competitorSlugs(input.competitors); + const docs: ConceptDoc[] = []; + const findings: LintFinding[] = []; + const rootEntries: IndexEntry[] = []; + + for (const competitorExport of input.competitors) { + const { competitor, dossierSections, battlecardSections, claims } = competitorExport; + const slug = slugs.get(competitor.id) ?? slugify(competitor.id); + const claimsById: ReadonlyMap = new Map(claims.map((c) => [c.id, c])); + const context = { competitor, competitorSlug: slug, baseUrl: input.baseUrl, claimsById }; + + const sectionDocs: ConceptDoc[] = []; + for (const head of headsByKind(dossierSections)) { + const result = sectionDoc(head, 'dossier', DOSSIER_KIND_TITLES[head.kind], context); + sectionDocs.push(result.doc); + findings.push(...result.findings); + } + for (const head of headsByKind(battlecardSections)) { + const result = sectionDoc(head, 'battlecard', BATTLECARD_KIND_TITLES[head.kind], context); + sectionDocs.push(result.doc); + findings.push(...result.findings); + } + + docs.push(competitorIndexDoc(competitor, slug, input.baseUrl, sectionDocs), ...sectionDocs); + rootEntries.push({ + path: `competitors/${slug}/index.md`, + title: competitor.name, + description: competitor.primaryDomain, + }); + } + + docs.push( + generateIndex({ + title: `${input.workspace.name} — competitive knowledge`, + description: + 'OKF bundle projected from published, claim-verified dossiers and battlecards. See log.md for the change history.', + entries: rootEntries, + }), + versionLog(input.competitors), + ); + + return { + files: buildBundle(docs), + findings: [...findings, ...validateBundle(docs)], + }; +}; diff --git a/packages/okf-export/src/publisher.ts b/packages/okf-export/src/publisher.ts new file mode 100644 index 0000000..a41319f --- /dev/null +++ b/packages/okf-export/src/publisher.ts @@ -0,0 +1,49 @@ +import type { BundleDiff } from './manifest'; + +/** + * A customer-designated git destination for a workspace's OKF bundle. Operator-configured for M2 + * (self-serve UI config deferred); the sweep receives a list of these. + */ +export interface GitBundleTarget { + readonly workspaceId: string; + readonly workspaceName: string; + /** Only 'github' is implemented; the field keeps the port provider-neutral. */ + readonly provider: 'github'; + /** `owner/name`. */ + readonly repo: string; + /** The branch the bundle commit lands on (created from `baseBranch` if absent). */ + readonly branch: string; + /** The branch a pull request targets; when null, the commit is pushed without opening a PR. */ + readonly baseBranch: string | null; + /** Directory the bundle is written under, e.g. `intel/acme` (no leading/trailing slash). */ + readonly subdir: string; +} + +/** A request to publish one bundle: the files (already under `target.subdir`) plus the diff context. */ +export interface BundlePublishRequest { + readonly target: GitBundleTarget; + /** Full current bundle as repo-relative path → content (paths already prefixed with subdir). */ + readonly files: ReadonlyMap; + /** Paths (repo-relative) to delete — the diff's removed set mapped under the subdir. */ + readonly deletions: readonly string[]; + readonly commitMessage: string; + /** The bundle diff, for a publisher that wants to describe the change (e.g. PR body). */ + readonly diff: BundleDiff; +} + +export type BundlePublishResult = + | { + readonly ok: true; + readonly commitSha: string; + readonly branchRef: string; + readonly pullRequestUrl: string | null; + } + | { readonly ok: false; readonly error: string }; + +/** + * Port for pushing a bundle to a git host. Concrete impls (GitHub) live in the pipeline package, + * mirroring the Notifier port/impl split; the pure delivery orchestration depends only on this. + */ +export interface BundlePublisher { + publish(request: BundlePublishRequest): Promise; +} diff --git a/packages/okf-export/src/sections.ts b/packages/okf-export/src/sections.ts new file mode 100644 index 0000000..c7f386e --- /dev/null +++ b/packages/okf-export/src/sections.ts @@ -0,0 +1,129 @@ +import type { + BattlecardSection, + BattlecardSectionKind, + Claim, + Competitor, + DossierSection, + DossierSectionKind, +} from '@flank/core'; +import { slugify, type ConceptDoc, type LintFinding } from '@okf/core'; + +export const DOSSIER_KIND_TITLES: Readonly> = { + overview: 'Overview', + pricing: 'Pricing', + product: 'Product', + gtm: 'Go-to-market', + team: 'Team', +}; + +export const BATTLECARD_KIND_TITLES: Readonly> = { + why_we_win: 'Why we win', + landmines: 'Landmines', + pricing_counter: 'Pricing counter', + objections: 'Objection handling', +}; + +type AnySection = DossierSection | BattlecardSection; + +/** Head of each (kind) version chain — the only version that renders as a doc. */ +export const headsByKind = (sections: readonly S[]): readonly S[] => { + const heads = new Map(); + for (const section of sections) { + const current = heads.get(section.kind); + if (current === undefined || section.version > current.version) { + heads.set(section.kind, section); + } + } + return [...heads.values()].sort((a, b) => (a.kind < b.kind ? -1 : 1)); +}; + +const collapseWhitespace = (text: string): string => text.replaceAll(/\s+/g, ' ').trim(); + +const hostOf = (url: string): string => { + try { + return new URL(url).host; + } catch { + return url; + } +}; + +const claimLine = (claim: Claim): string => { + const quote = collapseWhitespace(claim.quoteText); + const captured = claim.capturedAt.toISOString(); + const verified = claim.verifiedAt?.toISOString() ?? ''; + return `- "${quote}" — [${hostOf(claim.sourceUrl)}](${claim.sourceUrl}), captured ${captured}, verified ${verified}`; +}; + +export interface SectionDocResult { + readonly doc: ConceptDoc; + readonly findings: readonly LintFinding[]; +} + +export interface SectionDocContext { + readonly competitor: Competitor; + readonly competitorSlug: string; + readonly baseUrl: string; + readonly claimsById: ReadonlyMap; +} + +/** + * Render one section head as a concept doc. Only VERIFIED claims render into + * the Sources block (Invariant 1); a cited claim that is missing or + * unverified yields an UNVERIFIED_CITATION error finding — the projection + * gate that keeps unproven provenance out of shipped bundles. + */ +export const sectionDoc = ( + section: AnySection, + family: 'dossier' | 'battlecard', + kindTitle: string, + context: SectionDocContext, +): SectionDocResult => { + const path = `competitors/${context.competitorSlug}/${family}/${slugify(section.kind)}.md`; + + const findings: LintFinding[] = []; + const verified: Claim[] = []; + for (const claimId of section.claimIds) { + const claim = context.claimsById.get(claimId); + if (claim === undefined) { + findings.push({ + check: 'UNVERIFIED_CITATION', + severity: 'error', + path, + message: `cited claim ${claimId} could not be resolved`, + }); + } else if (claim.verifiedAt === null) { + findings.push({ + check: 'UNVERIFIED_CITATION', + severity: 'error', + path, + message: `cited claim ${claimId} is unverified`, + }); + } else { + verified.push(claim); + } + } + + const bodyParts = [ + section.contentMd.trim(), + ...(verified.length > 0 ? [`## Sources\n\n${verified.map(claimLine).join('\n')}`] : []), + `[← ${context.competitor.name}](../index.md)`, + ]; + + const doc: ConceptDoc = { + path, + frontmatter: { + type: family === 'dossier' ? 'Dossier Section' : 'Battlecard Section', + title: `${context.competitor.name} — ${kindTitle}`, + resource: `${context.baseUrl}/authed/c/${context.competitor.id}`, + timestamp: section.createdAt.toISOString(), + extra: { + kind: section.kind, + version: String(section.version), + ...('model' in section && section.model !== null && { model: section.model }), + }, + }, + body: bodyParts.join('\n\n'), + }; + + return { doc, findings }; +}; diff --git a/packages/okf-export/src/tar.test.ts b/packages/okf-export/src/tar.test.ts new file mode 100644 index 0000000..0471021 --- /dev/null +++ b/packages/okf-export/src/tar.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import { bundleToTar } from './tar'; + +const decoder = new TextDecoder(); + +/** Minimal ustar reader for assertions: returns [name, content] pairs. */ +const readTar = (archive: Uint8Array): [string, string][] => { + const entries: [string, string][] = []; + let offset = 0; + while (offset < archive.length) { + const header = archive.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) break; + const name = decoder.decode(header.subarray(0, 100)).replace(/\0.*$/, ''); + const size = Number.parseInt(decoder.decode(header.subarray(124, 136)).replace(/\0.*$/, ''), 8); + const content = decoder.decode(archive.subarray(offset + 512, offset + 512 + size)); + entries.push([name, content]); + offset += 512 + Math.ceil(size / 512) * 512; + } + return entries; +}; + +describe('bundleToTar', () => { + const files = new Map([ + ['index.md', '---\ntype: Index\n---\n'], + ['tables/orders.md', '---\ntype: Table\n---\n\nbody\n'], + ]); + + it('round-trips names and contents through a ustar reader', () => { + // Act + const archive = bundleToTar(files); + + // Assert + expect(readTar(archive)).toEqual([...files.entries()]); + }); + + it('carries valid header checksums', () => { + // Arrange + const archive = bundleToTar(files); + const header = archive.subarray(0, 512); + + // Act — recompute with the checksum field read as spaces. + const stored = Number.parseInt(decoder.decode(header.subarray(148, 155)), 8); + let sum = 0; + for (let i = 0; i < 512; i += 1) { + sum += i >= 148 && i < 156 ? 0x20 : header[i]; + } + + // Assert + expect(stored).toBe(sum); + }); + + it('is byte-deterministic and block-aligned with the end-of-archive marker', () => { + // Act + const first = bundleToTar(files); + const second = bundleToTar(files); + + // Assert + expect(first).toEqual(second); + expect(first.length % 512).toBe(0); + expect(first.subarray(first.length - 1024).every((byte) => byte === 0)).toBe(true); + }); + + it('rejects entry names over 100 bytes', () => { + // Arrange + const long = new Map([[`${'a/'.repeat(60)}x.md`, 'content']]); + + // Act & Assert + expect(() => bundleToTar(long)).toThrow(/100 bytes/); + }); +}); diff --git a/packages/okf-export/src/tar.ts b/packages/okf-export/src/tar.ts new file mode 100644 index 0000000..7f6a5bc --- /dev/null +++ b/packages/okf-export/src/tar.ts @@ -0,0 +1,61 @@ +/** + * Deterministic POSIX ustar archive of a bundle (path → content). Fixed + * metadata (mode 0644, uid/gid 0, mtime 0) and the bundle's sorted path order + * make the tarball byte-stable — same bundle, same tarball, diffable in + * object storage. Compression is the delivery layer's choice. + */ + +const BLOCK = 512; +const encoder = new TextEncoder(); + +const writeString = (block: Uint8Array, offset: number, value: string): void => { + block.set(encoder.encode(value), offset); +}; + +const writeOctal = (block: Uint8Array, offset: number, length: number, value: number): void => { + // Octal ASCII, zero-padded, NUL-terminated — the conventional ustar form. + writeString(block, offset, `${value.toString(8).padStart(length - 1, '0')}\0`); +}; + +const headerFor = (path: string, size: number): Uint8Array => { + const name = encoder.encode(path); + if (name.length > 100) { + throw new Error(`tar entry name exceeds 100 bytes: ${path}`); + } + const block = new Uint8Array(BLOCK); + block.set(name, 0); + writeOctal(block, 100, 8, 0o644); // mode + writeOctal(block, 108, 8, 0); // uid + writeOctal(block, 116, 8, 0); // gid + writeOctal(block, 124, 12, size); + writeOctal(block, 136, 12, 0); // mtime — fixed for determinism + writeString(block, 148, ' '); // checksum field counts as spaces + writeString(block, 156, '0'); // typeflag: regular file + writeString(block, 257, 'ustar\0'); + writeString(block, 263, '00'); + + const checksum = block.reduce((sum, byte) => sum + byte, 0); + writeString(block, 148, `${checksum.toString(8).padStart(6, '0')}\0 `); + return block; +}; + +export const bundleToTar = (files: ReadonlyMap): Uint8Array => { + const blocks: Uint8Array[] = []; + for (const [path, content] of files) { + const data = encoder.encode(content); + blocks.push(headerFor(path, data.length)); + const padded = new Uint8Array(Math.ceil(data.length / BLOCK) * BLOCK); + padded.set(data, 0); + blocks.push(padded); + } + blocks.push(new Uint8Array(BLOCK), new Uint8Array(BLOCK)); // end-of-archive marker + + const total = blocks.reduce((sum, b) => sum + b.length, 0); + const archive = new Uint8Array(total); + let offset = 0; + for (const block of blocks) { + archive.set(block, offset); + offset += block.length; + } + return archive; +}; diff --git a/packages/okf-export/src/types.ts b/packages/okf-export/src/types.ts new file mode 100644 index 0000000..611cbad --- /dev/null +++ b/packages/okf-export/src/types.ts @@ -0,0 +1,30 @@ +import type { BattlecardSection, Claim, Competitor, DossierSection } from '@flank/core'; +import type { LintFinding } from '@okf/core'; + +/** Everything the projector needs for one competitor, already workspace-scoped. */ +export interface CompetitorExport { + readonly competitor: Competitor; + /** Full version chains (all versions) — the log renders from them; docs render the head only. */ + readonly dossierSections: readonly DossierSection[]; + readonly battlecardSections: readonly BattlecardSection[]; + /** Resolved claim rows for every claimId cited by any exported section head. */ + readonly claims: readonly Claim[]; +} + +export interface WorkspaceExportInput { + readonly workspace: { readonly id: string; readonly name: string }; + /** App origin for `resource:` deep links, e.g. https://app.flank.example (no trailing slash). */ + readonly baseUrl: string; + readonly competitors: readonly CompetitorExport[]; +} + +/** + * The projection result. `files` is the deterministic bundle (path → content). + * `findings` merges structural checks with the semantic UNVERIFIED_CITATION + * gate; any `severity: 'error'` finding means the bundle MUST NOT ship + * (projection-only rule — the lint check guards the gate itself). + */ +export interface WorkspaceBundle { + readonly files: ReadonlyMap; + readonly findings: readonly LintFinding[]; +} diff --git a/packages/okf-export/tsconfig.json b/packages/okf-export/tsconfig.json new file mode 100644 index 0000000..824d57c --- /dev/null +++ b/packages/okf-export/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/okf/package.json b/packages/okf/package.json new file mode 100644 index 0000000..35a446b --- /dev/null +++ b/packages/okf/package.json @@ -0,0 +1,19 @@ +{ + "name": "@okf/core", + "version": "0.1.0", + "private": true, + "description": "App-neutral Open Knowledge Format layer: concept docs, deterministic frontmatter + bundle rendering, index/log generation, structural validation. No I/O. Built inside flank first, slated for extraction to .scaffold/ for the sibling apps.", + "type": "module", + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "^26.0.0", + "typescript": "^6.0.3", + "vitest": "^4.1.9" + } +} diff --git a/packages/okf/src/bundle.test.ts b/packages/okf/src/bundle.test.ts new file mode 100644 index 0000000..0b67512 --- /dev/null +++ b/packages/okf/src/bundle.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from 'vitest'; +import { buildBundle, renderConcept } from './bundle'; +import { generateIndex } from './index-gen'; +import { generateLog } from './log-gen'; +import type { ConceptDoc } from './concept'; + +/** The OKF blog post's own example, as a golden fixture. */ +const ordersDoc: ConceptDoc = { + path: 'tables/orders.md', + frontmatter: { + type: 'BigQuery Table', + title: 'Orders', + description: 'One row per completed customer order.', + resource: 'https://console.cloud.google.com/bigquery?p=acme&d=sales&t=orders', + tags: ['sales', 'revenue'], + timestamp: '2026-05-28T14:30:00Z', + }, + body: [ + '# Schema', + '', + '| Column | Type | Description |', + '|--------|------|-------------|', + '| `order_id` | STRING | Globally unique order identifier. |', + '| `customer_id` | STRING | FK to [customers](customers.md). |', + '', + '# Joins', + '', + 'Joined with [customers](customers.md) on `customer_id`.', + ].join('\n'), +}; + +describe('renderConcept', () => { + it('renders the golden fixture byte-exactly', () => { + // Act + const content = renderConcept(ordersDoc); + + // Assert + expect(content).toBe( + [ + '---', + 'type: BigQuery Table', + 'title: Orders', + 'description: One row per completed customer order.', + 'resource: https://console.cloud.google.com/bigquery?p=acme&d=sales&t=orders', + 'tags: [sales, revenue]', + 'timestamp: 2026-05-28T14:30:00Z', + '---', + '', + '# Schema', + '', + '| Column | Type | Description |', + '|--------|------|-------------|', + '| `order_id` | STRING | Globally unique order identifier. |', + '| `customer_id` | STRING | FK to [customers](customers.md). |', + '', + '# Joins', + '', + 'Joined with [customers](customers.md) on `customer_id`.', + '', + ].join('\n'), + ); + }); + + it('renders a bodyless concept as frontmatter only', () => { + // Arrange + const doc: ConceptDoc = { path: 'a.md', frontmatter: { type: 'Stub' }, body: ' \n' }; + + // Act & Assert + expect(renderConcept(doc)).toBe('---\ntype: Stub\n---\n'); + }); +}); + +describe('buildBundle', () => { + const customersDoc: ConceptDoc = { + path: 'tables/customers.md', + frontmatter: { type: 'BigQuery Table', title: 'Customers' }, + body: 'See [orders](orders.md).', + }; + + it('is byte-deterministic and path-sorted regardless of input order', () => { + // Arrange + const index = generateIndex({ + title: 'Sales', + entries: [ + { path: 'tables/orders.md', title: 'Orders' }, + { path: 'tables/customers.md', title: 'Customers' }, + ], + }); + const log = generateLog([ + { timestamp: '2026-05-28T14:30:00Z', summary: 'Initial capture.', details: ['2 tables'] }, + ]); + + // Act — two builds, different input order. + const first = buildBundle([ordersDoc, customersDoc, index, log]); + const second = buildBundle([log, index, customersDoc, ordersDoc]); + + // Assert — identical keys in sorted order, identical bytes. + expect([...first.keys()]).toEqual([ + 'index.md', + 'log.md', + 'tables/customers.md', + 'tables/orders.md', + ]); + expect([...first.entries()]).toEqual([...second.entries()]); + }); + + it('rejects duplicate paths', () => { + // Act & Assert + expect(() => buildBundle([ordersDoc, { ...ordersDoc, body: 'other' }])).toThrow(/duplicate/); + }); + + it('rejects absolute and non-markdown paths', () => { + // Act & Assert + expect(() => + buildBundle([{ path: '/etc/x.md', frontmatter: { type: 'T' }, body: '' }]), + ).toThrow(/invalid bundle path/); + expect(() => + buildBundle([{ path: 'tables/orders.txt', frontmatter: { type: 'T' }, body: '' }]), + ).toThrow(/invalid bundle path/); + }); +}); + +describe('generateIndex', () => { + it('groups entries by top-level directory with sorted sections', () => { + // Act + const index = generateIndex({ + title: 'Sales knowledge', + description: 'Bundle entry point.', + entries: [ + { path: 'tables/orders.md', title: 'Orders', description: 'One row per order.' }, + { path: 'metrics/wau.md', title: 'Weekly Active Users' }, + { path: 'tables/customers.md', title: 'Customers' }, + ], + }); + + // Assert + expect(index.path).toBe('index.md'); + expect(index.frontmatter.type).toBe('Index'); + expect(index.body).toBe( + [ + '## metrics', + '', + '- [Weekly Active Users](metrics/wau.md)', + '', + '## tables', + '', + '- [Customers](tables/customers.md)', + '- [Orders](tables/orders.md) — One row per order.', + ].join('\n'), + ); + }); +}); + +describe('generateLog', () => { + it('renders entries in caller order without reordering (append-only)', () => { + // Act + const log = generateLog([ + { timestamp: '2026-05-01T00:00:00Z', summary: 'First run.' }, + { timestamp: '2026-05-08T00:00:00Z', summary: 'Second run.', details: ['1 page touched'] }, + ]); + + // Assert + expect(log.body).toBe( + [ + '## 2026-05-01T00:00:00Z', + '', + 'First run.', + '', + '## 2026-05-08T00:00:00Z', + '', + 'Second run.', + '', + '- 1 page touched', + ].join('\n'), + ); + }); +}); diff --git a/packages/okf/src/bundle.ts b/packages/okf/src/bundle.ts new file mode 100644 index 0000000..367e16c --- /dev/null +++ b/packages/okf/src/bundle.ts @@ -0,0 +1,34 @@ +import type { ConceptDoc } from './concept'; +import { emitFrontmatter } from './frontmatter'; + +/** + * Render a concept doc to its exact file content. Byte-deterministic: fixed + * frontmatter key order, trimmed body, LF endings, single trailing newline. + */ +export const renderConcept = (doc: ConceptDoc): string => { + const header = `---\n${emitFrontmatter(doc.frontmatter)}\n---\n`; + const body = doc.body.trim(); + return body === '' ? header : `${header}\n${body}\n`; +}; + +const BUNDLE_PATH = /^[a-z0-9][a-z0-9/._-]*\.md$/; + +/** + * Build a bundle as an immutable path → content map, sorted by path so + * iteration (and any archive built from it) is deterministic. Duplicate or + * malformed paths are producer bugs and fail fast; content-level problems are + * the validator's job. + */ +export const buildBundle = (docs: readonly ConceptDoc[]): ReadonlyMap => { + const files = new Map(); + for (const doc of [...docs].sort((a, b) => (a.path < b.path ? -1 : 1))) { + if (!BUNDLE_PATH.test(doc.path)) { + throw new Error(`invalid bundle path (want relative kebab-case .md): ${doc.path}`); + } + if (files.has(doc.path)) { + throw new Error(`duplicate bundle path: ${doc.path}`); + } + files.set(doc.path, renderConcept(doc)); + } + return files; +}; diff --git a/packages/okf/src/concept.ts b/packages/okf/src/concept.ts new file mode 100644 index 0000000..16b4c32 --- /dev/null +++ b/packages/okf/src/concept.ts @@ -0,0 +1,47 @@ +/** + * OKF concept document model (spec v0.1). A bundle is a directory of markdown + * concept docs; `type` is the only mandatory frontmatter field. Everything in + * this package is a pure function over these types — writing to disk/S3/git is + * the consuming app's job (producer/consumer separation). + */ + +/** + * Fixed rendering order for the standard frontmatter keys so identical input + * always yields identical bytes (bundles must stay git-diffable). + */ +export const FRONTMATTER_KEY_ORDER = [ + 'type', + 'title', + 'description', + 'resource', + 'tags', + 'timestamp', +] as const; + +export interface ConceptFrontmatter { + /** The only mandatory OKF field, e.g. "Battlecard", "Postgres Table". */ + readonly type: string; + readonly title?: string; + readonly description?: string; + /** Deep link back into the producing app or source system. */ + readonly resource?: string; + readonly tags?: readonly string[]; + /** + * ISO-8601 instant carried from source data. Never wall-clock at render + * time — that would break byte-determinism. + */ + readonly timestamp?: string; + /** + * Producer-defined fields beyond the standard set, rendered after the + * standard keys in sorted key order. Values are always strings. + */ + readonly extra?: Readonly>; +} + +export interface ConceptDoc { + /** Bundle-relative POSIX path, e.g. `tables/orders.md`. */ + readonly path: string; + readonly frontmatter: ConceptFrontmatter; + /** Markdown body without the frontmatter block. */ + readonly body: string; +} diff --git a/packages/okf/src/frontmatter.test.ts b/packages/okf/src/frontmatter.test.ts new file mode 100644 index 0000000..ec14dd1 --- /dev/null +++ b/packages/okf/src/frontmatter.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import { emitFrontmatter, parseConcept } from './frontmatter'; +import { renderConcept } from './bundle'; +import type { ConceptFrontmatter } from './concept'; + +describe('emitFrontmatter', () => { + it('renders standard keys in fixed order regardless of object key order', () => { + // Arrange — keys deliberately out of order. + const frontmatter: ConceptFrontmatter = { + timestamp: '2026-05-28T14:30:00Z', + tags: ['sales', 'revenue'], + type: 'BigQuery Table', + title: 'Orders', + }; + + // Act + const yaml = emitFrontmatter(frontmatter); + + // Assert + expect(yaml).toBe( + [ + 'type: BigQuery Table', + 'title: Orders', + 'tags: [sales, revenue]', + // Unquoted like the spec's own example; parseConcept always returns strings. + 'timestamp: 2026-05-28T14:30:00Z', + ].join('\n'), + ); + }); + + it('renders extra keys after standard keys in sorted order', () => { + // Arrange + const frontmatter: ConceptFrontmatter = { + type: 'Battlecard', + extra: { workspace: 'acme', competitor: 'globex' }, + }; + + // Act & Assert + expect(emitFrontmatter(frontmatter)).toBe( + ['type: Battlecard', 'competitor: globex', 'workspace: acme'].join('\n'), + ); + }); + + it('quotes values a YAML parser would read as non-strings or syntax', () => { + // Arrange + const frontmatter: ConceptFrontmatter = { + type: 'Metric', + title: 'true', + description: 'ratio: definition pending', + extra: { note: "O'Brien's draft", version: '42' }, + }; + + // Act + const yaml = emitFrontmatter(frontmatter); + + // Assert + expect(yaml).toContain("title: 'true'"); + expect(yaml).toContain("description: 'ratio: definition pending'"); + expect(yaml).toContain("note: 'O''Brien''s draft'"); + expect(yaml).toContain("version: '42'"); + }); +}); + +describe('parseConcept', () => { + it('round-trips a rendered concept document', () => { + // Arrange + const doc = { + path: 'tables/orders.md', + frontmatter: { + type: 'BigQuery Table', + title: 'Orders', + description: 'One row per completed customer order.', + resource: 'https://example.com/orders', + tags: ['sales', "o'brien"], + timestamp: '2026-05-28T14:30:00Z', + extra: { owner: 'data-eng' }, + }, + body: '# Schema\n\nJoined with [customers](customers.md).', + }; + + // Act + const parsed = parseConcept(renderConcept(doc)); + + // Assert + expect(parsed.frontmatter).toEqual(doc.frontmatter); + expect(parsed.body).toBe(doc.body); + }); + + it('rejects content without a frontmatter block', () => { + // Act & Assert + expect(() => parseConcept('# Just markdown\n')).toThrow(/frontmatter/); + }); +}); diff --git a/packages/okf/src/frontmatter.ts b/packages/okf/src/frontmatter.ts new file mode 100644 index 0000000..7cc22fd --- /dev/null +++ b/packages/okf/src/frontmatter.ts @@ -0,0 +1,151 @@ +import { FRONTMATTER_KEY_ORDER, type ConceptFrontmatter } from './concept'; + +/** + * Deterministic emitter/parser for the small YAML subset OKF frontmatter + * needs: string scalars and flat string arrays. Hand-rolled instead of a YAML + * dependency so output bytes are fully under our control (key order, quoting) + * and the package stays dependency-free. + */ + +/** + * Values that a YAML parser would read as a non-string (bool, null, number) + * must be quoted so consumers always get strings back. + */ +const YAML_NON_STRING = /^(?:true|false|null|~|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)$/i; + +const needsQuoting = (value: string): boolean => { + if (value === '') return true; + if (YAML_NON_STRING.test(value)) return true; + if (value !== value.trim()) return true; + if (value.includes(': ') || value.endsWith(':')) return true; + if (value.includes(' #') || value.includes('\n')) return true; + // Leading characters that carry YAML syntax meaning. + if (/^[-?[\]{}>|*&!%@`"'#,]/.test(value)) return true; + return value.includes("'") || value.includes('"'); +}; + +const quoteScalar = (value: string): string => + needsQuoting(value) ? `'${value.replaceAll("'", "''")}'` : value; + +const unquoteScalar = (raw: string): string => { + const trimmed = raw.trim(); + if (trimmed.startsWith("'") && trimmed.endsWith("'") && trimmed.length >= 2) { + return trimmed.slice(1, -1).replaceAll("''", "'"); + } + if (trimmed.startsWith('"') && trimmed.endsWith('"') && trimmed.length >= 2) { + return trimmed.slice(1, -1); + } + return trimmed; +}; + +const emitArray = (values: readonly string[]): string => + `[${values.map(quoteScalar).join(', ')}]`; + +/** Split an inline YAML array body on top-level commas (quote-aware). */ +const splitInlineArray = (body: string): string[] => { + const items: string[] = []; + let current = ''; + let inQuote = false; + for (let i = 0; i < body.length; i += 1) { + const char = body[i]; + if (char === "'") { + // '' inside a quoted scalar is an escaped quote, not a boundary. + if (inQuote && body[i + 1] === "'") { + current += "''"; + i += 1; + continue; + } + inQuote = !inQuote; + current += char; + } else if (char === ',' && !inQuote) { + items.push(current); + current = ''; + } else { + current += char; + } + } + if (current.trim() !== '') items.push(current); + return items.map(unquoteScalar); +}; + +/** + * Render frontmatter to YAML lines: standard keys in FRONTMATTER_KEY_ORDER, + * then `extra` keys in sorted order. Deterministic by construction. + */ +export const emitFrontmatter = (frontmatter: ConceptFrontmatter): string => { + const lines: string[] = []; + for (const key of FRONTMATTER_KEY_ORDER) { + const value = frontmatter[key]; + if (value === undefined) continue; + lines.push( + Array.isArray(value) + ? `${key}: ${emitArray(value)}` + : `${key}: ${quoteScalar(value as string)}`, + ); + } + const extraEntries = Object.entries(frontmatter.extra ?? {}).sort(([a], [b]) => + a < b ? -1 : 1, + ); + for (const [key, value] of extraEntries) { + lines.push(`${key}: ${quoteScalar(value)}`); + } + return lines.join('\n'); +}; + +export interface ParsedConcept { + readonly frontmatter: ConceptFrontmatter; + readonly body: string; +} + +/** + * Parse a rendered concept document back into frontmatter + body. Inverse of + * the emitter for the subset it produces; used by consumers reading bundles + * and by round-trip tests. + */ +export const parseConcept = (content: string): ParsedConcept => { + if (!content.startsWith('---\n')) { + throw new Error('concept document must start with a frontmatter block (---)'); + } + const end = content.indexOf('\n---\n', 4); + if (end === -1) { + throw new Error('unterminated frontmatter block'); + } + const yamlBlock = content.slice(4, end); + const body = content.slice(end + '\n---\n'.length).replace(/^\n/, '').replace(/\n$/, ''); + + let type = ''; + const scalars: Record = {}; + let tags: readonly string[] | undefined; + for (const line of yamlBlock.split('\n')) { + if (line.trim() === '') continue; + const separator = line.indexOf(': '); + if (separator === -1) { + throw new Error(`unparseable frontmatter line: ${line}`); + } + const key = line.slice(0, separator); + const rawValue = line.slice(separator + 2); + if (rawValue.startsWith('[') && rawValue.endsWith(']')) { + if (key !== 'tags') { + throw new Error(`array values are only supported for tags, got: ${key}`); + } + tags = splitInlineArray(rawValue.slice(1, -1)); + } else if (key === 'type') { + type = unquoteScalar(rawValue); + } else { + scalars[key] = unquoteScalar(rawValue); + } + } + + const standard = new Set(FRONTMATTER_KEY_ORDER); + const extraEntries = Object.entries(scalars).filter(([key]) => !standard.has(key)); + const frontmatter: ConceptFrontmatter = { + type, + ...(scalars.title !== undefined && { title: scalars.title }), + ...(scalars.description !== undefined && { description: scalars.description }), + ...(scalars.resource !== undefined && { resource: scalars.resource }), + ...(tags !== undefined && { tags }), + ...(scalars.timestamp !== undefined && { timestamp: scalars.timestamp }), + ...(extraEntries.length > 0 && { extra: Object.fromEntries(extraEntries) }), + }; + return { frontmatter, body }; +}; diff --git a/packages/okf/src/index-gen.ts b/packages/okf/src/index-gen.ts new file mode 100644 index 0000000..9a05ef9 --- /dev/null +++ b/packages/okf/src/index-gen.ts @@ -0,0 +1,55 @@ +import type { ConceptDoc } from './concept'; +import { relativeLink } from './links'; + +/** + * `index.md` generation — the bundle's progressive-disclosure entry point. + * Entries are grouped by top-level directory and sorted by path, so the index + * is deterministic regardless of input order. + */ + +export interface IndexEntry { + readonly path: string; + readonly title: string; + readonly description?: string; +} + +export interface IndexOptions { + readonly title: string; + readonly description?: string; + readonly entries: readonly IndexEntry[]; +} + +const topLevelGroup = (path: string): string => { + const index = path.indexOf('/'); + return index === -1 ? '' : path.slice(0, index); +}; + +const entryLine = (entry: IndexEntry): string => { + const link = `[${entry.title}](${relativeLink('index.md', entry.path)})`; + return entry.description === undefined ? `- ${link}` : `- ${link} — ${entry.description}`; +}; + +export const generateIndex = (options: IndexOptions): ConceptDoc => { + const sorted = [...options.entries].sort((a, b) => (a.path < b.path ? -1 : 1)); + const groups = new Map(); + for (const entry of sorted) { + const group = topLevelGroup(entry.path); + groups.set(group, [...(groups.get(group) ?? []), entry]); + } + + const sections: string[] = []; + for (const [group, entries] of [...groups.entries()].sort(([a], [b]) => (a < b ? -1 : 1))) { + const lines = entries.map(entryLine).join('\n'); + sections.push(group === '' ? lines : `## ${group}\n\n${lines}`); + } + + return { + path: 'index.md', + frontmatter: { + type: 'Index', + title: options.title, + ...(options.description !== undefined && { description: options.description }), + }, + body: sections.join('\n\n'), + }; +}; diff --git a/packages/okf/src/index.ts b/packages/okf/src/index.ts new file mode 100644 index 0000000..c1d26f7 --- /dev/null +++ b/packages/okf/src/index.ts @@ -0,0 +1,18 @@ +export { FRONTMATTER_KEY_ORDER } from './concept'; +export type { ConceptDoc, ConceptFrontmatter } from './concept'; +export { emitFrontmatter, parseConcept } from './frontmatter'; +export type { ParsedConcept } from './frontmatter'; +export { + extractInternalLinks, + markdownLink, + relativeLink, + resolveLink, + slugify, +} from './links'; +export { buildBundle, renderConcept } from './bundle'; +export { generateIndex } from './index-gen'; +export type { IndexEntry, IndexOptions } from './index-gen'; +export { generateLog } from './log-gen'; +export type { LogEntry } from './log-gen'; +export { validateBundle } from './validate'; +export type { LintCheck, LintFinding, LintSeverity } from './validate'; diff --git a/packages/okf/src/links.test.ts b/packages/okf/src/links.test.ts new file mode 100644 index 0000000..8cc6667 --- /dev/null +++ b/packages/okf/src/links.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; +import { + extractInternalLinks, + markdownLink, + relativeLink, + resolveLink, + slugify, +} from './links'; + +describe('slugify', () => { + it('lowercases and collapses non-alphanumeric runs', () => { + expect(slugify('EU AI Act — Article 6(2)')).toBe('eu-ai-act-article-6-2'); + }); + + it('strips diacritics', () => { + expect(slugify('Zürich café résumé')).toBe('zurich-cafe-resume'); + }); +}); + +describe('relativeLink', () => { + it('links between siblings without a prefix', () => { + expect(relativeLink('tables/orders.md', 'tables/customers.md')).toBe('customers.md'); + }); + + it('links from the bundle root downward', () => { + expect(relativeLink('index.md', 'tables/orders.md')).toBe('tables/orders.md'); + }); + + it('links across directories via parent segments', () => { + expect(relativeLink('tables/orders.md', 'metrics/wau.md')).toBe('../metrics/wau.md'); + }); +}); + +describe('markdownLink', () => { + it('renders a relative markdown link', () => { + expect(markdownLink('Customers', 'tables/orders.md', 'tables/customers.md')).toBe( + '[Customers](customers.md)', + ); + }); +}); + +describe('extractInternalLinks', () => { + it('returns internal targets and skips external URLs and fragments', () => { + // Arrange + const body = [ + 'Joined with [customers](customers.md) on `customer_id`.', + 'See [console](https://console.example.com/x) and [docs](mailto:a@b.c).', + 'Also [metric](../metrics/wau.md#definition).', + ].join('\n'); + + // Act & Assert + expect(extractInternalLinks(body)).toEqual(['customers.md', '../metrics/wau.md']); + }); +}); + +describe('resolveLink', () => { + it('resolves relative links against the linking file', () => { + expect(resolveLink('tables/orders.md', '../metrics/wau.md')).toBe('metrics/wau.md'); + }); + + it('resolves the spec root-absolute form', () => { + expect(resolveLink('tables/orders.md', '/tables/customers.md')).toBe('tables/customers.md'); + }); + + it('rejects links escaping the bundle root', () => { + expect(() => resolveLink('index.md', '../../etc/passwd.md')).toThrow(/escapes/); + }); +}); diff --git a/packages/okf/src/links.ts b/packages/okf/src/links.ts new file mode 100644 index 0000000..7fc31d3 --- /dev/null +++ b/packages/okf/src/links.ts @@ -0,0 +1,86 @@ +/** + * Bundle-internal linking. Concepts link to each other with normal markdown + * links — that graph IS the OKF knowledge graph, so link construction and + * resolution must be exact and deterministic. + */ + +/** Stable, filesystem-safe slug for concept file names. */ +export const slugify = (input: string): string => + input + .normalize('NFKD') + .replace(/[̀-ͯ]/g, '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + +const dirname = (path: string): string => { + const index = path.lastIndexOf('/'); + return index === -1 ? '' : path.slice(0, index); +}; + +/** Normalize `.` / `..` segments in a bundle-relative POSIX path. */ +const normalizeSegments = (path: string): string => { + const output: string[] = []; + for (const segment of path.split('/')) { + if (segment === '' || segment === '.') continue; + if (segment === '..') { + if (output.length === 0) { + throw new Error(`link escapes the bundle root: ${path}`); + } + output.pop(); + } else { + output.push(segment); + } + } + return output.join('/'); +}; + +/** + * Relative link target from one bundle file to another, e.g. + * `tables/orders.md` → `metrics/wau.md` yields `../metrics/wau.md`. + */ +export const relativeLink = (fromPath: string, toPath: string): string => { + const fromSegments = dirname(fromPath) === '' ? [] : dirname(fromPath).split('/'); + const toSegments = toPath.split('/'); + let common = 0; + while ( + common < fromSegments.length && + common < toSegments.length - 1 && + fromSegments[common] === toSegments[common] + ) { + common += 1; + } + const ups = fromSegments.length - common; + return [...Array(ups).fill('..'), ...toSegments.slice(common)].join('/'); +}; + +/** Markdown link to another concept, relative to the linking file. */ +export const markdownLink = (title: string, fromPath: string, toPath: string): string => + `[${title}](${relativeLink(fromPath, toPath)})`; + +const EXTERNAL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; + +/** + * Extract bundle-internal link targets from a markdown body. External URLs + * (http:, mailto:, …) are skipped; fragments and queries are stripped. + */ +export const extractInternalLinks = (body: string): string[] => { + const targets: string[] = []; + for (const match of body.matchAll(/\]\(([^()\s]+)\)/g)) { + const href = (match[1] ?? '').replace(/[#?].*$/, ''); + if (href === '' || EXTERNAL_SCHEME.test(href)) continue; + targets.push(href); + } + return targets; +}; + +/** + * Resolve a link found in `fromPath` to a normalized bundle-relative path. + * Supports both relative links (`../tables/orders.md`) and the spec's + * root-absolute form (`/tables/customers.md`). + */ +export const resolveLink = (fromPath: string, href: string): string => { + if (href.startsWith('/')) return normalizeSegments(href.slice(1)); + const base = dirname(fromPath); + return normalizeSegments(base === '' ? href : `${base}/${href}`); +}; diff --git a/packages/okf/src/log-gen.ts b/packages/okf/src/log-gen.ts new file mode 100644 index 0000000..2e47bc1 --- /dev/null +++ b/packages/okf/src/log-gen.ts @@ -0,0 +1,29 @@ +import type { ConceptDoc } from './concept'; + +/** + * `log.md` generation — the bundle's append-only change history. Callers pass + * entries in chronological order (oldest first); rendering never reorders, + * so successive bundle versions only ever append lines (git-diff friendly). + */ + +export interface LogEntry { + /** ISO-8601 instant carried from source data (run/event tables). */ + readonly timestamp: string; + readonly summary: string; + /** Optional detail bullets, e.g. pages touched or event hashes. */ + readonly details?: readonly string[]; +} + +const renderEntry = (entry: LogEntry): string => { + const heading = `## ${entry.timestamp}`; + const details = (entry.details ?? []).map((detail) => `- ${detail}`).join('\n'); + return details === '' + ? `${heading}\n\n${entry.summary}` + : `${heading}\n\n${entry.summary}\n\n${details}`; +}; + +export const generateLog = (entries: readonly LogEntry[]): ConceptDoc => ({ + path: 'log.md', + frontmatter: { type: 'Log', title: 'Change log' }, + body: entries.map(renderEntry).join('\n\n'), +}); diff --git a/packages/okf/src/validate.test.ts b/packages/okf/src/validate.test.ts new file mode 100644 index 0000000..00e273b --- /dev/null +++ b/packages/okf/src/validate.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from 'vitest'; +import { validateBundle } from './validate'; +import type { ConceptDoc } from './concept'; + +const doc = (path: string, body: string, type = 'Note'): ConceptDoc => ({ + path, + frontmatter: { type }, + body, +}); + +describe('validateBundle', () => { + it('flags an empty bundle', () => { + // Act + const findings = validateBundle([]); + + // Assert + expect(findings).toEqual([ + expect.objectContaining({ check: 'EMPTY_BUNDLE', severity: 'error' }), + ]); + }); + + it('passes a well-formed linked bundle without findings', () => { + // Arrange + const bundle = [ + doc('index.md', 'See [orders](tables/orders.md).', 'Index'), + doc('tables/orders.md', 'FK to [customers](customers.md).'), + doc('tables/customers.md', 'Referenced by [orders](/tables/orders.md).'), + ]; + + // Act & Assert + expect(validateBundle(bundle)).toEqual([]); + }); + + it('flags a missing type as an error', () => { + // Act + const findings = validateBundle([doc('index.md', '', ' ')]); + + // Assert + expect(findings).toEqual([ + expect.objectContaining({ check: 'MISSING_TYPE', path: 'index.md' }), + ]); + }); + + it('flags duplicate paths', () => { + // Act + const findings = validateBundle([doc('index.md', 'a'), doc('index.md', 'b')]); + + // Assert + expect(findings).toContainEqual( + expect.objectContaining({ check: 'DUPLICATE_PATH', path: 'index.md' }), + ); + }); + + it('flags links to missing targets and links escaping the bundle', () => { + // Arrange + const bundle = [ + doc('index.md', 'See [gone](tables/gone.md) and [out](../outside.md).', 'Index'), + ]; + + // Act + const findings = validateBundle(bundle); + + // Assert + expect(findings).toContainEqual( + expect.objectContaining({ + check: 'BROKEN_LINK', + message: expect.stringContaining('gone.md'), + }), + ); + expect(findings).toContainEqual( + expect.objectContaining({ + check: 'BROKEN_LINK', + message: expect.stringContaining('escapes'), + }), + ); + }); + + it('flags unlinked concepts as orphans but never index.md or log.md', () => { + // Arrange + const bundle = [ + doc('index.md', 'nothing linked', 'Index'), + doc('log.md', '', 'Log'), + doc('tables/orders.md', 'no inbound links'), + ]; + + // Act + const findings = validateBundle(bundle); + + // Assert + expect(findings).toEqual([ + expect.objectContaining({ check: 'ORPHAN', severity: 'warning', path: 'tables/orders.md' }), + ]); + }); +}); diff --git a/packages/okf/src/validate.ts b/packages/okf/src/validate.ts new file mode 100644 index 0000000..93c807b --- /dev/null +++ b/packages/okf/src/validate.ts @@ -0,0 +1,110 @@ +import type { ConceptDoc } from './concept'; +import { extractInternalLinks, resolveLink } from './links'; + +/** + * Structural bundle validation. Check IDs follow the shared lint spec + * (.scaffold/okf-lint-spec.md); semantic checks (UNVERIFIED_CITATION, + * CONTRADICTION, STALE_EFFECTIVE_DATE, SOURCE_GONE) need app/domain context + * and live in each app's pipeline, not here. + */ + +export type LintCheck = + | 'EMPTY_BUNDLE' + | 'DUPLICATE_PATH' + | 'MISSING_TYPE' + | 'BROKEN_LINK' + | 'ORPHAN'; + +export type LintSeverity = 'error' | 'warning'; + +export interface LintFinding { + /** + * Structural IDs come from {@link LintCheck}; apps may emit semantic IDs + * (UNVERIFIED_CITATION, …) per the shared lint spec in the same shape. + */ + readonly check: LintCheck | (string & {}); + readonly severity: LintSeverity; + /** Bundle path the finding is anchored to ('' for bundle-level findings). */ + readonly path: string; + readonly message: string; +} + +/** Entry points are reachable by convention, so they are never orphans. */ +const ENTRY_POINTS = new Set(['index.md', 'log.md']); + +export const validateBundle = (docs: readonly ConceptDoc[]): LintFinding[] => { + if (docs.length === 0) { + return [ + { + check: 'EMPTY_BUNDLE', + severity: 'error', + path: '', + message: 'bundle contains no concept documents', + }, + ]; + } + + const findings: LintFinding[] = []; + const paths = new Set(); + for (const doc of docs) { + if (paths.has(doc.path)) { + findings.push({ + check: 'DUPLICATE_PATH', + severity: 'error', + path: doc.path, + message: `more than one concept renders to ${doc.path}`, + }); + } + paths.add(doc.path); + if (doc.frontmatter.type.trim() === '') { + findings.push({ + check: 'MISSING_TYPE', + severity: 'error', + path: doc.path, + message: 'frontmatter is missing the mandatory type field', + }); + } + } + + const linkedTargets = new Set(); + for (const doc of docs) { + for (const href of extractInternalLinks(doc.body)) { + let target: string; + try { + target = resolveLink(doc.path, href); + } catch { + findings.push({ + check: 'BROKEN_LINK', + severity: 'error', + path: doc.path, + message: `link escapes the bundle root: ${href}`, + }); + continue; + } + if (paths.has(target)) { + linkedTargets.add(target); + } else { + findings.push({ + check: 'BROKEN_LINK', + severity: 'error', + path: doc.path, + message: `link target does not exist in bundle: ${href}`, + }); + } + } + } + + for (const doc of docs) { + if (ENTRY_POINTS.has(doc.path)) continue; + if (!linkedTargets.has(doc.path)) { + findings.push({ + check: 'ORPHAN', + severity: 'warning', + path: doc.path, + message: 'no other concept links to this document', + }); + } + } + + return findings; +}; diff --git a/packages/okf/tsconfig.json b/packages/okf/tsconfig.json new file mode 100644 index 0000000..9e25e6e --- /dev/null +++ b/packages/okf/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*.ts"] +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 4e0b440..0e4d097 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -16,6 +16,8 @@ "dependencies": { "@anthropic-ai/sdk": "^0.105.0", "@flank/core": "workspace:*", + "@flank/okf-export": "workspace:*", + "@okf/core": "workspace:*", "cron-parser": "^5.0.4", "inngest": "^4.7.0", "zod": "^4.4.3" diff --git a/packages/pipeline/src/github-publisher.test.ts b/packages/pipeline/src/github-publisher.test.ts new file mode 100644 index 0000000..737357e --- /dev/null +++ b/packages/pipeline/src/github-publisher.test.ts @@ -0,0 +1,173 @@ +import type { BundlePublishRequest, GitBundleTarget } from '@flank/okf-export'; +import { describe, expect, it } from 'vitest'; +import { + GitHubBundlePublisher, + LivePublishBannedError, + liveBanBundlePublisher, +} from './github-publisher'; + +const TARGET: GitBundleTarget = { + workspaceId: 'ws-a', + workspaceName: 'Acme', + provider: 'github', + repo: 'acme/intel', + branch: 'flank-okf', + baseBranch: 'main', + subdir: 'intel/acme', +}; + +const request = (over: Partial = {}): BundlePublishRequest => ({ + target: TARGET, + files: new Map([['intel/acme/index.md', 'hello']]), + deletions: ['intel/acme/competitors/gone/index.md'], + commitMessage: 'chore(okf): update Acme competitive bundle (+1 ~0 -1)', + diff: { added: ['index.md'], modified: [], removed: ['competitors/gone/index.md'], unchanged: [] }, + ...over, +}); + +interface Call { + readonly method: string; + readonly path: string; + readonly body: unknown; +} + +/** A scripted GitHub API: route on `METHOD /path` → [status, body]; records every call. */ +const fakeGitHub = (routes: Record) => { + const calls: Call[] = []; + const fetchImpl = async (url: string, init: RequestInit): Promise => { + const method = init.method ?? 'GET'; + const path = url.replace('https://api.github.test', ''); + calls.push({ + method, + path, + body: init.body === undefined ? undefined : JSON.parse(init.body as string), + }); + const route = routes[`${method} ${path}`]; + if (route === undefined) throw new Error(`unrouted ${method} ${path}`); + const [status, body] = route; + return new Response(body === null ? '' : JSON.stringify(body), { status }); + }; + return { fetchImpl, calls }; +}; + +const HAPPY_ROUTES: Record = { + 'GET /repos/acme/intel/git/ref/heads/flank-okf': [200, { object: { sha: 'headsha' } }], + 'GET /repos/acme/intel/git/commits/headsha': [200, { tree: { sha: 'basetree' } }], + 'POST /repos/acme/intel/git/trees': [201, { sha: 'newtree' }], + 'POST /repos/acme/intel/git/commits': [201, { sha: 'newcommit' }], + 'PATCH /repos/acme/intel/git/refs/heads/flank-okf': [200, {}], + 'POST /repos/acme/intel/pulls': [201, { html_url: 'https://github.com/acme/intel/pull/7' }], +}; + +describe('GitHubBundlePublisher', () => { + it('commits the bundle onto an existing branch and opens a PR', async () => { + // Arrange + const { fetchImpl, calls } = fakeGitHub(HAPPY_ROUTES); + const publisher = new GitHubBundlePublisher({ + token: 't', + fetchImpl, + apiBase: 'https://api.github.test', + }); + + // Act + const result = await publisher.publish(request()); + + // Assert + expect(result).toEqual({ + ok: true, + commitSha: 'newcommit', + branchRef: 'refs/heads/flank-okf', + pullRequestUrl: 'https://github.com/acme/intel/pull/7', + }); + // The tree is built on the branch's current tree, with content blobs and a null-sha deletion. + const treeCall = calls.find((c) => c.path === '/repos/acme/intel/git/trees'); + const tree = (treeCall?.body as { base_tree: string; tree: unknown[] }) ?? { base_tree: '', tree: [] }; + expect(tree.base_tree).toBe('basetree'); + expect(tree.tree).toContainEqual({ + path: 'intel/acme/index.md', + mode: '100644', + type: 'blob', + content: 'hello', + }); + expect(tree.tree).toContainEqual({ + path: 'intel/acme/competitors/gone/index.md', + mode: '100644', + type: 'blob', + sha: null, + }); + // The new commit's parent is the prior head. + const commitCall = calls.find((c) => c.path === '/repos/acme/intel/git/commits'); + expect((commitCall?.body as { parents: string[] }).parents).toEqual(['headsha']); + }); + + it('creates the head branch from the base branch when it is missing', async () => { + // Arrange — branch ref 404s; base ref + create-ref are scripted. + const { fetchImpl, calls } = fakeGitHub({ + ...HAPPY_ROUTES, + 'GET /repos/acme/intel/git/ref/heads/flank-okf': [404, { message: 'Not Found' }], + 'GET /repos/acme/intel/git/ref/heads/main': [200, { object: { sha: 'basesha' } }], + 'POST /repos/acme/intel/git/refs': [201, { ref: 'refs/heads/flank-okf' }], + 'GET /repos/acme/intel/git/commits/basesha': [200, { tree: { sha: 'basetree' } }], + }); + const publisher = new GitHubBundlePublisher({ + token: 't', + fetchImpl, + apiBase: 'https://api.github.test', + }); + + // Act + const result = await publisher.publish(request()); + + // Assert — created the branch off basesha and committed with basesha as parent. + expect(result.ok).toBe(true); + const createRef = calls.find((c) => c.path === '/repos/acme/intel/git/refs'); + expect(createRef?.body).toEqual({ ref: 'refs/heads/flank-okf', sha: 'basesha' }); + }); + + it('skips the PR when no base branch is configured', async () => { + // Arrange + const { fetchImpl, calls } = fakeGitHub(HAPPY_ROUTES); + const publisher = new GitHubBundlePublisher({ + token: 't', + fetchImpl, + apiBase: 'https://api.github.test', + }); + + // Act + const result = await publisher.publish( + request({ target: { ...TARGET, baseBranch: null } }), + ); + + // Assert + expect(result).toMatchObject({ ok: true, pullRequestUrl: null }); + expect(calls.some((c) => c.path === '/repos/acme/intel/pulls')).toBe(false); + }); + + it('returns ok:false with the status when the API errors, never throwing', async () => { + // Arrange — the tree call 500s. + const { fetchImpl } = fakeGitHub({ + ...HAPPY_ROUTES, + 'POST /repos/acme/intel/git/trees': [500, { message: 'boom' }], + }); + const publisher = new GitHubBundlePublisher({ + token: 't', + fetchImpl, + apiBase: 'https://api.github.test', + }); + + // Act + const result = await publisher.publish(request()); + + // Assert + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain('500'); + }); +}); + +describe('liveBanBundlePublisher', () => { + it('throws instead of dialing out', async () => { + await expect(liveBanBundlePublisher.publish(request())).rejects.toBeInstanceOf( + LivePublishBannedError, + ); + }); +}); diff --git a/packages/pipeline/src/github-publisher.ts b/packages/pipeline/src/github-publisher.ts new file mode 100644 index 0000000..9a62bd3 --- /dev/null +++ b/packages/pipeline/src/github-publisher.ts @@ -0,0 +1,242 @@ +import type { + BundlePublishRequest, + BundlePublishResult, + BundlePublisher, +} from '@flank/okf-export'; + +const DEFAULT_TIMEOUT_MS = 15_000; +const DEFAULT_API_BASE = 'https://api.github.com'; +const BLOB_MODE = '100644'; + +type FetchImpl = (url: string, init: RequestInit) => Promise; + +export interface GitHubPublisherOptions { + /** + * A token with contents:write (and pull-requests:write when opening PRs) on the customer repos — + * a GitHub App installation token or PAT. One token serves every configured target, mirroring the + * single-credential Notifier impls; per-target credentials can come later. + */ + readonly token: string; + readonly fetchImpl?: FetchImpl; + readonly timeoutMs?: number; + readonly apiBase?: string; +} + +/** Thrown by {@link liveBanBundlePublisher}: any push in a hermetic test is a breach. */ +export class LivePublishBannedError extends Error { + constructor(message: string) { + super(message); + this.name = 'LivePublishBannedError'; + } +} + +/** Default for unit tests: any publish throws instead of dialing out. Inject a fake to actually push. */ +export const liveBanBundlePublisher: BundlePublisher = { + async publish(): Promise { + throw new LivePublishBannedError( + 'refusing a live git push: tests/CI are hermetic — inject a fake publisher', + ); + }, +}; + +const messageOf = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +interface GhResponse { + readonly status: number; + readonly body: unknown; +} + +/** + * Publishes an OKF bundle to a GitHub repo via the Git Data API: build a tree on top of the head + * branch's current tree (unchanged files persist, deletions null out their blob), commit it, fast- + * forward the branch ref, and open/reuse a PR when a base branch is configured. All network failures + * are caught and returned as `{ ok: false }` so one target never aborts the delivery sweep. + */ +export class GitHubBundlePublisher implements BundlePublisher { + private readonly fetchImpl: FetchImpl; + private readonly timeoutMs: number; + private readonly apiBase: string; + + constructor(private readonly options: GitHubPublisherOptions) { + this.fetchImpl = options.fetchImpl ?? fetch; + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + this.apiBase = options.apiBase ?? DEFAULT_API_BASE; + } + + async publish(request: BundlePublishRequest): Promise { + const { target } = request; + try { + const baseBranchName = target.baseBranch ?? target.branch; + const headSha = await this.resolveHeadSha(target.repo, target.branch, baseBranchName); + const headTreeSha = await this.commitTreeSha(target.repo, headSha); + + const tree = [ + ...[...request.files.entries()].map(([path, content]) => ({ + path, + mode: BLOB_MODE, + type: 'blob' as const, + content, + })), + // A null sha removes the path from the tree (a deletion). + ...request.deletions.map((path) => ({ + path, + mode: BLOB_MODE, + type: 'blob' as const, + sha: null, + })), + ]; + + const newTree = await this.gh('POST', `/repos/${target.repo}/git/trees`, { + base_tree: headTreeSha, + tree, + }); + const newTreeSha = asString(readField(newTree.body, 'sha'), 'git/trees.sha'); + + const commit = await this.gh('POST', `/repos/${target.repo}/git/commits`, { + message: request.commitMessage, + tree: newTreeSha, + parents: [headSha], + }); + const commitSha = asString(readField(commit.body, 'sha'), 'git/commits.sha'); + + await this.gh('PATCH', `/repos/${target.repo}/git/refs/heads/${target.branch}`, { + sha: commitSha, + force: false, + }); + + const pullRequestUrl = + target.baseBranch !== null && target.baseBranch !== target.branch + ? await this.ensurePullRequest(target.repo, target.branch, target.baseBranch, request) + : null; + + return { + ok: true, + commitSha, + branchRef: `refs/heads/${target.branch}`, + pullRequestUrl, + }; + } catch (error) { + return { ok: false, error: messageOf(error) }; + } + } + + /** Current tip of `branch`, creating it from `baseBranch` when it does not yet exist. */ + private async resolveHeadSha(repo: string, branch: string, baseBranch: string): Promise { + const existing = await this.gh('GET', `/repos/${repo}/git/ref/heads/${branch}`, undefined, [ + 404, + ]); + if (existing.status !== 404) { + return asString(readField(existing.body, 'object', 'sha'), 'git/ref.object.sha'); + } + // Branch absent — branch off the base branch's tip. + const base = await this.gh('GET', `/repos/${repo}/git/ref/heads/${baseBranch}`); + const baseSha = asString(readField(base.body, 'object', 'sha'), 'git/ref.object.sha'); + await this.gh('POST', `/repos/${repo}/git/refs`, { + ref: `refs/heads/${branch}`, + sha: baseSha, + }); + return baseSha; + } + + private async commitTreeSha(repo: string, commitSha: string): Promise { + const commit = await this.gh('GET', `/repos/${repo}/git/commits/${commitSha}`); + return asString(readField(commit.body, 'tree', 'sha'), 'git/commits.tree.sha'); + } + + /** Open a PR head→base, or return the URL of the one already open for that head. */ + private async ensurePullRequest( + repo: string, + head: string, + base: string, + request: BundlePublishRequest, + ): Promise { + const created = await this.gh( + 'POST', + `/repos/${repo}/pulls`, + { + title: request.commitMessage, + head, + base, + body: pullRequestBody(request), + }, + [422], // 422 == a PR for this head already exists + ); + if (created.status !== 422) { + return asStringOrNull(readField(created.body, 'html_url')); + } + const owner = repo.split('/')[0]; + const existing = await this.gh( + 'GET', + `/repos/${repo}/pulls?state=open&head=${owner}:${head}&base=${base}`, + ); + return Array.isArray(existing.body) && existing.body.length > 0 + ? asStringOrNull(readField(existing.body[0], 'html_url')) + : null; + } + + private async gh( + method: string, + path: string, + body?: unknown, + allowStatuses: readonly number[] = [], + ): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), this.timeoutMs); + try { + const response = await this.fetchImpl(`${this.apiBase}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.options.token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: controller.signal, + }); + const text = await response.text(); + const parsed = text === '' ? null : JSON.parse(text); + if (!response.ok && !allowStatuses.includes(response.status)) { + throw new Error(`GitHub ${method} ${path} → ${response.status}: ${truncate(text)}`); + } + return { status: response.status, body: parsed }; + } finally { + clearTimeout(timer); + } + } +} + +const truncate = (text: string, max = 300): string => + text.length > max ? `${text.slice(0, max)}…` : text; + +const pullRequestBody = (request: BundlePublishRequest): string => + [ + `Automated OKF bundle update for **${request.target.workspaceName}**.`, + '', + `- ${request.diff.added.length} added`, + `- ${request.diff.modified.length} modified`, + `- ${request.diff.removed.length} removed`, + '', + 'Generated from published, claim-verified dossiers and battlecards. See `log.md` for history.', + ].join('\n'); + +/** Read a nested field from an unknown JSON body without asserting a shape up front. */ +const readField = (body: unknown, ...path: readonly string[]): unknown => { + let current = body; + for (const key of path) { + if (current === null || typeof current !== 'object') return undefined; + current = (current as Record)[key]; + } + return current; +}; + +const asString = (value: unknown, label: string): string => { + if (typeof value !== 'string' || value === '') { + throw new Error(`GitHub response missing ${label}`); + } + return value; +}; + +const asStringOrNull = (value: unknown): string | null => + typeof value === 'string' && value !== '' ? value : null; diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 163a486..277c252 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -15,5 +15,8 @@ export * from './alerts'; export * from './alert-render'; export * from './notifier-impls'; export * from './delivery'; +export * from './github-publisher'; +export * from './okf-config'; +export * from './okf-delivery'; export * from './fixture-run'; export * from './fixture-files'; diff --git a/packages/pipeline/src/inngest.ts b/packages/pipeline/src/inngest.ts index 3172eb4..2578fd2 100644 --- a/packages/pipeline/src/inngest.ts +++ b/packages/pipeline/src/inngest.ts @@ -1,8 +1,10 @@ import type { FlankStore, Notifier, SynthesisClient, TriageClient } from '@flank/core'; +import type { BundlePublisher, GitBundleTarget } from '@flank/okf-export'; import { Inngest } from 'inngest'; import { randomUUID } from 'node:crypto'; import { runDeliverySweep, type DeliveryOptions } from './delivery'; import { HttpFetcher } from './http-fetcher'; +import { runOkfDelivery } from './okf-delivery'; import { runScheduledTick, type SchedulerOptions } from './scheduler'; import { runNightlySynthesis } from './synthesis'; @@ -104,3 +106,44 @@ export const createDeliverySweepFunction = ( ); }, ); + +/** Concrete runtime the OKF delivery sweep needs: the store, a git publisher, and the targets. */ +export interface OkfDeliveryRuntime { + readonly store: FlankStore; + readonly publisher: BundlePublisher; + /** Operator-configured customer repos to deliver to (M2; self-serve config comes later). */ + readonly targets: readonly GitBundleTarget[]; + /** App origin for `resource:` deep links in the projected bundle. */ + readonly baseUrl: string; +} + +export interface OkfDeliveryConfig { + /** Cron cadence (default 05:00 daily — after nightly synthesis at 04:00 has republished sections). */ + readonly cron?: string; +} + +/** + * The OKF git-delivery cron (M2): after synthesis has refreshed sections, project each configured + * workspace's bundle and push the diff to its designated repo. Runs after nightly-synthesis so it + * ships the freshly published versions. Singleton so two ticks never double-push the same branch. + */ +export const createOkfDeliveryFunction = ( + buildRuntime: () => Promise, + config: OkfDeliveryConfig = {}, +) => + inngest.createFunction( + { + id: 'okf-delivery', + concurrency: { limit: 1 }, + triggers: [{ cron: config.cron ?? '0 5 * * *' }], + }, + async () => { + const { store, publisher, targets, baseUrl } = await buildRuntime(); + return runOkfDelivery( + { store, publisher, nextId: () => randomUUID() }, + targets, + new Date(), + { baseUrl }, + ); + }, + ); diff --git a/packages/pipeline/src/memory-store.ts b/packages/pipeline/src/memory-store.ts index 8f451eb..e7b268f 100644 --- a/packages/pipeline/src/memory-store.ts +++ b/packages/pipeline/src/memory-store.ts @@ -12,6 +12,7 @@ import { type AppUser, type BattlecardSection, type BattlecardSectionKind, + type BundleDelivery, type Claim, type Competitor, type CoverageRun, @@ -59,6 +60,7 @@ interface StoreState { readonly memberships: Map; readonly channelConfigs: Map; readonly alerts: Map; + readonly bundleDeliveries: Map; } /** @@ -88,6 +90,7 @@ export class MemoryFlankStore implements FlankStore { memberships: new Map(), channelConfigs: new Map(), alerts: new Map(), + bundleDeliveries: new Map(), }; private insertUnique( @@ -669,6 +672,30 @@ export class MemoryFlankStore implements FlankStore { ); } + // --- OKF bundle delivery (M2) --- + + async insertBundleDelivery(delivery: BundleDelivery): Promise { + if (!this.state.workspaces.has(delivery.workspaceId)) { + throw new UnknownEntityError(`workspace ${delivery.workspaceId} does not exist`); + } + return this.insertUnique(this.state.bundleDeliveries, delivery, 'bundle_delivery'); + } + + async latestPublishedBundleDelivery(workspaceId: string): Promise { + let best: BundleDelivery | null = null; + for (const delivery of this.state.bundleDeliveries.values()) { + if (delivery.workspaceId !== workspaceId || delivery.status !== 'published') continue; + // Latest = max by (createdAt, id) — the same deterministic ordering the SQL uses. + if ( + best === null || + byCreatedThenId(delivery.createdAt, delivery.id, best.createdAt, best.id) > 0 + ) { + best = delivery; + } + } + return best; + } + async withTransaction(fn: (tx: FlankStore) => Promise): Promise { const checkpoint = this.captureState(); try { @@ -702,6 +729,7 @@ export class MemoryFlankStore implements FlankStore { memberships: new Map(this.state.memberships), channelConfigs: new Map(this.state.channelConfigs), alerts: new Map(this.state.alerts), + bundleDeliveries: new Map(this.state.bundleDeliveries), }; } @@ -723,6 +751,7 @@ export class MemoryFlankStore implements FlankStore { replaceMap(this.state.memberships, checkpoint.memberships); replaceMap(this.state.channelConfigs, checkpoint.channelConfigs); replaceMap(this.state.alerts, checkpoint.alerts); + replaceMap(this.state.bundleDeliveries, checkpoint.bundleDeliveries); } } diff --git a/packages/pipeline/src/okf-config.test.ts b/packages/pipeline/src/okf-config.test.ts new file mode 100644 index 0000000..5b2771d --- /dev/null +++ b/packages/pipeline/src/okf-config.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; +import { GitHubBundlePublisher, liveBanBundlePublisher } from './github-publisher'; +import { createOkfPublisher, parseOkfTargets } from './okf-config'; + +describe('parseOkfTargets', () => { + it('returns no targets for absent or blank config', () => { + expect(parseOkfTargets(undefined)).toEqual([]); + expect(parseOkfTargets(' ')).toEqual([]); + }); + + it('parses and normalizes a valid target array', () => { + // Arrange + const raw = JSON.stringify([ + { + workspaceId: 'ws-a', + workspaceName: 'Acme', + provider: 'github', + repo: 'acme/intel', + branch: 'flank-okf', + subdir: '/intel/acme/', + }, + ]); + + // Act + const targets = parseOkfTargets(raw); + + // Assert — baseBranch defaults to null, subdir slashes stripped. + expect(targets).toEqual([ + { + workspaceId: 'ws-a', + workspaceName: 'Acme', + provider: 'github', + repo: 'acme/intel', + branch: 'flank-okf', + baseBranch: null, + subdir: 'intel/acme', + }, + ]); + }); + + it('throws on a malformed repo (fail closed, never deliver to an unintended repo)', () => { + const raw = JSON.stringify([ + { + workspaceId: 'ws-a', + workspaceName: 'Acme', + provider: 'github', + repo: 'not-a-repo', + branch: 'main', + }, + ]); + expect(() => parseOkfTargets(raw)).toThrow(); + }); +}); + +describe('createOkfPublisher', () => { + it('returns a real GitHub publisher when a token is set', () => { + expect(createOkfPublisher({ FLANK_OKF_GITHUB_TOKEN: 'ghp_x' })).toBeInstanceOf( + GitHubBundlePublisher, + ); + }); + + it('falls back to the live-ban publisher when no token is configured', () => { + expect(createOkfPublisher({})).toBe(liveBanBundlePublisher); + }); +}); diff --git a/packages/pipeline/src/okf-config.ts b/packages/pipeline/src/okf-config.ts new file mode 100644 index 0000000..6c0bcc1 --- /dev/null +++ b/packages/pipeline/src/okf-config.ts @@ -0,0 +1,41 @@ +import type { BundlePublisher, GitBundleTarget } from '@flank/okf-export'; +import { z } from 'zod'; +import { GitHubBundlePublisher, liveBanBundlePublisher } from './github-publisher'; + +/** Boundary validation for an operator-configured delivery target (M2 config via env JSON). */ +const GitBundleTargetSchema = z.object({ + workspaceId: z.string().min(1), + workspaceName: z.string().min(1), + provider: z.literal('github'), + repo: z.string().regex(/^[^/\s]+\/[^/\s]+$/, 'repo must be owner/name'), + branch: z.string().min(1), + baseBranch: z.string().min(1).nullable().default(null), + // Normalize away leading/trailing slashes so path-joining stays predictable. + subdir: z + .string() + .default('') + .transform((value) => value.replace(/^\/+|\/+$/g, '')), +}); + +/** + * Parse the `FLANK_OKF_TARGETS` env var (a JSON array) into validated targets. Absent/blank → no + * targets (the sweep no-ops). Malformed JSON or a bad target throws — fail closed, never deliver to + * an unintended repo. + */ +export const parseOkfTargets = (raw: string | undefined): readonly GitBundleTarget[] => { + if (raw === undefined || raw.trim() === '') return []; + const parsed = z.array(GitBundleTargetSchema).parse(JSON.parse(raw)); + return Object.freeze(parsed.map((target) => Object.freeze(target))); +}; + +/** + * Build the git publisher from env: a real {@link GitHubBundlePublisher} when + * `FLANK_OKF_GITHUB_TOKEN` is set, else the live-ban publisher (which throws if ever called — safe + * because with no token the operator also configures no targets, so the sweep never publishes). + */ +export const createOkfPublisher = (env: Readonly>): BundlePublisher => { + const token = env.FLANK_OKF_GITHUB_TOKEN; + return token !== undefined && token !== '' + ? new GitHubBundlePublisher({ token }) + : liveBanBundlePublisher; +}; diff --git a/packages/pipeline/src/okf-delivery.test.ts b/packages/pipeline/src/okf-delivery.test.ts new file mode 100644 index 0000000..cebac8a --- /dev/null +++ b/packages/pipeline/src/okf-delivery.test.ts @@ -0,0 +1,222 @@ +import type { + Claim, + Competitor, + DossierSection, + Snapshot, + Source, + Workspace, +} from '@flank/core'; +import type { + BundlePublishRequest, + BundlePublishResult, + BundlePublisher, + GitBundleTarget, +} from '@flank/okf-export'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { MemoryFlankStore } from './memory-store'; +import { runOkfDelivery } from './okf-delivery'; + +const NOW = new Date('2026-07-01T00:00:00Z'); +let counter = 0; +const nextId = () => `id-${(counter += 1)}`; + +const WS: Workspace = { id: 'ws-a', name: 'Acme', planTier: 'growth' }; +const COMP: Competitor = { + id: 'comp-a', + workspaceId: 'ws-a', + name: 'Globex', + primaryDomain: 'globex.com', +}; +const SRC: Source = { + id: 'src-a', + competitorId: 'comp-a', + type: 'pricing', + url: 'https://globex.com/pricing', + adapter: 'html', + cadence: '0 6 * * *', + legalStatus: 'open', +}; +const SNAP: Snapshot = { + id: 'snap-1', + sourceId: 'src-a', + contentHash: 'h', + normalizedText: 'Analyst $59 per month', + fetchedAt: NOW, + httpStatus: 200, + vantage: null, +}; + +const TARGET: GitBundleTarget = { + workspaceId: 'ws-a', + workspaceName: 'Acme', + provider: 'github', + repo: 'acme/intel', + branch: 'flank-okf', + baseBranch: 'main', + subdir: 'intel/acme', +}; + +const claim = (id: string, verified: boolean): Claim => ({ + id, + deltaId: 'd-1', + snapshotId: 'snap-1', + quoteText: 'Analyst $59 per month', + charStart: 0, + charEnd: 21, + sourceUrl: 'https://globex.com/pricing', + capturedAt: NOW, + verifiedAt: verified ? NOW : null, +}); + +const dossier = (claimIds: readonly string[]): DossierSection => ({ + id: 'ds-1', + competitorId: 'comp-a', + kind: 'pricing', + version: 1, + contentMd: '# Pricing\n\nAnalyst tier is $59/mo.', + claimIds, + model: 'claude-sonnet-4-6', + batchId: 'batch-1', + supersedesId: null, + createdAt: NOW, +}); + +/** A publisher that records requests and returns a scripted result. */ +const fakePublisher = ( + result: BundlePublishResult = { + ok: true, + commitSha: 'sha-1', + branchRef: 'refs/heads/flank-okf', + pullRequestUrl: 'https://github.com/acme/intel/pull/1', + }, +): { publisher: BundlePublisher; requests: BundlePublishRequest[] } => { + const requests: BundlePublishRequest[] = []; + return { + requests, + publisher: { + async publish(request) { + requests.push(request); + return result; + }, + }, + }; +}; + +const seedPublishable = async (store: MemoryFlankStore, claimIds: readonly string[], verified: boolean) => { + await store.seedWorkspace(WS); + await store.seedCompetitor(COMP); + await store.seedSource(SRC); + await store.insertSnapshot(WS.id, SNAP); + await store.insertDelta(WS.id, { + id: 'd-1', + sourceId: 'src-a', + fromSnapshotId: null, + toSnapshotId: 'snap-1', + changedSpans: [], + triageClass: 'pricing_change', + materiality: 2, + rationale: 'price change', + state: 'published', + confirmedBySnapshotId: null, + createdAt: NOW, + }); + for (const id of new Set(claimIds)) await store.insertClaim(WS.id, claim(id, verified)); + await store.insertDossierSection(WS.id, dossier(claimIds)); +}; + +describe('runOkfDelivery', () => { + let store: MemoryFlankStore; + beforeEach(() => { + store = new MemoryFlankStore(); + counter = 0; + }); + + it('publishes a changed bundle, records it, and prefixes files under the target subdir', async () => { + // Arrange + await seedPublishable(store, ['c-1'], true); + const { publisher, requests } = fakePublisher(); + + // Act + const report = await runOkfDelivery({ store, publisher, nextId }, [TARGET], NOW, { + baseUrl: 'https://app.flank.test', + }); + + // Assert + expect(report).toMatchObject({ targetsConsidered: 1, published: 1, unchanged: 0, blocked: 0 }); + // Files are pushed under the subdir. + expect([...requests[0].files.keys()].every((p) => p.startsWith('intel/acme/'))).toBe(true); + // A published delivery is recorded with the bundle manifest (bundle-relative paths). + const recorded = await store.latestPublishedBundleDelivery(WS.id); + expect(recorded?.status).toBe('published'); + expect(recorded?.commitSha).toBe('sha-1'); + expect(Object.keys(recorded?.manifest ?? {})).toContain('index.md'); + expect(recorded!.filesAdded).toBeGreaterThan(0); + }); + + it('skips an unchanged bundle without recording a second delivery', async () => { + // Arrange + await seedPublishable(store, ['c-1'], true); + const { publisher } = fakePublisher(); + + // Act — deliver twice against identical state. + await runOkfDelivery({ store, publisher, nextId }, [TARGET], NOW, { baseUrl: 'https://x' }); + const second = await runOkfDelivery({ store, publisher, nextId }, [TARGET], NOW, { + baseUrl: 'https://x', + }); + + // Assert — nothing new shipped, exactly one published record exists. + expect(second).toMatchObject({ published: 0, unchanged: 1 }); + const deliveries = await store.latestPublishedBundleDelivery(WS.id); + expect(deliveries?.id).toBe('id-1'); // still the first run's record + }); + + it('blocks and records a failed delivery when a cited claim is unverified (projection gate)', async () => { + // Arrange — the section cites an unverified claim, so the projection emits an error finding. + await seedPublishable(store, ['c-1'], false); + const { publisher, requests } = fakePublisher(); + + // Act + const report = await runOkfDelivery({ store, publisher, nextId }, [TARGET], NOW, { + baseUrl: 'https://x', + }); + + // Assert — nothing pushed; a failed record explains why. + expect(report).toMatchObject({ published: 0, blocked: 1 }); + expect(requests).toHaveLength(0); + expect(await store.latestPublishedBundleDelivery(WS.id)).toBeNull(); + }); + + it('records a failed delivery when the publisher errors, without aborting the sweep', async () => { + // Arrange + await seedPublishable(store, ['c-1'], true); + const { publisher } = fakePublisher({ ok: false, error: 'github 503' }); + + // Act + const report = await runOkfDelivery({ store, publisher, nextId }, [TARGET], NOW, { + baseUrl: 'https://x', + }); + + // Assert + expect(report).toMatchObject({ published: 0, failed: 1 }); + expect(await store.latestPublishedBundleDelivery(WS.id)).toBeNull(); + }); + + it('counts a target that throws as an error and keeps going', async () => { + // Arrange — a target for a workspace that does not exist makes loadWorkspaceExport throw nothing + // (listCompetitors returns []), so instead force an error via a broken publisher on a real ws. + await seedPublishable(store, ['c-1'], true); + const throwing: BundlePublisher = { + async publish() { + throw new Error('unexpected'); + }, + }; + + // Act + const report = await runOkfDelivery({ store, publisher: throwing, nextId }, [TARGET], NOW, { + baseUrl: 'https://x', + }); + + // Assert + expect(report).toMatchObject({ errors: 1, published: 0 }); + }); +}); diff --git a/packages/pipeline/src/okf-delivery.ts b/packages/pipeline/src/okf-delivery.ts new file mode 100644 index 0000000..ce4a04a --- /dev/null +++ b/packages/pipeline/src/okf-delivery.ts @@ -0,0 +1,147 @@ +import type { BundleDelivery, FlankStore } from '@flank/core'; +import { + loadWorkspaceExport, + manifestFromRecord, + manifestToRecord, + planDelivery, + projectWorkspaceBundle, + type BundlePublisher, + type GitBundleTarget, +} from '@flank/okf-export'; +import { liveBanBundlePublisher } from './github-publisher'; + +export interface OkfDeliveryDeps { + readonly store: FlankStore; + /** Defaults to {@link liveBanBundlePublisher} so a test that forgets to inject one fails loud. */ + readonly publisher?: BundlePublisher; + readonly nextId: () => string; +} + +export interface OkfDeliveryOptions { + /** App origin for `resource:` deep links in the projected bundle. */ + readonly baseUrl: string; +} + +export interface OkfDeliveryReport { + readonly targetsConsidered: number; + readonly published: number; + readonly unchanged: number; + readonly blocked: number; + readonly failed: number; + /** Targets that threw before a decision (load/project/store error) — never aborts the sweep. */ + readonly errors: number; +} + +const blockedError = (findings: readonly { readonly message: string }[]): string => + `blocked by ${findings.length} error finding(s): ${findings + .map((finding) => finding.message) + .slice(0, 5) + .join('; ')}`; + +/** + * One OKF delivery sweep: for each configured target, project the workspace's bundle, diff it + * against the last published delivery, and push only when something changed. Enforces the + * projection-only rule (an error finding blocks the push and records a failed attempt) and never + * lets one target abort the others. Only `published`/`failed` attempts are recorded — an unchanged + * bundle ships and records nothing. + */ +export const runOkfDelivery = async ( + deps: OkfDeliveryDeps, + targets: readonly GitBundleTarget[], + now: Date, + options: OkfDeliveryOptions, +): Promise => { + const publisher = deps.publisher ?? liveBanBundlePublisher; + let published = 0; + let unchanged = 0; + let blocked = 0; + let failed = 0; + let errors = 0; + + const record = (delivery: BundleDelivery): Promise => + deps.store.insertBundleDelivery(delivery); + + for (const target of targets) { + try { + const input = await loadWorkspaceExport( + deps.store, + { id: target.workspaceId, name: target.workspaceName }, + options.baseUrl, + ); + const { files, findings } = projectWorkspaceBundle(input); + + const previous = await deps.store.latestPublishedBundleDelivery(target.workspaceId); + const previousManifest = + previous === null ? new Map() : manifestFromRecord(previous.manifest); + + const plan = planDelivery({ target, files, findings, previousManifest }); + + if (plan.kind === 'unchanged') { + unchanged += 1; + continue; + } + + if (plan.kind === 'blocked') { + await record( + failedDelivery(deps.nextId(), target.workspaceId, now, blockedError(plan.blockingFindings)), + ); + blocked += 1; + continue; + } + + const result = await publisher.publish(plan.request); + if (result.ok) { + await record({ + id: deps.nextId(), + workspaceId: target.workspaceId, + status: 'published', + commitSha: result.commitSha, + branchRef: result.branchRef, + pullRequestUrl: result.pullRequestUrl, + manifest: manifestToRecord(plan.manifest), + filesAdded: plan.diff.added.length, + filesModified: plan.diff.modified.length, + filesRemoved: plan.diff.removed.length, + error: null, + createdAt: now, + }); + published += 1; + } else { + await record(failedDelivery(deps.nextId(), target.workspaceId, now, result.error)); + failed += 1; + } + } catch { + errors += 1; + } + } + + return Object.freeze({ + targetsConsidered: targets.length, + published, + unchanged, + blocked, + failed, + errors, + }); +}; + +const failedDelivery = ( + id: string, + workspaceId: string, + now: Date, + error: string, +): BundleDelivery => + Object.freeze({ + id, + workspaceId, + status: 'failed', + commitSha: null, + branchRef: null, + pullRequestUrl: null, + manifest: {}, + filesAdded: 0, + filesModified: 0, + filesRemoved: 0, + error, + createdAt: now, + }); diff --git a/packages/pipeline/src/store-contract.ts b/packages/pipeline/src/store-contract.ts index ae0f6b1..750e534 100644 --- a/packages/pipeline/src/store-contract.ts +++ b/packages/pipeline/src/store-contract.ts @@ -8,6 +8,7 @@ import { type AlertChannelConfig, type AppUser, type BattlecardSection, + type BundleDelivery, type Claim, type CoverageRun, type Delta, @@ -800,6 +801,68 @@ export const runFlankStoreContract = (label: string, makeStore: () => FlankStore }); }); + describe('okf bundle delivery (M2)', () => { + const deliveryOn = ( + workspaceId: string, + id: string, + over: Partial = {}, + ): BundleDelivery => ({ + id, + workspaceId, + status: 'published', + commitSha: `sha-${id}`, + branchRef: 'refs/heads/flank-okf', + pullRequestUrl: null, + manifest: { 'index.md': 'hash-index' }, + filesAdded: 1, + filesModified: 0, + filesRemoved: 0, + error: null, + createdAt: AT, + ...over, + }); + + it('records deliveries append-only and rejects a duplicate id', async () => { + await store.insertBundleDelivery(deliveryOn(WS_A.id, 'del-1')); + await expect( + store.insertBundleDelivery(deliveryOn(WS_A.id, 'del-1')), + ).rejects.toBeInstanceOf(AppendOnlyViolationError); + }); + + it('rejects a delivery for an unknown workspace', async () => { + await expect( + store.insertBundleDelivery(deliveryOn('ws-nope', 'del-x')), + ).rejects.toBeInstanceOf(UnknownEntityError); + }); + + it('returns the newest PUBLISHED delivery as the baseline, ignoring failed and other tenants', async () => { + // Older published, newer failed, plus another tenant's newer published — none should win. + await store.insertBundleDelivery( + deliveryOn(WS_A.id, 'del-old', { createdAt: new Date('2026-06-01T00:00:00Z') }), + ); + await store.insertBundleDelivery( + deliveryOn(WS_A.id, 'del-new', { createdAt: new Date('2026-06-08T00:00:00Z') }), + ); + await store.insertBundleDelivery( + deliveryOn(WS_A.id, 'del-failed', { + status: 'failed', + manifest: {}, + commitSha: null, + branchRef: null, + error: 'boom', + createdAt: new Date('2026-06-09T00:00:00Z'), + }), + ); + await store.insertBundleDelivery( + deliveryOn(WS_B.id, 'del-b', { createdAt: new Date('2026-06-10T00:00:00Z') }), + ); + + const baseline = await store.latestPublishedBundleDelivery(WS_A.id); + expect(baseline?.id).toBe('del-new'); + expect(await store.latestPublishedBundleDelivery('ws-empty')).toBeNull(); + }); + }); + describe('withTransaction is atomic', () => { it('rolls back every write when the transaction throws', async () => { await expect( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21bdfba..6c2373e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -63,6 +63,9 @@ importers: '@flank/db': specifier: workspace:* version: link:../../packages/db + '@flank/okf-export': + specifier: workspace:* + version: link:../../packages/okf-export '@flank/pipeline': specifier: workspace:* version: link:../../packages/pipeline @@ -145,6 +148,37 @@ importers: specifier: ^4.1.9 version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@26.0.0)(@vitest/coverage-v8@4.1.9)(vite@7.3.5(@types/node@26.0.0)(tsx@4.22.4)(yaml@2.9.0)) + packages/okf: + devDependencies: + '@types/node': + specifier: ^26.0.0 + version: 26.0.0 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.9 + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@26.0.0)(@vitest/coverage-v8@4.1.9)(vite@7.3.5(@types/node@26.0.0)(tsx@4.22.4)(yaml@2.9.0)) + + packages/okf-export: + dependencies: + '@flank/core': + specifier: workspace:* + version: link:../core + '@okf/core': + specifier: workspace:* + version: link:../okf + devDependencies: + '@types/node': + specifier: ^26.0.0 + version: 26.0.0 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.9 + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@26.0.0)(@vitest/coverage-v8@4.1.9)(vite@7.3.5(@types/node@26.0.0)(tsx@4.22.4)(yaml@2.9.0)) + packages/pipeline: dependencies: '@anthropic-ai/sdk': @@ -153,6 +187,12 @@ importers: '@flank/core': specifier: workspace:* version: link:../core + '@flank/okf-export': + specifier: workspace:* + version: link:../okf-export + '@okf/core': + specifier: workspace:* + version: link:../okf cron-parser: specifier: ^5.0.4 version: 5.5.0 From e7c1a2d22dd49f42a9eda9eb5488c310d8a5c0ab Mon Sep 17 00:00:00 2001 From: Andreas Mueller Date: Sun, 12 Jul 2026 20:06:39 +0200 Subject: [PATCH 2/2] cleanup --- .codex/hooks.json | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .codex/hooks.json diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 0000000..180ac02 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,29 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write|Edit", + "hooks": [ + { + "type": "command", + "command": "f=$(jq -r '.tool_input.file_path // empty'); case \"$f\" in *.ts|*.tsx|*.js|*.jsx|*.json|*.css|*.md) if [ -f \"$CLAUDE_PROJECT_DIR/package.json\" ]; then cd \"$CLAUDE_PROJECT_DIR\" && pnpm exec prettier --write \"$f\" >/dev/null 2>&1; fi ;; esac; exit 0" + }, + { + "type": "command", + "command": "f=$(jq -r '.tool_input.file_path // empty'); case \"$f\" in *.ts|*.tsx) if [ -f \"$CLAUDE_PROJECT_DIR/package.json\" ]; then cd \"$CLAUDE_PROJECT_DIR\" && pnpm exec eslint --fix \"$f\" >/dev/null 2>&1; fi ;; esac; exit 0" + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "if [ -f \"$CLAUDE_PROJECT_DIR/package.json\" ]; then cd \"$CLAUDE_PROJECT_DIR\" && pnpm exec tsc --noEmit --pretty false 2>&1 | tail -20; fi; exit 0" + } + ] + } + ] + } +}