From 91077128302785932b524fc60db4cbda3734685a Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 12:34:07 +0200 Subject: [PATCH 01/11] feat(shared): add committed/ephemeral typegen cache path resolvers Introduce getCommittedCacheDir() -> /.appkit and getEphemeralStateDir() -> /node_modules/.databricks/appkit as the single source of truth for typegen cache/state locations, co-located with spawn-lock.ts and re-exported from the shared root barrel. Rewire getSpawnLockPath to compose from getEphemeralStateDir (behavior-preserving, same resolved path). Foundation for relocating the typegen cache out of node_modules so it survives pnpm install on warehouse-less CI/CD. xavier loop: iteration 1 -- Phase 1 (resolvers) Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../src/cli/commands/cache-paths.test.ts | 28 +++++++++++++++++++ .../shared/src/cli/commands/cache-paths.ts | 25 +++++++++++++++++ .../shared/src/cli/commands/spawn-lock.ts | 6 ++-- packages/shared/src/index.ts | 1 + 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 packages/shared/src/cli/commands/cache-paths.test.ts create mode 100644 packages/shared/src/cli/commands/cache-paths.ts diff --git a/packages/shared/src/cli/commands/cache-paths.test.ts b/packages/shared/src/cli/commands/cache-paths.test.ts new file mode 100644 index 000000000..d6a6c139d --- /dev/null +++ b/packages/shared/src/cli/commands/cache-paths.test.ts @@ -0,0 +1,28 @@ +import os from "node:os"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { getCommittedCacheDir, getEphemeralStateDir } from "./cache-paths"; + +describe("cache-paths", () => { + test("getCommittedCacheDir returns the .appkit directory under root", () => { + const root = path.join(os.tmpdir(), "some-project"); + expect(getCommittedCacheDir(root)).toBe(path.join(root, ".appkit")); + }); + + test("getCommittedCacheDir defaults to process.cwd() when no argument is given", () => { + expect(getCommittedCacheDir()).toBe(path.join(process.cwd(), ".appkit")); + }); + + test("getEphemeralStateDir returns node_modules/.databricks/appkit under root", () => { + const root = path.join(os.tmpdir(), "some-project"); + expect(getEphemeralStateDir(root)).toBe( + path.join(root, "node_modules", ".databricks", "appkit"), + ); + }); + + test("getEphemeralStateDir defaults to process.cwd() when no argument is given", () => { + expect(getEphemeralStateDir()).toBe( + path.join(process.cwd(), "node_modules", ".databricks", "appkit"), + ); + }); +}); diff --git a/packages/shared/src/cli/commands/cache-paths.ts b/packages/shared/src/cli/commands/cache-paths.ts new file mode 100644 index 000000000..a370104a2 --- /dev/null +++ b/packages/shared/src/cli/commands/cache-paths.ts @@ -0,0 +1,25 @@ +import path from "node:path"; + +/** + * Home for typegen COMMITTED caches (build inputs that must survive `pnpm + * install` and travel with the app in git). Per-app, anchored at the app root. + * The two caches (query/metric + serving) live here as `.appkit/*.json`. + * + * @param rootDir - project root. Defaults to the current working directory. + * @returns absolute path to the committed cache directory. + */ +export function getCommittedCacheDir(rootDir: string = process.cwd()): string { + return path.join(rootDir, ".appkit"); +} + +/** + * Home for typegen EPHEMERAL coordination state (worker spawn lock, ui-variant + * choices). Stays under `node_modules/` — gitignored, cleared on clean install. + * Never committed. + * + * @param rootDir - project root. Defaults to the current working directory. + * @returns absolute path to the ephemeral state directory. + */ +export function getEphemeralStateDir(rootDir: string = process.cwd()): string { + return path.join(rootDir, "node_modules", ".databricks", "appkit"); +} diff --git a/packages/shared/src/cli/commands/spawn-lock.ts b/packages/shared/src/cli/commands/spawn-lock.ts index d4f86c4a5..f99eb7cba 100644 --- a/packages/shared/src/cli/commands/spawn-lock.ts +++ b/packages/shared/src/cli/commands/spawn-lock.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { getEphemeralStateDir } from "./cache-paths.js"; /** * How long a spawn lock is considered fresh. A held lock newer than this means a @@ -28,10 +29,7 @@ export const SPAWN_LOCK_STALE_MS = 6 * 60 * 1000; */ export function getSpawnLockPath(rootDir: string): string { return path.join( - rootDir, - "node_modules", - ".databricks", - "appkit", + getEphemeralStateDir(rootDir), ".appkit-typegen-worker.lock", ); } diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index d036e0dbd..b93aac683 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,5 +1,6 @@ export * from "./agent"; export * from "./cache"; +export * from "./cli/commands/cache-paths"; export * from "./execute"; export * from "./genie"; export * from "./plugin"; From 586cc8782e35cc83924846acaaa10568e83243d1 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 12:51:21 +0200 Subject: [PATCH 02/11] feat(appkit): relocate typegen caches to committed .appkit/ dir Move the query/metric cache (types-cache.json) and serving cache (serving-types-cache.json) out of node_modules/.databricks/appkit into the committed per-app .appkit/ directory via getCommittedCacheDir(), so they survive `pnpm install` and let a blocking CI/CD build succeed with no warehouse access. Flat filenames (the dir namespaces them). Both writers now sort top-level record keys for deterministic, merge-friendly diffs. The serving plugin's runtime read now resolves through the same exported getServingCachePath() the writer uses (reader == writer by construction, fixing the inline-path drift risk). Add queryCacheFileExists() for the blocking-mode bootstrap-vs-drift message (consumed next). Route the ephemeral ui-variant choices file through getEphemeralStateDir() (path unchanged; stays gitignored). Document .appkit/ as a committed artifact in the template. xavier loop: iteration 3 -- Phase 2 Wave 1 (2A, 2B+2C, 2D) + Phase 3A Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/serving/serving.ts | 11 +- .../src/plugins/ui-variants/choice-sink.ts | 43 ++-- packages/appkit/src/type-generator/cache.ts | 65 ++++-- .../src/type-generator/serving/cache.ts | 45 +++-- .../serving/tests/cache.test.ts | 24 ++- .../tests/cache-serialization.test.ts | 189 ++++++++++++++++++ .../shared/src/cli/commands/spawn-lock.ts | 7 +- template/README.md | 1 + 8 files changed, 328 insertions(+), 57 deletions(-) create mode 100644 packages/appkit/src/type-generator/tests/cache-serialization.test.ts diff --git a/packages/appkit/src/plugins/serving/serving.ts b/packages/appkit/src/plugins/serving/serving.ts index 37707778e..d9d0b51f7 100644 --- a/packages/appkit/src/plugins/serving/serving.ts +++ b/packages/appkit/src/plugins/serving/serving.ts @@ -1,4 +1,3 @@ -import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import type express from "express"; @@ -9,6 +8,7 @@ import { createLogger } from "../../logging"; import { type ExecutionResult, Plugin, toPlugin } from "../../plugin"; import type { PluginManifest, ResourceRequirement } from "../../registry"; import { ResourceType } from "../../registry"; +import { getServingCachePath } from "../../type-generator/serving/cache"; import { servingInvokeDefaults } from "./defaults"; import manifest from "./manifest.json"; import { filterRequestBody, loadEndpointSchemas } from "./schema-filter"; @@ -66,14 +66,7 @@ export class ServingPlugin extends Plugin { } async setup(): Promise { - const cacheFile = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", - ".appkit-serving-types-cache.json", - ); - this.schemaAllowlists = await loadEndpointSchemas(cacheFile); + this.schemaAllowlists = await loadEndpointSchemas(getServingCachePath()); if (this.schemaAllowlists.size > 0) { logger.debug( "Loaded schema allowlists for %d endpoint(s)", diff --git a/packages/appkit/src/plugins/ui-variants/choice-sink.ts b/packages/appkit/src/plugins/ui-variants/choice-sink.ts index 271700da8..5e22edd8a 100644 --- a/packages/appkit/src/plugins/ui-variants/choice-sink.ts +++ b/packages/appkit/src/plugins/ui-variants/choice-sink.ts @@ -1,22 +1,31 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { getEphemeralStateDir } from "shared"; /** - * Path of the JSONL choices file, relative to the app's working directory. - * A coding agent reads it to pick up the developer's in-browser confirmation - * and finalize the chosen variant. + * Filename of the JSONL choices file. The agent skill discovers this file at + * the ephemeral state directory (under node_modules/). + */ +const UI_CHOICES_FILENAME = ".appkit-ui-choices.jsonl"; + +/** + * Default absolute path to the JSONL choices file: ephemeral state dir + * (node_modules/.databricks/appkit/) + the contract filename. Kept + * ephemeral & gitignored; NOT part of the committed .appkit/ relocation. * - * Under `node_modules/`, so it's gitignored and cleared on a clean install. - * Each line is spent on read — the agent removes it once the choice is - * finalized. + * A coding agent reads it to pick up the developer's in-browser confirmation + * and finalize the chosen variant. Each line is spent on read — the agent + * removes it once the choice is finalized. * * CONTRACT: the `databricks-app-variants` agent skill (in the - * databricks-agent-skills repo) discovers the choices file at this path. - * Changing this value silently breaks that skill's file discovery — update the - * skill's `find` path in the same change. + * databricks-agent-skills repo) discovers the choices file at this absolute + * path. Changing this value silently breaks that skill's file discovery — + * update the skill's `find` path in the same change. */ -const UI_CHOICES_FILE = - "node_modules/.databricks/appkit/.appkit-ui-choices.jsonl"; +const DEFAULT_UI_CHOICES_PATH = path.join( + getEphemeralStateDir(), + UI_CHOICES_FILENAME, +); /** * One recorded variant choice. @@ -42,14 +51,16 @@ export interface UiChoiceRecord { /** * File store for confirmed variant choices: upserts choices into - * {@link UI_CHOICES_FILE}, one line per `` id. + * {@link DEFAULT_UI_CHOICES_PATH}, one line per `` id. * * The store is **keyed and latest-wins**: at most one record per `blockId`, and * recording an existing `blockId` replaces it rather than appending, so the file * always reflects the current choice for each block. * - * The file is resolved against `process.cwd()`, so it lands under whatever - * directory the dev server runs from. Concurrent confirms are serialized behind + * By default, the file lands at the ephemeral state directory under + * `node_modules/.databricks/appkit/`. Callers may override the path (e.g., for + * testing), in which case it is resolved against `process.cwd()`; absolute + * paths are passed through unchanged. Concurrent confirms are serialized behind * an internal queue so their read-modify-write can't interleave and lose an * update. * @@ -59,8 +70,8 @@ export class FileChoiceStore { private readonly filePath: string; private writeQueue: Promise = Promise.resolve(); - constructor(relativePath: string = UI_CHOICES_FILE) { - this.filePath = path.resolve(process.cwd(), relativePath); + constructor(pathArg: string = DEFAULT_UI_CHOICES_PATH) { + this.filePath = path.resolve(process.cwd(), pathArg); } record(record: UiChoiceRecord): Promise { diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index ac320f8bc..eeaa5b86b 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { getCommittedCacheDir } from "shared"; import { createLogger } from "../logging/logger"; import type { MetricSchema } from "./mv-registry/types"; @@ -38,7 +39,7 @@ export interface MetricCacheEntry { /** * Structural gate for reviving a cached metric entry at partition time. * - * The cache file lives in `node_modules/.databricks` and is plain JSON — + * The cache file lives in the committed `.appkit/` dir and is plain JSON — * hand-edits, truncation, or a stale writer can leave entries whose shape no * longer matches {@link MetricCacheEntry}. A malformed entry must read as a * cache MISS (re-describe) rather than crash the pass or render revived @@ -87,13 +88,7 @@ interface Cache { } export const CACHE_VERSION = "3"; -const CACHE_FILE = ".appkit-types-cache.json"; -const CACHE_DIR = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", -); +const CACHE_FILE = "types-cache.json"; /** * Hash the SQL query @@ -120,9 +115,10 @@ export function metricCacheHash(source: string, lane: string): string { * @returns - the cache */ export async function loadCache(): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); + const cacheDir = getCommittedCacheDir(); + const cachePath = path.join(cacheDir, CACHE_FILE); try { - await fs.mkdir(CACHE_DIR, { recursive: true }); + await fs.mkdir(cacheDir, { recursive: true }); const raw = await fs.readFile(cachePath, "utf8"); const cache = JSON.parse(raw) as Cache; @@ -138,10 +134,53 @@ export async function loadCache(): Promise { } /** - * Save the cache to the file system + * Save the cache to the file system with deterministic serialization + * (sorted top-level keys for no-op regen diffs and merge-friendly output). * @param cache - cache object to save */ export async function saveCache(cache: Cache): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); - await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); + const cacheDir = getCommittedCacheDir(); + const cachePath = path.join(cacheDir, CACHE_FILE); + + // Ensure the cache directory exists + await fs.mkdir(cacheDir, { recursive: true }); + + // Rebuild cache with sorted top-level record keys for deterministic output. + // This ensures a no-op regen produces an identical file (no spurious diffs) + // and unrelated query/metric changes don't collide in git merges. + const sorted: Cache = { + version: cache.version, + queries: Object.fromEntries( + Object.keys(cache.queries) + .sort() + .map((key) => [key, cache.queries[key]]), + ), + }; + + // Include sorted metrics if present + if (cache.metrics) { + const metrics = cache.metrics; + sorted.metrics = Object.fromEntries( + Object.keys(metrics) + .sort() + .map((key) => [key, metrics[key]]), + ); + } + + await fs.writeFile(cachePath, JSON.stringify(sorted, null, 2), "utf8"); +} + +/** + * Whether the committed query/metric cache file exists on disk. Lets the + * blocking-mode fatal path distinguish an uninitialized cache (bootstrap: + * run `generate-types --wait` against a warehouse and commit `.appkit/`) from + * a drifted key (regenerate the affected query/metric). + */ +export async function queryCacheFileExists(): Promise { + try { + await fs.access(path.join(getCommittedCacheDir(), CACHE_FILE)); + return true; + } catch { + return false; + } } diff --git a/packages/appkit/src/type-generator/serving/cache.ts b/packages/appkit/src/type-generator/serving/cache.ts index dc9bf7e27..7752bab8e 100644 --- a/packages/appkit/src/type-generator/serving/cache.ts +++ b/packages/appkit/src/type-generator/serving/cache.ts @@ -1,18 +1,13 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { getCommittedCacheDir } from "shared"; import { createLogger } from "../../logging/logger"; const logger = createLogger("type-generator:serving:cache"); export const CACHE_VERSION = "1"; -const CACHE_FILE = ".appkit-serving-types-cache.json"; -const CACHE_DIR = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", -); +const CACHE_FILE = "serving-types-cache.json"; export interface ServingCacheEntry { hash: string; @@ -27,14 +22,24 @@ export interface ServingCache { endpoints: Record; } +/** + * Absolute path of the committed serving-types cache. The single source of + * truth shared by the writer (this module) and the runtime reader + * (serving plugin `setup()`), so a relocation can never desync them. + */ +export function getServingCachePath(): string { + return path.join(getCommittedCacheDir(), CACHE_FILE); +} + export function hashSchema(schemaJson: string): string { return crypto.createHash("sha256").update(schemaJson).digest("hex"); } export async function loadServingCache(): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); + const cachePath = getServingCachePath(); + const cacheDir = getCommittedCacheDir(); try { - await fs.mkdir(CACHE_DIR, { recursive: true }); + await fs.mkdir(cacheDir, { recursive: true }); const raw = await fs.readFile(cachePath, "utf8"); const cache = JSON.parse(raw) as ServingCache; if (cache.version === CACHE_VERSION) { @@ -50,7 +55,23 @@ export async function loadServingCache(): Promise { } export async function saveServingCache(cache: ServingCache): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); - await fs.mkdir(CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); + const cachePath = getServingCachePath(); + const cacheDir = getCommittedCacheDir(); + await fs.mkdir(cacheDir, { recursive: true }); + + // Sort endpoint keys for deterministic output (merge-friendly diffs) + const sortedCache: ServingCache = { + version: cache.version, + endpoints: Object.keys(cache.endpoints) + .sort() + .reduce( + (acc, key) => { + acc[key] = cache.endpoints[key]; + return acc; + }, + {} as Record, + ), + }; + + await fs.writeFile(cachePath, JSON.stringify(sortedCache, null, 2), "utf8"); } diff --git a/packages/appkit/src/type-generator/serving/tests/cache.test.ts b/packages/appkit/src/type-generator/serving/tests/cache.test.ts index 0c99c997f..107a680f4 100644 --- a/packages/appkit/src/type-generator/serving/tests/cache.test.ts +++ b/packages/appkit/src/type-generator/serving/tests/cache.test.ts @@ -81,27 +81,43 @@ describe("serving cache", () => { }); describe("saveServingCache", () => { - test("writes cache to file", async () => { + test("writes cache to file with sorted endpoints", async () => { vi.mocked(fs.writeFile).mockResolvedValue(); const cache: ServingCache = { version: CACHE_VERSION, endpoints: { - test: { + zebra: { hash: "xyz", requestType: "{}", responseType: "{}", chunkType: null, requestKeys: [], }, + apple: { + hash: "abc", + requestType: "{}", + responseType: "{}", + chunkType: null, + requestKeys: [], + }, }, }; await saveServingCache(cache); + // Endpoints should be sorted alphabetically in the output + const expectedSorted: ServingCache = { + version: CACHE_VERSION, + endpoints: { + apple: cache.endpoints.apple, + zebra: cache.endpoints.zebra, + }, + }; + expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining(".appkit-serving-types-cache.json"), - JSON.stringify(cache, null, 2), + expect.stringContaining("serving-types-cache.json"), + JSON.stringify(expectedSorted, null, 2), "utf8", ); }); diff --git a/packages/appkit/src/type-generator/tests/cache-serialization.test.ts b/packages/appkit/src/type-generator/tests/cache-serialization.test.ts new file mode 100644 index 000000000..15b84701a --- /dev/null +++ b/packages/appkit/src/type-generator/tests/cache-serialization.test.ts @@ -0,0 +1,189 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + access: vi.fn(), + mkdir: vi.fn(), + writeFile: vi.fn(), +})); + +vi.mock("node:fs/promises", () => ({ + default: { + access: mocks.access, + mkdir: mocks.mkdir, + writeFile: mocks.writeFile, + }, +})); + +// Mock getCommittedCacheDir to return a consistent test path +vi.mock("shared", () => ({ + getCommittedCacheDir: () => "/test/app/.appkit", +})); + +const { saveCache, queryCacheFileExists } = await import("../cache"); + +describe("cache serialization", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test("saves cache with sorted top-level query keys", async () => { + const cache = { + version: "3", + queries: { + zeta: { hash: "hash-z", type: "type-z", retry: false }, + alpha: { hash: "hash-a", type: "type-a", retry: false }, + mid: { hash: "hash-m", type: "type-m", retry: false }, + }, + }; + + mocks.mkdir.mockResolvedValue(undefined); + mocks.writeFile.mockResolvedValue(undefined); + + await saveCache(cache); + + // Verify mkdir was called with the correct directory + expect(mocks.mkdir).toHaveBeenCalledWith("/test/app/.appkit", { + recursive: true, + }); + + // Capture the JSON string passed to writeFile + const writeFileCall = mocks.writeFile.mock.calls[0]; + expect(writeFileCall).toBeDefined(); + const writtenJson = writeFileCall[1] as string; + const parsed = JSON.parse(writtenJson); + + // Assert top-level query keys are sorted ascending + const queryKeys = Object.keys(parsed.queries); + expect(queryKeys).toEqual(["alpha", "mid", "zeta"]); + + // Assert each key's value is preserved + expect(parsed.queries.alpha.type).toBe("type-a"); + expect(parsed.queries.mid.type).toBe("type-m"); + expect(parsed.queries.zeta.type).toBe("type-z"); + }); + + test("saves cache with sorted top-level metric keys", async () => { + const cache = { + version: "3", + queries: { + queryA: { hash: "hash-qa", type: "type-qa", retry: false }, + }, + metrics: { + zeta_metric: { + hash: "hash-zm", + schema: { + key: "zeta_metric", + source: "schema1", + lane: "sp" as const, + measures: [{ name: "meas1", type: "double", isMeasure: true }], + dimensions: [{ name: "dim1", type: "string", isMeasure: false }], + }, + retry: false, + }, + alpha_metric: { + hash: "hash-am", + schema: { + key: "alpha_metric", + source: "schema2", + lane: "obo" as const, + measures: [{ name: "meas2", type: "int", isMeasure: true }], + dimensions: [{ name: "dim2", type: "timestamp", isMeasure: false }], + }, + retry: false, + }, + }, + }; + + mocks.mkdir.mockResolvedValue(undefined); + mocks.writeFile.mockResolvedValue(undefined); + + await saveCache(cache); + + const writeFileCall = mocks.writeFile.mock.calls[0]; + const writtenJson = writeFileCall[1] as string; + const parsed = JSON.parse(writtenJson); + + // Assert top-level metric keys are sorted ascending + const metricKeys = Object.keys(parsed.metrics); + expect(metricKeys).toEqual(["alpha_metric", "zeta_metric"]); + + // Assert each metric's values are preserved + expect(parsed.metrics.alpha_metric.schema.key).toBe("alpha_metric"); + expect(parsed.metrics.zeta_metric.schema.key).toBe("zeta_metric"); + }); + + test("preserves nested entry values (measures, dimensions, schema fields) in original order", async () => { + const measureArray = [ + { name: "measure1", type: "double", isMeasure: true }, + { name: "measure2", type: "int", isMeasure: true }, + ]; + const dimensionArray = [ + { name: "dim1", type: "string", isMeasure: false }, + { name: "dim2", type: "timestamp", isMeasure: false }, + ]; + + const cache = { + version: "3", + queries: {}, + metrics: { + my_metric: { + hash: "test-hash", + schema: { + key: "my_metric", + source: "test_source", + lane: "sp" as const, + measures: measureArray, + dimensions: dimensionArray, + degraded: false, + }, + retry: false, + }, + }, + }; + + mocks.mkdir.mockResolvedValue(undefined); + mocks.writeFile.mockResolvedValue(undefined); + + await saveCache(cache); + + const writeFileCall = mocks.writeFile.mock.calls[0]; + const writtenJson = writeFileCall[1] as string; + const parsed = JSON.parse(writtenJson); + + // Assert nested array order is preserved + expect(parsed.metrics.my_metric.schema.measures).toEqual(measureArray); + expect(parsed.metrics.my_metric.schema.dimensions).toEqual(dimensionArray); + + // Assert other schema fields are preserved + expect(parsed.metrics.my_metric.schema.degraded).toBe(false); + }); + + test("queryCacheFileExists returns true when file exists", async () => { + mocks.access.mockResolvedValue(undefined); + + const result = await queryCacheFileExists(); + + expect(result).toBe(true); + expect(mocks.access).toHaveBeenCalledWith( + "/test/app/.appkit/types-cache.json", + ); + }); + + test("queryCacheFileExists returns false when file does not exist", async () => { + const error = new Error("ENOENT"); + (error as NodeJS.ErrnoException).code = "ENOENT"; + mocks.access.mockRejectedValue(error); + + const result = await queryCacheFileExists(); + + expect(result).toBe(false); + }); + + test("queryCacheFileExists returns false for any error", async () => { + mocks.access.mockRejectedValue(new Error("Some other error")); + + const result = await queryCacheFileExists(); + + expect(result).toBe(false); + }); +}); diff --git a/packages/shared/src/cli/commands/spawn-lock.ts b/packages/shared/src/cli/commands/spawn-lock.ts index f99eb7cba..95d197300 100644 --- a/packages/shared/src/cli/commands/spawn-lock.ts +++ b/packages/shared/src/cli/commands/spawn-lock.ts @@ -18,9 +18,10 @@ export const SPAWN_LOCK_STALE_MS = 6 * 60 * 1000; /** * Resolve the on-disk path of the single-flight spawn lock for a project. * - * Lives alongside the type-generator cache (`node_modules/.databricks/appkit/`) - * so it shares the same already-creatable, gitignored, per-project location and - * doesn't introduce a new directory. The lock is keyed only by project root, so + * Lives in the ephemeral state dir (`node_modules/.databricks/appkit/`) — a + * gitignored, per-project location cleared on a clean install. This is + * transient coordination state, deliberately kept separate from the committed + * `.appkit/` type cache. The lock is keyed only by project root, so * concurrent `generate-types` invocations for the same project (postinstall + * predev, say) contend for one lock and only one wins the spawn. * diff --git a/template/README.md b/template/README.md index 621631c13..985998a6d 100644 --- a/template/README.md +++ b/template/README.md @@ -110,6 +110,7 @@ This creates: - `dist/server.js` - Compiled server bundle - `client/dist/` - Bundled client assets +- `.appkit/` - TypeScript type definitions cache (committed to git; do not ignore) ### Production From efe7314fdf8d87b09726274bdb8e83d8ae3e2935 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 12:57:31 +0200 Subject: [PATCH 03/11] feat(appkit): distinguish bootstrap vs drift in blocking-mode typegen fatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When blocking-mode typegen fails fatally (a cache MISS with no reachable/ authorized warehouse), branch the fatal remedy on whether the committed .appkit/ cache file exists: bootstrap (never initialized → run `generate-types --wait` against a warehouse and commit .appkit/) vs drift (cache present but stale/missing the failing key → regenerate it). Same crash, clearer next step. queryCacheFileExists() is called only on the fatal path, so a warm all-hit run does no extra work. The write-then-throw contract (always emit the .d.ts, then throw to fail the build) is unchanged. Tests: all-hit blocking run makes ZERO warehouse round-trips (getWarehouse never probed); both fatal message branches asserted. Verify-at-implementation findings (no fix needed): (A) new WorkspaceClient({}) in query-registry constructs unconditionally but is lazy (no eager auth/network) — all-hit runs make zero calls, confirmed by test; (B) SDK auth errors (401/403) classify as fatal (not connectivity) via isConnectivityError, so the authz cutover crashes the build loud as intended. xavier loop: iteration 4 -- Phase 2 Wave 2 (2E) Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/appkit/src/type-generator/index.ts | 24 +++++++-- .../tests/generate-queries.test.ts | 31 +++++++++++ .../src/type-generator/tests/index.test.ts | 53 +++++++++++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index 2ce7f611d..86b49768a 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -9,6 +9,7 @@ import { loadCache, type MetricCacheEntry, metricCacheHash, + queryCacheFileExists, saveCache, } from "./cache"; import { getErrorDiagnostic, isConnectivityError } from "./errors"; @@ -174,7 +175,21 @@ export class TypegenSyntaxError extends Error { export class TypegenFatalError extends Error { readonly queries: QueryFatalError[]; - constructor(queries: QueryFatalError[], warehouseId?: string) { + constructor( + queries: QueryFatalError[], + warehouseId?: string, + cacheInitialized: boolean = true, + ) { + // Distinguish bootstrap (no committed cache yet) from drift (cache exists but is stale/missing key). + // Bootstrap → operator needs to initialize; drift → operator needs to regenerate the affected key. + const nextStep = cacheInitialized + ? // Drift: cache file exists but is missing/stale for the failing query/metric. + warehouseId + ? `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against warehouse ${pc.bold(warehouseId)} and commit ${pc.bold(".appkit/")}.` + : `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")}.` + : // Bootstrap: no committed cache found; operator needs to initialize from scratch. + `No committed type cache found (${pc.bold(".appkit/")}). Run ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")} before building without warehouse access.`; + super( formatTypegenFailureMessage({ syntaxErrors: [], @@ -187,9 +202,7 @@ export class TypegenFatalError extends Error { "authentication failure", "SDK configuration errors", ], - nextStep: warehouseId - ? `Verify access to warehouse ${pc.bold(warehouseId)} and rerun type generation.` - : "Verify warehouse access and rerun type generation.", + nextStep, }), ); this.name = "TypegenFatalError"; @@ -391,7 +404,8 @@ export async function generateFromEntryPoint(options: { throw new TypegenSyntaxError(syntaxErrors, warehouseId, fatalErrors); } if (fatalErrors.length > 0) { - throw new TypegenFatalError(fatalErrors, warehouseId); + const cacheExists = await queryCacheFileExists(); + throw new TypegenFatalError(fatalErrors, warehouseId, cacheExists); } logger.debug("Type generation complete!"); diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index 22a054f82..c8208e3cb 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -712,6 +712,37 @@ describe("generateQueriesFromDescribe", () => { expect(syntaxErrors).toEqual([]); }); + test("blocking mode all-hit: makes ZERO warehouse round-trips on cache HIT", async () => { + const sql1 = "SELECT id FROM t1"; + const sql2 = "SELECT name FROM t2"; + mocks.readdir.mockResolvedValue(["t1.sql", "t2.sql"]); + mocks.readFile.mockResolvedValueOnce(sql1).mockResolvedValueOnce(sql2); + // All queries hit the cache with matching hashes. + mocks.loadCache.mockReturnValueOnce({ + version: CACHE_VERSION, + queries: { + t1: { hash: hashSQL(sql1), type: CACHED_GOOD_TYPE, retry: false }, + t2: { hash: hashSQL(sql2), type: CACHED_GOOD_TYPE, retry: false }, + }, + }); + + const { schemas, syntaxErrors, fatalErrors } = await describeQueries( + "/queries", + "wh-123", + ); + + // ZERO warehouse round-trips: no preflight probe (getWarehouse never called), + // no warehouse start, no DESCRIBE executions. All queries served from cache. + expect(mocks.getWarehouse).not.toHaveBeenCalled(); + expect(mocks.startWarehouse).not.toHaveBeenCalled(); + expect(mocks.executeStatement).not.toHaveBeenCalled(); + expect(schemas).toHaveLength(2); + expect(schemas[0].type).toBe(CACHED_GOOD_TYPE); + expect(schemas[1].type).toBe(CACHED_GOOD_TYPE); + expect(syntaxErrors).toEqual([]); + expect(fatalErrors).toEqual([]); + }); + test("stale retry-flagged cache entry is re-described, not reused", async () => { const sql = "SELECT id FROM t"; mocks.readdir.mockResolvedValue(["t.sql"]); diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index efcc60f44..03ac6dc51 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => ({ startWarehouse: vi.fn(), waitUntilRunning: vi.fn(), executeStatement: vi.fn(), + queryCacheFileExists: vi.fn(), // In-memory stand-in for the on-disk typegen cache file. `undefined` means // "no file yet"; otherwise it holds the serialized JSON exactly as // saveCache would have written it, so load/save round-trips behave like @@ -64,6 +65,7 @@ vi.mock("../cache", async (importOriginal) => { saveCache: vi.fn(async (cache: unknown) => { mocks.cacheFile.contents = JSON.stringify(cache, null, 2); }), + queryCacheFileExists: mocks.queryCacheFileExists, }; }); @@ -270,6 +272,57 @@ describe("generateFromEntryPoint — query failure handling", () => { expect(fs.existsSync(outFile)).toBe(true); expect(fs.readFileSync(outFile, "utf-8")).toContain("bad_auth"); }); + + test("bootstrap case: distinguishes missing cache from drift in fatal message", async () => { + mocks.generateQueriesFromDescribe.mockResolvedValue({ + schemas: [unknownSchema("query1")], + syntaxErrors: [], + fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], + }); + // Simulate first checkout: no committed cache file yet. + mocks.queryCacheFileExists.mockResolvedValue(false); + + try { + await generateFromEntryPoint({ + outFile, + queryFolder: "/queries", + warehouseId: "wh-1", + }); + expect.fail("should have thrown TypegenFatalError"); + } catch (err) { + expect(err).toBeInstanceOf(TypegenFatalError); + expect((err as Error).message).toMatch(/No committed type cache found/i); + expect((err as Error).message).toMatch(/generate-types --wait/i); + expect((err as Error).message).toMatch(/commit \.appkit\//i); + } + }); + + test("drift case: cache exists but is stale/missing for failing query", async () => { + mocks.generateQueriesFromDescribe.mockResolvedValue({ + schemas: [unknownSchema("query1")], + syntaxErrors: [], + fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], + }); + // Cache file exists but is out of date (the query's key is missing or stale). + mocks.queryCacheFileExists.mockResolvedValue(true); + + try { + await generateFromEntryPoint({ + outFile, + queryFolder: "/queries", + warehouseId: "wh-1", + }); + expect.fail("should have thrown TypegenFatalError"); + } catch (err) { + expect(err).toBeInstanceOf(TypegenFatalError); + expect((err as Error).message).toMatch(/missing or stale/i); + expect((err as Error).message).toMatch(/Regenerate with/i); + expect((err as Error).message).toMatch(/generate-types --wait/i); + expect((err as Error).message).not.toMatch( + /No committed type cache found/i, + ); + } + }); }); describe("generateFromEntryPoint — metric-view emission", () => { From 992dd66d4de4500f817da62ce3ef865ff71ab61c Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 13:31:39 +0200 Subject: [PATCH 04/11] fix(appkit): snapshot cache existence before generation; seed dev-playground MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a bootstrap-vs-drift misclassification found in live warehouse testing: generateFromEntryPoint read queryCacheFileExists() at the fatal throw site, but generateQueriesFromDescribe and syncMetricViewsTypes both persist the cache during the run — so the file always existed by then and the fatal message always said "drift", making the bootstrap branch dead code. Capture the existence snapshot at function entry (before any generation writes) and thread it to the throw. The mocked unit test passed while the bug was live because it stubbed the predicate directly; the test now ties the mock to the in-memory cache file so a regression (moving the check back) fails. Exclude .appkit/ from Biome: the committed cache is a generated artifact whose serialization saveCache owns, so the formatter must not govern it (a regen would otherwise trip lint). Seed apps/dev-playground/.appkit/ from a live blocking run against a warehouse (16 queries + 2 metrics + 1 serving endpoint). Verified end-to-end: a warm all-hit blocking re-run makes ZERO warehouse round-trips (0 new, 16 from cache, 0.00s) and the cache is byte-stable across a no-op regen (deterministic diff). xavier loop: iteration 5 -- Phase 3B integration + 2E ordering bugfix Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../.appkit/serving-types-cache.json | 21 ++ apps/dev-playground/.appkit/types-cache.json | 207 ++++++++++++++++++ biome.json | 1 + packages/appkit/src/type-generator/index.ts | 10 +- .../src/type-generator/tests/index.test.ts | 42 +++- 5 files changed, 268 insertions(+), 13 deletions(-) create mode 100644 apps/dev-playground/.appkit/serving-types-cache.json create mode 100644 apps/dev-playground/.appkit/types-cache.json diff --git a/apps/dev-playground/.appkit/serving-types-cache.json b/apps/dev-playground/.appkit/serving-types-cache.json new file mode 100644 index 000000000..26a3312a1 --- /dev/null +++ b/apps/dev-playground/.appkit/serving-types-cache.json @@ -0,0 +1,21 @@ +{ + "version": "1", + "endpoints": { + "default": { + "hash": "e66c960c5d57436d7fc4e21086650a534892a1548ccf2b333db2af6046832471", + "requestType": "{\n messages?: {\n role?: \"user\" | \"assistant\";\n content?: string;\n }[];\n /** @openapi integer, nullable */\n n?: number | null;\n max_tokens?: number;\n /** @openapi double, nullable */\n top_p?: number | null;\n reasoning_effort?: \"low\" | \"medium\" | \"high\" | \"max\" | null;\n /** @openapi double, nullable */\n temperature?: number | null;\n stop?: string | string[] | null;\n /** @openapi integer, nullable */\n top_k?: number | null;\n}", + "responseType": "{\n model?: string;\n choices?: {\n index?: number;\n message?: {\n role?: \"user\" | \"assistant\";\n content?: string;\n };\n finish_reason?: string;\n }[];\n usage?: {\n prompt_tokens?: number;\n completion_tokens?: number;\n total_tokens?: number;\n } | null;\n object?: string;\n id?: string;\n created?: number;\n}", + "chunkType": "{\n model?: string;\n choices?: {\n index?: number;\n delta?: {\n role?: \"user\" | \"assistant\";\n content?: string;\n };\n finish_reason?: string | null;\n }[];\n object?: string;\n id?: string;\n created?: number;\n}", + "requestKeys": [ + "messages", + "n", + "max_tokens", + "top_p", + "reasoning_effort", + "temperature", + "stop", + "top_k" + ] + } + } +} \ No newline at end of file diff --git a/apps/dev-playground/.appkit/types-cache.json b/apps/dev-playground/.appkit/types-cache.json new file mode 100644 index 000000000..a91799298 --- /dev/null +++ b/apps/dev-playground/.appkit/types-cache.json @@ -0,0 +1,207 @@ +{ + "version": "3", + "queries": { + "app_activity_heatmap": { + "hash": "744281be86b14c2b14ed76b94060e4af", + "type": "{\n name: \"app_activity_heatmap\";\n parameters: {\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType STRING */\n day_of_week: string;\n /** @sqlType DECIMAL(35,2) */\n spend: number;\n }>;\n }", + "retry": false + }, + "apps_list": { + "hash": "4b7393b43f8e7beaa52174ed4d918c04", + "type": "{\n name: \"apps_list\";\n parameters: Record;\n result: Array<{\n /** @sqlType STRING */\n id: string;\n /** @sqlType STRING */\n name: string;\n /** @sqlType STRING */\n creator: string;\n /** @sqlType STRING */\n tags: string;\n /** @sqlType DECIMAL(38,6) */\n totalSpend: number;\n /** @sqlType DATE */\n createdAt: string;\n }>;\n }", + "retry": false + }, + "cost_recommendations": { + "hash": "730c7d8b5e2726981088b5975157b0da", + "type": "{\n name: \"cost_recommendations\";\n parameters: Record;\n result: Array<{\n /** @sqlType INT */\n dummy: number;\n }>;\n }", + "retry": false + }, + "dashboard_fare_distribution": { + "hash": "d280e96e06e1b5189d8fe734901e6e72", + "type": "{\n name: \"dashboard_fare_distribution\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n fare_bucket: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n avg_distance: number;\n }>;\n }", + "retry": false + }, + "dashboard_hourly_heatmap": { + "hash": "827ca45db36ccf6f9c86c653c6175d69", + "type": "{\n name: \"dashboard_hourly_heatmap\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType INT */\n day_of_week: number;\n /** @sqlType INT */\n hour_of_day: number;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n }>;\n }", + "retry": false + }, + "dashboard_kpi_sparklines": { + "hash": "1b3c14b03e1de455ff78f2d3feaec047", + "type": "{\n name: \"dashboard_kpi_sparklines\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType DATE */\n trip_date: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n total_revenue: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n /** @sqlType DOUBLE */\n avg_distance: number;\n }>;\n }", + "retry": false + }, + "dashboard_kpis": { + "hash": "24ef19b8b1218edb8f247f6d79c915ff", + "type": "{\n name: \"dashboard_kpis\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType BIGINT */\n total_trips: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n /** @sqlType DOUBLE */\n avg_distance: number;\n /** @sqlType DOUBLE */\n max_fare: number;\n /** @sqlType DOUBLE */\n min_fare: number;\n }>;\n }", + "retry": false + }, + "dashboard_top_zone": { + "hash": "fe3ed4ef76e4e7eff329c88072a44fe9", + "type": "{\n name: \"dashboard_top_zone\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType INT */\n pickup_zip: number;\n /** @sqlType BIGINT */\n trip_count: number;\n }>;\n }", + "retry": false + }, + "dashboard_top_zones": { + "hash": "363c554c5a060ee2236f4e9fa1cf3c61", + "type": "{\n name: \"dashboard_top_zones\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n pickup_zip: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n total_revenue: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n }>;\n }", + "retry": false + }, + "dashboard_trips_over_time": { + "hash": "eb8bed65ed02787ed050a016472b72f6", + "type": "{\n name: \"dashboard_trips_over_time\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType DATE */\n trip_date: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n /** @sqlType DOUBLE */\n total_revenue: number;\n }>;\n }", + "retry": false + }, + "example": { + "hash": "aeb02ed3e8a6c77279099406f8709543", + "type": "{\n name: \"example\";\n parameters: Record;\n result: Array<{\n /** @sqlType BOOLEAN */\n \"(1 = 1)\": boolean;\n }>;\n }", + "retry": false + }, + "spend_data": { + "hash": "caa0430652fe15eff658e48e6dac2446", + "type": "{\n name: \"spend_data\";\n parameters: {\n /** STRING - use sql.string() */\n groupBy: SQLStringMarker;\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n /** STRING - use sql.string() */\n appId: SQLStringMarker;\n /** STRING - use sql.string() */\n creator: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n group_key: string;\n /** @sqlType TIMESTAMP */\n aggregation_period: string;\n /** @sqlType DECIMAL(38,6) */\n cost_usd: number;\n }>;\n }", + "retry": false + }, + "spend_summary": { + "hash": "bbe188624c3f5904c3a7593cb32982d5", + "type": "{\n name: \"spend_summary\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType DECIMAL(33,0) */\n total: number;\n /** @sqlType DECIMAL(33,0) */\n average: number;\n /** @sqlType DECIMAL(33,0) */\n forecasted: number;\n }>;\n }", + "retry": false + }, + "sql_helpers_test": { + "hash": "1322df4ba9c107e8d23e2a04bae860c5", + "type": "{\n name: \"sql_helpers_test\";\n parameters: {\n /** STRING - use sql.string() */\n stringParam: SQLStringMarker;\n /** NUMERIC - use sql.numeric() */\n numberParam: SQLNumberMarker;\n /** BOOLEAN - use sql.boolean() */\n booleanParam: SQLBooleanMarker;\n /** DATE - use sql.date() */\n dateParam: SQLDateMarker;\n /** TIMESTAMP - use sql.timestamp() */\n timestampParam: SQLTimestampMarker;\n /** STRING - use sql.string() */\n binaryParam: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n string_value: string;\n /** @sqlType INT */\n number_value: number;\n /** @sqlType BOOLEAN */\n boolean_value: boolean;\n /** @sqlType STRING */\n date_value: string;\n /** @sqlType STRING */\n timestamp_value: string;\n /** @sqlType BINARY */\n binary_value: string;\n /** @sqlType STRING */\n binary_hex: string;\n /** @sqlType INT */\n binary_length: number;\n }>;\n }", + "retry": false + }, + "top_contributors": { + "hash": "2d58759cca2fe31dae06475a23080120", + "type": "{\n name: \"top_contributors\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType DECIMAL(38,6) */\n total_cost_usd: number;\n }>;\n }", + "retry": false + }, + "untagged_apps": { + "hash": "5946262b49710b8ab458d1bf950ff8c9", + "type": "{\n name: \"untagged_apps\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType STRING */\n creator: string;\n /** @sqlType DECIMAL(38,6) */\n total_cost_usd: number;\n /** @sqlType DECIMAL(38,10) */\n avg_period_cost_usd: number;\n }>;\n }", + "retry": false + } + }, + "metrics": { + "customers": { + "hash": "26269cc37c4ea648a8a9cafb66a402dd", + "schema": { + "key": "customers", + "source": "appkit_demo.public.customer_metrics", + "lane": "obo", + "measures": [ + { + "name": "active_accounts", + "type": "bigint", + "isMeasure": true, + "displayName": "Active Accounts", + "format": "#,##0" + }, + { + "name": "churn_rate", + "type": "decimal", + "isMeasure": true, + "displayName": "Churn Rate" + }, + { + "name": "avg_ltv", + "type": "double", + "isMeasure": true, + "displayName": "Average LTV", + "format": "$#,##0.00" + } + ], + "dimensions": [ + { + "name": "segment", + "type": "string", + "isMeasure": false, + "displayName": "Customer Segment" + }, + { + "name": "region", + "type": "string", + "isMeasure": false, + "displayName": "Region" + }, + { + "name": "csm_email", + "type": "string", + "isMeasure": false, + "displayName": "CSM Email" + } + ] + }, + "retry": false + }, + "revenue": { + "hash": "8d0592a864778470213ca22517f21fae", + "schema": { + "key": "revenue", + "source": "appkit_demo.public.revenue_metrics", + "lane": "sp", + "measures": [ + { + "name": "mrr", + "type": "double", + "isMeasure": true, + "displayName": "Monthly Recurring Revenue", + "format": "$#,##0.00" + }, + { + "name": "arr", + "type": "double", + "isMeasure": true, + "description": "Annualized contract value across all active subscriptions", + "displayName": "Annual Recurring Revenue", + "format": "$#,##0.00" + }, + { + "name": "new_arr", + "type": "double", + "isMeasure": true, + "displayName": "New ARR", + "format": "$#,##0.00" + }, + { + "name": "churned_arr", + "type": "double", + "isMeasure": true, + "displayName": "Churned ARR", + "format": "$#,##0.00" + } + ], + "dimensions": [ + { + "name": "region", + "type": "string", + "isMeasure": false, + "displayName": "Region" + }, + { + "name": "segment", + "type": "string", + "isMeasure": false, + "displayName": "Customer Segment" + }, + { + "name": "created_at", + "type": "timestamp_ltz", + "isMeasure": false, + "displayName": "Subscription Start", + "timeGrains": [ + "day", + "hour", + "minute", + "month", + "quarter", + "week", + "year" + ] + } + ] + }, + "retry": false + } + } +} \ No newline at end of file diff --git a/biome.json b/biome.json index c9b8a4a1f..24ddeb018 100644 --- a/biome.json +++ b/biome.json @@ -14,6 +14,7 @@ "!**/*.d.ts", "!**/build", "!**/coverage", + "!**/.appkit", "!packages/appkit-ui/src/react/ui", "!**/routeTree.gen.ts", "!docs/.docusaurus", diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index 86b49768a..92bd4bd3f 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -329,6 +329,13 @@ export async function generateFromEntryPoint(options: { logger.debug("Starting type generation..."); + // Snapshot BEFORE any generation writes the cache: generateQueriesFromDescribe + // and syncMetricViewsTypes both persist to .appkit/types-cache.json during the + // run, so reading existence at the throw site would always report "exists". + // Capturing here is what lets the fatal message tell bootstrap (no cache yet) + // apart from drift (cache present but stale/missing the failing key). + const cacheExistedAtStart = await queryCacheFileExists(); + let queryRegistry: QuerySchema[] = []; let syntaxErrors: QuerySyntaxError[] = []; let fatalErrors: QueryFatalError[] = []; @@ -404,8 +411,7 @@ export async function generateFromEntryPoint(options: { throw new TypegenSyntaxError(syntaxErrors, warehouseId, fatalErrors); } if (fatalErrors.length > 0) { - const cacheExists = await queryCacheFileExists(); - throw new TypegenFatalError(fatalErrors, warehouseId, cacheExists); + throw new TypegenFatalError(fatalErrors, warehouseId, cacheExistedAtStart); } logger.debug("Type generation complete!"); diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index 03ac6dc51..8c1319d2f 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -65,7 +65,9 @@ vi.mock("../cache", async (importOriginal) => { saveCache: vi.fn(async (cache: unknown) => { mocks.cacheFile.contents = JSON.stringify(cache, null, 2); }), - queryCacheFileExists: mocks.queryCacheFileExists, + queryCacheFileExists: vi.fn( + async () => mocks.cacheFile.contents !== undefined, + ), }; }); @@ -273,14 +275,23 @@ describe("generateFromEntryPoint — query failure handling", () => { expect(fs.readFileSync(outFile, "utf-8")).toContain("bad_auth"); }); - test("bootstrap case: distinguishes missing cache from drift in fatal message", async () => { - mocks.generateQueriesFromDescribe.mockResolvedValue({ - schemas: [unknownSchema("query1")], - syntaxErrors: [], - fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], + test("bootstrap case: no cache at start → reports bootstrap message even if cache is written during generation", async () => { + // Start with no cache file (fresh checkout). + mocks.cacheFile.contents = undefined; + mocks.generateQueriesFromDescribe.mockImplementation(async () => { + // During generation, the query path's saveCache writes to the in-memory file. + // This simulates the bug scenario: if we checked existence AFTER this runs, + // we'd incorrectly report "drift" instead of "bootstrap". + mocks.cacheFile.contents = JSON.stringify({ + version: "3", + queries: {}, + }); + return { + schemas: [unknownSchema("query1")], + syntaxErrors: [], + fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], + }; }); - // Simulate first checkout: no committed cache file yet. - mocks.queryCacheFileExists.mockResolvedValue(false); try { await generateFromEntryPoint({ @@ -291,20 +302,27 @@ describe("generateFromEntryPoint — query failure handling", () => { expect.fail("should have thrown TypegenFatalError"); } catch (err) { expect(err).toBeInstanceOf(TypegenFatalError); + // This is the critical assertion: the message must reflect the state + // BEFORE generation, not after (when the cache file now exists). expect((err as Error).message).toMatch(/No committed type cache found/i); expect((err as Error).message).toMatch(/generate-types --wait/i); expect((err as Error).message).toMatch(/commit \.appkit\//i); + // Must NOT say "missing or stale" (the drift message): + expect((err as Error).message).not.toMatch(/missing or stale/i); } }); - test("drift case: cache exists but is stale/missing for failing query", async () => { + test("drift case: cache exists at start → reports drift message", async () => { + // Pre-seed the cache (simulating a previous successful run). + mocks.cacheFile.contents = JSON.stringify({ + version: "3", + queries: { existing_query: { hash: "abc", type: "{}", retry: false } }, + }); mocks.generateQueriesFromDescribe.mockResolvedValue({ schemas: [unknownSchema("query1")], syntaxErrors: [], fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], }); - // Cache file exists but is out of date (the query's key is missing or stale). - mocks.queryCacheFileExists.mockResolvedValue(true); try { await generateFromEntryPoint({ @@ -315,9 +333,11 @@ describe("generateFromEntryPoint — query failure handling", () => { expect.fail("should have thrown TypegenFatalError"); } catch (err) { expect(err).toBeInstanceOf(TypegenFatalError); + // Cache existed at start → report drift (stale/missing key). expect((err as Error).message).toMatch(/missing or stale/i); expect((err as Error).message).toMatch(/Regenerate with/i); expect((err as Error).message).toMatch(/generate-types --wait/i); + // Must NOT say "No committed type cache found" (the bootstrap message): expect((err as Error).message).not.toMatch( /No committed type cache found/i, ); From bccf63090c6ea4bc7ba38a0fcae77d4e97f8a93e Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 14:08:54 +0200 Subject: [PATCH 05/11] fix(appkit): keep node: cache-path resolver out of the client-safe shared barrel CI surfaced two failures from routing the cache-path resolvers through the shared root barrel: 1. Docs Build (+ PR Template): the root barrel `shared/src/index.ts` is client-safe (docs webpack aliases shared->src and pulls it via appkit-ui), but cache-paths imports `node:path`, so the docs client bundle died with UnhandledSchemeError. Move the resolvers off the root barrel onto a Node-only subpath and import them there from appkit's cache/serving/ choice-sink modules. tsdown owns shared's generated `exports` (devExports), so the subpath is added as a tsdown entry and keyed by its src-relative path: `shared/cli/commands/cache-paths`. 2. Unit Tests: the new bootstrap-vs-drift message assertions matched raw text, but picocolors emits ANSI codes when CI is set, splitting the strings. Strip ANSI (incl. the ESC byte) before matching. Also repoint the cache serialization test's `vi.mock` to the new subpath specifier. Verified with the gates CI runs: `CI=true pnpm test` (3725 pass), `pnpm docs:build` (client compiles, no node: error), typecheck, build, check, knip. xavier loop: iteration 6 -- CI fixes (docs node: leak + ANSI-brittle tests) Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../src/plugins/ui-variants/choice-sink.ts | 2 +- packages/appkit/src/type-generator/cache.ts | 2 +- .../src/type-generator/serving/cache.ts | 2 +- .../tests/cache-serialization.test.ts | 7 +++-- .../src/type-generator/tests/index.test.ts | 26 ++++++++++++------- packages/appkit/tsconfig.json | 3 +++ packages/shared/package.json | 5 ++++ packages/shared/src/index.ts | 1 - packages/shared/tsdown.config.ts | 10 ++++++- tsconfig.json | 3 +++ 10 files changed, 44 insertions(+), 17 deletions(-) diff --git a/packages/appkit/src/plugins/ui-variants/choice-sink.ts b/packages/appkit/src/plugins/ui-variants/choice-sink.ts index 5e22edd8a..7ff98d2b5 100644 --- a/packages/appkit/src/plugins/ui-variants/choice-sink.ts +++ b/packages/appkit/src/plugins/ui-variants/choice-sink.ts @@ -1,6 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { getEphemeralStateDir } from "shared"; +import { getEphemeralStateDir } from "shared/cli/commands/cache-paths"; /** * Filename of the JSONL choices file. The agent skill discovers this file at diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index eeaa5b86b..800a96bee 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; -import { getCommittedCacheDir } from "shared"; +import { getCommittedCacheDir } from "shared/cli/commands/cache-paths"; import { createLogger } from "../logging/logger"; import type { MetricSchema } from "./mv-registry/types"; diff --git a/packages/appkit/src/type-generator/serving/cache.ts b/packages/appkit/src/type-generator/serving/cache.ts index 7752bab8e..91effeb9d 100644 --- a/packages/appkit/src/type-generator/serving/cache.ts +++ b/packages/appkit/src/type-generator/serving/cache.ts @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; -import { getCommittedCacheDir } from "shared"; +import { getCommittedCacheDir } from "shared/cli/commands/cache-paths"; import { createLogger } from "../../logging/logger"; const logger = createLogger("type-generator:serving:cache"); diff --git a/packages/appkit/src/type-generator/tests/cache-serialization.test.ts b/packages/appkit/src/type-generator/tests/cache-serialization.test.ts index 15b84701a..c67a4e8f6 100644 --- a/packages/appkit/src/type-generator/tests/cache-serialization.test.ts +++ b/packages/appkit/src/type-generator/tests/cache-serialization.test.ts @@ -14,8 +14,11 @@ vi.mock("node:fs/promises", () => ({ }, })); -// Mock getCommittedCacheDir to return a consistent test path -vi.mock("shared", () => ({ +// Mock getCommittedCacheDir to return a consistent test path. +// cache.ts imports it from the Node-only "shared/cli/commands/cache-paths" +// subpath (kept out of the client-safe "shared" root barrel), so the mock must +// target that exact specifier. +vi.mock("shared/cli/commands/cache-paths", () => ({ getCommittedCacheDir: () => "/test/app/.appkit", })); diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index 8c1319d2f..2807da943 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -11,6 +11,12 @@ import { } from "vitest"; import type { DatabricksStatementExecutionResponse } from "../types"; +// picocolors emits ANSI color codes when CI is set; strip them (ESC + the +// bracketed SGR sequence) so message-content assertions match regardless of +// the color environment. +// biome-ignore lint/suspicious/noControlCharactersInRegex: matching the ESC (\x1b) byte is the point. +const stripAnsi = (s: string): string => s.replace(/\x1b\[[0-9;]*m/g, ""); + const mocks = vi.hoisted(() => ({ generateQueriesFromDescribe: vi.fn(), getWarehouseState: vi.fn(), @@ -304,11 +310,12 @@ describe("generateFromEntryPoint — query failure handling", () => { expect(err).toBeInstanceOf(TypegenFatalError); // This is the critical assertion: the message must reflect the state // BEFORE generation, not after (when the cache file now exists). - expect((err as Error).message).toMatch(/No committed type cache found/i); - expect((err as Error).message).toMatch(/generate-types --wait/i); - expect((err as Error).message).toMatch(/commit \.appkit\//i); + const message = stripAnsi((err as Error).message); + expect(message).toMatch(/No committed type cache found/i); + expect(message).toMatch(/generate-types --wait/i); + expect(message).toMatch(/commit \.appkit\//i); // Must NOT say "missing or stale" (the drift message): - expect((err as Error).message).not.toMatch(/missing or stale/i); + expect(message).not.toMatch(/missing or stale/i); } }); @@ -334,13 +341,12 @@ describe("generateFromEntryPoint — query failure handling", () => { } catch (err) { expect(err).toBeInstanceOf(TypegenFatalError); // Cache existed at start → report drift (stale/missing key). - expect((err as Error).message).toMatch(/missing or stale/i); - expect((err as Error).message).toMatch(/Regenerate with/i); - expect((err as Error).message).toMatch(/generate-types --wait/i); + const message = stripAnsi((err as Error).message); + expect(message).toMatch(/missing or stale/i); + expect(message).toMatch(/Regenerate with/i); + expect(message).toMatch(/generate-types --wait/i); // Must NOT say "No committed type cache found" (the bootstrap message): - expect((err as Error).message).not.toMatch( - /No committed type cache found/i, - ); + expect(message).not.toMatch(/No committed type cache found/i); } }); }); diff --git a/packages/appkit/tsconfig.json b/packages/appkit/tsconfig.json index 5265a6881..a673208b0 100644 --- a/packages/appkit/tsconfig.json +++ b/packages/appkit/tsconfig.json @@ -7,6 +7,9 @@ "@/*": ["src/*"], "@tools/*": ["../../tools/*"], "shared": ["../../packages/shared/src"], + "shared/cli/commands/cache-paths": [ + "../../packages/shared/src/cli/commands/cache-paths" + ], "@databricks/lakebase": ["../../packages/lakebase/src"] } }, diff --git a/packages/shared/package.json b/packages/shared/package.json index 7045336fc..2483ca3e1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -15,6 +15,10 @@ "development": "./src/cli/index.ts", "default": "./dist/cli/index.js" }, + "./cli/commands/cache-paths": { + "development": "./src/cli/commands/cache-paths.ts", + "default": "./dist/cli/commands/cache-paths.js" + }, "./package.json": "./package.json" }, "scripts": { @@ -35,6 +39,7 @@ "exports": { ".": "./dist/index.js", "./cli": "./dist/cli/index.js", + "./cli/commands/cache-paths": "./dist/cli/commands/cache-paths.js", "./package.json": "./package.json" } }, diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index b93aac683..d036e0dbd 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,6 +1,5 @@ export * from "./agent"; export * from "./cache"; -export * from "./cli/commands/cache-paths"; export * from "./execute"; export * from "./genie"; export * from "./plugin"; diff --git a/packages/shared/tsdown.config.ts b/packages/shared/tsdown.config.ts index 9db6274a7..07f6e3562 100644 --- a/packages/shared/tsdown.config.ts +++ b/packages/shared/tsdown.config.ts @@ -2,7 +2,15 @@ import { defineConfig } from "tsdown"; export default defineConfig({ name: "shared", - entry: ["src/index.ts", "src/cli/index.ts"], + entry: [ + "src/index.ts", + "src/cli/index.ts", + // Node-only path resolvers, exposed as the "shared/cache-paths" subpath so + // appkit can import them without pulling node: builtins into the + // client-safe root barrel. Must be a tsdown entry or the exports plugin + // (exports.devExports below) drops the "./cache-paths" export on rebuild. + "src/cli/commands/cache-paths.ts", + ], outDir: "dist", minify: false, format: "esm", diff --git a/tsconfig.json b/tsconfig.json index 1da2d8e99..d53ce07b7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,9 @@ "@databricks/appkit": ["packages/appkit/*"], "@databricks/appkit-ui": ["packages/appkit-ui/*"], "shared": ["packages/shared/src"], + "shared/cli/commands/cache-paths": [ + "packages/shared/src/cli/commands/cache-paths" + ], "@tools/*": ["tools/*"] } }, From 640d4642b09a15ad3fd4d8d2f1ef1bba5a4ad6b4 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 14:21:06 +0200 Subject: [PATCH 06/11] refactor(shared): keep cache-paths internal (drop public subpath export) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache-path resolvers are an internal monorepo helper, not part of the shared package's public API. The previous commit exposed them as a "./cli/commands/cache-paths" export (a side effect of adding a tsdown entry, which makes the exports plugin generate the mapping) — leaking shared's internal file layout for no benefit. appkit resolves the import via its tsconfig `paths` alias and bundles shared inline (noExternal), so the export is not load-bearing: dropping the tsdown entry (and thus the generated export) keeps typecheck, appkit build, vitest, and docs:build all green. shared stays private; cache-paths stays internal. xavier loop: iteration 7 -- keep cache-paths internal Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/shared/package.json | 5 ----- packages/shared/tsdown.config.ts | 10 +--------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/packages/shared/package.json b/packages/shared/package.json index 2483ca3e1..7045336fc 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -15,10 +15,6 @@ "development": "./src/cli/index.ts", "default": "./dist/cli/index.js" }, - "./cli/commands/cache-paths": { - "development": "./src/cli/commands/cache-paths.ts", - "default": "./dist/cli/commands/cache-paths.js" - }, "./package.json": "./package.json" }, "scripts": { @@ -39,7 +35,6 @@ "exports": { ".": "./dist/index.js", "./cli": "./dist/cli/index.js", - "./cli/commands/cache-paths": "./dist/cli/commands/cache-paths.js", "./package.json": "./package.json" } }, diff --git a/packages/shared/tsdown.config.ts b/packages/shared/tsdown.config.ts index 07f6e3562..9db6274a7 100644 --- a/packages/shared/tsdown.config.ts +++ b/packages/shared/tsdown.config.ts @@ -2,15 +2,7 @@ import { defineConfig } from "tsdown"; export default defineConfig({ name: "shared", - entry: [ - "src/index.ts", - "src/cli/index.ts", - // Node-only path resolvers, exposed as the "shared/cache-paths" subpath so - // appkit can import them without pulling node: builtins into the - // client-safe root barrel. Must be a tsdown entry or the exports plugin - // (exports.devExports below) drops the "./cache-paths" export on rebuild. - "src/cli/commands/cache-paths.ts", - ], + entry: ["src/index.ts", "src/cli/index.ts"], outDir: "dist", minify: false, format: "esm", From fa4cf8412291c7780af5933fd3126fe49cf78437 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 14:34:26 +0200 Subject: [PATCH 07/11] fix(shared): restore cache-paths subpath export (required at runtime) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the previous "keep cache-paths internal" change, which broke the Playground Integration Tests with ERR_PACKAGE_PATH_NOT_EXPORTED. The export IS load-bearing on a path the earlier verification missed: dev-playground runs its server via `tsx` under the `development` export condition, so @databricks/appkit resolves to SOURCE, whose `import "shared/cli/commands/cache-paths"` is a bare cross-package specifier that tsx resolves through shared's package.json exports map at runtime — tsconfig `paths` do not apply there. Without the export the server never boots and Playwright times out. Re-add the tsdown entry (which regenerates the export) with a comment explaining the runtime dependency so it isn't dropped again. The export is monorepo-internal only: shared is private and appkit bundles it inline (noExternal), so it never reaches published-appkit consumers. Verified by booting the dev-playground server under the development condition (starts clean, no export error). xavier loop: iteration 8 -- restore runtime-required cache-paths export Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/shared/package.json | 5 +++++ packages/shared/tsdown.config.ts | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/shared/package.json b/packages/shared/package.json index 7045336fc..2483ca3e1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -15,6 +15,10 @@ "development": "./src/cli/index.ts", "default": "./dist/cli/index.js" }, + "./cli/commands/cache-paths": { + "development": "./src/cli/commands/cache-paths.ts", + "default": "./dist/cli/commands/cache-paths.js" + }, "./package.json": "./package.json" }, "scripts": { @@ -35,6 +39,7 @@ "exports": { ".": "./dist/index.js", "./cli": "./dist/cli/index.js", + "./cli/commands/cache-paths": "./dist/cli/commands/cache-paths.js", "./package.json": "./package.json" } }, diff --git a/packages/shared/tsdown.config.ts b/packages/shared/tsdown.config.ts index 9db6274a7..19ccf1d81 100644 --- a/packages/shared/tsdown.config.ts +++ b/packages/shared/tsdown.config.ts @@ -2,7 +2,17 @@ import { defineConfig } from "tsdown"; export default defineConfig({ name: "shared", - entry: ["src/index.ts", "src/cli/index.ts"], + entry: [ + "src/index.ts", + "src/cli/index.ts", + // Node-only path resolvers. Needs its own entry (→ generated + // "./cli/commands/cache-paths" export) because appkit imports it as a bare + // cross-package specifier that tsx resolves via shared's exports map at + // runtime (dev-playground's `development`-condition source run) — tsconfig + // paths don't apply there. Kept OUT of the client-safe root barrel because + // it imports node:path (would break the docs client webpack bundle). + "src/cli/commands/cache-paths.ts", + ], outDir: "dist", minify: false, format: "esm", From 868240331d6395d4674a6620eaf26b48850d9d19 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 14:37:22 +0200 Subject: [PATCH 08/11] docs(shared): note cache-paths export is required by Playground Integration Tests Expand the tsdown entry comment to explicitly warn that removing the cache-paths export breaks the "Playground Integration Tests" CI job (the dev-playground server fails to boot with ERR_PACKAGE_PATH_NOT_EXPORTED and Playwright times out), and that build/typecheck/unit tests do NOT catch it. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/shared/tsdown.config.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/shared/tsdown.config.ts b/packages/shared/tsdown.config.ts index 19ccf1d81..1a6684384 100644 --- a/packages/shared/tsdown.config.ts +++ b/packages/shared/tsdown.config.ts @@ -11,6 +11,11 @@ export default defineConfig({ // runtime (dev-playground's `development`-condition source run) — tsconfig // paths don't apply there. Kept OUT of the client-safe root barrel because // it imports node:path (would break the docs client webpack bundle). + // + // DO NOT REMOVE this export: without it the dev-playground server fails to + // boot with ERR_PACKAGE_PATH_NOT_EXPORTED, which times out the "Playground + // Integration Tests" (Playwright) CI job. Build/typecheck/unit tests all + // still pass, so only that job catches its absence. "src/cli/commands/cache-paths.ts", ], outDir: "dist", From 2baef04634195cae444f617f219b446f2fc9672c Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 14:45:39 +0200 Subject: [PATCH 09/11] refactor(appkit): trim ui-variants choice-sink to minimal helper adoption Route the ui-variants choices JSONL path through getEphemeralStateDir() without the incidental churn: keep the single UI_CHOICES_FILE constant and the original constructor signature, and restore the main docstrings (with only the accuracy fix the now-absolute path requires). Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../src/plugins/ui-variants/choice-sink.ts | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/packages/appkit/src/plugins/ui-variants/choice-sink.ts b/packages/appkit/src/plugins/ui-variants/choice-sink.ts index 7ff98d2b5..2fa966e67 100644 --- a/packages/appkit/src/plugins/ui-variants/choice-sink.ts +++ b/packages/appkit/src/plugins/ui-variants/choice-sink.ts @@ -3,28 +3,22 @@ import path from "node:path"; import { getEphemeralStateDir } from "shared/cli/commands/cache-paths"; /** - * Filename of the JSONL choices file. The agent skill discovers this file at - * the ephemeral state directory (under node_modules/). - */ -const UI_CHOICES_FILENAME = ".appkit-ui-choices.jsonl"; - -/** - * Default absolute path to the JSONL choices file: ephemeral state dir - * (node_modules/.databricks/appkit/) + the contract filename. Kept - * ephemeral & gitignored; NOT part of the committed .appkit/ relocation. - * + * Absolute path of the JSONL choices file, under `node_modules/`. * A coding agent reads it to pick up the developer's in-browser confirmation - * and finalize the chosen variant. Each line is spent on read — the agent - * removes it once the choice is finalized. + * and finalize the chosen variant. + * + * Under `node_modules/`, so it's gitignored and cleared on a clean install. + * Each line is spent on read — the agent removes it once the choice is + * finalized. * * CONTRACT: the `databricks-app-variants` agent skill (in the - * databricks-agent-skills repo) discovers the choices file at this absolute - * path. Changing this value silently breaks that skill's file discovery — - * update the skill's `find` path in the same change. + * databricks-agent-skills repo) discovers the choices file at this path. + * Changing this value silently breaks that skill's file discovery — update the + * skill's `find` path in the same change. */ -const DEFAULT_UI_CHOICES_PATH = path.join( +const UI_CHOICES_FILE = path.join( getEphemeralStateDir(), - UI_CHOICES_FILENAME, + ".appkit-ui-choices.jsonl", ); /** @@ -51,16 +45,14 @@ export interface UiChoiceRecord { /** * File store for confirmed variant choices: upserts choices into - * {@link DEFAULT_UI_CHOICES_PATH}, one line per `` id. + * {@link UI_CHOICES_FILE}, one line per `` id. * * The store is **keyed and latest-wins**: at most one record per `blockId`, and * recording an existing `blockId` replaces it rather than appending, so the file * always reflects the current choice for each block. * - * By default, the file lands at the ephemeral state directory under - * `node_modules/.databricks/appkit/`. Callers may override the path (e.g., for - * testing), in which case it is resolved against `process.cwd()`; absolute - * paths are passed through unchanged. Concurrent confirms are serialized behind + * The file is resolved against `process.cwd()`, so it lands under whatever + * directory the dev server runs from. Concurrent confirms are serialized behind * an internal queue so their read-modify-write can't interleave and lose an * update. * @@ -70,8 +62,8 @@ export class FileChoiceStore { private readonly filePath: string; private writeQueue: Promise = Promise.resolve(); - constructor(pathArg: string = DEFAULT_UI_CHOICES_PATH) { - this.filePath = path.resolve(process.cwd(), pathArg); + constructor(relativePath: string = UI_CHOICES_FILE) { + this.filePath = path.resolve(process.cwd(), relativePath); } record(record: UiChoiceRecord): Promise { From c9ede7b2742441103485aee0ce1e121e0780ead1 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 15:28:20 +0200 Subject: [PATCH 10/11] fix(appkit): don't misdirect cache-unrelated typegen fatals to .appkit/ Make TypegenFatalError's cacheInitialized snapshot optional. A cache-unrelated fatal (e.g. a malformed definitions.json) now omits it and gets generic "Fix the FATAL error(s) above, then re-run generate-types" wording, instead of the bootstrap/drift message telling the operator to regenerate and commit .appkit/ (which only applies to failures tied to the type cache). Co-authored-by: Isaac Signed-off-by: Atila Fassina --- packages/appkit/src/type-generator/index.ts | 28 ++++++++++++------- .../src/type-generator/tests/index.test.ts | 5 ++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index 92bd4bd3f..237125482 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -178,17 +178,25 @@ export class TypegenFatalError extends Error { constructor( queries: QueryFatalError[], warehouseId?: string, - cacheInitialized: boolean = true, + cacheInitialized?: boolean, ) { - // Distinguish bootstrap (no committed cache yet) from drift (cache exists but is stale/missing key). - // Bootstrap → operator needs to initialize; drift → operator needs to regenerate the affected key. - const nextStep = cacheInitialized - ? // Drift: cache file exists but is missing/stale for the failing query/metric. - warehouseId - ? `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against warehouse ${pc.bold(warehouseId)} and commit ${pc.bold(".appkit/")}.` - : `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")}.` - : // Bootstrap: no committed cache found; operator needs to initialize from scratch. - `No committed type cache found (${pc.bold(".appkit/")}). Run ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")} before building without warehouse access.`; + // The cache snapshot is optional: callers whose failure relates to the type + // cache pass it to get bootstrap/drift remediation; callers with an unrelated + // fatal (e.g. a malformed definitions.json) omit it and get generic wording. + const nextStep = + cacheInitialized === undefined + ? // No cache snapshot: the failure is unrelated to the cache — don't + // misdirect the operator toward regenerating/committing `.appkit/`. + `Fix the ${pc.bold("FATAL")} error(s) above, then re-run ${pc.bold("generate-types")}.` + : // Distinguish bootstrap (no committed cache yet) from drift (cache exists but is stale/missing key). + // Bootstrap → operator needs to initialize; drift → operator needs to regenerate the affected key. + cacheInitialized + ? // Drift: cache file exists but is missing/stale for the failing query/metric. + warehouseId + ? `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against warehouse ${pc.bold(warehouseId)} and commit ${pc.bold(".appkit/")}.` + : `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")}.` + : // Bootstrap: no committed cache found; operator needs to initialize from scratch. + `No committed type cache found (${pc.bold(".appkit/")}). Run ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")} before building without warehouse access.`; super( formatTypegenFailureMessage({ diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index 2807da943..ae3ae46f5 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -727,6 +727,11 @@ describe("generateFromEntryPoint — metric-view emission", () => { expect(error).toBeInstanceOf(TypegenFatalError); expect((error as Error).message).toContain("definitions.json"); + // A config parse error is unrelated to the type cache: the remediation must + // NOT tell the operator to regenerate/commit `.appkit/` (that's the cache + // bootstrap/drift wording, reserved for callers that pass the cache snapshot). + expect((error as Error).message).not.toContain(".appkit/"); + expect((error as Error).message).toContain("Fix the"); // Query types were written before the metric config was read. expect(fs.existsSync(outFile)).toBe(true); }); From 67f1aa781645c624ee8d0c02dafb45808492bd2a Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 31 Jul 2026 16:39:16 +0200 Subject: [PATCH 11/11] docs: committed .appkit/ type cache and warehouse-less deploys (#503) * docs: document committed .appkit/ type cache and warehouse-less deploys Reflect the typegen cache relocation in the docs: - type-generation.md: add a "Type cache" section (cache now lives in the committed .appkit/ dir, not node_modules; what to commit; deterministic serialization) and a "Deploying without warehouse access" subsection covering the rollout ordering (seed + commit .appkit/ while the warehouse is reachable, then cut access) and the bootstrap-vs-drift failure messages. Update the --wait section to describe hit-skips-warehouse / miss-attempts / unresolved-miss-fails semantics. - project-setup.md: add .appkit/ to the canonical layout as a committed artifact (do not gitignore). - app-management.mdx: deploy-guide admonition that a warehouse-less deploy needs a committed .appkit/ covering every query. Co-authored-by: Isaac Signed-off-by: Atila Fassina * docs: match typegen cache-failure labels to actual CLI messages Align the warehouse-less deploy failure bullets with the strings the build actually prints ('No committed type cache found' vs 'The committed .appkit/ cache is missing or stale') so they are greppable in CI logs, and broaden the drift case to cover added/missing queries, not just changed ones. Co-authored-by: Isaac Signed-off-by: Atila Fassina --------- Signed-off-by: Atila Fassina --- docs/docs/app-management.mdx | 6 ++++++ docs/docs/development/project-setup.md | 1 + docs/docs/development/type-generation.md | 22 +++++++++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/docs/app-management.mdx b/docs/docs/app-management.mdx index 4aeffebba..5369783d2 100644 --- a/docs/docs/app-management.mdx +++ b/docs/docs/app-management.mdx @@ -42,6 +42,12 @@ databricks apps deploy --help ``` ::: +:::info Type cache and warehouse-less deploys + +The build step runs type generation in blocking mode. If your deploy pipeline cannot reach a SQL warehouse, the build still succeeds **only if the committed `.appkit/` type cache covers every query** — otherwise it fails rather than shipping degraded types. Commit `.appkit/` while the warehouse is still reachable, before removing warehouse access from CI/CD. See [Type generation → Deploying without warehouse access](./development/type-generation.md#deploying-without-warehouse-access). + +::: + ### Examples - Deploy to a specific target: diff --git a/docs/docs/development/project-setup.md b/docs/docs/development/project-setup.md index 8aea2e14c..97d5fb2b8 100644 --- a/docs/docs/development/project-setup.md +++ b/docs/docs/development/project-setup.md @@ -24,6 +24,7 @@ my-app/ ├── config/ │ └── queries/ │ └── my_query.sql +├── .appkit/ # committed type cache (do NOT gitignore) ├── app.yaml ├── package.json └── tsconfig.json diff --git a/docs/docs/development/type-generation.md b/docs/docs/development/type-generation.md index 04b2091c6..94199cd6e 100644 --- a/docs/docs/development/type-generation.md +++ b/docs/docs/development/type-generation.md @@ -82,7 +82,27 @@ Pass `--wait` for CI and production builds, where accurate types must be present npx @databricks/appkit generate-types --wait ``` -In blocking mode the generator starts a stopped warehouse, waits (bounded) for it to reach `RUNNING`, and then describes your queries. It fails only when the configured warehouse no longer exists (deleted/deleting), so a transient outage or a cold warehouse degrades gracefully rather than breaking the build. The app template wires this up for you: `postinstall` and `predev` run the non-blocking default, while `prebuild` runs `--wait`. +In blocking mode the generator skips the warehouse entirely for any query whose types are already in the [type cache](#type-cache) (a cache **hit** makes zero warehouse round-trips). Only a cache **miss** contacts the warehouse: it starts a stopped warehouse, waits (bounded) for it to reach `RUNNING`, and then describes the query. A miss that cannot be resolved to a real type — a deleted warehouse, or (see below) a CI environment with no warehouse access — **fails the build** rather than shipping `result: unknown`. A transient connectivity blip still degrades gracefully. The app template wires this up for you: `postinstall` and `predev` run the non-blocking default, while `prebuild` runs `--wait`. + +## Type cache + +Type generation is backed by a per-app cache so that unchanged queries never pay for a repeat `DESCRIBE`. The cache lives in a committed `.appkit/` directory at your app root: + +- `.appkit/types-cache.json` — query and metric-view DESCRIBE results +- `.appkit/serving-types-cache.json` — model-serving endpoint schemas + +**`.appkit/` is a committed build artifact — check it into git; do not add it to `.gitignore`.** Committing it is what lets a blocking (`--wait`) build succeed with **no warehouse access**: on a fresh `git checkout` + `pnpm install`, every query is already a cache hit, so the build makes zero warehouse round-trips. (Ephemeral coordination state — a background-worker lock and the UI-variant choices file — stays under `node_modules/.databricks/appkit/` and is *not* committed.) + +The cache serializes deterministically (top-level keys sorted), so a no-op regeneration produces a no-op diff and unrelated query changes don't collide in merges. + +### Deploying without warehouse access + +If your CI/CD environment cannot reach a SQL warehouse (for example, an authorization boundary removes warehouse access from the deploy pipeline), a blocking build still works **as long as the committed `.appkit/` cache covers every query** — all hits, no warehouse needed. + +This imposes a rollout ordering: **produce and commit `.appkit/` while the warehouse is still reachable, then remove warehouse access.** A build that runs after warehouse access is gone with an uninitialized or incomplete cache fails loudly (never silently degrades to `unknown`), and the error message tells you which case you hit: + +- **`No committed type cache found`** (no `.appkit/` yet) — run `generate-types --wait` against a warehouse and commit `.appkit/`. +- **`The committed .appkit/ cache is missing or stale`** (a query was added or changed since the cache was built, or its entry is missing) — regenerate against a warehouse and commit the updated `.appkit/`. ## Metric-view types