From 428e24567a7e5a169aa4f1ee1c86499899dd2408 Mon Sep 17 00:00:00 2001 From: Jorge Calvar Date: Thu, 30 Jul 2026 18:06:23 +0200 Subject: [PATCH] feat(appkit): embed typegen cache in generated .d.ts files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The typegen cache lived in node_modules, which npm install wipes on every deploy — so a deployed app started cold and re-ran every workspace call (DESCRIBE / serving getOpenApi) at build time. Under the assumed-role authz model the Apps service principal running the build may lack access those calls need, which fails the deployment. Move the cache into the committed generated .d.ts files: a header comment holds one change-detector hash per entry, the body holds the rendered type block. On a later run, an entry whose source hash matches the header is a cache hit and its block is reused verbatim with no workspace call. A deploy with unchanged types therefore makes zero workspace calls; a genuinely changed entry is still described (and fails loudly if unreachable). - New embedded-cache module: render/parse header, split body into per-entry blocks (string/comment-aware), preserve timestamp on unchanged runs. - Serving re-keyed on a local identity hash (alias|endpointName) computed before fetching, so a hit skips getOpenApi; serving.d.ts is now committed and the runtime request-param allowlist reads it directly. - Remove the node_modules JSON caches. --no-cache stays as the force-refresh escape hatch. Co-authored-by: Isaac Signed-off-by: Jorge Calvar --- apps/dev-playground/.gitignore | 5 +- .../src/plugins/serving/schema-filter.ts | 54 +- .../appkit/src/plugins/serving/serving.ts | 13 +- .../serving/tests/schema-filter.test.ts | 71 +-- packages/appkit/src/type-generator/cache.ts | 204 ++++--- .../src/type-generator/embedded-cache.ts | 557 ++++++++++++++++++ packages/appkit/src/type-generator/index.ts | 185 +++--- .../mv-registry/render-types.ts | 87 ++- .../src/type-generator/query-registry.ts | 27 +- .../src/type-generator/serving/cache.ts | 85 ++- .../src/type-generator/serving/generator.ts | 117 ++-- .../serving/tests/cache.test.ts | 128 ++-- .../serving/tests/generator.test.ts | 49 +- .../__snapshots__/mv-registry.test.ts.snap | 37 +- .../tests/embedded-cache.test.ts | 251 ++++++++ .../tests/generate-queries.test.ts | 108 ++-- .../src/type-generator/tests/index.test.ts | 337 +++++------ .../type-generator/tests/mv-registry.test.ts | 20 +- .../tests/sync-metric-views-types.test.ts | 55 +- packages/appkit/src/type-generator/types.ts | 7 + template/_gitignore | 3 - 21 files changed, 1699 insertions(+), 701 deletions(-) create mode 100644 packages/appkit/src/type-generator/embedded-cache.ts create mode 100644 packages/appkit/src/type-generator/tests/embedded-cache.test.ts diff --git a/apps/dev-playground/.gitignore b/apps/dev-playground/.gitignore index d79530cf5..5c957fcd6 100644 --- a/apps/dev-playground/.gitignore +++ b/apps/dev-playground/.gitignore @@ -1,6 +1,3 @@ # Playwright test-results/ -playwright-report/ - -# Auto-generated types (regenerated on `pnpm dev` by appKitTypesPlugin) -shared/appkit-types/serving.d.ts \ No newline at end of file +playwright-report/ \ No newline at end of file diff --git a/packages/appkit/src/plugins/serving/schema-filter.ts b/packages/appkit/src/plugins/serving/schema-filter.ts index 92a25c69f..88e234b39 100644 --- a/packages/appkit/src/plugins/serving/schema-filter.ts +++ b/packages/appkit/src/plugins/serving/schema-filter.ts @@ -1,54 +1,50 @@ import fs from "node:fs/promises"; import { createLogger } from "../../logging/logger"; import { - CACHE_VERSION, - type ServingCache, -} from "../../type-generator/serving/cache"; + objectMemberValue, + objectTopLevelKeys, + splitEntryBlocks, +} from "../../type-generator/embedded-cache"; const logger = createLogger("serving:schema-filter"); -function isValidCache(data: unknown): data is ServingCache { - return ( - typeof data === "object" && - data !== null && - "version" in data && - (data as ServingCache).version === CACHE_VERSION && - "endpoints" in data && - typeof (data as ServingCache).endpoints === "object" - ); -} +/** Registry interface name whose members hold the serving type blocks. */ +const SERVING_INTERFACE = "ServingEndpointRegistry"; /** - * Loads endpoint schemas from the type generation cache file. - * Returns a map of alias → allowed parameter keys. + * Load per-endpoint request-parameter allowlists from the committed generated + * `serving.d.ts`. The allowlist for an alias is the set of top-level keys of + * its rendered `request` object type (`stream` is already excluded at + * generation time). A generic `Record` request has no keys, so + * it yields no allowlist (passthrough). A missing file → no filtering + * (passthrough). */ export async function loadEndpointSchemas( - cacheFile: string, + typesFile: string, ): Promise>> { const allowlists = new Map>(); try { - const raw = await fs.readFile(cacheFile, "utf8"); - const parsed: unknown = JSON.parse(raw); - if (!isValidCache(parsed)) { - logger.warn("Serving types cache has invalid structure, skipping"); - return allowlists; - } - const cache = parsed; - - for (const [alias, entry] of Object.entries(cache.endpoints)) { - if (entry.requestKeys && entry.requestKeys.length > 0) { - allowlists.set(alias, new Set(entry.requestKeys)); + const source = await fs.readFile(typesFile, "utf8"); + const blocks = splitEntryBlocks(source, SERVING_INTERFACE); + for (const [alias, block] of Object.entries(blocks)) { + // Each member is `{ request: ; response: ...; chunk: ...; }`. + // Extract the `request` value, then its top-level object keys. + const requestType = objectMemberValue(block, "request"); + if (!requestType) continue; + const keys = objectTopLevelKeys(requestType); + if (keys.length > 0) { + allowlists.set(alias, new Set(keys)); } } } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ENOENT") { logger.warn( - "Failed to load serving types cache: %s", + "Failed to load serving types for request filtering: %s", (err as Error).message, ); } - // No cache → no filtering, passthrough mode + // No file → no filtering, passthrough mode } return allowlists; diff --git a/packages/appkit/src/plugins/serving/serving.ts b/packages/appkit/src/plugins/serving/serving.ts index 37707778e..41959fed6 100644 --- a/packages/appkit/src/plugins/serving/serving.ts +++ b/packages/appkit/src/plugins/serving/serving.ts @@ -66,14 +66,15 @@ export class ServingPlugin extends Plugin { } async setup(): Promise { - const cacheFile = path.join( + // The request-parameter allowlist is derived from the committed generated + // serving types. + const typesFile = path.join( process.cwd(), - "node_modules", - ".databricks", - "appkit", - ".appkit-serving-types-cache.json", + "shared", + "appkit-types", + "serving.d.ts", ); - this.schemaAllowlists = await loadEndpointSchemas(cacheFile); + this.schemaAllowlists = await loadEndpointSchemas(typesFile); if (this.schemaAllowlists.size > 0) { logger.debug( "Loaded schema allowlists for %d endpoint(s)", diff --git a/packages/appkit/src/plugins/serving/tests/schema-filter.test.ts b/packages/appkit/src/plugins/serving/tests/schema-filter.test.ts index 4fc030d83..436f464b7 100644 --- a/packages/appkit/src/plugins/serving/tests/schema-filter.test.ts +++ b/packages/appkit/src/plugins/serving/tests/schema-filter.test.ts @@ -109,50 +109,53 @@ describe("schema-filter", () => { expect(result.size).toBe(0); }); - test("reads requestKeys from cache entries", async () => { + test("derives the allowlist from the request object's top-level keys", async () => { const fs = (await import("node:fs/promises")).default; - vi.mocked(fs.readFile).mockResolvedValue( - JSON.stringify({ - version: "1", - endpoints: { - default: { - hash: "abc", - requestType: "{}", - responseType: "{}", - chunkType: null, - requestKeys: ["messages", "temperature", "max_tokens"], - }, - }, - }), - ); - - const result = await loadEndpointSchemas("/some/path"); + // A committed serving.d.ts: the `request` type's top-level keys are the + // allowlist for the alias. + vi.mocked(fs.readFile).mockResolvedValue(`import "@databricks/appkit"; +declare module "@databricks/appkit" { + interface ServingEndpointRegistry { + default: { + request: { + messages: Array<{ role: string; content: string }>; + temperature: number; + max_tokens: number; + }; + response: { model: string }; + chunk: unknown; + }; + } +} +`); + + const result = await loadEndpointSchemas("/some/path/serving.d.ts"); expect(result.size).toBe(1); const keys = result.get("default"); expect(keys).toBeDefined(); expect(keys?.has("messages")).toBe(true); expect(keys?.has("temperature")).toBe(true); expect(keys?.has("max_tokens")).toBe(true); + // Nested keys (e.g. `role`, `content`) are NOT part of the top-level allowlist. + expect(keys?.has("role")).toBe(false); }); - test("skips entries without requestKeys (backwards compat)", async () => { + test("a generic Record request yields no allowlist (passthrough mode)", async () => { const fs = (await import("node:fs/promises")).default; - vi.mocked(fs.readFile).mockResolvedValue( - JSON.stringify({ - version: "1", - endpoints: { - default: { - hash: "abc", - requestType: "{ messages: string[] }", - responseType: "{}", - chunkType: null, - }, - }, - }), - ); - - const result = await loadEndpointSchemas("/some/path"); - // No requestKeys → passthrough mode (no allowlist) + vi.mocked(fs.readFile).mockResolvedValue(`import "@databricks/appkit"; +declare module "@databricks/appkit" { + interface ServingEndpointRegistry { + default: { + request: Record; + response: unknown; + chunk: unknown; + }; + } +} +`); + + const result = await loadEndpointSchemas("/some/path/serving.d.ts"); + // Record has no top-level keys → passthrough (no allowlist). expect(result.size).toBe(0); }); }); diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index ac320f8bc..4d73a676e 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -1,99 +1,51 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; -import path from "node:path"; import { createLogger } from "../logging/logger"; -import type { MetricSchema } from "./mv-registry/types"; +import { + deindentBlock, + parseCacheHeader, + splitEntryBlocks, +} from "./embedded-cache"; const logger = createLogger("type-generator:cache"); /** - * Cache types + * Cache entry for a single query. * @property hash - the hash of the SQL query - * @property type - the type of the query + * @property type - the rendered TypeScript type for the query * @property retry - when true the entry never satisfies a cache hit, so the * query is re-described on the next pass; fresh successful describes * persist `retry: false` */ -interface CacheEntry { +export interface CacheEntry { hash: string; type: string; retry: boolean; } /** - * One cached metric-view DESCRIBE outcome. + * In-memory query cache, reconstructed from a committed generated `analytics.d.ts`. * - * `hash` is md5 over `"|"` — the two config inputs that - * determine a DESCRIBE — so editing either invalidates the entry. `schema` - * is the full {@link MetricSchema} persisted verbatim (it is JSON-safe by - * design), letting a warm pass regenerate the metric artifact without a - * single warehouse call. + * The generated file carries one hash per entry in its header comment and the + * rendered type block in its body; {@link loadQueryCache} pairs the two back + * into this shape, which the query hit/degrade logic consumes. */ -export interface MetricCacheEntry { - hash: string; - schema: MetricSchema; - retry: boolean; -} - -/** - * Structural gate for reviving a cached metric entry at partition time. - * - * The cache file lives in `node_modules/.databricks` and is plain JSON — - * hand-edits, truncation, or a stale writer can leave entries whose shape no - * longer matches {@link MetricCacheEntry}. A malformed entry must read as a - * cache MISS (re-describe) rather than crash the pass or render revived - * garbage into the artifacts. - */ -export function isRevivableMetricCacheEntry(entry: unknown): boolean { - if (typeof entry !== "object" || entry === null) return false; - const e = entry as Record; - if (typeof e.hash !== "string" || typeof e.retry !== "boolean") { - return false; - } - const schema = e.schema; - if (typeof schema !== "object" || schema === null || Array.isArray(schema)) { - return false; - } - const s = schema as Record; - const isColumnArray = (value: unknown): boolean => - Array.isArray(value) && - value.every( - (col) => - typeof col === "object" && - col !== null && - typeof (col as Record).name === "string" && - typeof (col as Record).type === "string", - ); - return ( - typeof s.key === "string" && - typeof s.source === "string" && - (s.lane === "sp" || s.lane === "obo") && - (s.degraded === undefined || typeof s.degraded === "boolean") && - isColumnArray(s.measures) && - isColumnArray(s.dimensions) - ); -} - -/** - * Cache interface - * @property version - the version of the cache - * @property queries - the queries in the cache - * @property metrics - cached metric-view schemas keyed by metric key. - */ -interface Cache { +export interface Cache { version: string; queries: Record; - metrics?: Record; } export const CACHE_VERSION = "3"; -const CACHE_FILE = ".appkit-types-cache.json"; -const CACHE_DIR = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", -); + +/** Registry interface name whose members hold the query type blocks. */ +const QUERY_INTERFACE = "QueryRegistry"; + +/** + * Indentation (in spaces) that {@link generateTypeDeclarations} applies to + * every non-first line of a query type block at assembly time. Reversed here so + * a block extracted from a committed file re-renders byte-identically. + */ +const QUERY_BLOCK_INDENT = 4; /** * Hash the SQL query @@ -106,42 +58,108 @@ export function hashSQL(sql: string): string { } /** - * Change detector stored on {@link MetricCacheEntry.hash}: md5 over - * `"|"` — the two config inputs that determine a DESCRIBE — - * so editing either invalidates the entry. + * Change detector for a metric-view entry: md5 over `"|"` — the + * two config inputs that determine a DESCRIBE — so editing either invalidates + * the entry. */ export function metricCacheHash(source: string, lane: string): string { return hashSQL(`${source}|${lane}`); } /** - * Load the cache from the file system - * If the cache is not found, run the query explain - * @returns - the cache + * Reconstruct the query cache from a committed `analytics.d.ts`. + * + * Reads the header hash table and pairs each entry with its rendered type block + * from the file body. A missing file, missing header, or a version mismatch all + * yield an empty cache (every query reads as a MISS and is re-described). */ -export async function loadCache(): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); +export async function loadQueryCache(outFile: string): Promise { + let source: string; try { - await fs.mkdir(CACHE_DIR, { recursive: true }); - - const raw = await fs.readFile(cachePath, "utf8"); - const cache = JSON.parse(raw) as Cache; - if (cache.version === CACHE_VERSION) { - return cache; - } + source = await fs.readFile(outFile, "utf8"); } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - logger.warn("Cache file is corrupted, flushing cache completely."); + logger.warn( + "Could not read generated types at %s, treating as empty cache.", + outFile, + ); } + return { version: CACHE_VERSION, queries: {} }; } - return { version: CACHE_VERSION, queries: {} }; + + const header = parseCacheHeader(source); + if (header.version !== CACHE_VERSION) { + return { version: CACHE_VERSION, queries: {} }; + } + + const blocks = splitEntryBlocks(source, QUERY_INTERFACE); + const queries: Record = Object.create(null); + for (const [name, hash] of Object.entries(header.hashes)) { + const block = blocks[name]; + if (block === undefined) continue; // header/body drift → miss + queries[name] = { + hash, + type: deindentBlock(block, QUERY_BLOCK_INDENT), + retry: false, + }; + } + + return { version: CACHE_VERSION, queries }; } +/** Registry interface name whose members hold the metric type blocks. */ +const METRIC_INTERFACE = "MetricRegistry"; + /** - * Save the cache to the file system - * @param cache - cache object to save + * One reconstructed metric cache entry. + * @property hash - md5 of `"|"` at generation time + * @property member - the full rendered member string (` "key": { ... }`) as + * produced by the metric renderer, reused verbatim on a cache hit */ -export async function saveCache(cache: Cache): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); - await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); +export interface MetricCacheEntry { + hash: string; + member: string; +} + +interface MetricCache { + version: string; + entries: Record; +} + +/** + * Reconstruct the metric-view cache from a committed `metric-views.d.ts`. + * + * The value block extracted from the file body is exactly what the metric + * renderer emits for a metric's value, so the full member is reassembled as + * ` "": ` — reusable verbatim without reconstructing the + * underlying `MetricSchema`. Missing file / header / version mismatch → empty. + */ +export async function loadMetricCache(mvOutFile: string): Promise { + let source: string; + try { + source = await fs.readFile(mvOutFile, "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") { + logger.warn( + "Could not read generated metric types at %s, treating as empty cache.", + mvOutFile, + ); + } + return { version: CACHE_VERSION, entries: {} }; + } + + const header = parseCacheHeader(source); + if (header.version !== CACHE_VERSION) { + return { version: CACHE_VERSION, entries: {} }; + } + + const blocks = splitEntryBlocks(source, METRIC_INTERFACE); + const entries: Record = Object.create(null); + for (const [key, hash] of Object.entries(header.hashes)) { + const block = blocks[key]; + if (block === undefined) continue; // header/body drift → miss + entries[key] = { hash, member: ` ${JSON.stringify(key)}: ${block}` }; + } + + return { version: CACHE_VERSION, entries }; } diff --git a/packages/appkit/src/type-generator/embedded-cache.ts b/packages/appkit/src/type-generator/embedded-cache.ts new file mode 100644 index 000000000..3b4aad99a --- /dev/null +++ b/packages/appkit/src/type-generator/embedded-cache.ts @@ -0,0 +1,557 @@ +import fsSync from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * The typegen cache, embedded in the generated `.d.ts` files themselves. + * + * A generated file has two parts this module reads and writes: + * + * 1. A **header comment** listing one `name → hash` per generated entry. + * The hash is a change detector: it matches iff the entry's source + * (SQL text / `source|lane` / endpoint identity) is unchanged. + * 2. The **body**, a `declare module { interface { ... } }` whose + * members are the rendered type blocks, one per entry. + * + * Together they act as a cache: on a later run, an entry whose current hash + * matches the header is a HIT — its rendered block is reused verbatim from the + * body, with no workspace call. Because the file is committed to the app repo, + * an unchanged run touches the workspace zero times. Degraded / `unknown` + * entries are omitted from the header, so they always read as a MISS and get + * re-described on the next successful pass. + * + * Entry points (each subsystem's generator/loader calls these): + * - {@link renderCacheHeader} build the header block for a fresh file + * - {@link resolveHeaderTimestamp} keep the timestamp stable on unchanged runs + * - {@link parseCacheHeader} read `name → hash` back from a committed file + * - {@link splitEntryBlocks} read `name → rendered block` from the body + * - {@link objectMemberValue} / {@link objectTopLevelKeys} + * inspect a rendered block (serving allowlist) + * + * The rest of the module is a small string/comment-aware TypeScript scanner + * used by those functions so braces inside strings or comments never corrupt + * member boundaries. + */ + +// Marker lines that bound the machine-readable region of the header. The +// leading rule embeds the cache version + tool version + timestamp; the +// trailing rule closes the entry table. Parsing keys off these exact prefixes. +const OPEN_RULE_PREFIX = "// ── typegen cache"; +const CLOSE_RULE = `// ${"─".repeat(60)}`; + +export interface CacheHeaderEntry { + /** Logical entry name (query key, metric key, serving alias). */ + name: string; + /** Local change-detector hash (hex). */ + hash: string; + /** + * Optional decorative middle columns shown between name and hash (e.g. + * `source · lane` for metric views). Ignored by the parser, which only + * reads the first (name) and last (hash) tokens of each entry line. + */ + detail?: string; +} + +interface RenderCacheHeaderOptions { + /** Cache-format version for this subsystem (queries/metrics "3", serving "1"). */ + version: string; + /** One-line human explanation of what a matching hash skips. */ + explainer: string; + /** Successfully-generated entries (name + hash [+ decorative detail]). */ + entries: CacheHeaderEntry[]; + /** ISO timestamp; defaults to now. Only changes when the file is rewritten. */ + timestamp?: string; + /** Tool version string; defaults to the resolved `@databricks/appkit` version. */ + toolVersion?: string; +} + +/** + * Render the cache header comment block that precedes the imports in every + * generated `.d.ts`. Ends with a trailing newline so it can be prepended + * directly to the file body. + */ +export function renderCacheHeader(opts: RenderCacheHeaderOptions): string { + const timestamp = opts.timestamp ?? new Date().toISOString(); + const toolVersion = opts.toolVersion ?? getAppkitVersion(); + + const lines: string[] = [ + "// Auto-generated by AppKit — DO NOT EDIT", + "// Regenerate with: npx @databricks/appkit generate-types", + "//", + `${OPEN_RULE_PREFIX} · v${opts.version} · appkit@${toolVersion} · ${timestamp} ──`, + `// ${opts.explainer}`, + ]; + + if (opts.entries.length > 0) { + // Align the hash column: pad the visible "name + detail" prefix to a + // common width so the table scans cleanly in diffs. + const prefixes = opts.entries.map((e) => + e.detail ? `${e.name} ${e.detail}` : e.name, + ); + const width = Math.max(...prefixes.map((p) => p.length)); + for (let i = 0; i < opts.entries.length; i++) { + lines.push( + `// ${prefixes[i].padEnd(width)} ${opts.entries[i].hash}`, + ); + } + } else { + lines.push("// (no cached entries)"); + } + + lines.push(CLOSE_RULE); + return `${lines.join("\n")}\n`; +} + +interface ParsedCacheHeader { + /** Version parsed from the open rule, or null if no header was found. */ + version: string | null; + /** Generation timestamp parsed from the open rule, or null if absent. */ + timestamp: string | null; + /** Map of entry name → hash for every parsed entry line. */ + hashes: Record; +} + +/** + * Decide the timestamp to stamp on a freshly-rendered file, preserving the + * committed file's timestamp when nothing changed so an all-hit regeneration is + * byte-identical (no git churn). Returns the prior timestamp iff the file's + * existing header carries the exact same set of entry→hash pairs; otherwise a + * fresh `now` ISO string (a real change → new stamp). + */ +export function resolveHeaderTimestamp( + priorSource: string, + nextEntries: CacheHeaderEntry[], +): string { + const prior = parseCacheHeader(priorSource); + const now = new Date().toISOString(); + if (prior.timestamp === null) return now; + const priorKeys = Object.keys(prior.hashes); + if (priorKeys.length !== nextEntries.length) return now; + for (const e of nextEntries) { + if (prior.hashes[e.name] !== e.hash) return now; + } + return prior.timestamp; +} + +const HEX_RE = /^[0-9a-f]+$/i; + +/** + * Parse the cache header out of a generated `.d.ts` file's contents. Returns an + * empty result when no (or a malformed) header is present, so a missing header + * safely reads as "every entry is a MISS". + * + * Robustness: an entry line is `// [] `. Only the + * first token (name) and the last token (hash, validated as hex) are read; any + * middle columns are decorative and ignored. This keeps parsing stable across + * the differing per-subsystem line layouts. + */ +export function parseCacheHeader(source: string): ParsedCacheHeader { + const lines = source.split("\n"); + let version: string | null = null; + let timestamp: string | null = null; + // Null-prototype map so an entry named "__proto__" is an own key, not a + // prototype write (no prototype pollution from a committed file). + const hashes: Record = Object.create(null); + + let inTable = false; + for (const raw of lines) { + const line = raw.trimEnd(); + if (!inTable) { + if (line.startsWith(OPEN_RULE_PREFIX)) { + const m = line.match(/·\s*v([^\s·]+)/); + version = m ? m[1] : null; + // Timestamp is the last "· ··" segment before the closing "──". + const ts = line.match(/·\s*([0-9T:.-]+Z)\s*──\s*$/); + timestamp = ts ? ts[1] : null; + inTable = true; + } + continue; + } + // Inside the table: the close rule ends it. + if (line === CLOSE_RULE) break; + // The explainer and "(no cached entries)" lines have no trailing hex hash. + const body = line.replace(/^\/\/\s*/, ""); + const tokens = body.split(/\s+/).filter(Boolean); + if (tokens.length < 2) continue; + const last = tokens[tokens.length - 1]; + if (!HEX_RE.test(last)) continue; + const name = tokens[0]; + hashes[name] = last; + } + + return { version, timestamp, hashes }; +} + +/** + * Extract the balanced body (between the outermost `{` and its matching `}`) of + * `interface { ... }` from the source, or null if not found. + */ +function extractInterfaceBody( + source: string, + interfaceName: string, +): string | null { + const marker = `interface ${interfaceName}`; + const at = source.indexOf(marker); + if (at === -1) return null; + const open = source.indexOf("{", at + marker.length); + if (open === -1) return null; + const close = captureBalanced(source, open); + if (close === -1) return null; + return source.slice(open + 1, close); +} + +/** + * Split the members of a generated registry interface into a map of + * `entryName → renderedValueBlock`, where the value block is the balanced + * `{ ... }` object literal that follows `name:`. A cache hit reuses the entry's + * block verbatim, so the regenerated type is identical without re-describing. + * + * String literals, `//` line comments, and `/* *​/` block comments are skipped + * so braces inside them never corrupt the member boundaries. Quoted member + * names are unquoted so the key matches the logical entry name used elsewhere. + */ +export function splitEntryBlocks( + source: string, + interfaceName: string, +): Record { + const body = extractInterfaceBody(source, interfaceName); + if (body === null) return {}; + + // Null-prototype map so a member literally named "__proto__" is stored as an + // own key rather than hitting the prototype setter (no prototype pollution). + const blocks: Record = Object.create(null); + let i = 0; + const n = body.length; + while (i < n) { + i = skipTrivia(body, i); + if (i >= n) break; + + // Read the member key: a quoted string or a bare identifier. + const key = readKey(body, i); + if (!key) break; + i = key.next; + + i = skipTrivia(body, i); + if (body[i] !== ":") break; + i = skipTrivia(body, i + 1); + + if (body[i] !== "{") break; + const close = captureBalanced(body, i); + if (close === -1) break; + blocks[key.name] = body.slice(i, close + 1); + i = close + 1; + + // Skip an optional member separator. + i = skipTrivia(body, i); + if (body[i] === ";" || body[i] === ",") i += 1; + } + + return blocks; +} + +/** + * Reverse the fixed indentation that a renderer applies to every non-first line + * of a value block (queries indent members by 4 spaces at assembly time). Given + * a final-form block extracted from a committed file, strip up to `n` leading + * spaces from lines 2..N to recover the renderer's pre-indent input, so the + * reused block re-renders byte-identically. The first line is left untouched. + */ +export function deindentBlock(block: string, n: number): string { + const strip = new RegExp(`^ {1,${n}}`); + return block + .split("\n") + .map((line, i) => (i === 0 ? line : line.replace(strip, ""))) + .join("\n"); +} + +/** + * Given a `{ key: value; ... }` object-type block (including the outer braces), + * return its top-level member key names in order. Nested objects, string + * literals, and comments are skipped, so only the outermost members are + * returned. Quoted keys are unquoted. Returns [] if `block` isn't an object. + */ +export function objectTopLevelKeys(block: string): string[] { + const open = block.indexOf("{"); + if (open === -1) return []; + const close = captureBalanced(block, open); + if (close === -1) return []; + const body = block.slice(open + 1, close); + + const keys: string[] = []; + let i = 0; + const n = body.length; + while (i < n) { + i = skipTrivia(body, i); + if (i >= n) break; + const key = readKey(body, i); + if (!key) break; + i = skipTrivia(body, key.next); + if (body[i] !== ":") { + // Not a `key:` member (e.g. an index signature or union) — bail out to + // avoid mis-parsing; the caller treats an empty/partial list as "no + // allowlist" (passthrough), which is safe. + break; + } + keys.push(key.name); + i = skipTrivia(body, i + 1); + // Skip this member's value: a balanced object, or up to the next top-level + // `;` / `,`. + if (body[i] === "{") { + const vClose = captureBalanced(body, i); + if (vClose === -1) break; + i = vClose + 1; + } else { + i = skipToTopLevelSeparator(body, i); + } + i = skipTrivia(body, i); + if (body[i] === ";" || body[i] === ",") i += 1; + } + return keys; +} + +/** + * From a `{ member: ; ... }` object-type block, return the raw text of + * `` for the named top-level member, or undefined if absent. The value + * runs from just after `member:` to (but not including) its top-level `;`/`,`. + */ +export function objectMemberValue( + block: string, + member: string, +): string | undefined { + const open = block.indexOf("{"); + if (open === -1) return undefined; + const close = captureBalanced(block, open); + if (close === -1) return undefined; + const body = block.slice(open + 1, close); + + let i = 0; + const n = body.length; + while (i < n) { + i = skipTrivia(body, i); + if (i >= n) break; + const key = readKey(body, i); + if (!key) break; + i = skipTrivia(body, key.next); + if (body[i] !== ":") break; + i = skipTrivia(body, i + 1); + const valueStart = i; + if (body[i] === "{") { + const vClose = captureBalanced(body, i); + if (vClose === -1) break; + i = vClose + 1; + } else { + i = skipToTopLevelSeparator(body, i); + } + if (key.name === member) { + return body.slice(valueStart, i).trim(); + } + i = skipTrivia(body, i); + if (body[i] === ";" || body[i] === ",") i += 1; + } + return undefined; +} + +/** + * Advance past a member value that is not a brace-object, stopping at the next + * top-level `;` or `,`. Skips nested `<...>`, `(...)`, `[...]`, `{...}`, strings + * and comments so separators inside them don't end the value early. + */ +function skipToTopLevelSeparator(source: string, i: number): number { + const n = source.length; + let angle = 0; + while (i < n) { + const c = source[i]; + if (c === '"' || c === "'" || c === "`") { + i = skipString(source, i); + continue; + } + if (c === "/" && source[i + 1] === "/") { + i = skipLineComment(source, i); + continue; + } + if (c === "/" && source[i + 1] === "*") { + i = skipBlockComment(source, i); + continue; + } + if (c === "{" || c === "(" || c === "[") { + i = captureBalancedGeneric(source, i); + continue; + } + if (c === "<") angle += 1; + else if (c === ">") { + if (angle > 0) angle -= 1; + } else if (angle === 0 && (c === ";" || c === ",")) { + return i; + } + i += 1; + } + return n; +} + +/** + * Like {@link captureBalanced} but for any of `{}`, `()`, `[]`; returns the + * index just past the matching close (or end of input). + */ +function captureBalancedGeneric(source: string, open: number): number { + const closeOf: Record = { "{": "}", "(": ")", "[": "]" }; + const want = closeOf[source[open]]; + let depth = 0; + let i = open; + const n = source.length; + while (i < n) { + const c = source[i]; + if (c === '"' || c === "'" || c === "`") { + i = skipString(source, i); + continue; + } + if (c === "/" && source[i + 1] === "/") { + i = skipLineComment(source, i); + continue; + } + if (c === "/" && source[i + 1] === "*") { + i = skipBlockComment(source, i); + continue; + } + if (c === source[open]) depth += 1; + else if (c === want) { + depth -= 1; + if (depth === 0) return i + 1; + } + i += 1; + } + return n; +} + +// ── low-level scanners ───────────────────────────────────────────────────── + +/** + * Given `source[open] === "{"`, return the index of the matching `}` (or -1). + * Skips string literals (", ', `) and `//` / block comments so their contents + * cannot throw off the brace depth. + */ +function captureBalanced(source: string, open: number): number { + let depth = 0; + let i = open; + const n = source.length; + while (i < n) { + const c = source[i]; + if (c === '"' || c === "'" || c === "`") { + i = skipString(source, i); + continue; + } + if (c === "/" && source[i + 1] === "/") { + i = skipLineComment(source, i); + continue; + } + if (c === "/" && source[i + 1] === "*") { + i = skipBlockComment(source, i); + continue; + } + if (c === "{") depth += 1; + else if (c === "}") { + depth -= 1; + if (depth === 0) return i; + } + i += 1; + } + return -1; +} + +/** Skip whitespace and both comment styles, returning the next content index. */ +function skipTrivia(source: string, i: number): number { + const n = source.length; + while (i < n) { + const c = source[i]; + if (c === " " || c === "\t" || c === "\n" || c === "\r") { + i += 1; + } else if (c === "/" && source[i + 1] === "/") { + i = skipLineComment(source, i); + } else if (c === "/" && source[i + 1] === "*") { + i = skipBlockComment(source, i); + } else { + break; + } + } + return i; +} + +/** `source[i]` is a quote char; return the index just past the closing quote. */ +function skipString(source: string, i: number): number { + const quote = source[i]; + i += 1; + const n = source.length; + while (i < n) { + const c = source[i]; + if (c === "\\") { + i += 2; + continue; + } + if (c === quote) return i + 1; + i += 1; + } + return n; +} + +/** `source[i]` starts a `//` comment; return the index of the newline/EOF. */ +function skipLineComment(source: string, i: number): number { + const nl = source.indexOf("\n", i); + return nl === -1 ? source.length : nl; +} + +/** `source[i]` starts a block comment; return the index just past its close. */ +function skipBlockComment(source: string, i: number): number { + const end = source.indexOf("*/", i + 2); + return end === -1 ? source.length : end + 2; +} + +/** Read a member key (quoted string or bare identifier) at `source[i]`. */ +function readKey( + source: string, + i: number, +): { name: string; next: number } | null { + const c = source[i]; + if (c === '"' || c === "'") { + const end = skipString(source, i); + // Strip the surrounding quotes to recover the logical name. + return { name: source.slice(i + 1, end - 1), next: end }; + } + const m = /^[A-Za-z0-9_$]+/.exec(source.slice(i)); + if (!m) return null; + return { name: m[0], next: i + m[0].length }; +} + +// ── package version resolution ─────────────────────────────────────────── + +let cachedVersion: string | undefined; + +/** + * Resolve the installed `@databricks/appkit` version for the header stamp. + * Walks up from this module looking for the package's own package.json; + * returns "unknown" if it can't be found (never throws). + */ +function getAppkitVersion(): string { + if (cachedVersion !== undefined) return cachedVersion; + cachedVersion = "unknown"; + try { + let dir = path.dirname(fileURLToPath(import.meta.url)); + for (let depth = 0; depth < 8; depth++) { + const pkgPath = path.join(dir, "package.json"); + if (fsSync.existsSync(pkgPath)) { + const pkg = JSON.parse(fsSync.readFileSync(pkgPath, "utf8")) as { + name?: string; + version?: string; + }; + if (pkg.name === "@databricks/appkit" && pkg.version) { + cachedVersion = pkg.version; + break; + } + } + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + } catch { + // Fall back to "unknown" — the version stamp is informational only. + } + return cachedVersion; +} diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index 2ce7f611d..031db79f9 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -5,12 +5,12 @@ import dotenv from "dotenv"; import pc from "picocolors"; import { createLogger } from "../logging/logger"; import { - isRevivableMetricCacheEntry, - loadCache, + CACHE_VERSION, + loadMetricCache, type MetricCacheEntry, metricCacheHash, - saveCache, } from "./cache"; +import { renderCacheHeader, resolveHeaderTimestamp } from "./embedded-cache"; import { getErrorDiagnostic, isConnectivityError } from "./errors"; import { migrateProjectConfig, @@ -19,7 +19,11 @@ import { } from "./migration"; import { readMetricConfig, resolveMetricConfig } from "./mv-registry/config"; import { createWorkspaceDescribeFetcher } from "./mv-registry/describe"; -import { generateMetricTypeDeclarations } from "./mv-registry/render-types"; +import { + generateMetricTypeDeclarationsFromMembers, + type MetricMember, + renderMetricEntry, +} from "./mv-registry/render-types"; import { emptyMetricSchema, syncMetrics } from "./mv-registry/sync"; import type { DescribeFetcher, @@ -203,7 +207,10 @@ export class TypegenFatalError extends Error { * @param querySchemas - the list of query schemas * @returns - the type declarations as a string */ -function generateTypeDeclarations(querySchemas: QuerySchema[] = []): string { +function generateTypeDeclarations( + querySchemas: QuerySchema[] = [], + priorSource = "", +): string { const queryEntries = querySchemas .map(({ name, type }) => { const indentedType = type @@ -216,9 +223,22 @@ function generateTypeDeclarations(querySchemas: QuerySchema[] = []): string { const querySection = queryEntries ? `\n${queryEntries};\n ` : ""; - return `// Auto-generated by AppKit - DO NOT EDIT -// Generated by 'npx @databricks/appkit generate-types' or Vite plugin during build -import "@databricks/appkit-ui/react"; + // The cache header records one hash per query that was successfully generated + // (a cache hit or a fresh successful describe). Degraded / `unknown` schemas + // carry no hash and are omitted, so they re-read as a MISS next pass. + const headerEntries = querySchemas + .filter((s): s is QuerySchema & { hash: string } => Boolean(s.hash)) + .map((s) => ({ name: s.name, hash: s.hash })); + const header = renderCacheHeader({ + version: CACHE_VERSION, + explainer: + "A matching source hash skips the warehouse DESCRIBE on build & deploy.", + entries: headerEntries, + // Preserve the prior timestamp when nothing changed → no git churn. + timestamp: resolveHeaderTimestamp(priorSource, headerEntries), + }); + + return `${header}import "@databricks/appkit-ui/react"; import type { SQLTypeMarker, SQLStringMarker, SQLNumberMarker, SQLBooleanMarker, SQLBinaryMarker, SQLDateMarker, SQLTimestampMarker } from "@databricks/appkit-ui/js"; declare module "@databricks/appkit-ui/react" { @@ -323,13 +343,17 @@ export async function generateFromEntryPoint(options: { const result = await generateQueriesFromDescribe(queryFolder, warehouseId, { noCache, mode, + cacheFile: outFile, }); queryRegistry = result.schemas; syntaxErrors = result.syntaxErrors ?? []; fatalErrors = result.fatalErrors ?? []; } - const typeDeclarations = generateTypeDeclarations(queryRegistry); + // Read the prior committed file (if any) so an unchanged run preserves its + // generation timestamp → the .d.ts stays byte-identical (no git churn). + const priorOut = await fs.readFile(outFile, "utf-8").catch(() => ""); + const typeDeclarations = generateTypeDeclarations(queryRegistry, priorOut); await fs.mkdir(path.dirname(outFile), { recursive: true }); await fs.writeFile(outFile, typeDeclarations, "utf-8"); @@ -467,35 +491,33 @@ export async function syncMetricViewsTypes(options: { const fatalErrors: Array<{ name: string; message: string }> = []; - // Load the shared typegen cache and copy its `metrics` section into a null-prototype map. - const cache = await loadCache(); - const mvCacheSection: Record = Object.create(null); - if (!noCache && cache.metrics) { - for (const key of Object.keys(cache.metrics)) { - mvCacheSection[key] = cache.metrics[key]; - } - } - - // Partition BEFORE any gate/preflight decision: a hit (a structurally valid, - // hash-matching, NON-degraded cached entry) is served from cache no matter - // what the warehouse is doing. The cache only ever holds successful describes - // (a degraded outcome is never persisted — see the write block below), so the - // `degraded !== true` guard is normally moot; it also defends against a stale - // degraded entry left by an older writer, which re-describes instead of - // serving. Everything else (new, edited, unrevivable, or degraded) is eligible - // for DESCRIBE, so a fully-warm pass makes zero warehouse calls and constructs - // zero clients. Mirrors the query path: only a good result is cache-servable. - const hitSchemas = new Map(); + // Reconstruct the metric cache from the committed metric-views.d.ts (its + // header hashes + body member blocks). Missing/old file → empty (all MISS). + const metricCache = noCache + ? { + version: CACHE_VERSION, + entries: {} as Record, + } + : await loadMetricCache(metricOutFile); + + // Partition BEFORE any gate/preflight decision: a hit (a hash-matching cached + // entry) is served from cache no matter what the warehouse is doing. Only + // non-degraded members are ever recorded in the header (see the renderer), so + // a committed hit is by construction a good result. Everything else (new, + // edited, or absent) is eligible for DESCRIBE, so a fully-warm pass makes zero + // warehouse calls and constructs zero clients. Mirrors the query path. + // + // `hitMembers` holds the previously-rendered member string, reused verbatim so + // the output round-trips byte-identically without reconstructing a MetricSchema. + const hitMembers = new Map(); const describeNeeded: typeof resolution.entries = []; for (const entry of resolution.entries) { - const prior = mvCacheSection[entry.key]; + const prior = metricCache.entries[entry.key]; if ( prior !== undefined && - isRevivableMetricCacheEntry(prior) && - prior.hash === metricCacheHash(entry.source, entry.lane) && - prior.schema.degraded !== true + prior.hash === metricCacheHash(entry.source, entry.lane) ) { - hitSchemas.set(entry.key, prior.schema); + hitMembers.set(entry.key, prior.member); } else { describeNeeded.push(entry); } @@ -633,49 +655,61 @@ export async function syncMetricViewsTypes(options: { ); } - // Cache only successful schema results for describe-needed keys; remove stale cache for degraded ones. - for (let i = 0; i < describeNeeded.length; i++) { - // syncMetrics return one schema per entry in entry order, so described[i] always belongs to describeNeeded[i]. - const entry = describeNeeded[i]; - if (described[i].degraded === true) { - delete mvCacheSection[entry.key]; - continue; - } - mvCacheSection[entry.key] = { - hash: metricCacheHash(entry.source, entry.lane), - schema: described[i], - // Vestigial, mirrors the query path's only cache write (always false): a - // persisted entry is by construction a good result, so it never needs a - // re-describe flag. Kept for on-disk shape compatibility with existing - // version-3 caches (isRevivableMetricCacheEntry gates on a boolean). - retry: false, - }; - } - - // Prune entries whose key is no longer configured - const configuredKeys = new Set(resolution.entries.map((e) => e.key)); - let prunedCount = 0; - for (const key of Object.keys(mvCacheSection)) { - if (!configuredKeys.has(key)) { - delete mvCacheSection[key]; - prunedCount++; - } - } - - // Save when this pass produced outcomes, bypassed the cache, or pruned. - if (describeNeeded.length > 0 || noCache || prunedCount > 0) { - cache.metrics = mvCacheSection; - await saveCache(cache); - } - - // Merge cached hits with fresh results back into config order. + // Freshly-described schemas by key (one per describe-needed entry, in order). const describedByKey = new Map(); for (const schema of described) { describedByKey.set(schema.key, schema); } + + // Assemble the file members in config order. A cache hit reuses its committed + // member string verbatim (round-trips byte-identically without reconstructing + // a MetricSchema); a describe-needed key renders freshly. Only non-degraded + // members record a header hash, so a degraded key re-reads as a MISS next + // pass. Stale keys drop out implicitly — only configured entries are emitted, + // so the committed cache is rewritten to exactly the current config. + const members: MetricMember[] = resolution.entries.map((entry) => { + const hit = hitMembers.get(entry.key); + if (hit !== undefined) { + return { + key: entry.key, + member: hit, + headerEntry: { + name: entry.key, + detail: `${entry.source} · ${entry.lane}`, + hash: metricCacheHash(entry.source, entry.lane), + }, + }; + } + const schema = describedByKey.get(entry.key) ?? emptyMetricSchema(entry); + return { + key: entry.key, + member: renderMetricEntry(schema), + headerEntry: + schema.degraded === true + ? undefined + : { + name: entry.key, + detail: `${entry.source} · ${entry.lane}`, + hash: metricCacheHash(entry.source, entry.lane), + }, + }; + }); + + // Result schemas (metadata for the caller). Cache hits don't reconstruct a + // full MetricSchema — they surface a lightweight non-degraded placeholder + // carrying the key/source/lane, which is all any caller reads. const schemas = resolution.entries.map((entry) => { - const schema = hitSchemas.get(entry.key) ?? describedByKey.get(entry.key); - if (schema !== undefined) return schema; + const described = describedByKey.get(entry.key); + if (described !== undefined) return described; + if (hitMembers.has(entry.key)) { + return { + key: entry.key, + source: entry.source, + lane: entry.lane, + measures: [], + dimensions: [], + } satisfies MetricSchema; + } logger.warn( "no schema resolved for metric key %s — emitting degraded types (should not happen)", entry.key, @@ -683,10 +717,17 @@ export async function syncMetricViewsTypes(options: { return emptyMetricSchema(entry); }); + // Preserve the prior file's timestamp when the entry set is unchanged → the + // .d.ts stays byte-identical on an all-hit pass (no git churn). + const priorMetricOut = await fs + .readFile(metricOutFile, "utf-8") + .catch(() => ""); await fs.mkdir(path.dirname(metricOutFile), { recursive: true }); await fs.writeFile( metricOutFile, - generateMetricTypeDeclarations(schemas), + generateMetricTypeDeclarationsFromMembers(members, { + priorSource: priorMetricOut, + }), "utf-8", ); diff --git a/packages/appkit/src/type-generator/mv-registry/render-types.ts b/packages/appkit/src/type-generator/mv-registry/render-types.ts index f70e584c8..821db2b35 100644 --- a/packages/appkit/src/type-generator/mv-registry/render-types.ts +++ b/packages/appkit/src/type-generator/mv-registry/render-types.ts @@ -1,3 +1,5 @@ +import { CACHE_VERSION, metricCacheHash } from "../cache"; +import { renderCacheHeader, resolveHeaderTimestamp } from "../embedded-cache"; import type { MetricColumnMetadata, MetricSchema } from "./types"; /** @@ -32,7 +34,7 @@ function tsTypeFor(sqlType: string): string { } // Render a MetricRegistry interface entry from a MetricSchema. -function renderMetricEntry(schema: MetricSchema): string { +export function renderMetricEntry(schema: MetricSchema): string { if (schema.degraded) { return renderDegradedMetricEntry(schema); } @@ -155,29 +157,92 @@ ${inner}; }`; } -// Render the augmentation block for the appkit-ui MetricRegistry interface. -function renderMetricRegistry(schemas: MetricSchema[]): string { - if (schemas.length === 0) { +// Render the augmentation block for the appkit-ui MetricRegistry interface from +// pre-rendered members (one ` "": { ... }` string each). Members come +// either from a fresh `renderMetricEntry` or, on a cache hit, verbatim from the +// previously-committed file — the two are interchangeable strings here. +function renderMetricRegistryFromMembers(members: string[]): string { + if (members.length === 0) { return `declare module "@databricks/appkit-ui/react" { interface MetricRegistry {} } `; } - const entries = schemas.map(renderMetricEntry).join(";\n"); return `declare module "@databricks/appkit-ui/react" { interface MetricRegistry { -${entries}; +${members.join(";\n")}; } } `; } -// Build the full metric-views.d.ts file from a list of metric schemas. +/** + * A metric member ready to assemble into the registry, paired with the header + * metadata needed to record it in the cache header. `member` is the rendered + * ` "": { ... }` string; `headerEntry` is present only for non-degraded + * members (degraded members are omitted from the header so they re-read as a + * MISS). + */ +export interface MetricMember { + key: string; + member: string; + headerEntry?: { name: string; detail: string; hash: string }; +} + +/** + * Build the full metric-views.d.ts file from pre-rendered members (the + * cache-hit-aware assembly path used by `syncMetricViewsTypes`). + * + * `timestamp` is injectable so tests can produce a deterministic snapshot; in + * production it defaults to now inside renderCacheHeader. + */ +export function generateMetricTypeDeclarationsFromMembers( + members: MetricMember[], + opts?: { timestamp?: string; priorSource?: string }, +): string { + const entries = members + .map((m) => m.headerEntry) + .filter((e): e is NonNullable => e !== undefined); + // Explicit timestamp (tests) wins; otherwise preserve the prior file's + // timestamp when the entry set is unchanged → byte-identical, no git churn. + const timestamp = + opts?.timestamp ?? + (opts?.priorSource !== undefined + ? resolveHeaderTimestamp(opts.priorSource, entries) + : undefined); + const header = renderCacheHeader({ + version: CACHE_VERSION, + explainer: + "A matching source|lane hash skips the warehouse DESCRIBE on build & deploy.", + timestamp, + entries, + }); + return `${header}import "@databricks/appkit-ui/react"; +${renderMetricRegistryFromMembers(members.map((m) => m.member))}`; +} + +/** + * Build the full metric-views.d.ts file directly from a list of metric schemas. + * Convenience wrapper (used by tests and any always-fresh caller) that renders + * every schema and delegates to {@link generateMetricTypeDeclarationsFromMembers}. + */ export function generateMetricTypeDeclarations( schemas: MetricSchema[], + opts?: { timestamp?: string }, ): string { - return `// Auto-generated by AppKit - DO NOT EDIT -// Generated by 'npx @databricks/appkit generate-types' or Vite plugin during build -import "@databricks/appkit-ui/react"; -${renderMetricRegistry(schemas)}`; + return generateMetricTypeDeclarationsFromMembers( + schemas.map((s) => ({ + key: s.key, + member: renderMetricEntry(s), + headerEntry: + s.degraded === true + ? undefined + : { + name: s.key, + detail: `${s.source} · ${s.lane}`, + hash: metricCacheHash(s.source, s.lane), + }, + })), + opts, + ); } diff --git a/packages/appkit/src/type-generator/query-registry.ts b/packages/appkit/src/type-generator/query-registry.ts index fb239487a..806ad3308 100644 --- a/packages/appkit/src/type-generator/query-registry.ts +++ b/packages/appkit/src/type-generator/query-registry.ts @@ -4,7 +4,7 @@ import { WorkspaceClient } from "@databricks/sdk-experimental"; import { tableFromIPC } from "apache-arrow"; import pc from "picocolors"; import { createLogger } from "../logging/logger"; -import { CACHE_VERSION, hashSQL, loadCache, saveCache } from "./cache"; +import { CACHE_VERSION, type Cache, hashSQL, loadQueryCache } from "./cache"; import { getErrorDiagnostic, isConnectivityError } from "./errors"; import { decidePreflight, type PreflightMode } from "./preflight"; import { Spinner } from "./spinner"; @@ -268,7 +268,7 @@ function generateUnknownResultQuery(sql: string, queryName: string): string { * `unknown` from SQL alone. Never persists `result: unknown`. */ function degradedType( - cache: Awaited>, + cache: Cache, queryName: string, sql: string, sqlHash: string, @@ -539,12 +539,18 @@ export async function generateQueriesFromDescribe( noCache?: boolean; concurrency?: number; mode?: PreflightMode; + /** + * Path to the committed generated `analytics.d.ts`. Its header + body are + * the cache source. Absent → empty cache (every query is a MISS). + */ + cacheFile?: string; } = {}, ): Promise { const { noCache = false, concurrency: rawConcurrency = 10, mode = "non-blocking", + cacheFile, } = options; const concurrency = typeof rawConcurrency === "number" && Number.isFinite(rawConcurrency) @@ -554,11 +560,9 @@ export async function generateQueriesFromDescribe( // read all query files and cache in parallel const [allFiles, cache] = await Promise.all([ fs.readdir(queryFolder), - noCache - ? ({ version: CACHE_VERSION, queries: {} } as Awaited< - ReturnType - >) - : loadCache(), + noCache || !cacheFile + ? Promise.resolve({ version: CACHE_VERSION, queries: {} } as Cache) + : loadQueryCache(cacheFile), ]); const queryFiles = allFiles.filter((file) => file.endsWith(".sql")); @@ -605,7 +609,7 @@ export async function generateQueriesFromDescribe( if (cached && cached.hash === sqlHash && !cached.retry) { cachedResults.push({ index: i, - schema: { name: queryName, type: cached.type }, + schema: { name: queryName, type: cached.type, hash: sqlHash }, }); logEntries.push({ queryName, status: "HIT" }); } else { @@ -850,7 +854,7 @@ export async function generateQueriesFromDescribe( return { status: "ok", index, - schema: { name: queryName, type }, + schema: { name: queryName, type, hash: sqlHash }, cacheEntry: { hash: sqlHash, type, retry: false }, }; }; @@ -946,19 +950,20 @@ export async function generateQueriesFromDescribe( } }; + // The cache is persisted by the caller writing the generated `.d.ts` + // (its header records each entry's hash; its body holds the type block), + // so there is no separate cache file to save between batches here. if (uncachedQueries.length > concurrency) { for (let b = 0; b < uncachedQueries.length; b += concurrency) { const batch = uncachedQueries.slice(b, b + concurrency); const batchResults = await Promise.allSettled(batch.map(describeOne)); processBatchResults(batchResults, b); - await saveCache(cache); } } else { const settled = await Promise.allSettled( uncachedQueries.map(describeOne), ); processBatchResults(settled, 0); - await saveCache(cache); } spinner.stop(""); diff --git a/packages/appkit/src/type-generator/serving/cache.ts b/packages/appkit/src/type-generator/serving/cache.ts index dc9bf7e27..32f995521 100644 --- a/packages/appkit/src/type-generator/serving/cache.ts +++ b/packages/appkit/src/type-generator/serving/cache.ts @@ -1,25 +1,26 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; -import path from "node:path"; import { createLogger } from "../../logging/logger"; +import { parseCacheHeader, splitEntryBlocks } from "../embedded-cache"; const logger = createLogger("type-generator:serving:cache"); export const CACHE_VERSION = "1"; -const CACHE_FILE = ".appkit-serving-types-cache.json"; -const CACHE_DIR = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", -); + +/** Registry interface name whose members hold the serving type blocks. */ +const SERVING_INTERFACE = "ServingEndpointRegistry"; export interface ServingCacheEntry { + /** + * Local identity hash: sha256 of `"|"`. It is + * computable WITHOUT fetching the OpenAPI schema, so a matching committed + * entry lets a build/deploy skip the network fetch entirely. Upstream schema + * drift (same endpoint, changed model) is not detected — use `--no-cache` to + * force a refresh. + */ hash: string; - requestType: string; - responseType: string; - chunkType: string | null; - requestKeys: string[]; + /** Rendered registry member body (`{ request: ...; response: ...; chunk: ...; }`). */ + member: string; } export interface ServingCache { @@ -27,30 +28,54 @@ export interface ServingCache { endpoints: Record; } -export function hashSchema(schemaJson: string): string { - return crypto.createHash("sha256").update(schemaJson).digest("hex"); +/** + * Local identity hash for a serving endpoint. Computed from the alias and the + * resolved endpoint name (from the config's env var) — both known locally, so + * a cache hit needs no `getOpenApi` call. + */ +export function endpointIdentityHash( + alias: string, + endpointName: string, +): string { + return crypto + .createHash("sha256") + .update(`${alias}|${endpointName}`) + .digest("hex"); } -export async function loadServingCache(): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); +/** + * Reconstruct the serving cache from a committed `serving.d.ts`. + * + * Reads the header hash table and pairs each alias with its rendered member + * body from the file (both `declare module` blocks are identical, so the first + * found suffices). Missing file / header / version mismatch → empty cache. + */ +export async function loadServingCache(outFile: string): Promise { + let source: string; try { - await fs.mkdir(CACHE_DIR, { recursive: true }); - const raw = await fs.readFile(cachePath, "utf8"); - const cache = JSON.parse(raw) as ServingCache; - if (cache.version === CACHE_VERSION) { - return cache; - } - logger.debug("Cache version mismatch, starting fresh"); + source = await fs.readFile(outFile, "utf8"); } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - logger.warn("Cache file is corrupted, flushing cache completely."); + logger.warn( + "Could not read generated serving types at %s, treating as empty cache.", + outFile, + ); } + return { version: CACHE_VERSION, endpoints: {} }; + } + + const header = parseCacheHeader(source); + if (header.version !== CACHE_VERSION) { + return { version: CACHE_VERSION, endpoints: {} }; + } + + const blocks = splitEntryBlocks(source, SERVING_INTERFACE); + const endpoints: Record = {}; + for (const [alias, hash] of Object.entries(header.hashes)) { + const block = blocks[alias]; + if (block === undefined) continue; // header/body drift → miss + endpoints[alias] = { hash, member: block }; } - return { version: CACHE_VERSION, endpoints: {} }; -} -export async function saveServingCache(cache: ServingCache): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); - await fs.mkdir(CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); + return { version: CACHE_VERSION, endpoints }; } diff --git a/packages/appkit/src/type-generator/serving/generator.ts b/packages/appkit/src/type-generator/serving/generator.ts index b377c6597..4d0e692f0 100644 --- a/packages/appkit/src/type-generator/serving/generator.ts +++ b/packages/appkit/src/type-generator/serving/generator.ts @@ -4,6 +4,7 @@ import { WorkspaceClient } from "@databricks/sdk-experimental"; import pc from "picocolors"; import { createLogger } from "../../logging/logger"; import type { EndpointConfig } from "../../plugins/serving/types"; +import { renderCacheHeader, resolveHeaderTimestamp } from "../embedded-cache"; import { migrateProjectConfig, removeOldGeneratedTypes, @@ -11,16 +12,14 @@ import { } from "../migration"; import { CACHE_VERSION, - hashSchema, + endpointIdentityHash, loadServingCache, type ServingCache, - saveServingCache, } from "./cache"; import { convertRequestSchema, convertResponseSchema, deriveChunkType, - extractRequestKeys, } from "./converter"; import { fetchOpenApiSchema } from "./fetcher"; import { @@ -66,14 +65,25 @@ export async function generateServingTypes( const startTime = performance.now(); - const cache = noCache + // Read the committed serving.d.ts once: it's both the cache source and the + // prior file we diff against to preserve the timestamp on an unchanged run. + const priorSource = await fs.readFile(outFile, "utf-8").catch(() => ""); + + // Reconstruct the serving cache from the committed serving.d.ts (its header + // identity hashes + body member blocks). Missing/old file → empty. + const cache: ServingCache = noCache ? { version: CACHE_VERSION, endpoints: {} } - : await loadServingCache(); + : await loadServingCache(outFile); let client: WorkspaceClient | undefined; - let updated = false; + const getClient = (): WorkspaceClient => { + client ??= new WorkspaceClient({}); + return client; + }; const registryEntries: string[] = []; + const headerEntries: Array<{ name: string; hash: string; detail: string }> = + []; const logEntries: Array<{ alias: string; status: "HIT" | "MISS"; @@ -81,16 +91,19 @@ export async function generateServingTypes( }> = []; for (const [alias, config] of Object.entries(endpoints)) { - client ??= new WorkspaceClient({}); - const result = await processEndpoint(alias, config, client, cache); - if (result.cacheUpdated) updated = true; + const result = await processEndpoint(alias, config, cache, getClient); registryEntries.push(result.entry); + if (result.headerEntry) headerEntries.push(result.headerEntry); logEntries.push(result.log); } printLogTable(logEntries, startTime); - const output = generateTypeDeclarations(registryEntries); + const output = generateTypeDeclarations( + registryEntries, + headerEntries, + priorSource, + ); await fs.mkdir(path.dirname(outFile), { recursive: true }); await fs.writeFile(outFile, output, "utf-8"); @@ -106,16 +119,17 @@ export async function generateServingTypes( } else { logger.debug("Wrote serving types to %s", outFile); } - - if (updated) { - await saveServingCache(cache as ServingCache); - } } interface EndpointResult { entry: string; log: { alias: string; status: "HIT" | "MISS"; error?: string }; - cacheUpdated: boolean; + /** + * Present only for a fully-resolved endpoint (cache hit or a fresh successful + * conversion), recording its identity hash in the file header. Omitted for a + * permissive/generic fallback so it re-reads as a MISS next pass. + */ + headerEntry?: { name: string; hash: string; detail: string }; } function genericEntry(alias: string): string { @@ -130,20 +144,37 @@ function genericEntry(alias: string): string { async function processEndpoint( alias: string, config: EndpointConfig, - client: WorkspaceClient, - cache: { endpoints: Record }, + cache: ServingCache, + getClient: () => WorkspaceClient, ): Promise { const endpointName = process.env[config.env]; if (!endpointName) { + // No resolved endpoint name → nothing to key on. Emit a generic entry and + // leave it out of the header so it re-reads as a MISS once configured. return { entry: genericEntry(alias), log: { alias, status: "MISS", error: `env ${config.env} not set` }, - cacheUpdated: false, }; } + // Local identity hash — computed WITHOUT any network call, so a committed hit + // skips the fetch entirely. + const hash = endpointIdentityHash(alias, endpointName); + const detail = endpointName; + + // Cache hit: reuse the committed member verbatim. No getOpenApi call. + const cached = cache.endpoints[alias]; + if (cached && cached.hash === hash) { + return { + entry: ` ${alias}: ${cached.member};`, + log: { alias, status: "HIT" }, + headerEntry: { name: alias, hash, detail }, + }; + } + + // Cache miss: fetch the OpenAPI schema and convert it. const result = await fetchOpenApiSchema( - client, + getClient(), endpointName, config.servedModel, ); @@ -151,27 +182,10 @@ async function processEndpoint( return { entry: genericEntry(alias), log: { alias, status: "MISS", error: "schema fetch failed" }, - cacheUpdated: false, }; } const { spec, pathKey } = result; - const hash = hashSchema(JSON.stringify(spec)); - - // Cache hit - const cached = cache.endpoints[alias]; - if (cached && cached.hash === hash) { - return { - entry: buildRegistryEntry( - alias, - cached.requestType, - cached.responseType, - cached.chunkType, - ), - log: { alias, status: "HIT" }, - cacheUpdated: false, - }; - } // Cache miss — convert schema to types const operation = spec.paths[pathKey]?.post; @@ -179,7 +193,6 @@ async function processEndpoint( return { entry: genericEntry(alias), log: { alias, status: "MISS", error: "no POST operation" }, - cacheUpdated: false, }; } @@ -187,20 +200,11 @@ async function processEndpoint( const requestType = convertRequestSchema(operation); const responseType = convertResponseSchema(operation); const chunkType = deriveChunkType(operation); - const requestKeys = extractRequestKeys(operation); - - cache.endpoints[alias] = { - hash, - requestType, - responseType, - chunkType, - requestKeys, - }; return { entry: buildRegistryEntry(alias, requestType, responseType, chunkType), log: { alias, status: "MISS" }, - cacheUpdated: true, + headerEntry: { name: alias, hash, detail }, }; } catch (convErr) { logger.warn( @@ -211,7 +215,6 @@ async function processEndpoint( return { entry: genericEntry(alias), log: { alias, status: "MISS", error: "schema conversion failed" }, - cacheUpdated: false, }; } } @@ -294,10 +297,20 @@ function indentType(typeStr: string, baseIndent: string): string { .join("\n"); } -function generateTypeDeclarations(entries: string[]): string { - return `// Auto-generated by AppKit - DO NOT EDIT -// Generated from serving endpoint OpenAPI schemas -import "@databricks/appkit"; +function generateTypeDeclarations( + entries: string[], + headerEntries: Array<{ name: string; hash: string; detail: string }>, + priorSource = "", +): string { + const header = renderCacheHeader({ + version: CACHE_VERSION, + explainer: + "A matching endpoint-identity hash skips the serving fetch on build & deploy.", + entries: headerEntries, + // Preserve the prior timestamp when the entry set is unchanged → no churn. + timestamp: resolveHeaderTimestamp(priorSource, headerEntries), + }); + return `${header}import "@databricks/appkit"; import "@databricks/appkit-ui/react"; declare module "@databricks/appkit" { diff --git a/packages/appkit/src/type-generator/serving/tests/cache.test.ts b/packages/appkit/src/type-generator/serving/tests/cache.test.ts index 0c99c997f..4b7dae90c 100644 --- a/packages/appkit/src/type-generator/serving/tests/cache.test.ts +++ b/packages/appkit/src/type-generator/serving/tests/cache.test.ts @@ -2,10 +2,8 @@ import fs from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { CACHE_VERSION, - hashSchema, + endpointIdentityHash, loadServingCache, - type ServingCache, - saveServingCache, } from "../cache"; vi.mock("node:fs/promises"); @@ -19,18 +17,21 @@ describe("serving cache", () => { vi.restoreAllMocks(); }); - describe("hashSchema", () => { - test("returns consistent SHA256 hash", () => { - const hash1 = hashSchema('{"openapi": "3.1.0"}'); - const hash2 = hashSchema('{"openapi": "3.1.0"}'); + describe("endpointIdentityHash", () => { + test("returns a consistent SHA256 hash for the same alias + endpoint", () => { + const hash1 = endpointIdentityHash("chat", "my-endpoint"); + const hash2 = endpointIdentityHash("chat", "my-endpoint"); expect(hash1).toBe(hash2); expect(hash1).toHaveLength(64); // SHA256 hex }); - test("different inputs produce different hashes", () => { - const hash1 = hashSchema('{"a": 1}'); - const hash2 = hashSchema('{"a": 2}'); - expect(hash1).not.toBe(hash2); + test("different alias or endpoint produces different hashes", () => { + expect(endpointIdentityHash("chat", "a")).not.toBe( + endpointIdentityHash("chat", "b"), + ); + expect(endpointIdentityHash("a", "ep")).not.toBe( + endpointIdentityHash("b", "ep"), + ); }); }); @@ -40,70 +41,69 @@ describe("serving cache", () => { Object.assign(new Error("ENOENT"), { code: "ENOENT" }), ); - const cache = await loadServingCache(); + const cache = await loadServingCache("/out/serving.d.ts"); expect(cache).toEqual({ version: CACHE_VERSION, endpoints: {} }); }); - test("returns parsed cache when file exists with correct version", async () => { - const cached: ServingCache = { - version: CACHE_VERSION, - endpoints: { - llm: { - hash: "abc", - requestType: "{ messages: string[] }", - responseType: "{ model: string }", - chunkType: null, - requestKeys: ["messages"], - }, - }, - }; - vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(cached)); - - const cache = await loadServingCache(); - expect(cache).toEqual(cached); + test("reconstructs the cache from a committed serving.d.ts header + body", async () => { + const hash = endpointIdentityHash("chat", "my-endpoint"); + const file = `// Auto-generated by AppKit — DO NOT EDIT +// Regenerate with: npx @databricks/appkit generate-types +// +// ── typegen cache · v${CACHE_VERSION} · appkit@0.0.0 · 2026-01-01T00:00:00Z ── +// A matching endpoint-identity hash skips the serving fetch on build & deploy. +// chat my-endpoint ${hash} +// ${"─".repeat(60)} +import "@databricks/appkit"; +import "@databricks/appkit-ui/react"; + +declare module "@databricks/appkit" { + interface ServingEndpointRegistry { + chat: { + request: { messages: string[] }; + response: { model: string }; + chunk: unknown; + }; + } +} + +declare module "@databricks/appkit-ui/react" { + interface ServingEndpointRegistry { + chat: { + request: { messages: string[] }; + response: { model: string }; + chunk: unknown; + }; + } +} +`; + vi.mocked(fs.readFile).mockResolvedValue(file); + + const cache = await loadServingCache("/out/serving.d.ts"); + expect(cache.version).toBe(CACHE_VERSION); + expect(cache.endpoints.chat.hash).toBe(hash); + expect(cache.endpoints.chat.member).toContain("request: {"); + expect(cache.endpoints.chat.member).toContain("messages: string[]"); }); - test("flushes cache when version mismatches", async () => { - vi.mocked(fs.readFile).mockResolvedValue( - JSON.stringify({ version: "0", endpoints: { old: {} } }), - ); + test("returns empty cache when the header version mismatches", async () => { + const file = `// ── typegen cache · v0 · appkit@0.0.0 · 2026-01-01T00:00:00Z ── +// explainer +// chat my-endpoint deadbeef +// ${"─".repeat(60)} +import "@databricks/appkit"; +`; + vi.mocked(fs.readFile).mockResolvedValue(file); - const cache = await loadServingCache(); + const cache = await loadServingCache("/out/serving.d.ts"); expect(cache).toEqual({ version: CACHE_VERSION, endpoints: {} }); }); - test("flushes cache when file is corrupted", async () => { - vi.mocked(fs.readFile).mockResolvedValue("not json"); + test("returns empty cache when the file has no header", async () => { + vi.mocked(fs.readFile).mockResolvedValue("export const x = 1;\n"); - const cache = await loadServingCache(); + const cache = await loadServingCache("/out/serving.d.ts"); expect(cache).toEqual({ version: CACHE_VERSION, endpoints: {} }); }); }); - - describe("saveServingCache", () => { - test("writes cache to file", async () => { - vi.mocked(fs.writeFile).mockResolvedValue(); - - const cache: ServingCache = { - version: CACHE_VERSION, - endpoints: { - test: { - hash: "xyz", - requestType: "{}", - responseType: "{}", - chunkType: null, - requestKeys: [], - }, - }, - }; - - await saveServingCache(cache); - - expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining(".appkit-serving-types-cache.json"), - JSON.stringify(cache, null, 2), - "utf8", - ); - }); - }); }); diff --git a/packages/appkit/src/type-generator/serving/tests/generator.test.ts b/packages/appkit/src/type-generator/serving/tests/generator.test.ts index edd325909..a8d649cce 100644 --- a/packages/appkit/src/type-generator/serving/tests/generator.test.ts +++ b/packages/appkit/src/type-generator/serving/tests/generator.test.ts @@ -4,13 +4,15 @@ import { generateServingTypes } from "../generator"; vi.mock("node:fs/promises"); -// Mock cache module -vi.mock("../cache", () => ({ - CACHE_VERSION: "1", - hashSchema: vi.fn(() => "mock-hash"), - loadServingCache: vi.fn(async () => ({ version: "1", endpoints: {} })), - saveServingCache: vi.fn(async () => {}), -})); +// Mock cache module — keep the real identity-hash + loader shape, stub load to +// an empty cache so these tests exercise the fetch/convert (MISS) path. +vi.mock("../cache", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + loadServingCache: vi.fn(async () => ({ version: "1", endpoints: {} })), + }; +}); // Mock fetcher const mockFetchOpenApiSchema = vi.fn(); @@ -93,6 +95,10 @@ describe("generateServingTypes", () => { beforeEach(() => { vi.mocked(fs.mkdir).mockResolvedValue(undefined); vi.mocked(fs.writeFile).mockResolvedValue(); + // No committed file on disk in these unit tests → prior-source read misses. + vi.mocked(fs.readFile).mockRejectedValue( + Object.assign(new Error("ENOENT"), { code: "ENOENT" }), + ); process.env.TEST_SERVING_ENDPOINT = "my-endpoint"; }); @@ -122,7 +128,7 @@ describe("generateServingTypes", () => { const output = vi.mocked(fs.writeFile).mock.calls[0][1] as string; // Verify module augmentation structure - expect(output).toContain("// Auto-generated by AppKit - DO NOT EDIT"); + expect(output).toContain("// Auto-generated by AppKit — DO NOT EDIT"); expect(output).toContain('import "@databricks/appkit"'); expect(output).toContain('import "@databricks/appkit-ui/react"'); expect(output).toContain('declare module "@databricks/appkit"'); @@ -213,4 +219,31 @@ describe("generateServingTypes", () => { const output = vi.mocked(fs.writeFile).mock.calls[0][1] as string; expect(output).toContain("default:"); }); + + test("cache hit: reuses the committed member without fetching", async () => { + const { endpointIdentityHash } = await import("../cache"); + const cacheModule = await import("../cache"); + const hash = endpointIdentityHash("llm", "my-endpoint"); + vi.mocked(cacheModule.loadServingCache).mockResolvedValueOnce({ + version: "1", + endpoints: { + llm: { + hash, + member: + "{\n request: { foo: string };\n response: unknown;\n chunk: unknown;\n }", + }, + }, + }); + + await generateServingTypes({ + outFile, + endpoints: { llm: { env: "TEST_SERVING_ENDPOINT" } }, + }); + + expect(mockFetchOpenApiSchema).not.toHaveBeenCalled(); + const output = vi.mocked(fs.writeFile).mock.calls[0][1] as string; + // Reused member body appears verbatim, and the header records the hash. + expect(output).toContain("request: { foo: string }"); + expect(output).toContain(hash); + }); }); diff --git a/packages/appkit/src/type-generator/tests/__snapshots__/mv-registry.test.ts.snap b/packages/appkit/src/type-generator/tests/__snapshots__/mv-registry.test.ts.snap index 6970320c2..912196b41 100644 --- a/packages/appkit/src/type-generator/tests/__snapshots__/mv-registry.test.ts.snap +++ b/packages/appkit/src/type-generator/tests/__snapshots__/mv-registry.test.ts.snap @@ -1,8 +1,13 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`generateMetricTypeDeclarations — snapshot > emits TimeGrain union for a metric view with time-typed + regular dimensions 1`] = ` -"// Auto-generated by AppKit - DO NOT EDIT -// Generated by 'npx @databricks/appkit generate-types' or Vite plugin during build +"// Auto-generated by AppKit — DO NOT EDIT +// Regenerate with: npx @databricks/appkit generate-types +// +// ── typegen cache · v3 · appkit@0.48.0 · 2026-01-01T00:00:00.000Z ── +// A matching source|lane hash skips the warehouse DESCRIBE on build & deploy. +// revenue appkit_demo.public.revenue_metrics_v2 · sp f7c03ea86a5812c61d414e070ce8b487 +// ──────────────────────────────────────────────────────────── import "@databricks/appkit-ui/react"; declare module "@databricks/appkit-ui/react" { interface MetricRegistry { @@ -52,8 +57,14 @@ declare module "@databricks/appkit-ui/react" { `; exports[`generateMetricTypeDeclarations — snapshot > emits a stable MetricRegistry augmentation for a mixed sp + obo input 1`] = ` -"// Auto-generated by AppKit - DO NOT EDIT -// Generated by 'npx @databricks/appkit generate-types' or Vite plugin during build +"// Auto-generated by AppKit — DO NOT EDIT +// Regenerate with: npx @databricks/appkit generate-types +// +// ── typegen cache · v3 · appkit@0.48.0 · 2026-01-01T00:00:00.000Z ── +// A matching source|lane hash skips the warehouse DESCRIBE on build & deploy. +// customer_metrics appkit_demo.public.customer_metrics · obo 26269cc37c4ea648a8a9cafb66a402dd +// revenue appkit_demo.public.revenue_metrics · sp 8d0592a864778470213ca22517f21fae +// ──────────────────────────────────────────────────────────── import "@databricks/appkit-ui/react"; declare module "@databricks/appkit-ui/react" { interface MetricRegistry { @@ -142,8 +153,13 @@ declare module "@databricks/appkit-ui/react" { `; exports[`generateMetricTypeDeclarations — snapshot > emits an empty MetricRegistry interface when no metrics are registered 1`] = ` -"// Auto-generated by AppKit - DO NOT EDIT -// Generated by 'npx @databricks/appkit generate-types' or Vite plugin during build +"// Auto-generated by AppKit — DO NOT EDIT +// Regenerate with: npx @databricks/appkit generate-types +// +// ── typegen cache · v3 · appkit@0.48.0 · 2026-01-01T00:00:00.000Z ── +// A matching source|lane hash skips the warehouse DESCRIBE on build & deploy. +// (no cached entries) +// ──────────────────────────────────────────────────────────── import "@databricks/appkit-ui/react"; declare module "@databricks/appkit-ui/react" { interface MetricRegistry {} @@ -152,8 +168,13 @@ declare module "@databricks/appkit-ui/react" { `; exports[`generateMetricTypeDeclarations — snapshot > emits permissive types for a degraded entry and accurate empty unions for a confirmed-empty entry 1`] = ` -"// Auto-generated by AppKit - DO NOT EDIT -// Generated by 'npx @databricks/appkit generate-types' or Vite plugin during build +"// Auto-generated by AppKit — DO NOT EDIT +// Regenerate with: npx @databricks/appkit generate-types +// +// ── typegen cache · v3 · appkit@0.48.0 · 2026-01-01T00:00:00.000Z ── +// A matching source|lane hash skips the warehouse DESCRIBE on build & deploy. +// dims_only appkit_demo.public.dims_only · obo c2082b3d3e9c0d88b1afdf344d7db063 +// ──────────────────────────────────────────────────────────── import "@databricks/appkit-ui/react"; declare module "@databricks/appkit-ui/react" { interface MetricRegistry { diff --git a/packages/appkit/src/type-generator/tests/embedded-cache.test.ts b/packages/appkit/src/type-generator/tests/embedded-cache.test.ts new file mode 100644 index 000000000..af73a878b --- /dev/null +++ b/packages/appkit/src/type-generator/tests/embedded-cache.test.ts @@ -0,0 +1,251 @@ +import { describe, expect, test } from "vitest"; +import { + type CacheHeaderEntry, + deindentBlock, + objectMemberValue, + objectTopLevelKeys, + parseCacheHeader, + renderCacheHeader, + resolveHeaderTimestamp, + splitEntryBlocks, +} from "../embedded-cache"; + +/** + * This module encodes the typegen cache INTO a generated `.d.ts` and decodes it + * back out. The block below is what a real committed `analytics.d.ts` looks + * like — a header comment (the cache) plus the module augmentation (the body): + * + * // Auto-generated by AppKit — DO NOT EDIT + * // Regenerate with: npx @databricks/appkit generate-types + * // + * // ── typegen cache · v3 · appkit@0.48.0 · 2026-01-01T00:00:00.000Z ── + * // A matching source hash skips the warehouse DESCRIBE on build & deploy. + * // apps_list 9f0e7a21c4d3b2a1 + * // revenue 44de1b90c7f2e3a1 + * // ──────────────────────────────────────────────────────────── + * import "@databricks/appkit-ui/react"; + * declare module "@databricks/appkit-ui/react" { + * interface QueryRegistry { + * apps_list: { name: "apps_list"; result: Array<{ id: string }>; }; + * revenue: { name: "revenue"; result: Array<{ arr: number }>; }; + * } + * } + * + * The lifecycle test below reproduces exactly how a subsystem generator uses + * these functions: render a file, then on the next run parse the header + split + * the body to serve cache hits without re-describing. + */ + +const TS = "2026-01-01T00:00:00.000Z"; + +// Build a full generated-file string: header (from renderCacheHeader) + body. +const sampleFile = (entries: CacheHeaderEntry[], body: string) => + `${renderCacheHeader({ + version: "3", + explainer: "A matching source hash skips the DESCRIBE.", + timestamp: TS, + toolVersion: "9.9.9", + entries, + })}${body}`; + +describe("cache lifecycle: encode a generated file, then decode it back", () => { + test("render → parse header + split body recovers each entry's hash and block", () => { + // ── Run 1: the generator renders a file for two described queries. ── + // Each entry contributes a header line (name + hash) and a body member + // (the rendered TypeScript block). + const entries: CacheHeaderEntry[] = [ + { name: "apps_list", hash: "9f0e7a21c4d3b2a1" }, + { name: "revenue", hash: "44de1b90c7f2e3a1" }, + ]; + const body = `import "@databricks/appkit-ui/react"; +declare module "@databricks/appkit-ui/react" { + interface QueryRegistry { + apps_list: { + name: "apps_list"; + result: Array<{ id: string }>; + }; + revenue: { + name: "revenue"; + result: Array<{ arr: number }>; + }; + } +} +`; + const committedFile = sampleFile(entries, body); + + // ── Run 2: the generator reads the committed file back. ── + // Step A — parseCacheHeader recovers the name → hash map (the change + // detector). A generator compares each of these to the current source hash. + const header = parseCacheHeader(committedFile); + expect(header.version).toBe("3"); + expect(header.timestamp).toBe(TS); + expect(header.hashes).toEqual({ + apps_list: "9f0e7a21c4d3b2a1", + revenue: "44de1b90c7f2e3a1", + }); + + // Step B — splitEntryBlocks recovers each entry's rendered block. On a + // cache HIT the generator reuses this verbatim instead of re-describing. + const blocks = splitEntryBlocks(committedFile, "QueryRegistry"); + expect(Object.keys(blocks).sort()).toEqual(["apps_list", "revenue"]); + expect(blocks.apps_list).toContain('name: "apps_list"'); + expect(blocks.revenue).toContain("arr: number"); + + // Step C — the generator's cache lookup: for a given query name, a HIT is + // "header hash matches the current source AND a body block exists". + const isHit = (name: string, currentHash: string) => + header.hashes[name] === currentHash && blocks[name] !== undefined; + expect(isHit("apps_list", "9f0e7a21c4d3b2a1")).toBe(true); // unchanged + expect(isHit("revenue", "different-hash")).toBe(false); // edited → re-describe + expect(isHit("new_query", "whatever")).toBe(false); // not in cache + }); +}); + +describe("renderCacheHeader / parseCacheHeader", () => { + test("an empty entry list renders and parses with no hashes", () => { + const file = sampleFile([], "export {};\n"); + expect(file).toContain("(no cached entries)"); + const parsed = parseCacheHeader(file); + expect(parsed.version).toBe("3"); + expect(parsed.hashes).toEqual({}); + }); + + test("a decorative detail column is preserved in the render but ignored on parse", () => { + const file = sampleFile( + [ + { + name: "revenue", + detail: "cat.sch.rev · sp", + hash: "44de1b90c7f2e3a1", + }, + ], + "", + ); + // The detail is shown to human readers ... + expect(file).toContain("cat.sch.rev · sp"); + // ... but the parser reads only the first (name) and last (hash) tokens. + expect(parseCacheHeader(file).hashes).toEqual({ + revenue: "44de1b90c7f2e3a1", + }); + }); + + test("a file with no header parses as empty (all-miss), never throws", () => { + const parsed = parseCacheHeader("// just some code\nexport const x = 1;\n"); + expect(parsed.version).toBeNull(); + expect(parsed.hashes).toEqual({}); + }); + + test("the explainer line is never mistaken for an entry", () => { + const parsed = parseCacheHeader( + sampleFile([{ name: "q", hash: "abcd12" }], ""), + ); + expect(Object.keys(parsed.hashes)).toEqual(["q"]); + }); +}); + +describe("splitEntryBlocks", () => { + const body = `import "x"; +declare module "m" { + interface QueryRegistry { + apps_list: { + name: "apps_list"; + result: Array<{ id: string; note: "a }{ brace in string" }>; + }; + other: { + name: "other"; + result: unknown; + }; + } +} +`; + + test("splits members by top-level key, ignoring braces inside strings", () => { + const blocks = splitEntryBlocks(body, "QueryRegistry"); + expect(Object.keys(blocks).sort()).toEqual(["apps_list", "other"]); + expect(blocks.apps_list).toContain('name: "apps_list"'); + // The `}{` inside the string literal must not end the member early. + expect(blocks.apps_list).toContain("a }{ brace in string"); + }); + + test("returns {} when the interface is absent", () => { + expect(splitEntryBlocks(body, "NopeRegistry")).toEqual({}); + }); + + test("a member literally named __proto__ is an own key (no pollution)", () => { + const proto = `declare module "m" { + interface R { + "__proto__": { a: 1 }; + normal: { b: 2 }; + } +}`; + const blocks = splitEntryBlocks(proto, "R"); + expect(Object.hasOwn(blocks, "__proto__")).toBe(true); + expect(Object.hasOwn(blocks, "normal")).toBe(true); + // Object.prototype was not mutated. + expect(({} as Record).a).toBeUndefined(); + }); +}); + +describe("objectTopLevelKeys / objectMemberValue (serving request allowlist)", () => { + // A serving member's rendered block: the request param allowlist is the set + // of top-level keys of its `request` object type. + const block = `{ + request: { messages: string[]; temperature: number }; + response: { model: string }; + chunk: unknown; + }`; + + test("objectMemberValue returns the raw value text of a member", () => { + expect(objectMemberValue(block, "request")).toBe( + "{ messages: string[]; temperature: number }", + ); + expect(objectMemberValue(block, "chunk")).toBe("unknown"); + expect(objectMemberValue(block, "nope")).toBeUndefined(); + }); + + test("objectTopLevelKeys returns only the outermost keys", () => { + const request = objectMemberValue(block, "request") ?? ""; + // `messages` and `temperature` — NOT the nested keys inside them. + expect(objectTopLevelKeys(request)).toEqual(["messages", "temperature"]); + }); + + test("a generic Record request has no top-level keys (passthrough)", () => { + expect(objectTopLevelKeys("Record")).toEqual([]); + }); +}); + +describe("deindentBlock", () => { + test("strips up to n leading spaces from lines 2..N, leaving line 1 intact", () => { + const block = "{\n x: 1;\n }"; + // n=4: line 2 " x" loses 4 spaces → " x"; line 3 " }" loses 4 → "}". + expect(deindentBlock(block, 4)).toBe("{\n x: 1;\n}"); + }); +}); + +describe("resolveHeaderTimestamp (stable timestamp ⇒ byte-identical unchanged runs)", () => { + const entries: CacheHeaderEntry[] = [{ name: "q", hash: "abcd12" }]; + + test("preserves the prior timestamp when the entry set is unchanged", () => { + const prior = sampleFile(entries, ""); + expect(resolveHeaderTimestamp(prior, entries)).toBe(TS); + }); + + test("returns a fresh timestamp when a hash changed", () => { + const prior = sampleFile(entries, ""); + const next = resolveHeaderTimestamp(prior, [{ name: "q", hash: "ffff99" }]); + expect(next).not.toBe(TS); + }); + + test("returns a fresh timestamp when an entry is added", () => { + const prior = sampleFile(entries, ""); + const next = resolveHeaderTimestamp(prior, [ + ...entries, + { name: "q2", hash: "0011aa" }, + ]); + expect(next).not.toBe(TS); + }); + + test("returns a fresh timestamp when there is no prior header", () => { + expect(resolveHeaderTimestamp("", entries)).not.toBe(TS); + }); +}); diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index 22a054f82..e358f6681 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -12,8 +12,9 @@ const mocks = vi.hoisted(() => ({ startWarehouse: vi.fn(), spinnerStop: vi.fn(), spinnerPrintDetail: vi.fn(), - loadCache: vi.fn(() => ({ version: "2", queries: {} })), - saveCache: vi.fn(), + // The query cache now loads from the committed analytics.d.ts (via a path). + // Default: empty cache (every query a MISS). Seed per-test for cache-hit cases. + loadQueryCache: vi.fn(async () => ({ version: "3", queries: {} })), })); vi.mock("node:fs/promises", () => ({ @@ -41,12 +42,25 @@ vi.mock("../spinner", () => ({ vi.mock("../cache", async (importOriginal) => { const actual = (await importOriginal()) as Record; - return { ...actual, loadCache: mocks.loadCache, saveCache: mocks.saveCache }; + return { ...actual, loadQueryCache: mocks.loadQueryCache }; }); const { generateQueriesFromDescribe } = await import("../query-registry"); const { CACHE_VERSION, hashSQL } = await import("../cache"); +// Query generation no longer persists a separate cache file — the caller +// persists by writing the .d.ts. So tests pass a cacheFile path (its presence +// enables cache loading via the mocked loadQueryCache) and assert the returned +// schemas' `.hash`, which is set exactly for entries that should be recorded in +// the committed header (cache hits + fresh successful describes). +const CACHE_FILE = "/out/analytics.d.ts"; + +// Which query names carry a hash in the returned schemas — i.e. what the caller +// would record in the committed cache header this run (the new equivalent of +// "what got persisted"). +const persistedNames = (schemas: Array<{ name: string; hash?: string }>) => + schemas.filter((s) => s.hash !== undefined).map((s) => s.name); + // The default mode is "non-blocking", which never probes the warehouse and never // describes. The bulk of these tests exercise the DESCRIBE / classify path, so // they run in "blocking" mode (probe → proceed → describe) by default. Tests @@ -58,6 +72,7 @@ function describeQueries( ) { return generateQueriesFromDescribe(queryFolder, warehouseId, { mode: "blocking", + cacheFile: CACHE_FILE, ...options, }); } @@ -66,14 +81,15 @@ function describeQueries( // through verbatim, so equality proves reuse rather than regeneration. const CACHED_GOOD_TYPE = "RESULT_REUSED_FROM_CACHE"; -// The `queries` map of the cache object last handed to saveCache — i.e. what -// actually got persisted this run. -const lastSavedQueries = () => - ( - mocks.saveCache.mock.calls.at(-1)?.[0] as - | { queries: Record } - | undefined - )?.queries; +// Schemas returned this run keyed by name, restricted to entries that carry a +// hash (i.e. would be written to the committed cache header) — the new +// equivalent of "what got persisted". +const persistedByName = ( + schemas: Array<{ name: string; type: string; hash?: string }>, +) => + Object.fromEntries( + schemas.filter((s) => s.hash !== undefined).map((s) => [s.name, s]), + ); function succeededResult(columns: [string, string, string | null][]) { return { @@ -142,11 +158,10 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].type).toContain("id: number"); expect(schemas[0].type).toContain("name: string"); expect(mocks.spinnerStop).toHaveBeenCalledWith(""); - expect(mocks.saveCache).toHaveBeenCalledTimes(1); - // clean success: cached, and not flagged as a syntax error + // clean success: recorded in the header (carries a hash), not a syntax error expect(syntaxErrors).toEqual([]); expect(fatalErrors).toEqual([]); - expect(lastSavedQueries()?.users.type).toContain("id: number"); + expect(persistedByName(schemas).users.type).toContain("id: number"); }); test("ARROW attachment path — decodes Arrow rows into a real query schema", async () => { @@ -178,8 +193,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].type).not.toContain("result: unknown"); expect(syntaxErrors).toEqual([]); expect(fatalErrors).toEqual([]); - // A resolved schema is cached (we only persist non-unknown results). - expect(lastSavedQueries()?.users.type).toContain("id: number"); + // A resolved schema carries a hash (only non-unknown results are recorded). + expect(persistedByName(schemas).users.type).toContain("id: number"); }); test("DESCRIBE request — tries JSON_ARRAY/INLINE first with a 30s wait", async () => { @@ -221,7 +236,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].name).toBe("bad_table"); expect(schemas[0].type).toContain("result: unknown"); expect(mocks.spinnerStop).toHaveBeenCalledWith(""); - expect(mocks.saveCache).toHaveBeenCalledTimes(1); + // A genuine SQL error is not recorded in the header (no hash). + expect(persistedNames(schemas)).not.toContain("bad_table"); }); test("FAILED status without error message — uses fallback message and produces unknown result type", async () => { @@ -238,7 +254,7 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].name).toBe("query"); expect(schemas[0].type).toContain("result: unknown"); expect(mocks.spinnerStop).toHaveBeenCalledWith(""); - expect(mocks.saveCache).toHaveBeenCalledTimes(1); + expect(persistedNames(schemas)).not.toContain("query"); }); test("partial failure — caches success, unknown result for failure, output includes both", async () => { @@ -269,8 +285,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[1].name).toBe("bad"); expect(schemas[1].type).toContain("result: unknown"); - // saveCache called once after all parallel queries complete - expect(mocks.saveCache).toHaveBeenCalledTimes(1); + // Only the success entry is recorded in the header. + expect(persistedNames(schemas)).toEqual(["good"]); }); test("all queries fail (connectivity + syntax) — all produce unknown result types", async () => { @@ -297,19 +313,16 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[1].name).toBe("b"); expect(schemas[1].type).toContain("result: unknown"); - // saveCache called once after all parallel queries complete - expect(mocks.saveCache).toHaveBeenCalledTimes(1); // a = connectivity (rejected) → NOT a syntax error; b = FAILED → syntax error expect(syntaxErrors).toEqual([{ name: "b", message: "Table not found" }]); // neither a connectivity failure nor a SQL error is classified as fatal expect(fatalErrors).toEqual([]); - // neither failure is persisted to the cache - expect(lastSavedQueries()).not.toHaveProperty("a"); - expect(lastSavedQueries()).not.toHaveProperty("b"); + // neither failure is recorded in the header + expect(persistedNames(schemas)).toEqual([]); }); - test("concurrency batching — saves cache after each batch", async () => { - // 3 queries with concurrency=2 → 2 batches (2 + 1), saveCache called twice + test("concurrency batching — all batches' successes are recorded", async () => { + // 3 queries with concurrency=2 → 2 batches (2 + 1); all successes recorded mocks.readdir.mockResolvedValue(["q1.sql", "q2.sql", "q3.sql"]); mocks.readFile .mockResolvedValueOnce("SELECT id FROM t1") @@ -330,8 +343,8 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[1].name).toBe("q2"); expect(schemas[2].name).toBe("q3"); - // 2 batches → 2 saveCache calls - expect(mocks.saveCache).toHaveBeenCalledTimes(2); + // All successes across both batches are recorded in the header. + expect(persistedNames(schemas).sort()).toEqual(["q1", "q2", "q3"]); }); test("unknown result type includes parameters from SQL", async () => { @@ -358,7 +371,7 @@ describe("generateQueriesFromDescribe", () => { // A prior good type cached under a STALE hash: the query is a cache MISS // (so DESCRIBE is attempted). If the warehouse is unreachable, do not // publish the stale result columns for different SQL text. - mocks.loadCache.mockReturnValueOnce({ + mocks.loadQueryCache.mockResolvedValueOnce({ version: CACHE_VERSION, queries: { users: { hash: "stale-hash", type: CACHED_GOOD_TYPE, retry: false }, @@ -380,12 +393,9 @@ describe("generateQueriesFromDescribe", () => { // connectivity is never recorded as a syntax error expect(syntaxErrors).toEqual([]); expect(fatalErrors).toEqual([]); - // the existing good entry is left intact (not overwritten with unknown) - expect(lastSavedQueries()?.users).toEqual({ - hash: "stale-hash", - type: CACHED_GOOD_TYPE, - retry: false, - }); + // The stale entry was for different SQL, so it can't be published for the + // new SQL: the degraded result carries no hash and is not recorded. + expect(persistedNames(schemas)).not.toContain("users"); }); test("fatal rejected DESCRIBE request is not downgraded to offline", async () => { @@ -407,8 +417,7 @@ describe("generateQueriesFromDescribe", () => { message: "PERMISSION_DENIED: missing warehouse permission", }, ]); - expect(mocks.saveCache).toHaveBeenCalledTimes(1); - expect(lastSavedQueries()).not.toHaveProperty("users"); + expect(persistedNames(schemas)).not.toContain("users"); }); test("HTTP 503 wrapper error is classified as connectivity", async () => { @@ -589,8 +598,8 @@ describe("generateQueriesFromDescribe", () => { expect(fatalErrors).toEqual([ { name: "bad_auth", message: "PERMISSION_DENIED" }, ]); - expect(lastSavedQueries()?.good.type).toContain("id: number"); - expect(lastSavedQueries()).not.toHaveProperty("bad_auth"); + expect(persistedByName(schemas).good.type).toContain("id: number"); + expect(persistedNames(schemas)).not.toContain("bad_auth"); }); test("empty result (described, no columns) is unknown, not a syntax error, not cached", async () => { @@ -605,7 +614,7 @@ describe("generateQueriesFromDescribe", () => { expect(schemas[0].type).toContain("result: unknown"); expect(syntaxErrors).toEqual([]); - expect(lastSavedQueries()).not.toHaveProperty("empty"); + expect(persistedNames(schemas)).not.toContain("empty"); }); test("PENDING (non-terminal, warehouse not ready) degrades to unknown, not empty, not cached", async () => { @@ -630,7 +639,7 @@ describe("generateQueriesFromDescribe", () => { expect(syntaxErrors).toEqual([]); expect(fatalErrors).toEqual([]); // a non-ready warehouse must never persist `result: unknown` - expect(lastSavedQueries()).not.toHaveProperty("users"); + expect(persistedNames(schemas)).not.toContain("users"); }); test("PENDING reuses a prior good cached type when the SQL hash matches", async () => { @@ -665,7 +674,7 @@ describe("generateQueriesFromDescribe", () => { expect(syntaxErrors).toEqual([]); expect(fatalErrors).toEqual([]); // the good cached type persists; PENDING never overwrites it with unknown - expect(lastSavedQueries()?.users.type).toContain("id: number"); + expect(persistedByName(schemas).users.type).toContain("id: number"); }); test("syntax error (FAILED) is recorded in syntaxErrors and not cached", async () => { @@ -688,14 +697,14 @@ describe("generateQueriesFromDescribe", () => { expect(syntaxErrors).toEqual([ { name: "broken", message: "Table or view not found: missing" }, ]); - expect(lastSavedQueries()).not.toHaveProperty("broken"); + expect(persistedNames(schemas)).not.toContain("broken"); }); test("cache HIT serves the stored type without calling the warehouse", async () => { const sql = "SELECT id FROM t"; mocks.readdir.mockResolvedValue(["t.sql"]); mocks.readFile.mockResolvedValue(sql); - mocks.loadCache.mockReturnValueOnce({ + mocks.loadQueryCache.mockResolvedValueOnce({ version: CACHE_VERSION, queries: { t: { hash: hashSQL(sql), type: CACHED_GOOD_TYPE, retry: false }, @@ -717,7 +726,7 @@ describe("generateQueriesFromDescribe", () => { mocks.readdir.mockResolvedValue(["t.sql"]); mocks.readFile.mockResolvedValue(sql); // Matching hash but retry:true (legacy poisoned entry) → must NOT be a HIT. - mocks.loadCache.mockReturnValueOnce({ + mocks.loadQueryCache.mockResolvedValueOnce({ version: CACHE_VERSION, queries: { t: { hash: hashSQL(sql), type: "STALE_UNKNOWN", retry: true }, @@ -854,8 +863,8 @@ describe("generateQueriesFromDescribe", () => { expect(fatalErrors).toEqual([]); expect(syntaxErrors).toEqual([]); expect(schemas[0].type).toContain("result: unknown"); - // degraded, never a fatal failure - expect(lastSavedQueries()).toBeUndefined(); + // degraded, never a fatal failure — nothing recorded in the header + expect(persistedNames(schemas)).toEqual([]); }); test("STARTING + blocking — waits for RUNNING, then describes normally", async () => { @@ -966,7 +975,7 @@ describe("generateQueriesFromDescribe", () => { mocks.readFile.mockResolvedValue(sql); // Seed a last-good cached type under the current SQL hash. non-blocking // serves it via the normal cache HIT path — still no probe, no DESCRIBE. - mocks.loadCache.mockReturnValueOnce({ + mocks.loadQueryCache.mockResolvedValueOnce({ version: CACHE_VERSION, queries: { users: { hash: hashSQL(sql), type: CACHED_GOOD_TYPE, retry: false }, @@ -977,6 +986,7 @@ describe("generateQueriesFromDescribe", () => { const { schemas, syntaxErrors, fatalErrors } = await generateQueriesFromDescribe("/queries", "wh-123", { mode: "non-blocking", + cacheFile: CACHE_FILE, }); expect(mocks.getWarehouse).not.toHaveBeenCalled(); diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index efcc60f44..addb49f44 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -17,12 +17,6 @@ const mocks = vi.hoisted(() => ({ startWarehouse: vi.fn(), waitUntilRunning: vi.fn(), executeStatement: vi.fn(), - // In-memory stand-in for the on-disk typegen cache file. `undefined` means - // "no file yet"; otherwise it holds the serialized JSON exactly as - // saveCache would have written it, so load/save round-trips behave like - // the real implementation (string parse, own-property semantics, unknown - // sibling keys preserved) without touching node_modules/.databricks. - cacheFile: { contents: undefined as string | undefined }, })); // Mock only the warehouse-describe step; index.ts owns the throw decision we @@ -35,37 +29,10 @@ vi.mock("../query-registry", async (importOriginal) => { }; }); -// The metric path persists schemas in the shared typegen cache; redirect -// loadCache/saveCache to the in-memory `cacheFile` above so tests control -// cache state per test and nothing leaks to the real cache file (which would -// make DESCRIBE-count assertions order- and rerun-dependent). hashSQL and -// CACHE_VERSION pass through unmocked. -vi.mock("../cache", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - loadCache: vi.fn(async () => { - const raw = mocks.cacheFile.contents; - if (raw !== undefined) { - try { - const parsed = JSON.parse(raw) as Awaited< - ReturnType - >; - if (parsed.version === actual.CACHE_VERSION) { - return parsed; - } - } catch { - // Corrupted "file": fall through to the fresh-cache default, same - // as the real loadCache. - } - } - return { version: actual.CACHE_VERSION, queries: {} }; - }), - saveCache: vi.fn(async (cache: unknown) => { - mocks.cacheFile.contents = JSON.stringify(cache, null, 2); - }), - }; -}); +// The metric cache now travels IN the committed metric-views.d.ts, which each +// test writes to a real path — so it round-trips through that file with no +// module mocking. `savedCache()` below reconstructs the small shape these tests +// assert against directly from that file. // The metric gate's status-only probe and the metric blocking preflight // resolve through getWarehouseState/startWarehouse/waitUntilRunning; stub all @@ -96,9 +63,6 @@ vi.mock("@databricks/sdk-experimental", () => ({ const { WorkspaceClient } = await import("@databricks/sdk-experimental"); const { generateFromEntryPoint, TypegenFatalError, TypegenSyntaxError } = await import("../index"); -// The "../cache" mock spreads the actual module, so this is the real hashSQL — -// used to seed cache entries whose hash genuinely matches the config. -const { hashSQL } = await import("../cache"); const outputDir = path.join(__dirname, "__output__"); @@ -313,9 +277,25 @@ describe("generateFromEntryPoint — metric-view emission", () => { ); }; + // A metric key is "cached" iff it appears in the committed metric file's + // cache header (degraded keys carry no header hash). Reads the real file. + const cachedMetricKeys = (): Set => { + let source = ""; + try { + source = fs.readFileSync(metricFile, "utf-8"); + } catch { + return new Set(); + } + const keys = new Set(); + for (const line of source.split("\n")) { + const m = line.match(/^\/\/\s{3}(\S+)\s+.*\s+[0-9a-f]{6,}$/); + if (m) keys.add(m[1]); + } + return keys; + }; + beforeEach(() => { vi.clearAllMocks(); - mocks.cacheFile.contents = undefined; fs.rmSync(metricsDir, { recursive: true, force: true }); fs.mkdirSync(queryFolder, { recursive: true }); fs.mkdirSync(metricViewsFolder, { recursive: true }); @@ -734,8 +714,7 @@ describe("generateFromEntryPoint — metric-view emission", () => { // The degraded outcome is NEVER cached (mirrors the query path): the key is // left uncached so a later pass re-probes, and no stale/sticky entry can be // served on a subsequent --wait run. - const metrics = JSON.parse(mocks.cacheFile.contents ?? "{}").metrics ?? {}; - expect(metrics.revenue).toBeUndefined(); + expect(cachedMetricKeys().has("revenue")).toBe(false); }); test("blocking + preflight wait rejects with a timeout: fatal after artifacts (no silent stall)", async () => { @@ -789,8 +768,7 @@ describe("generateFromEntryPoint — metric-view emission", () => { // The degraded outcome is not cached — the key stays uncached for the next // pass to re-probe. - const metrics = JSON.parse(mocks.cacheFile.contents ?? "{}").metrics ?? {}; - expect(metrics.revenue).toBeUndefined(); + expect(cachedMetricKeys().has("revenue")).toBe(false); }); test("blocking + preflight wait resolves non-RUNNING (STOPPED): degrades, does not throw", async () => { @@ -846,8 +824,7 @@ describe("generateFromEntryPoint — metric-view emission", () => { // The degraded outcome is not cached; the key stays uncached and the next // describe-capable pass re-probes it (convergence via re-describe, not via a // cached retry flag). - const metrics = JSON.parse(mocks.cacheFile.contents ?? "{}").metrics ?? {}; - expect(metrics.revenue).toBeUndefined(); + expect(cachedMetricKeys().has("revenue")).toBe(false); }); test.each<[string, boolean]>([ @@ -897,9 +874,7 @@ describe("generateFromEntryPoint — metric-view emission", () => { expect(declarations).toContain("measureKeys: string"); // The degraded outcome is not cached — no sticky entry to serve later. - const metrics = - JSON.parse(mocks.cacheFile.contents ?? "{}").metrics ?? {}; - expect(metrics.revenue).toBeUndefined(); + expect(cachedMetricKeys().has("revenue")).toBe(false); }, ); @@ -1105,8 +1080,71 @@ describe("generateFromEntryPoint — metric cache section", () => { ); }; - // Parse the in-memory "cache file" the way the next pass's loadCache would. - const savedCache = () => JSON.parse(mocks.cacheFile.contents ?? "{}"); + // Reconstruct the small cache shape these tests assert against, directly from + // the committed metric-views.d.ts (the cache now lives there). A metric key is + // "cached" iff it appears in the file's cache header AND its rendered member + // is non-degraded. We surface just the fields the assertions read: + // metrics[key].retry → always false for a present entry + // metrics[key].schema.source → parsed from the member's `source:` field + // metrics[key].schema.degraded → true when the member is the permissive form + // metrics[key].schema.measures[0].name → first measure key parsed from the member + type SavedMetric = { + retry: boolean; + schema: { + source?: string; + degraded?: true; + measures: Array<{ name: string }>; + }; + }; + const savedCache = (): { + version: string; + queries: Record; + metrics: Record; + } => { + let source = ""; + try { + source = fs.readFileSync(metricFile, "utf-8"); + } catch { + return { version: "3", queries: {}, metrics: Object.create(null) }; + } + // Header hash table lines: `// · `. + const headerKeys = new Set(); + for (const line of source.split("\n")) { + const m = line.match(/^\/\/\s{3}(\S+)\s+.*\s+[0-9a-f]{6,}$/); + if (m) headerKeys.add(m[1]); + } + const metrics: Record = Object.create(null); + for (const key of headerKeys) { + // Scope to the slice starting at this member's `"": {` declaration; + // the fields the tests read (source, first measure, degraded form) all + // appear before the next member, so a scoped forward search suffices. + const keyDeclRe = new RegExp( + `${JSON.stringify(key).replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}:\\s*\\{`, + ); + const at = source.search(keyDeclRe); + // Bound the slice to THIS member: from its declaration up to the member's + // own `measures:`/`dimensions:`/`measureKeys:` region — enough to read the + // fields the tests check without spilling into the next member. + const rawSlice = at === -1 ? "" : source.slice(at); + const boundEnd = rawSlice.indexOf("measureKeys:"); + const slice = boundEnd === -1 ? rawSlice : rawSlice.slice(0, boundEnd); + const src = slice.match(/source:\s*"([^"]*)"/)?.[1]; + // A degraded member renders permissive `measures: Record`. + const isDegraded = /measures:\s*Record/.test(slice); + // First measure key: the first `"":` after the `measures: {` token. + const afterMeasures = slice.slice(slice.indexOf("measures: {")); + const firstMeasure = afterMeasures.match(/"([^"]+)":/)?.[1]; + metrics[key] = { + retry: false, + schema: { + source: src, + degraded: isDegraded ? true : undefined, + measures: firstMeasure ? [{ name: firstMeasure }] : [], + }, + }; + } + return { version: "3", queries: {}, metrics }; + }; const run = ( overrides: Partial[0]> = {}, @@ -1121,7 +1159,6 @@ describe("generateFromEntryPoint — metric cache section", () => { beforeEach(() => { vi.clearAllMocks(); - mocks.cacheFile.contents = undefined; fs.rmSync(cacheTestDir, { recursive: true, force: true }); fs.mkdirSync(queryFolder, { recursive: true }); fs.mkdirSync(metricViewsFolder, { recursive: true }); @@ -1147,8 +1184,8 @@ describe("generateFromEntryPoint — metric cache section", () => { expect(mocks.executeStatement).toHaveBeenCalledTimes(1); const firstDeclarations = fs.readFileSync(metricFile, "utf-8"); - // Wipe the artifact so pass 2 provably rewrites it from cache alone. - fs.rmSync(metricFile); + // Pass 2 reconstructs the cache from the committed metric file (the file IS + // the cache now, so it must NOT be wiped) and serves every key as a HIT. vi.clearAllMocks(); await expect(run()).resolves.toBeUndefined(); @@ -1157,6 +1194,8 @@ describe("generateFromEntryPoint — metric cache section", () => { expect(mocks.getWarehouseState).not.toHaveBeenCalled(); // ... and the whole pass constructed zero SDK clients. expect(vi.mocked(WorkspaceClient)).not.toHaveBeenCalled(); + // The .d.ts is rewritten byte-identical from cache (timestamp unchanged + // because no describe occurred → the renderer reuses the same members). expect(fs.readFileSync(metricFile, "utf-8")).toBe(firstDeclarations); }); @@ -1254,9 +1293,10 @@ describe("generateFromEntryPoint — metric cache section", () => { ); await expect(run()).resolves.toBeUndefined(); + // The committed metric file IS the cache — keep it. A warehouse-down pass + // reconstructs the cached real schema from it and serves it (no describe). vi.clearAllMocks(); mocks.getWarehouseState.mockResolvedValue("STOPPED"); - fs.rmSync(metricFile); await expect(run()).resolves.toBeUndefined(); expect(mocks.executeStatement).not.toHaveBeenCalled(); @@ -1302,20 +1342,10 @@ describe("generateFromEntryPoint — metric cache section", () => { ]); }); - test("metric-path save preserves the queries section byte-for-byte, and a metrics-less cache file loads fine", async () => { - const seededQueries = { - my_query: { - hash: "abc123", - type: '{ name: "my_query"; parameters: Record; result: unknown; }', - retry: false, - }, - }; - // Pre-metrics cache file: version "3" with no `metrics` section at all. - mocks.cacheFile.contents = JSON.stringify( - { version: "3", queries: seededQueries }, - null, - 2, - ); + test("metric and query caches live in separate files (metric write lands only in the metric file)", async () => { + // The query cache travels in analytics.d.ts and the metric cache in + // metric-views.d.ts — two independent files. The metric member is written + // only to the metric file, never leaking into the analytics file. writeConfig({ revenue: { source: "demo.sales.revenue" } }); mocks.getWarehouseState.mockResolvedValue("RUNNING"); mocks.executeStatement.mockResolvedValue( @@ -1325,39 +1355,18 @@ describe("generateFromEntryPoint — metric cache section", () => { await expect(run()).resolves.toBeUndefined(); expect(mocks.executeStatement).toHaveBeenCalledTimes(1); + // Metric cache landed in its own file with a real (non-degraded) schema. const saved = savedCache(); - // Same version (no bump), queries byte-identical, metrics added beside. - expect(saved.version).toBe("3"); - expect(JSON.stringify(saved.queries)).toBe(JSON.stringify(seededQueries)); expect(saved.metrics.revenue.retry).toBe(false); expect(saved.metrics.revenue.schema.measures[0].name).toBe("total_revenue"); + // The metric member appears only in the metric file, not the analytics one. + expect(fs.readFileSync(metricFile, "utf-8")).toContain('"revenue"'); + expect(fs.readFileSync(outFile, "utf-8")).not.toContain('"revenue"'); }); - test("a configured metric key named __proto__ neither pollutes prototypes nor vanishes on save", async () => { - const protoEntry = { - hash: "deadbeef", - schema: { - key: "__proto__", - source: "demo.evil.proto", - lane: "sp", - measures: [], - dimensions: [], - degraded: true, - }, - retry: true, - }; - // Computed key keeps "__proto__" an own property of the literal, so the - // serialized cache file genuinely contains a "__proto__" metrics key. - mocks.cacheFile.contents = JSON.stringify({ - version: "3", - queries: {}, - metrics: { ["__proto__"]: protoEntry }, - }); - expect(mocks.cacheFile.contents).toContain('"__proto__"'); - - // "__proto__" passes the metric key regex, so a config can genuinely - // declare it. Keeping it CONFIGURED is what exempts it from pruning — - // the unconfigured-key case is covered by the prune tests. + test("a configured metric key named __proto__ round-trips through the committed cache without polluting prototypes", async () => { + // Pass 1: describe both keys and write the committed metric file, whose + // header + body will carry a literal "__proto__" entry. writeConfig({ ["__proto__"]: { source: "demo.evil.proto" }, revenue: { source: "demo.sales.revenue" }, @@ -1366,32 +1375,23 @@ describe("generateFromEntryPoint — metric cache section", () => { mocks.executeStatement.mockResolvedValue( describeResponseFor("total_revenue"), ); + await expect(run()).resolves.toBeUndefined(); + expect(mocks.executeStatement).toHaveBeenCalledTimes(2); + // Pass 2: warm. Reconstructing the cache from the committed file parses a + // "__proto__" entry — the loader uses null-prototype maps, so it's stored as + // an own key and never mutates Object.prototype. Both keys are cache HITs. + vi.clearAllMocks(); await expect(run()).resolves.toBeUndefined(); + expect(mocks.executeStatement).not.toHaveBeenCalled(); - // The seeded hash mismatches the configured source, so the key was - // re-described alongside revenue. - expect(mocks.executeStatement).toHaveBeenCalledTimes(2); + // No prototype pollution from parsing the "__proto__" key. + expect(({} as Record).source).toBeUndefined(); + expect(Object.prototype).not.toHaveProperty("source"); - // No prototype pollution: the entry's fields never leaked onto plain - // objects via an Object.prototype mutation — neither on the load copy - // nor on the describe-result write into the section. - expect(({} as Record).hash).toBeUndefined(); - expect(({} as Record).retry).toBeUndefined(); - expect(Object.prototype).not.toHaveProperty("hash"); - - // The entry survived load → null-prototype copy → write → save as an - // OWN key of the section (a plain-object section would have hit the - // __proto__ setter and silently dropped it from the serialized output). - expect(mocks.cacheFile.contents).toContain('"__proto__"'); + // Both keys survived as own entries of the reconstructed cache. const metrics = savedCache().metrics; expect(Object.hasOwn(metrics, "__proto__")).toBe(true); - const protoSaved = Object.getOwnPropertyDescriptor( - metrics, - "__proto__", - )?.value; - expect(protoSaved.retry).toBe(false); - expect(protoSaved.schema.measures[0].name).toBe("total_revenue"); expect(metrics.revenue.retry).toBe(false); }); @@ -1704,91 +1704,70 @@ describe("generateFromEntryPoint — metric cache section", () => { expect(Object.keys(savedCache().metrics)).toEqual(["revenue"]); }); - // ── Revival validation: malformed cache entries are misses, not crashes ── - - const revivableSchema = { - key: "revenue", - source: "demo.sales.revenue", - lane: "sp", - measures: [{ name: "m", type: "BIGINT", isMeasure: true }], - dimensions: [{ name: "region", type: "STRING", isMeasure: false }], - }; + // ── Committed-file reconstruction: malformed files are misses, not crashes ── - test("revival control: a well-formed seeded entry with a matching hash is served without describing", async () => { - // Control for the malformed matrix below: same hash/retry mechanics, - // valid shape ⇒ HIT. Proves the matrix's misses come from validation, - // not from a hash mismatch. - mocks.cacheFile.contents = JSON.stringify({ - version: "3", - queries: {}, - metrics: { - revenue: { - hash: hashSQL("demo.sales.revenue|sp"), - retry: false, - schema: revivableSchema, - }, - }, - }); + test("hit control: a warm committed file with a matching hash is served without describing", async () => { + // Pass 1 writes the committed metric file with a real schema. writeConfig({ revenue: { source: "demo.sales.revenue" } }); + mocks.getWarehouseState.mockResolvedValue("RUNNING"); + mocks.executeStatement.mockResolvedValue( + describeResponseFor("total_revenue"), + ); + await expect(run()).resolves.toBeUndefined(); + // Pass 2 reconstructs the cache from that file → HIT, no describe, no probe. + vi.clearAllMocks(); await expect(run()).resolves.toBeUndefined(); expect(mocks.executeStatement).not.toHaveBeenCalled(); expect(mocks.getWarehouseState).not.toHaveBeenCalled(); - expect(fs.readFileSync(metricFile, "utf-8")).toContain('"m": number'); + expect(fs.readFileSync(metricFile, "utf-8")).toContain( + '"total_revenue": number', + ); }); - test.each<[string, Record]>([ - ["schema is null", { schema: null }], - ["schema is an array", { schema: [] }], - [ - "schema missing measures", - { schema: { ...revivableSchema, measures: undefined } }, - ], - ["invalid lane", { schema: { ...revivableSchema, lane: "x" } }], - [ - "measures not an array", - { schema: { ...revivableSchema, measures: "nope" } }, - ], + test.each<[string, (file: string) => string]>([ [ - "column element missing type", - { schema: { ...revivableSchema, measures: [{ name: "m" }] } }, + "no cache header at all", + () => "// hand-written, no header\nexport {};\n", ], + ["header version mismatch", (file) => file.replace(/· v\d+ ·/, "· v0 ·")], [ - "non-boolean degraded", - { schema: { ...revivableSchema, degraded: "yep" } }, + "header hash present but body member removed", + (file) => + // Drop the rendered "revenue" member from the body, leaving a dangling + // header hash. Reconstruction must treat this as a MISS (drift), not + // serve a phantom entry or crash. + file.replace(/ {4}"revenue":[\s\S]*?\n {4}\};\n/, ""), ], - ["non-string hash", { hash: 42 }], - ["non-boolean retry", { retry: "yes" }], + ["truncated mid-member", (file) => file.slice(0, file.length / 2)], ])( - "revival validation: %s is a cache miss (re-described), never a crash", - async (_label, overrides) => { - mocks.cacheFile.contents = JSON.stringify({ - version: "3", - queries: {}, - metrics: { - revenue: { - hash: hashSQL("demo.sales.revenue|sp"), - retry: false, - schema: revivableSchema, - ...overrides, - }, - }, - }); + "reconstruction: %s is a cache miss (re-described), never a crash", + async (_label, mangle) => { + // Pass 1: produce a valid committed file. writeConfig({ revenue: { source: "demo.sales.revenue" } }); mocks.getWarehouseState.mockResolvedValue("RUNNING"); mocks.executeStatement.mockResolvedValue( describeResponseFor("total_revenue"), ); + await expect(run()).resolves.toBeUndefined(); + + // Corrupt the committed file, then re-run. + const mangled = mangle(fs.readFileSync(metricFile, "utf-8")); + fs.writeFileSync(metricFile, mangled); + + vi.clearAllMocks(); + mocks.getWarehouseState.mockResolvedValue("RUNNING"); + mocks.executeStatement.mockResolvedValue( + describeResponseFor("total_revenue"), + ); await expect(run()).resolves.toBeUndefined(); - // The malformed entry was not revived: the key was re-described and - // the cache healed with the fresh result. + // The mangled entry was not revived: the key was re-described and the + // file healed with the fresh result. expect(mocks.executeStatement).toHaveBeenCalledTimes(1); - expect(savedCache().metrics.revenue.retry).toBe(false); expect(savedCache().metrics.revenue.schema.measures[0].name).toBe( "total_revenue", ); - // The artifacts render the fresh schema — never the revived garbage. expect(fs.readFileSync(metricFile, "utf-8")).toContain( '"total_revenue": number', ); diff --git a/packages/appkit/src/type-generator/tests/mv-registry.test.ts b/packages/appkit/src/type-generator/tests/mv-registry.test.ts index 6fa77693e..12d0c1a55 100644 --- a/packages/appkit/src/type-generator/tests/mv-registry.test.ts +++ b/packages/appkit/src/type-generator/tests/mv-registry.test.ts @@ -1522,7 +1522,9 @@ describe("generateMetricTypeDeclarations — snapshot", () => { }); const { schemas } = await syncMetrics(resolution, fetcher); - const output = generateMetricTypeDeclarations(schemas); + const output = generateMetricTypeDeclarations(schemas, { + timestamp: "2026-01-01T00:00:00.000Z", + }); expect(output).toMatchSnapshot(); // Sanity assertions in addition to the snapshot, so future drift surfaces @@ -1535,7 +1537,9 @@ describe("generateMetricTypeDeclarations — snapshot", () => { }); test("emits an empty MetricRegistry interface when no metrics are registered", () => { - const output = generateMetricTypeDeclarations([]); + const output = generateMetricTypeDeclarations([], { + timestamp: "2026-01-01T00:00:00.000Z", + }); expect(output).toMatchSnapshot(); }); @@ -1566,7 +1570,9 @@ describe("generateMetricTypeDeclarations — snapshot", () => { const { schemas, failures } = await syncMetrics(resolution, fetcher); expect(failures).toEqual([]); - const output = generateMetricTypeDeclarations(schemas); + const output = generateMetricTypeDeclarations(schemas, { + timestamp: "2026-01-01T00:00:00.000Z", + }); expect(output).toMatchSnapshot(); // Sanity assertions in addition to the snapshot, so future drift surfaces @@ -1607,7 +1613,9 @@ describe("generateMetricTypeDeclarations — snapshot", () => { }); const { schemas } = await syncMetrics(resolution, fetcher); - const output = generateMetricTypeDeclarations(schemas); + const output = generateMetricTypeDeclarations(schemas, { + timestamp: "2026-01-01T00:00:00.000Z", + }); expect(output).toMatchSnapshot(); // Sanity assertions in addition to the snapshot, so future drift surfaces @@ -1975,7 +1983,9 @@ describe("artifact key-order determinism", () => { // Entry keys in the .d.ts appear as ` "": {` lines (4-space // indent — metadata column maps sit deeper and don't match). - const declarations = generateMetricTypeDeclarations(schemas); + const declarations = generateMetricTypeDeclarations(schemas, { + timestamp: "2026-01-01T00:00:00.000Z", + }); const dtsKeys = [...declarations.matchAll(/^ {4}"([^"]+)": \{$/gm)].map( (m) => m[1], ); diff --git a/packages/appkit/src/type-generator/tests/sync-metric-views-types.test.ts b/packages/appkit/src/type-generator/tests/sync-metric-views-types.test.ts index 337892aa8..99fd96ad6 100644 --- a/packages/appkit/src/type-generator/tests/sync-metric-views-types.test.ts +++ b/packages/appkit/src/type-generator/tests/sync-metric-views-types.test.ts @@ -17,40 +17,10 @@ import type { DatabricksStatementExecutionResponse } from "../types"; * shared typegen cache is honored (default) / bypassed (`cache: false`). */ -// In-memory stand-in for the on-disk typegen cache file so the focused metric -// sync's loadCache/saveCache never touch node_modules/.databricks and each test -// controls cache state. hashSQL / metricCacheHash / isRevivableMetricCacheEntry -// / CACHE_VERSION pass through unmocked (mirrors index.test.ts). -const mocks = vi.hoisted(() => ({ - cacheFile: { contents: undefined as string | undefined }, -})); - -vi.mock("../cache", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - loadCache: vi.fn(async () => { - const raw = mocks.cacheFile.contents; - if (raw !== undefined) { - try { - const parsed = JSON.parse(raw) as Awaited< - ReturnType - >; - if (parsed.version === actual.CACHE_VERSION) { - return parsed; - } - } catch { - // Corrupted "file": fall through to the fresh-cache default. - } - } - return { version: actual.CACHE_VERSION, queries: {} }; - }), - saveCache: vi.fn(async (cache: unknown) => { - mocks.cacheFile.contents = JSON.stringify(cache, null, 2); - }), - }; -}); - +// The metric cache now travels IN the committed metric-views.d.ts. Each test +// writes to a real tmp `metricOutFile`, so the cache round-trips through that +// file with no module mocking — a warm second run reconstructs the cache from +// the file the first run wrote. const { syncMetricViewsTypes } = await import("../index"); /** @@ -120,7 +90,6 @@ describe("syncMetricViewsTypes", () => { beforeEach(() => { fetcher.mockClear(); - mocks.cacheFile.contents = undefined; tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "sync-metric-types-")); metricViewsFolder = path.join(tmpRoot, "config", "metric-views"); fs.mkdirSync(metricViewsFolder, { recursive: true }); @@ -290,7 +259,7 @@ describe("syncMetricViewsTypes", () => { expect(declarations).toContain('"total_revenue": number'); }); - test("a removed metric key is pruned from the cache section", async () => { + test("a removed metric key is pruned from the generated file (and its cache header)", async () => { writeMixedConfig(); // Warm both keys. @@ -300,11 +269,9 @@ describe("syncMetricViewsTypes", () => { metricOutFile, metricFetcher: fetcher, }); - const afterFirst = JSON.parse(mocks.cacheFile.contents ?? "{}"); - expect(Object.keys(afterFirst.metrics).sort()).toEqual([ - "churn", - "revenue", - ]); + const afterFirst = fs.readFileSync(metricOutFile, "utf-8"); + expect(afterFirst).toContain('"revenue"'); + expect(afterFirst).toContain('"churn"'); // Shrink the config to a single key. fs.writeFileSync( @@ -321,7 +288,9 @@ describe("syncMetricViewsTypes", () => { metricFetcher: fetcher, }); - const afterSecond = JSON.parse(mocks.cacheFile.contents ?? "{}"); - expect(Object.keys(afterSecond.metrics)).toEqual(["revenue"]); + // The stale key is gone from both the body and the cache header. + const afterSecond = fs.readFileSync(metricOutFile, "utf-8"); + expect(afterSecond).toContain('"revenue"'); + expect(afterSecond).not.toContain('"churn"'); }); }); diff --git a/packages/appkit/src/type-generator/types.ts b/packages/appkit/src/type-generator/types.ts index e947bb530..ebbb16859 100644 --- a/packages/appkit/src/type-generator/types.ts +++ b/packages/appkit/src/type-generator/types.ts @@ -103,6 +103,13 @@ export const sqlTypeToHelper: Record = { export interface QuerySchema { name: string; type: string; + /** + * Local SQL change-detector hash (md5 of the source SQL), present only for + * entries that should be recorded in the generated file's cache header — i.e. + * cache hits and fresh successful describes. Omitted for degraded / `unknown` + * results so they always re-read as a MISS on the next pass. + */ + hash?: string; } /** diff --git a/template/_gitignore b/template/_gitignore index 29895e7c0..b02241f41 100644 --- a/template/_gitignore +++ b/template/_gitignore @@ -9,6 +9,3 @@ build/ test-results/ playwright-report/ -# Auto-generated types (endpoint-specific, varies per developer) -shared/appkit-types/serving.d.ts -