diff --git a/apps/dev-playground/.appkit/serving-types-cache.json b/apps/dev-playground/.appkit/serving-types-cache.json new file mode 100644 index 000000000..26a3312a1 --- /dev/null +++ b/apps/dev-playground/.appkit/serving-types-cache.json @@ -0,0 +1,21 @@ +{ + "version": "1", + "endpoints": { + "default": { + "hash": "e66c960c5d57436d7fc4e21086650a534892a1548ccf2b333db2af6046832471", + "requestType": "{\n messages?: {\n role?: \"user\" | \"assistant\";\n content?: string;\n }[];\n /** @openapi integer, nullable */\n n?: number | null;\n max_tokens?: number;\n /** @openapi double, nullable */\n top_p?: number | null;\n reasoning_effort?: \"low\" | \"medium\" | \"high\" | \"max\" | null;\n /** @openapi double, nullable */\n temperature?: number | null;\n stop?: string | string[] | null;\n /** @openapi integer, nullable */\n top_k?: number | null;\n}", + "responseType": "{\n model?: string;\n choices?: {\n index?: number;\n message?: {\n role?: \"user\" | \"assistant\";\n content?: string;\n };\n finish_reason?: string;\n }[];\n usage?: {\n prompt_tokens?: number;\n completion_tokens?: number;\n total_tokens?: number;\n } | null;\n object?: string;\n id?: string;\n created?: number;\n}", + "chunkType": "{\n model?: string;\n choices?: {\n index?: number;\n delta?: {\n role?: \"user\" | \"assistant\";\n content?: string;\n };\n finish_reason?: string | null;\n }[];\n object?: string;\n id?: string;\n created?: number;\n}", + "requestKeys": [ + "messages", + "n", + "max_tokens", + "top_p", + "reasoning_effort", + "temperature", + "stop", + "top_k" + ] + } + } +} \ No newline at end of file diff --git a/apps/dev-playground/.appkit/types-cache.json b/apps/dev-playground/.appkit/types-cache.json new file mode 100644 index 000000000..a91799298 --- /dev/null +++ b/apps/dev-playground/.appkit/types-cache.json @@ -0,0 +1,207 @@ +{ + "version": "3", + "queries": { + "app_activity_heatmap": { + "hash": "744281be86b14c2b14ed76b94060e4af", + "type": "{\n name: \"app_activity_heatmap\";\n parameters: {\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType STRING */\n day_of_week: string;\n /** @sqlType DECIMAL(35,2) */\n spend: number;\n }>;\n }", + "retry": false + }, + "apps_list": { + "hash": "4b7393b43f8e7beaa52174ed4d918c04", + "type": "{\n name: \"apps_list\";\n parameters: Record;\n result: Array<{\n /** @sqlType STRING */\n id: string;\n /** @sqlType STRING */\n name: string;\n /** @sqlType STRING */\n creator: string;\n /** @sqlType STRING */\n tags: string;\n /** @sqlType DECIMAL(38,6) */\n totalSpend: number;\n /** @sqlType DATE */\n createdAt: string;\n }>;\n }", + "retry": false + }, + "cost_recommendations": { + "hash": "730c7d8b5e2726981088b5975157b0da", + "type": "{\n name: \"cost_recommendations\";\n parameters: Record;\n result: Array<{\n /** @sqlType INT */\n dummy: number;\n }>;\n }", + "retry": false + }, + "dashboard_fare_distribution": { + "hash": "d280e96e06e1b5189d8fe734901e6e72", + "type": "{\n name: \"dashboard_fare_distribution\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n fare_bucket: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n avg_distance: number;\n }>;\n }", + "retry": false + }, + "dashboard_hourly_heatmap": { + "hash": "827ca45db36ccf6f9c86c653c6175d69", + "type": "{\n name: \"dashboard_hourly_heatmap\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType INT */\n day_of_week: number;\n /** @sqlType INT */\n hour_of_day: number;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n }>;\n }", + "retry": false + }, + "dashboard_kpi_sparklines": { + "hash": "1b3c14b03e1de455ff78f2d3feaec047", + "type": "{\n name: \"dashboard_kpi_sparklines\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType DATE */\n trip_date: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n total_revenue: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n /** @sqlType DOUBLE */\n avg_distance: number;\n }>;\n }", + "retry": false + }, + "dashboard_kpis": { + "hash": "24ef19b8b1218edb8f247f6d79c915ff", + "type": "{\n name: \"dashboard_kpis\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType BIGINT */\n total_trips: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n /** @sqlType DOUBLE */\n avg_distance: number;\n /** @sqlType DOUBLE */\n max_fare: number;\n /** @sqlType DOUBLE */\n min_fare: number;\n }>;\n }", + "retry": false + }, + "dashboard_top_zone": { + "hash": "fe3ed4ef76e4e7eff329c88072a44fe9", + "type": "{\n name: \"dashboard_top_zone\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType INT */\n pickup_zip: number;\n /** @sqlType BIGINT */\n trip_count: number;\n }>;\n }", + "retry": false + }, + "dashboard_top_zones": { + "hash": "363c554c5a060ee2236f4e9fa1cf3c61", + "type": "{\n name: \"dashboard_top_zones\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMin: SQLStringMarker;\n /** STRING - use sql.string() */\n fareMax: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n pickup_zip: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n total_revenue: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n }>;\n }", + "retry": false + }, + "dashboard_trips_over_time": { + "hash": "eb8bed65ed02787ed050a016472b72f6", + "type": "{\n name: \"dashboard_trips_over_time\";\n parameters: {\n /** STRING - use sql.string() */\n dateFrom: SQLStringMarker;\n /** STRING - use sql.string() */\n dateTo: SQLStringMarker;\n /** STRING - use sql.string() */\n pickupZip: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType DATE */\n trip_date: string;\n /** @sqlType BIGINT */\n trip_count: number;\n /** @sqlType DOUBLE */\n avg_fare: number;\n /** @sqlType DOUBLE */\n total_revenue: number;\n }>;\n }", + "retry": false + }, + "example": { + "hash": "aeb02ed3e8a6c77279099406f8709543", + "type": "{\n name: \"example\";\n parameters: Record;\n result: Array<{\n /** @sqlType BOOLEAN */\n \"(1 = 1)\": boolean;\n }>;\n }", + "retry": false + }, + "spend_data": { + "hash": "caa0430652fe15eff658e48e6dac2446", + "type": "{\n name: \"spend_data\";\n parameters: {\n /** STRING - use sql.string() */\n groupBy: SQLStringMarker;\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n /** STRING - use sql.string() */\n appId: SQLStringMarker;\n /** STRING - use sql.string() */\n creator: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n group_key: string;\n /** @sqlType TIMESTAMP */\n aggregation_period: string;\n /** @sqlType DECIMAL(38,6) */\n cost_usd: number;\n }>;\n }", + "retry": false + }, + "spend_summary": { + "hash": "bbe188624c3f5904c3a7593cb32982d5", + "type": "{\n name: \"spend_summary\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType DECIMAL(33,0) */\n total: number;\n /** @sqlType DECIMAL(33,0) */\n average: number;\n /** @sqlType DECIMAL(33,0) */\n forecasted: number;\n }>;\n }", + "retry": false + }, + "sql_helpers_test": { + "hash": "1322df4ba9c107e8d23e2a04bae860c5", + "type": "{\n name: \"sql_helpers_test\";\n parameters: {\n /** STRING - use sql.string() */\n stringParam: SQLStringMarker;\n /** NUMERIC - use sql.numeric() */\n numberParam: SQLNumberMarker;\n /** BOOLEAN - use sql.boolean() */\n booleanParam: SQLBooleanMarker;\n /** DATE - use sql.date() */\n dateParam: SQLDateMarker;\n /** TIMESTAMP - use sql.timestamp() */\n timestampParam: SQLTimestampMarker;\n /** STRING - use sql.string() */\n binaryParam: SQLStringMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n string_value: string;\n /** @sqlType INT */\n number_value: number;\n /** @sqlType BOOLEAN */\n boolean_value: boolean;\n /** @sqlType STRING */\n date_value: string;\n /** @sqlType STRING */\n timestamp_value: string;\n /** @sqlType BINARY */\n binary_value: string;\n /** @sqlType STRING */\n binary_hex: string;\n /** @sqlType INT */\n binary_length: number;\n }>;\n }", + "retry": false + }, + "top_contributors": { + "hash": "2d58759cca2fe31dae06475a23080120", + "type": "{\n name: \"top_contributors\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType DECIMAL(38,6) */\n total_cost_usd: number;\n }>;\n }", + "retry": false + }, + "untagged_apps": { + "hash": "5946262b49710b8ab458d1bf950ff8c9", + "type": "{\n name: \"untagged_apps\";\n parameters: {\n /** STRING - use sql.string() */\n aggregationLevel: SQLStringMarker;\n /** DATE - use sql.date() */\n startDate: SQLDateMarker;\n /** DATE - use sql.date() */\n endDate: SQLDateMarker;\n };\n result: Array<{\n /** @sqlType STRING */\n app_name: string;\n /** @sqlType STRING */\n creator: string;\n /** @sqlType DECIMAL(38,6) */\n total_cost_usd: number;\n /** @sqlType DECIMAL(38,10) */\n avg_period_cost_usd: number;\n }>;\n }", + "retry": false + } + }, + "metrics": { + "customers": { + "hash": "26269cc37c4ea648a8a9cafb66a402dd", + "schema": { + "key": "customers", + "source": "appkit_demo.public.customer_metrics", + "lane": "obo", + "measures": [ + { + "name": "active_accounts", + "type": "bigint", + "isMeasure": true, + "displayName": "Active Accounts", + "format": "#,##0" + }, + { + "name": "churn_rate", + "type": "decimal", + "isMeasure": true, + "displayName": "Churn Rate" + }, + { + "name": "avg_ltv", + "type": "double", + "isMeasure": true, + "displayName": "Average LTV", + "format": "$#,##0.00" + } + ], + "dimensions": [ + { + "name": "segment", + "type": "string", + "isMeasure": false, + "displayName": "Customer Segment" + }, + { + "name": "region", + "type": "string", + "isMeasure": false, + "displayName": "Region" + }, + { + "name": "csm_email", + "type": "string", + "isMeasure": false, + "displayName": "CSM Email" + } + ] + }, + "retry": false + }, + "revenue": { + "hash": "8d0592a864778470213ca22517f21fae", + "schema": { + "key": "revenue", + "source": "appkit_demo.public.revenue_metrics", + "lane": "sp", + "measures": [ + { + "name": "mrr", + "type": "double", + "isMeasure": true, + "displayName": "Monthly Recurring Revenue", + "format": "$#,##0.00" + }, + { + "name": "arr", + "type": "double", + "isMeasure": true, + "description": "Annualized contract value across all active subscriptions", + "displayName": "Annual Recurring Revenue", + "format": "$#,##0.00" + }, + { + "name": "new_arr", + "type": "double", + "isMeasure": true, + "displayName": "New ARR", + "format": "$#,##0.00" + }, + { + "name": "churned_arr", + "type": "double", + "isMeasure": true, + "displayName": "Churned ARR", + "format": "$#,##0.00" + } + ], + "dimensions": [ + { + "name": "region", + "type": "string", + "isMeasure": false, + "displayName": "Region" + }, + { + "name": "segment", + "type": "string", + "isMeasure": false, + "displayName": "Customer Segment" + }, + { + "name": "created_at", + "type": "timestamp_ltz", + "isMeasure": false, + "displayName": "Subscription Start", + "timeGrains": [ + "day", + "hour", + "minute", + "month", + "quarter", + "week", + "year" + ] + } + ] + }, + "retry": false + } + } +} \ No newline at end of file diff --git a/biome.json b/biome.json index c9b8a4a1f..24ddeb018 100644 --- a/biome.json +++ b/biome.json @@ -14,6 +14,7 @@ "!**/*.d.ts", "!**/build", "!**/coverage", + "!**/.appkit", "!packages/appkit-ui/src/react/ui", "!**/routeTree.gen.ts", "!docs/.docusaurus", diff --git a/docs/docs/app-management.mdx b/docs/docs/app-management.mdx index 4aeffebba..5369783d2 100644 --- a/docs/docs/app-management.mdx +++ b/docs/docs/app-management.mdx @@ -42,6 +42,12 @@ databricks apps deploy --help ``` ::: +:::info Type cache and warehouse-less deploys + +The build step runs type generation in blocking mode. If your deploy pipeline cannot reach a SQL warehouse, the build still succeeds **only if the committed `.appkit/` type cache covers every query** — otherwise it fails rather than shipping degraded types. Commit `.appkit/` while the warehouse is still reachable, before removing warehouse access from CI/CD. See [Type generation → Deploying without warehouse access](./development/type-generation.md#deploying-without-warehouse-access). + +::: + ### Examples - Deploy to a specific target: diff --git a/docs/docs/development/project-setup.md b/docs/docs/development/project-setup.md index 8aea2e14c..97d5fb2b8 100644 --- a/docs/docs/development/project-setup.md +++ b/docs/docs/development/project-setup.md @@ -24,6 +24,7 @@ my-app/ ├── config/ │ └── queries/ │ └── my_query.sql +├── .appkit/ # committed type cache (do NOT gitignore) ├── app.yaml ├── package.json └── tsconfig.json diff --git a/docs/docs/development/type-generation.md b/docs/docs/development/type-generation.md index 04b2091c6..94199cd6e 100644 --- a/docs/docs/development/type-generation.md +++ b/docs/docs/development/type-generation.md @@ -82,7 +82,27 @@ Pass `--wait` for CI and production builds, where accurate types must be present npx @databricks/appkit generate-types --wait ``` -In blocking mode the generator starts a stopped warehouse, waits (bounded) for it to reach `RUNNING`, and then describes your queries. It fails only when the configured warehouse no longer exists (deleted/deleting), so a transient outage or a cold warehouse degrades gracefully rather than breaking the build. The app template wires this up for you: `postinstall` and `predev` run the non-blocking default, while `prebuild` runs `--wait`. +In blocking mode the generator skips the warehouse entirely for any query whose types are already in the [type cache](#type-cache) (a cache **hit** makes zero warehouse round-trips). Only a cache **miss** contacts the warehouse: it starts a stopped warehouse, waits (bounded) for it to reach `RUNNING`, and then describes the query. A miss that cannot be resolved to a real type — a deleted warehouse, or (see below) a CI environment with no warehouse access — **fails the build** rather than shipping `result: unknown`. A transient connectivity blip still degrades gracefully. The app template wires this up for you: `postinstall` and `predev` run the non-blocking default, while `prebuild` runs `--wait`. + +## Type cache + +Type generation is backed by a per-app cache so that unchanged queries never pay for a repeat `DESCRIBE`. The cache lives in a committed `.appkit/` directory at your app root: + +- `.appkit/types-cache.json` — query and metric-view DESCRIBE results +- `.appkit/serving-types-cache.json` — model-serving endpoint schemas + +**`.appkit/` is a committed build artifact — check it into git; do not add it to `.gitignore`.** Committing it is what lets a blocking (`--wait`) build succeed with **no warehouse access**: on a fresh `git checkout` + `pnpm install`, every query is already a cache hit, so the build makes zero warehouse round-trips. (Ephemeral coordination state — a background-worker lock and the UI-variant choices file — stays under `node_modules/.databricks/appkit/` and is *not* committed.) + +The cache serializes deterministically (top-level keys sorted), so a no-op regeneration produces a no-op diff and unrelated query changes don't collide in merges. + +### Deploying without warehouse access + +If your CI/CD environment cannot reach a SQL warehouse (for example, an authorization boundary removes warehouse access from the deploy pipeline), a blocking build still works **as long as the committed `.appkit/` cache covers every query** — all hits, no warehouse needed. + +This imposes a rollout ordering: **produce and commit `.appkit/` while the warehouse is still reachable, then remove warehouse access.** A build that runs after warehouse access is gone with an uninitialized or incomplete cache fails loudly (never silently degrades to `unknown`), and the error message tells you which case you hit: + +- **`No committed type cache found`** (no `.appkit/` yet) — run `generate-types --wait` against a warehouse and commit `.appkit/`. +- **`The committed .appkit/ cache is missing or stale`** (a query was added or changed since the cache was built, or its entry is missing) — regenerate against a warehouse and commit the updated `.appkit/`. ## Metric-view types diff --git a/packages/appkit/src/plugins/serving/serving.ts b/packages/appkit/src/plugins/serving/serving.ts index 37707778e..d9d0b51f7 100644 --- a/packages/appkit/src/plugins/serving/serving.ts +++ b/packages/appkit/src/plugins/serving/serving.ts @@ -1,4 +1,3 @@ -import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import type express from "express"; @@ -9,6 +8,7 @@ import { createLogger } from "../../logging"; import { type ExecutionResult, Plugin, toPlugin } from "../../plugin"; import type { PluginManifest, ResourceRequirement } from "../../registry"; import { ResourceType } from "../../registry"; +import { getServingCachePath } from "../../type-generator/serving/cache"; import { servingInvokeDefaults } from "./defaults"; import manifest from "./manifest.json"; import { filterRequestBody, loadEndpointSchemas } from "./schema-filter"; @@ -66,14 +66,7 @@ export class ServingPlugin extends Plugin { } async setup(): Promise { - const cacheFile = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", - ".appkit-serving-types-cache.json", - ); - this.schemaAllowlists = await loadEndpointSchemas(cacheFile); + this.schemaAllowlists = await loadEndpointSchemas(getServingCachePath()); if (this.schemaAllowlists.size > 0) { logger.debug( "Loaded schema allowlists for %d endpoint(s)", diff --git a/packages/appkit/src/plugins/ui-variants/choice-sink.ts b/packages/appkit/src/plugins/ui-variants/choice-sink.ts index 271700da8..2fa966e67 100644 --- a/packages/appkit/src/plugins/ui-variants/choice-sink.ts +++ b/packages/appkit/src/plugins/ui-variants/choice-sink.ts @@ -1,8 +1,9 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { getEphemeralStateDir } from "shared/cli/commands/cache-paths"; /** - * Path of the JSONL choices file, relative to the app's working directory. + * Absolute path of the JSONL choices file, under `node_modules/`. * A coding agent reads it to pick up the developer's in-browser confirmation * and finalize the chosen variant. * @@ -15,8 +16,10 @@ import path from "node:path"; * Changing this value silently breaks that skill's file discovery — update the * skill's `find` path in the same change. */ -const UI_CHOICES_FILE = - "node_modules/.databricks/appkit/.appkit-ui-choices.jsonl"; +const UI_CHOICES_FILE = path.join( + getEphemeralStateDir(), + ".appkit-ui-choices.jsonl", +); /** * One recorded variant choice. diff --git a/packages/appkit/src/type-generator/cache.ts b/packages/appkit/src/type-generator/cache.ts index ac320f8bc..800a96bee 100644 --- a/packages/appkit/src/type-generator/cache.ts +++ b/packages/appkit/src/type-generator/cache.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { getCommittedCacheDir } from "shared/cli/commands/cache-paths"; import { createLogger } from "../logging/logger"; import type { MetricSchema } from "./mv-registry/types"; @@ -38,7 +39,7 @@ export interface MetricCacheEntry { /** * Structural gate for reviving a cached metric entry at partition time. * - * The cache file lives in `node_modules/.databricks` and is plain JSON — + * The cache file lives in the committed `.appkit/` dir and is plain JSON — * hand-edits, truncation, or a stale writer can leave entries whose shape no * longer matches {@link MetricCacheEntry}. A malformed entry must read as a * cache MISS (re-describe) rather than crash the pass or render revived @@ -87,13 +88,7 @@ interface Cache { } export const CACHE_VERSION = "3"; -const CACHE_FILE = ".appkit-types-cache.json"; -const CACHE_DIR = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", -); +const CACHE_FILE = "types-cache.json"; /** * Hash the SQL query @@ -120,9 +115,10 @@ export function metricCacheHash(source: string, lane: string): string { * @returns - the cache */ export async function loadCache(): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); + const cacheDir = getCommittedCacheDir(); + const cachePath = path.join(cacheDir, CACHE_FILE); try { - await fs.mkdir(CACHE_DIR, { recursive: true }); + await fs.mkdir(cacheDir, { recursive: true }); const raw = await fs.readFile(cachePath, "utf8"); const cache = JSON.parse(raw) as Cache; @@ -138,10 +134,53 @@ export async function loadCache(): Promise { } /** - * Save the cache to the file system + * Save the cache to the file system with deterministic serialization + * (sorted top-level keys for no-op regen diffs and merge-friendly output). * @param cache - cache object to save */ export async function saveCache(cache: Cache): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); - await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); + const cacheDir = getCommittedCacheDir(); + const cachePath = path.join(cacheDir, CACHE_FILE); + + // Ensure the cache directory exists + await fs.mkdir(cacheDir, { recursive: true }); + + // Rebuild cache with sorted top-level record keys for deterministic output. + // This ensures a no-op regen produces an identical file (no spurious diffs) + // and unrelated query/metric changes don't collide in git merges. + const sorted: Cache = { + version: cache.version, + queries: Object.fromEntries( + Object.keys(cache.queries) + .sort() + .map((key) => [key, cache.queries[key]]), + ), + }; + + // Include sorted metrics if present + if (cache.metrics) { + const metrics = cache.metrics; + sorted.metrics = Object.fromEntries( + Object.keys(metrics) + .sort() + .map((key) => [key, metrics[key]]), + ); + } + + await fs.writeFile(cachePath, JSON.stringify(sorted, null, 2), "utf8"); +} + +/** + * Whether the committed query/metric cache file exists on disk. Lets the + * blocking-mode fatal path distinguish an uninitialized cache (bootstrap: + * run `generate-types --wait` against a warehouse and commit `.appkit/`) from + * a drifted key (regenerate the affected query/metric). + */ +export async function queryCacheFileExists(): Promise { + try { + await fs.access(path.join(getCommittedCacheDir(), CACHE_FILE)); + return true; + } catch { + return false; + } } diff --git a/packages/appkit/src/type-generator/index.ts b/packages/appkit/src/type-generator/index.ts index 2ce7f611d..237125482 100644 --- a/packages/appkit/src/type-generator/index.ts +++ b/packages/appkit/src/type-generator/index.ts @@ -9,6 +9,7 @@ import { loadCache, type MetricCacheEntry, metricCacheHash, + queryCacheFileExists, saveCache, } from "./cache"; import { getErrorDiagnostic, isConnectivityError } from "./errors"; @@ -174,7 +175,29 @@ export class TypegenSyntaxError extends Error { export class TypegenFatalError extends Error { readonly queries: QueryFatalError[]; - constructor(queries: QueryFatalError[], warehouseId?: string) { + constructor( + queries: QueryFatalError[], + warehouseId?: string, + cacheInitialized?: boolean, + ) { + // The cache snapshot is optional: callers whose failure relates to the type + // cache pass it to get bootstrap/drift remediation; callers with an unrelated + // fatal (e.g. a malformed definitions.json) omit it and get generic wording. + const nextStep = + cacheInitialized === undefined + ? // No cache snapshot: the failure is unrelated to the cache — don't + // misdirect the operator toward regenerating/committing `.appkit/`. + `Fix the ${pc.bold("FATAL")} error(s) above, then re-run ${pc.bold("generate-types")}.` + : // Distinguish bootstrap (no committed cache yet) from drift (cache exists but is stale/missing key). + // Bootstrap → operator needs to initialize; drift → operator needs to regenerate the affected key. + cacheInitialized + ? // Drift: cache file exists but is missing/stale for the failing query/metric. + warehouseId + ? `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against warehouse ${pc.bold(warehouseId)} and commit ${pc.bold(".appkit/")}.` + : `The committed ${pc.bold(".appkit/")} cache is missing or stale for the failed ${plural(queries.length, "query", "queries")}. Regenerate with ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")}.` + : // Bootstrap: no committed cache found; operator needs to initialize from scratch. + `No committed type cache found (${pc.bold(".appkit/")}). Run ${pc.bold("generate-types --wait")} against a warehouse and commit ${pc.bold(".appkit/")} before building without warehouse access.`; + super( formatTypegenFailureMessage({ syntaxErrors: [], @@ -187,9 +210,7 @@ export class TypegenFatalError extends Error { "authentication failure", "SDK configuration errors", ], - nextStep: warehouseId - ? `Verify access to warehouse ${pc.bold(warehouseId)} and rerun type generation.` - : "Verify warehouse access and rerun type generation.", + nextStep, }), ); this.name = "TypegenFatalError"; @@ -316,6 +337,13 @@ export async function generateFromEntryPoint(options: { logger.debug("Starting type generation..."); + // Snapshot BEFORE any generation writes the cache: generateQueriesFromDescribe + // and syncMetricViewsTypes both persist to .appkit/types-cache.json during the + // run, so reading existence at the throw site would always report "exists". + // Capturing here is what lets the fatal message tell bootstrap (no cache yet) + // apart from drift (cache present but stale/missing the failing key). + const cacheExistedAtStart = await queryCacheFileExists(); + let queryRegistry: QuerySchema[] = []; let syntaxErrors: QuerySyntaxError[] = []; let fatalErrors: QueryFatalError[] = []; @@ -391,7 +419,7 @@ export async function generateFromEntryPoint(options: { throw new TypegenSyntaxError(syntaxErrors, warehouseId, fatalErrors); } if (fatalErrors.length > 0) { - throw new TypegenFatalError(fatalErrors, warehouseId); + throw new TypegenFatalError(fatalErrors, warehouseId, cacheExistedAtStart); } logger.debug("Type generation complete!"); diff --git a/packages/appkit/src/type-generator/serving/cache.ts b/packages/appkit/src/type-generator/serving/cache.ts index dc9bf7e27..91effeb9d 100644 --- a/packages/appkit/src/type-generator/serving/cache.ts +++ b/packages/appkit/src/type-generator/serving/cache.ts @@ -1,18 +1,13 @@ import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { getCommittedCacheDir } from "shared/cli/commands/cache-paths"; import { createLogger } from "../../logging/logger"; const logger = createLogger("type-generator:serving:cache"); export const CACHE_VERSION = "1"; -const CACHE_FILE = ".appkit-serving-types-cache.json"; -const CACHE_DIR = path.join( - process.cwd(), - "node_modules", - ".databricks", - "appkit", -); +const CACHE_FILE = "serving-types-cache.json"; export interface ServingCacheEntry { hash: string; @@ -27,14 +22,24 @@ export interface ServingCache { endpoints: Record; } +/** + * Absolute path of the committed serving-types cache. The single source of + * truth shared by the writer (this module) and the runtime reader + * (serving plugin `setup()`), so a relocation can never desync them. + */ +export function getServingCachePath(): string { + return path.join(getCommittedCacheDir(), CACHE_FILE); +} + export function hashSchema(schemaJson: string): string { return crypto.createHash("sha256").update(schemaJson).digest("hex"); } export async function loadServingCache(): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); + const cachePath = getServingCachePath(); + const cacheDir = getCommittedCacheDir(); try { - await fs.mkdir(CACHE_DIR, { recursive: true }); + await fs.mkdir(cacheDir, { recursive: true }); const raw = await fs.readFile(cachePath, "utf8"); const cache = JSON.parse(raw) as ServingCache; if (cache.version === CACHE_VERSION) { @@ -50,7 +55,23 @@ export async function loadServingCache(): Promise { } export async function saveServingCache(cache: ServingCache): Promise { - const cachePath = path.join(CACHE_DIR, CACHE_FILE); - await fs.mkdir(CACHE_DIR, { recursive: true }); - await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8"); + const cachePath = getServingCachePath(); + const cacheDir = getCommittedCacheDir(); + await fs.mkdir(cacheDir, { recursive: true }); + + // Sort endpoint keys for deterministic output (merge-friendly diffs) + const sortedCache: ServingCache = { + version: cache.version, + endpoints: Object.keys(cache.endpoints) + .sort() + .reduce( + (acc, key) => { + acc[key] = cache.endpoints[key]; + return acc; + }, + {} as Record, + ), + }; + + await fs.writeFile(cachePath, JSON.stringify(sortedCache, null, 2), "utf8"); } diff --git a/packages/appkit/src/type-generator/serving/tests/cache.test.ts b/packages/appkit/src/type-generator/serving/tests/cache.test.ts index 0c99c997f..107a680f4 100644 --- a/packages/appkit/src/type-generator/serving/tests/cache.test.ts +++ b/packages/appkit/src/type-generator/serving/tests/cache.test.ts @@ -81,27 +81,43 @@ describe("serving cache", () => { }); describe("saveServingCache", () => { - test("writes cache to file", async () => { + test("writes cache to file with sorted endpoints", async () => { vi.mocked(fs.writeFile).mockResolvedValue(); const cache: ServingCache = { version: CACHE_VERSION, endpoints: { - test: { + zebra: { hash: "xyz", requestType: "{}", responseType: "{}", chunkType: null, requestKeys: [], }, + apple: { + hash: "abc", + requestType: "{}", + responseType: "{}", + chunkType: null, + requestKeys: [], + }, }, }; await saveServingCache(cache); + // Endpoints should be sorted alphabetically in the output + const expectedSorted: ServingCache = { + version: CACHE_VERSION, + endpoints: { + apple: cache.endpoints.apple, + zebra: cache.endpoints.zebra, + }, + }; + expect(fs.writeFile).toHaveBeenCalledWith( - expect.stringContaining(".appkit-serving-types-cache.json"), - JSON.stringify(cache, null, 2), + expect.stringContaining("serving-types-cache.json"), + JSON.stringify(expectedSorted, null, 2), "utf8", ); }); diff --git a/packages/appkit/src/type-generator/tests/cache-serialization.test.ts b/packages/appkit/src/type-generator/tests/cache-serialization.test.ts new file mode 100644 index 000000000..c67a4e8f6 --- /dev/null +++ b/packages/appkit/src/type-generator/tests/cache-serialization.test.ts @@ -0,0 +1,192 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + access: vi.fn(), + mkdir: vi.fn(), + writeFile: vi.fn(), +})); + +vi.mock("node:fs/promises", () => ({ + default: { + access: mocks.access, + mkdir: mocks.mkdir, + writeFile: mocks.writeFile, + }, +})); + +// Mock getCommittedCacheDir to return a consistent test path. +// cache.ts imports it from the Node-only "shared/cli/commands/cache-paths" +// subpath (kept out of the client-safe "shared" root barrel), so the mock must +// target that exact specifier. +vi.mock("shared/cli/commands/cache-paths", () => ({ + getCommittedCacheDir: () => "/test/app/.appkit", +})); + +const { saveCache, queryCacheFileExists } = await import("../cache"); + +describe("cache serialization", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test("saves cache with sorted top-level query keys", async () => { + const cache = { + version: "3", + queries: { + zeta: { hash: "hash-z", type: "type-z", retry: false }, + alpha: { hash: "hash-a", type: "type-a", retry: false }, + mid: { hash: "hash-m", type: "type-m", retry: false }, + }, + }; + + mocks.mkdir.mockResolvedValue(undefined); + mocks.writeFile.mockResolvedValue(undefined); + + await saveCache(cache); + + // Verify mkdir was called with the correct directory + expect(mocks.mkdir).toHaveBeenCalledWith("/test/app/.appkit", { + recursive: true, + }); + + // Capture the JSON string passed to writeFile + const writeFileCall = mocks.writeFile.mock.calls[0]; + expect(writeFileCall).toBeDefined(); + const writtenJson = writeFileCall[1] as string; + const parsed = JSON.parse(writtenJson); + + // Assert top-level query keys are sorted ascending + const queryKeys = Object.keys(parsed.queries); + expect(queryKeys).toEqual(["alpha", "mid", "zeta"]); + + // Assert each key's value is preserved + expect(parsed.queries.alpha.type).toBe("type-a"); + expect(parsed.queries.mid.type).toBe("type-m"); + expect(parsed.queries.zeta.type).toBe("type-z"); + }); + + test("saves cache with sorted top-level metric keys", async () => { + const cache = { + version: "3", + queries: { + queryA: { hash: "hash-qa", type: "type-qa", retry: false }, + }, + metrics: { + zeta_metric: { + hash: "hash-zm", + schema: { + key: "zeta_metric", + source: "schema1", + lane: "sp" as const, + measures: [{ name: "meas1", type: "double", isMeasure: true }], + dimensions: [{ name: "dim1", type: "string", isMeasure: false }], + }, + retry: false, + }, + alpha_metric: { + hash: "hash-am", + schema: { + key: "alpha_metric", + source: "schema2", + lane: "obo" as const, + measures: [{ name: "meas2", type: "int", isMeasure: true }], + dimensions: [{ name: "dim2", type: "timestamp", isMeasure: false }], + }, + retry: false, + }, + }, + }; + + mocks.mkdir.mockResolvedValue(undefined); + mocks.writeFile.mockResolvedValue(undefined); + + await saveCache(cache); + + const writeFileCall = mocks.writeFile.mock.calls[0]; + const writtenJson = writeFileCall[1] as string; + const parsed = JSON.parse(writtenJson); + + // Assert top-level metric keys are sorted ascending + const metricKeys = Object.keys(parsed.metrics); + expect(metricKeys).toEqual(["alpha_metric", "zeta_metric"]); + + // Assert each metric's values are preserved + expect(parsed.metrics.alpha_metric.schema.key).toBe("alpha_metric"); + expect(parsed.metrics.zeta_metric.schema.key).toBe("zeta_metric"); + }); + + test("preserves nested entry values (measures, dimensions, schema fields) in original order", async () => { + const measureArray = [ + { name: "measure1", type: "double", isMeasure: true }, + { name: "measure2", type: "int", isMeasure: true }, + ]; + const dimensionArray = [ + { name: "dim1", type: "string", isMeasure: false }, + { name: "dim2", type: "timestamp", isMeasure: false }, + ]; + + const cache = { + version: "3", + queries: {}, + metrics: { + my_metric: { + hash: "test-hash", + schema: { + key: "my_metric", + source: "test_source", + lane: "sp" as const, + measures: measureArray, + dimensions: dimensionArray, + degraded: false, + }, + retry: false, + }, + }, + }; + + mocks.mkdir.mockResolvedValue(undefined); + mocks.writeFile.mockResolvedValue(undefined); + + await saveCache(cache); + + const writeFileCall = mocks.writeFile.mock.calls[0]; + const writtenJson = writeFileCall[1] as string; + const parsed = JSON.parse(writtenJson); + + // Assert nested array order is preserved + expect(parsed.metrics.my_metric.schema.measures).toEqual(measureArray); + expect(parsed.metrics.my_metric.schema.dimensions).toEqual(dimensionArray); + + // Assert other schema fields are preserved + expect(parsed.metrics.my_metric.schema.degraded).toBe(false); + }); + + test("queryCacheFileExists returns true when file exists", async () => { + mocks.access.mockResolvedValue(undefined); + + const result = await queryCacheFileExists(); + + expect(result).toBe(true); + expect(mocks.access).toHaveBeenCalledWith( + "/test/app/.appkit/types-cache.json", + ); + }); + + test("queryCacheFileExists returns false when file does not exist", async () => { + const error = new Error("ENOENT"); + (error as NodeJS.ErrnoException).code = "ENOENT"; + mocks.access.mockRejectedValue(error); + + const result = await queryCacheFileExists(); + + expect(result).toBe(false); + }); + + test("queryCacheFileExists returns false for any error", async () => { + mocks.access.mockRejectedValue(new Error("Some other error")); + + const result = await queryCacheFileExists(); + + expect(result).toBe(false); + }); +}); diff --git a/packages/appkit/src/type-generator/tests/generate-queries.test.ts b/packages/appkit/src/type-generator/tests/generate-queries.test.ts index 22a054f82..c8208e3cb 100644 --- a/packages/appkit/src/type-generator/tests/generate-queries.test.ts +++ b/packages/appkit/src/type-generator/tests/generate-queries.test.ts @@ -712,6 +712,37 @@ describe("generateQueriesFromDescribe", () => { expect(syntaxErrors).toEqual([]); }); + test("blocking mode all-hit: makes ZERO warehouse round-trips on cache HIT", async () => { + const sql1 = "SELECT id FROM t1"; + const sql2 = "SELECT name FROM t2"; + mocks.readdir.mockResolvedValue(["t1.sql", "t2.sql"]); + mocks.readFile.mockResolvedValueOnce(sql1).mockResolvedValueOnce(sql2); + // All queries hit the cache with matching hashes. + mocks.loadCache.mockReturnValueOnce({ + version: CACHE_VERSION, + queries: { + t1: { hash: hashSQL(sql1), type: CACHED_GOOD_TYPE, retry: false }, + t2: { hash: hashSQL(sql2), type: CACHED_GOOD_TYPE, retry: false }, + }, + }); + + const { schemas, syntaxErrors, fatalErrors } = await describeQueries( + "/queries", + "wh-123", + ); + + // ZERO warehouse round-trips: no preflight probe (getWarehouse never called), + // no warehouse start, no DESCRIBE executions. All queries served from cache. + expect(mocks.getWarehouse).not.toHaveBeenCalled(); + expect(mocks.startWarehouse).not.toHaveBeenCalled(); + expect(mocks.executeStatement).not.toHaveBeenCalled(); + expect(schemas).toHaveLength(2); + expect(schemas[0].type).toBe(CACHED_GOOD_TYPE); + expect(schemas[1].type).toBe(CACHED_GOOD_TYPE); + expect(syntaxErrors).toEqual([]); + expect(fatalErrors).toEqual([]); + }); + test("stale retry-flagged cache entry is re-described, not reused", async () => { const sql = "SELECT id FROM t"; mocks.readdir.mockResolvedValue(["t.sql"]); diff --git a/packages/appkit/src/type-generator/tests/index.test.ts b/packages/appkit/src/type-generator/tests/index.test.ts index efcc60f44..ae3ae46f5 100644 --- a/packages/appkit/src/type-generator/tests/index.test.ts +++ b/packages/appkit/src/type-generator/tests/index.test.ts @@ -11,12 +11,19 @@ import { } from "vitest"; import type { DatabricksStatementExecutionResponse } from "../types"; +// picocolors emits ANSI color codes when CI is set; strip them (ESC + the +// bracketed SGR sequence) so message-content assertions match regardless of +// the color environment. +// biome-ignore lint/suspicious/noControlCharactersInRegex: matching the ESC (\x1b) byte is the point. +const stripAnsi = (s: string): string => s.replace(/\x1b\[[0-9;]*m/g, ""); + const mocks = vi.hoisted(() => ({ generateQueriesFromDescribe: vi.fn(), getWarehouseState: vi.fn(), startWarehouse: vi.fn(), waitUntilRunning: vi.fn(), executeStatement: vi.fn(), + queryCacheFileExists: vi.fn(), // In-memory stand-in for the on-disk typegen cache file. `undefined` means // "no file yet"; otherwise it holds the serialized JSON exactly as // saveCache would have written it, so load/save round-trips behave like @@ -64,6 +71,9 @@ vi.mock("../cache", async (importOriginal) => { saveCache: vi.fn(async (cache: unknown) => { mocks.cacheFile.contents = JSON.stringify(cache, null, 2); }), + queryCacheFileExists: vi.fn( + async () => mocks.cacheFile.contents !== undefined, + ), }; }); @@ -270,6 +280,75 @@ describe("generateFromEntryPoint — query failure handling", () => { expect(fs.existsSync(outFile)).toBe(true); expect(fs.readFileSync(outFile, "utf-8")).toContain("bad_auth"); }); + + test("bootstrap case: no cache at start → reports bootstrap message even if cache is written during generation", async () => { + // Start with no cache file (fresh checkout). + mocks.cacheFile.contents = undefined; + mocks.generateQueriesFromDescribe.mockImplementation(async () => { + // During generation, the query path's saveCache writes to the in-memory file. + // This simulates the bug scenario: if we checked existence AFTER this runs, + // we'd incorrectly report "drift" instead of "bootstrap". + mocks.cacheFile.contents = JSON.stringify({ + version: "3", + queries: {}, + }); + return { + schemas: [unknownSchema("query1")], + syntaxErrors: [], + fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], + }; + }); + + try { + await generateFromEntryPoint({ + outFile, + queryFolder: "/queries", + warehouseId: "wh-1", + }); + expect.fail("should have thrown TypegenFatalError"); + } catch (err) { + expect(err).toBeInstanceOf(TypegenFatalError); + // This is the critical assertion: the message must reflect the state + // BEFORE generation, not after (when the cache file now exists). + const message = stripAnsi((err as Error).message); + expect(message).toMatch(/No committed type cache found/i); + expect(message).toMatch(/generate-types --wait/i); + expect(message).toMatch(/commit \.appkit\//i); + // Must NOT say "missing or stale" (the drift message): + expect(message).not.toMatch(/missing or stale/i); + } + }); + + test("drift case: cache exists at start → reports drift message", async () => { + // Pre-seed the cache (simulating a previous successful run). + mocks.cacheFile.contents = JSON.stringify({ + version: "3", + queries: { existing_query: { hash: "abc", type: "{}", retry: false } }, + }); + mocks.generateQueriesFromDescribe.mockResolvedValue({ + schemas: [unknownSchema("query1")], + syntaxErrors: [], + fatalErrors: [{ name: "query1", message: "PERMISSION_DENIED" }], + }); + + try { + await generateFromEntryPoint({ + outFile, + queryFolder: "/queries", + warehouseId: "wh-1", + }); + expect.fail("should have thrown TypegenFatalError"); + } catch (err) { + expect(err).toBeInstanceOf(TypegenFatalError); + // Cache existed at start → report drift (stale/missing key). + const message = stripAnsi((err as Error).message); + expect(message).toMatch(/missing or stale/i); + expect(message).toMatch(/Regenerate with/i); + expect(message).toMatch(/generate-types --wait/i); + // Must NOT say "No committed type cache found" (the bootstrap message): + expect(message).not.toMatch(/No committed type cache found/i); + } + }); }); describe("generateFromEntryPoint — metric-view emission", () => { @@ -648,6 +727,11 @@ describe("generateFromEntryPoint — metric-view emission", () => { expect(error).toBeInstanceOf(TypegenFatalError); expect((error as Error).message).toContain("definitions.json"); + // A config parse error is unrelated to the type cache: the remediation must + // NOT tell the operator to regenerate/commit `.appkit/` (that's the cache + // bootstrap/drift wording, reserved for callers that pass the cache snapshot). + expect((error as Error).message).not.toContain(".appkit/"); + expect((error as Error).message).toContain("Fix the"); // Query types were written before the metric config was read. expect(fs.existsSync(outFile)).toBe(true); }); diff --git a/packages/appkit/tsconfig.json b/packages/appkit/tsconfig.json index 5265a6881..a673208b0 100644 --- a/packages/appkit/tsconfig.json +++ b/packages/appkit/tsconfig.json @@ -7,6 +7,9 @@ "@/*": ["src/*"], "@tools/*": ["../../tools/*"], "shared": ["../../packages/shared/src"], + "shared/cli/commands/cache-paths": [ + "../../packages/shared/src/cli/commands/cache-paths" + ], "@databricks/lakebase": ["../../packages/lakebase/src"] } }, diff --git a/packages/shared/package.json b/packages/shared/package.json index 7045336fc..2483ca3e1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -15,6 +15,10 @@ "development": "./src/cli/index.ts", "default": "./dist/cli/index.js" }, + "./cli/commands/cache-paths": { + "development": "./src/cli/commands/cache-paths.ts", + "default": "./dist/cli/commands/cache-paths.js" + }, "./package.json": "./package.json" }, "scripts": { @@ -35,6 +39,7 @@ "exports": { ".": "./dist/index.js", "./cli": "./dist/cli/index.js", + "./cli/commands/cache-paths": "./dist/cli/commands/cache-paths.js", "./package.json": "./package.json" } }, diff --git a/packages/shared/src/cli/commands/cache-paths.test.ts b/packages/shared/src/cli/commands/cache-paths.test.ts new file mode 100644 index 000000000..d6a6c139d --- /dev/null +++ b/packages/shared/src/cli/commands/cache-paths.test.ts @@ -0,0 +1,28 @@ +import os from "node:os"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { getCommittedCacheDir, getEphemeralStateDir } from "./cache-paths"; + +describe("cache-paths", () => { + test("getCommittedCacheDir returns the .appkit directory under root", () => { + const root = path.join(os.tmpdir(), "some-project"); + expect(getCommittedCacheDir(root)).toBe(path.join(root, ".appkit")); + }); + + test("getCommittedCacheDir defaults to process.cwd() when no argument is given", () => { + expect(getCommittedCacheDir()).toBe(path.join(process.cwd(), ".appkit")); + }); + + test("getEphemeralStateDir returns node_modules/.databricks/appkit under root", () => { + const root = path.join(os.tmpdir(), "some-project"); + expect(getEphemeralStateDir(root)).toBe( + path.join(root, "node_modules", ".databricks", "appkit"), + ); + }); + + test("getEphemeralStateDir defaults to process.cwd() when no argument is given", () => { + expect(getEphemeralStateDir()).toBe( + path.join(process.cwd(), "node_modules", ".databricks", "appkit"), + ); + }); +}); diff --git a/packages/shared/src/cli/commands/cache-paths.ts b/packages/shared/src/cli/commands/cache-paths.ts new file mode 100644 index 000000000..a370104a2 --- /dev/null +++ b/packages/shared/src/cli/commands/cache-paths.ts @@ -0,0 +1,25 @@ +import path from "node:path"; + +/** + * Home for typegen COMMITTED caches (build inputs that must survive `pnpm + * install` and travel with the app in git). Per-app, anchored at the app root. + * The two caches (query/metric + serving) live here as `.appkit/*.json`. + * + * @param rootDir - project root. Defaults to the current working directory. + * @returns absolute path to the committed cache directory. + */ +export function getCommittedCacheDir(rootDir: string = process.cwd()): string { + return path.join(rootDir, ".appkit"); +} + +/** + * Home for typegen EPHEMERAL coordination state (worker spawn lock, ui-variant + * choices). Stays under `node_modules/` — gitignored, cleared on clean install. + * Never committed. + * + * @param rootDir - project root. Defaults to the current working directory. + * @returns absolute path to the ephemeral state directory. + */ +export function getEphemeralStateDir(rootDir: string = process.cwd()): string { + return path.join(rootDir, "node_modules", ".databricks", "appkit"); +} diff --git a/packages/shared/src/cli/commands/spawn-lock.ts b/packages/shared/src/cli/commands/spawn-lock.ts index d4f86c4a5..95d197300 100644 --- a/packages/shared/src/cli/commands/spawn-lock.ts +++ b/packages/shared/src/cli/commands/spawn-lock.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { getEphemeralStateDir } from "./cache-paths.js"; /** * How long a spawn lock is considered fresh. A held lock newer than this means a @@ -17,9 +18,10 @@ export const SPAWN_LOCK_STALE_MS = 6 * 60 * 1000; /** * Resolve the on-disk path of the single-flight spawn lock for a project. * - * Lives alongside the type-generator cache (`node_modules/.databricks/appkit/`) - * so it shares the same already-creatable, gitignored, per-project location and - * doesn't introduce a new directory. The lock is keyed only by project root, so + * Lives in the ephemeral state dir (`node_modules/.databricks/appkit/`) — a + * gitignored, per-project location cleared on a clean install. This is + * transient coordination state, deliberately kept separate from the committed + * `.appkit/` type cache. The lock is keyed only by project root, so * concurrent `generate-types` invocations for the same project (postinstall + * predev, say) contend for one lock and only one wins the spawn. * @@ -28,10 +30,7 @@ export const SPAWN_LOCK_STALE_MS = 6 * 60 * 1000; */ export function getSpawnLockPath(rootDir: string): string { return path.join( - rootDir, - "node_modules", - ".databricks", - "appkit", + getEphemeralStateDir(rootDir), ".appkit-typegen-worker.lock", ); } diff --git a/packages/shared/tsdown.config.ts b/packages/shared/tsdown.config.ts index 9db6274a7..1a6684384 100644 --- a/packages/shared/tsdown.config.ts +++ b/packages/shared/tsdown.config.ts @@ -2,7 +2,22 @@ import { defineConfig } from "tsdown"; export default defineConfig({ name: "shared", - entry: ["src/index.ts", "src/cli/index.ts"], + entry: [ + "src/index.ts", + "src/cli/index.ts", + // Node-only path resolvers. Needs its own entry (→ generated + // "./cli/commands/cache-paths" export) because appkit imports it as a bare + // cross-package specifier that tsx resolves via shared's exports map at + // runtime (dev-playground's `development`-condition source run) — tsconfig + // paths don't apply there. Kept OUT of the client-safe root barrel because + // it imports node:path (would break the docs client webpack bundle). + // + // DO NOT REMOVE this export: without it the dev-playground server fails to + // boot with ERR_PACKAGE_PATH_NOT_EXPORTED, which times out the "Playground + // Integration Tests" (Playwright) CI job. Build/typecheck/unit tests all + // still pass, so only that job catches its absence. + "src/cli/commands/cache-paths.ts", + ], outDir: "dist", minify: false, format: "esm", diff --git a/template/README.md b/template/README.md index 621631c13..985998a6d 100644 --- a/template/README.md +++ b/template/README.md @@ -110,6 +110,7 @@ This creates: - `dist/server.js` - Compiled server bundle - `client/dist/` - Bundled client assets +- `.appkit/` - TypeScript type definitions cache (committed to git; do not ignore) ### Production diff --git a/tsconfig.json b/tsconfig.json index 1da2d8e99..d53ce07b7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,6 +16,9 @@ "@databricks/appkit": ["packages/appkit/*"], "@databricks/appkit-ui": ["packages/appkit-ui/*"], "shared": ["packages/shared/src"], + "shared/cli/commands/cache-paths": [ + "packages/shared/src/cli/commands/cache-paths" + ], "@tools/*": ["tools/*"] } },