From 8a109f40b16e0a65a682ce634ad526d5a3c97598 Mon Sep 17 00:00:00 2001
From: Srinivas Vaddi <38348871+vaddisrinivas@users.noreply.github.com>
Date: Wed, 5 Aug 2026 11:06:11 -0400
Subject: [PATCH] feat(kernel): split runtime modules and add
transport/rollback checks
---
.github/workflows/expo-quality.yml | 9 +
README.md | 399 +---------
app/_layout.tsx | 19 +-
app/apps/[installationId].tsx | 86 ++-
app/index.tsx | 8 +-
cloudflare/utopia-registry-worker.ts | 208 +++---
package.json | 5 +-
scripts/import-corpus.ts | 374 ++++------
scripts/quality/check-doc-links.mjs | 58 ++
scripts/quality/check-kernel-v2-size.mjs | 2 +-
scripts/quality/report-catalog-similarity.mjs | 402 ++++------
src/kernel/capabilities.tsx | 493 ++++++++++---
src/kernel/capability-state.ts | 7 +-
src/kernel/computed.ts | 579 +++++++++++++++
src/kernel/data-home.ts | 698 +++++++++++++++---
src/kernel/layout.ts | 72 +-
src/kernel/operations.ts | 238 ++++++
src/kernel/persistence.ts | 151 +++-
src/kernel/policy.ts | 85 ++-
src/kernel/query.ts | 89 ++-
src/kernel/record-widgets.tsx | 89 ++-
src/kernel/registry.ts | 45 +-
src/kernel/render.tsx | 60 +-
src/kernel/runtime.ts | 16 +-
src/kernel/security.ts | 154 ++++
src/kernel/services.ts | 338 ++++-----
src/kernel/standard-widgets.tsx | 205 +++--
src/kernel/storage.macos.ts | 13 +-
src/kernel/storage.native.ts | 18 +-
src/kernel/store.tsx | 2 +-
src/kernel/theme.tsx | 48 +-
src/kernel/widget-support.ts | 35 +-
src/kernel/workflows.ts | 154 ++++
tests/kernel-v2/actions-query.test.ts | 50 ++
tests/kernel-v2/capability-flows.test.tsx | 10 +
...t.ts => chat-rollback-idempotency.test.ts} | 26 +-
tests/kernel-v2/chat-send.test.ts | 36 +
tests/kernel-v2/computed.test.ts | 56 ++
tests/kernel-v2/data-homes.test.ts | 211 ++++--
tests/kernel-v2/legacy-parity.test.ts | 16 +-
tests/kernel-v2/operations.test.ts | 81 ++
tests/kernel-v2/permission-bootstrap.test.ts | 173 +++++
tests/kernel-v2/persistence.test.ts | 29 +
tests/kernel-v2/policy.test.ts | 14 +-
tests/kernel-v2/registry-worker.test.ts | 84 ++-
tests/kernel-v2/registry.test.ts | 20 +-
tests/kernel-v2/services.test.ts | 8 +-
tests/kernel-v2/shell-route.test.ts | 13 +
tests/kernel-v2/visual-parity.test.ts | 73 +-
tests/kernel-v2/widget-primitives.test.ts | 81 ++
tests/kernel-v2/workflow.test.ts | 75 ++
tests/mocks/expo-crypto.ts | 10 +-
52 files changed, 4468 insertions(+), 1757 deletions(-)
create mode 100644 scripts/quality/check-doc-links.mjs
create mode 100644 src/kernel/computed.ts
create mode 100644 src/kernel/operations.ts
create mode 100644 src/kernel/security.ts
create mode 100644 src/kernel/workflows.ts
rename tests/kernel-v2/{server.test.ts => chat-rollback-idempotency.test.ts} (69%)
create mode 100644 tests/kernel-v2/chat-send.test.ts
create mode 100644 tests/kernel-v2/computed.test.ts
create mode 100644 tests/kernel-v2/operations.test.ts
create mode 100644 tests/kernel-v2/permission-bootstrap.test.ts
create mode 100644 tests/kernel-v2/shell-route.test.ts
create mode 100644 tests/kernel-v2/widget-primitives.test.ts
create mode 100644 tests/kernel-v2/workflow.test.ts
diff --git a/.github/workflows/expo-quality.yml b/.github/workflows/expo-quality.yml
index db853eb..677aff2 100644
--- a/.github/workflows/expo-quality.yml
+++ b/.github/workflows/expo-quality.yml
@@ -70,12 +70,21 @@ jobs:
- name: Validate domain package
run: npm run config:validate
+ - name: Check markdown links
+ run: npm run check:doc-links
+
- name: Type-check
run: npm run typecheck
- name: Run unit tests
run: npm test
+ - name: Run Phase 3 chat send gate
+ run: npm run phase3:check:chat-send
+
+ - name: Run Phase 3 chat rollback/idempotency gate
+ run: npm run phase3:check:chat-rollback-idempotency
+
- name: Validate Expo dependencies
run: npm run doctor
diff --git a/README.md b/README.md
index ca30502..e958a2d 100644
--- a/README.md
+++ b/README.md
@@ -1,383 +1,66 @@
# Utopia
-Utopia is a package-driven app platform for personal software.
+Utopia is a JSON-driven app shell for reusable product runtimes.
-The core idea is simple: a small native shell runs many useful apps from validated JSON packages. Data, screens, widgets, actions, permissions, provider connections, and AI behavior are described by app config instead of scattered bespoke screens.
+- JSON + registry entries define app behavior.
+- A shared kernel renders packages through generic widgets.
+- Tests and registry checks are the admission gate.
-Utopia is for one person first, then families, groups, and small companies as sync, sharing, roles, and recovery harden.
+Licensed under [PolyForm Noncommercial 1.0.0](./LICENSE).
-## License
-
-Utopia is source-available under the [PolyForm Noncommercial License 1.0.0](./LICENSE).
-
-Noncommercial personal, educational, research, charitable, government, and hobby use is permitted. Commercial use requires separate permission.
-
-Earlier public revisions released under Apache-2.0 remain under their original terms; this license applies from the commit that changed `LICENSE` forward.
-
-## Why this exists
-
-Most personal software is trapped between two bad choices:
-
-- rigid SaaS apps that almost fit;
-- custom code that becomes expensive to maintain.
-
-Utopia aims for a third shape:
-
-- install an app package;
-- connect the places where your data already lives;
-- let AI help change the package safely;
-- keep the native shell boring, stable, and reusable.
-
-The long-term target is a platform where each new app package makes the shell more reusable, not more bespoke. The measured goal is simple: more apps should ship as package-only JSON, and any shell growth should become a reusable runtime capability.
-
-## What it can build
-
-Utopia is strongest today for structured personal database and workflow apps:
-
-- food, pantry, recipes, meal planning, shopping;
-- home inventory;
-- plant care;
-- personal chores;
-- habit tracking;
-- trip planning;
-- small-team operating dashboards;
-- lightweight CRM;
-- collections, wishlists, reviews;
-- routines, checklists, logs, calendars, feeds, boards, charts.
-
-It is expanding into tool-shaped apps and lightweight interactive widgets, but “any app” is not proven yet.
-
-Bundled app packages:
-
-- [apps/food/food.v1.json](./apps/food/food.v1.json) — Food reference app.
-- [apps/scientific-calculator/scientific-calculator.v1.json](./apps/scientific-calculator/scientific-calculator.v1.json) — calculator tool app.
-- [apps/audio-loop-108/audio-loop-108.v1.json](./apps/audio-loop-108/audio-loop-108.v1.json) — local audio loop tool app.
-- [apps/habit-grid/habit-grid.v1.json](./apps/habit-grid/habit-grid.v1.json) — package-only habit tracker.
-- [apps/expense-splitter/expense-splitter.v1.json](./apps/expense-splitter/expense-splitter.v1.json) — package-only grouped balances and settlement proof.
-- [apps/split-rent/split-rent.v1.json](./apps/split-rent/split-rent.v1.json) — package-only weighted allocation proof.
-- [apps/workout-logger/workout-logger.v1.json](./apps/workout-logger/workout-logger.v1.json) — package-only persisted timed-flow proof.
-- [apps/focus-intervals/focus-intervals.v1.json](./apps/focus-intervals/focus-intervals.v1.json) — package-only interval-cycle proof.
-
-The 50-app adversarial suite lives under
-[`tests/fixtures/adversarial-apps/`](./tests/fixtures/adversarial-apps) as test
-input, not bundled products.
-
-The platform generalization scorecard tracks whether new apps need domain-specific renderer work or reusable shell capabilities: [docs/platform-generalization-scorecard.md](./docs/platform-generalization-scorecard.md).
-
-## The model
-
-```mermaid
-flowchart LR
- Registry["App registry URL"] --> Package["Validated app package JSON"]
- Package --> Shell["Utopia native shell"]
- Package --> Renderer["JSON-render UI"]
- Package --> Data["Local SQLite records"]
- Package --> Providers["Notion / Sheets / Drive-style homes"]
- Package --> AI["AI assistant + proposals"]
- AI --> Approval["Review / approval"]
- Approval --> Kernel["Canonical operation kernel"]
- Kernel --> Data
- Kernel --> Providers
-```
-
-## Current shape
-
-| Layer | Current status |
-|---|---|
-| Android | Native Expo / React Native shell |
-| iOS | Native Xcode project generated |
-| Web | Static Expo web export |
-| macOS | Native React Native macOS shell/prototype with JSON rendering and local media bridge; not release-proven yet |
-| UI | JSON-render powered surfaces with a widget registry |
-| Data | Local SQLite operation store |
-| Providers | Notion / Google Sheets style external homes |
-| AI | Assistant and package/data proposal path |
-| App install | Registry URL, app list, preview, approval, launch |
-
-## Important files
-
-### App JSON
-
-- [apps/food/food.v1.json](./apps/food/food.v1.json) — first app package/domain.
-- [apps/scientific-calculator/scientific-calculator.v1.json](./apps/scientific-calculator/scientific-calculator.v1.json) — non-Food tool package.
-- [apps/audio-loop-108/audio-loop-108.v1.json](./apps/audio-loop-108/audio-loop-108.v1.json) — non-Food media tool package.
-- [apps/habit-grid/habit-grid.v1.json](./apps/habit-grid/habit-grid.v1.json) — first package-only proof app.
-- [apps/expense-splitter/expense-splitter.v1.json](./apps/expense-splitter/expense-splitter.v1.json) — grouped balances and deterministic settlement proof.
-- [apps/split-rent/split-rent.v1.json](./apps/split-rent/split-rent.v1.json) — exact weighted-allocation proof.
-- [apps/workout-logger/workout-logger.v1.json](./apps/workout-logger/workout-logger.v1.json) — persisted flow/timer proof.
-- [packages/domain-config/domain-catalog.v1.json](./packages/domain-config/domain-catalog.v1.json) — active catalog and shell tabs.
-- [packages/domain-config/domains/food.v1.json](./packages/domain-config/domains/food.v1.json) — bundled Food domain config.
-- [packages/domain-config/domains/health.v1.json](./packages/domain-config/domains/health.v1.json) — Health preview.
-- [packages/domain-config/domains/plants.v1.json](./packages/domain-config/domains/plants.v1.json) — Plants preview.
-
-### Contracts
-
-- [packages/domain-config/schemas/domain.v1.schema.json](./packages/domain-config/schemas/domain.v1.schema.json)
-- [packages/domain-config/schemas/domain-catalog.v1.schema.json](./packages/domain-config/schemas/domain-catalog.v1.schema.json)
-- [packages/domain-config/schemas/workflow.v1.schema.json](./packages/domain-config/schemas/workflow.v1.schema.json)
-- [packages/domain-config/schemas/record.v1.schema.json](./packages/domain-config/schemas/record.v1.schema.json)
-
-### App factory pieces
-
-- [docs/github-app-factory.md](./docs/github-app-factory.md) — fork + `OPENAI_API_KEY` + natural-language app generation workflow.
-- [docs/adversarial-app-tests.md](./docs/adversarial-app-tests.md) — adversarial runtime probes and current partial results.
-- [docs/adversarial-app-matrix.json](./docs/adversarial-app-matrix.json) — checked 50-app falsification matrix; every row points to a test fixture.
-- [requests/app-idea.md](./requests/app-idea.md) — plain-English request template for the GitHub workflow.
-- [scripts/factory/generate-app-from-prompt.ts](./scripts/factory/generate-app-from-prompt.ts) — OpenAI structured-output generator for reviewable app packages.
-- [packages/domain-config/templates/utopia-data-plane-template.v1.json](./packages/domain-config/templates/utopia-data-plane-template.v1.json)
-- [packages/domain-config/templates/package-change-templates/package-change-blueprints.v1.json](./packages/domain-config/templates/package-change-templates/package-change-blueprints.v1.json)
-- [packages/domain-config/templates/package-change-templates/widget-screen-intents.v1.json](./packages/domain-config/templates/package-change-templates/widget-screen-intents.v1.json)
-
-### Registry install
-
-- [app/install.tsx](./app/install.tsx) — app picker / install screen.
-- [src/domain/package-install.ts](./src/domain/package-install.ts) — registry fetch, package preview, trust labels.
-- [tests/fixtures/package-install/registry.json](./tests/fixtures/package-install/registry.json) — registry fixture.
-- [tests/fixtures/package-install/valid-package.json](./tests/fixtures/package-install/valid-package.json) — installable package fixture.
-
-## Food app
-
-Food is the proof app.
-
-It is intended to feel like a focused AI-native kitchen system:
-
-- today’s meal plan;
-- seven-day planning;
-- pantry/fridge/freezer/shelf views;
-- use-first food;
-- recipes and recipe revisions;
-- shopping list and receipts;
-- nutrition observations;
-- Notion / Sheets data homes;
-- assistant workflows for “what can I cook tonight?” and “use these first.”
-
-### Data-home selection behavior
-
-Data-home options come from two checks only:
-
-- declared by app manifest (`data_homes` / `data-home:*` capability);
-- runtime availability from the adapter registry (`ready`, `requires_auth`, `offline`, `blocked`, or `unsupported`).
-
-Rows that are unavailable must remain visible with truthy reasons and `canSelect: false`.
-
-- not configured by app -> `... is not configured by this app.`
-- runtime offline -> `... is currently offline.`
-- auth required -> `... needs sign-in before use.`
-- unsupported at runtime -> `... is not supported by this runtime.`
-
-This contract is deterministic metadata; it does not assert live upstream provider proof by itself.
-
-## Widget surface
-
-The renderer supports a growing widget catalog. The product goal is not “more widgets forever”; it is fewer domain-specific widgets, stronger generic primitives, and explicit reusable runtime capabilities when JSON alone is not enough.
-
-Current generic/package-level widgets include:
-
-- assistant chat;
-- calendar block;
-- smart capture;
-- provider status;
-- data home settings;
-- AI provider settings;
-- theme and density controls;
-- health permissions;
-- posts;
-- polls;
-- feeds;
-- checklist cards;
-- charts;
-- galleries;
-- schema editor;
-- widget catalog;
-- file picker/export;
-- video player;
-- camera scanner;
-- location/map;
-- sensor readout;
-- local notifications;
-- contact picker;
-- calendar event;
-- biometric gate;
-- health status;
-- speech tool.
-
-Known renderer debt:
-
-- Food now uses shared `recordHeroSummary`, `groupedRecordShelf`, `horizontalRecordCarousel`, `recordTimeline`, `recordContentCard`, and `recordReviewCard` primitives; `askFoodBar` remains a separate assistant surface.
-- Calculator and Audio Loop prove non-Food apps, but each uses a specialized reusable runtime widget: `scientificCalculator`, `audioLoopPlayer`.
-- Habit Grid is the first package-only proof app: it uses existing `chartBlock`, `checklistCard`, `dataTable`, and `recordList` primitives.
-- Expense Splitter and Split Rent are package-only expression proofs using the
- shared deterministic kernel and existing `recordList` and `dataTable` UI.
-- Workout Logger uses generic `stepFlow` and `durationTimer` widgets backed by
- the persisted workflow journal.
-- Focus Intervals reuses the same flow/timer contract for a second domain with
- no new runtime primitive.
-- The scorecard must trend toward package-only apps and reusable capabilities, not app-specific shell growth.
-
-Rule of thumb: JSON can configure any capability the renderer already exposes. New behavior belongs in generic widgets, not one-off app screens.
-
-## App registry
-
-Utopia already has the basic install path:
-
-1. open Install;
-2. set a registry URL;
-3. fetch available app packages;
-4. preview screens, collections, widgets, providers, permissions, plugins;
-5. approve install;
-6. launch the installed app.
-
-This should evolve into a polished App Library with screenshots, categories, trust badges, permissions, install/open/remove, and shareable registries.
-
-## Platforms
-
-### Android
-
-```bash
-npm run android:dev
-```
-
-### iOS
-
-```bash
-npm run ios
-```
-
-Native project:
-
-```bash
-open ios/Utopia.xcodeproj
-```
-
-### Web
-
-```bash
-npm run web
-```
-
-Static export:
-
-```bash
-npm run export:web
-```
-
-### macOS
-
-There is a native React Native macOS shell/prototype.
-
-Useful commands:
-
-```bash
-npm run macos
-npm run macos:build
-```
-
-Current macOS bridge scope:
-
-- render package JSON surfaces;
-- pick/open/save local files;
-- open local video files through the native workspace bridge.
-
-It is not release-proven like Android signed build/export proof yet.
-
-## Development
-
-Install dependencies:
+## Current command surface
```bash
npm install
-```
-
-Validate config and contracts:
-
-```bash
npm run config:validate
-```
-
-Typecheck:
-
-```bash
npm run typecheck
-```
-
-Run tests:
-
-```bash
npm run test
-```
-
-Build exports:
-
-```bash
+npm run doctor
npm run export:web
npm run export:android
npm run export:ios
+npm run check:doc-links
+npm run phase3:check:chat-send
+npm run phase3:check:chat-rollback-idempotency
+npm run check:kernel-v2
```
-## Quality gates
-
-Useful focused gates:
-
-```bash
-npm run check:widget-catalog
-npm run check:platform-generalization
-npm run check:adversarial-app-matrix
-npm run materialize:adversarial-apps
-npm run check:native-capability-contract
-npm run check:package-owned-routes
-npm run check:food-app-vibe
-npm run check:link-install
-npm run check:json-render-only-ui
-```
-
-Broader gate:
-
-```bash
-npm run quality
-```
-
-## Design laws
-
-1. App behavior should live in app packages when possible.
-2. The native shell should stay small and reusable.
-3. The renderer should expose generic capabilities, not domain-specific hacks.
-4. The operation kernel is the only writer.
-5. AI proposes changes; validated contracts decide what can run.
-6. Provider sync should feel invisible, but remain verifiable.
-7. Secrets never belong in git.
-8. Prefer proven libraries over custom platform code.
-9. Generated apps should feel like real products, not config demos.
+## Current proof-facing files
-## What is not done
+- `app/_layout.tsx` — app root, theming, safe-area, router wiring.
+- `app/index.tsx` — registry catalog UI and install/launch flow.
+- `app/apps/[installationId].tsx` — installed app shell route.
+- `src/kernel/catalog.ts` — package catalog state and metadata.
+- `src/kernel/registry.ts` — catalog loading/install helpers.
+- `src/kernel/render.tsx` — generic JSON render path.
+- `src/kernel/*` — package runtime helpers.
+- `scripts/import-corpus.ts` — registry import and migration entry.
+- `scripts/validate-catalog.ts`, `scripts/generate-catalog.mjs` — canonical catalog generation.
+- `scripts/quality/check-doc-links.mjs` — local markdown link checker.
+- `scripts/quality/check-kernel-v2-size.mjs` — size/coverage budget check.
+- `scripts/quality/report-catalog-similarity.mjs` — similarity/de-duplication reports.
+- `tests/kernel-v2/*` — kernel, registry, and shell regression tests.
+- `tests/kernel-v2/chat-send.test.ts` — chat send contract gate.
+- `tests/kernel-v2/chat-rollback-idempotency.test.ts` — rollback/idempotency/security gate.
+- `tests/kernel-v2/shell-route.test.ts` — shell wrapper height regression gate.
-Utopia is not yet a finished app factory.
+## Quality gates (local truth)
-Still needed:
+`npm run config:validate` runs kernel-sized checks.
+`npm run check:kernel-v2` runs kernel tests + size guard.
-- richer App Library UX;
-- stronger package authoring flow;
-- better visual editor / AI package editor;
-- stronger generic widgets and less domain-specific renderer code;
-- declarative native permission flows per package;
-- enforced package capability boundary for untrusted registries;
-- generated-app quality evals beyond schema validity;
-- family/group sync, sharing, roles, recovery, and conflict UX proof;
-- polished provider connection UX;
-- production dependency/security cleanup;
-- physical-device release proof;
-- native HealthKit entitlement bridge;
-- speech-to-text bridge.
+Phase-3 gates are now separated:
-## Product thesis
+- `phase3:check:chat-send` validates send contract only.
+- `phase3:check:chat-rollback-idempotency` validates rollback/idempotency and scope guards.
-Utopia is not trying to be another notes app, database app, or chatbot wrapper.
+## What is not claimed
-It is trying to become a personal software substrate:
+- This README only claims files/commands that currently exist in this checkout.
+- Claims outside this list (for example, unproven platform claims, release-complete status, or extra app factories) are intentionally omitted.
-- JSON packages define apps;
-- the renderer makes them native and useful;
-- providers keep them connected to real data;
-- AI helps reshape them;
-- the kernel keeps writes safe.
+## Notes
-The dream: create one excellent shell, then ship endless excellent apps through packages.
+- Temporary size guard is set to 12k LOC (not a final target); current authored LOC is ~11.8k, so 10k target is deferred, not achieved.
+- Use the app catalog to view active/inactive apps.
+- Do not copy links to files that do not exist in-tree.
diff --git a/app/_layout.tsx b/app/_layout.tsx
index 0c9300e..6be4184 100644
--- a/app/_layout.tsx
+++ b/app/_layout.tsx
@@ -3,18 +3,21 @@ import 'react-native-gesture-handler';
import { Stack } from 'expo-router';
import { StatusBar } from 'expo-status-bar';
import { GestureHandlerRootView } from 'react-native-gesture-handler';
+import { SafeAreaProvider } from 'react-native-safe-area-context';
import { Theme } from '@/src/kernel/theme';
export default function RootLayout() {
return
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
;
}
diff --git a/app/apps/[installationId].tsx b/app/apps/[installationId].tsx
index 1d0a335..b6d85b1 100644
--- a/app/apps/[installationId].tsx
+++ b/app/apps/[installationId].tsx
@@ -2,25 +2,105 @@ import { useLocalSearchParams } from 'expo-router';
import { useRouter } from 'expo-router';
import { useEffect, useState } from 'react';
import { Button, H2, Paragraph, Spinner, YStack } from 'tamagui';
+import { Linking } from 'react-native';
+import { SafeAreaView } from 'react-native-safe-area-context';
+import { collectPendingRuntimePermissions, requestBootPermission, type PermissionRequest } from '@/src/kernel/capabilities';
import { findPackage } from '@/src/kernel/catalog';
import { PackageApp } from '@/src/kernel/render';
import type { AppPackage } from '@/src/kernel/schema';
import { AppStore } from '@/src/kernel/store';
+import { recordConsent } from '@/src/kernel/policy';
export default function AppRoute() {
const router = useRouter();
const params = useLocalSearchParams<{ installationId?: string | string[]; screen?: string | string[] }>();
const id = typeof params.installationId === 'string' ? params.installationId : '';
const screen = typeof params.screen === 'string' ? params.screen : undefined;
+ const installationId = id;
const [pkg, setPackage] = useState();
const [ready, setReady] = useState(false);
+ const [bootPermissions, setBootPermissions] = useState([]);
+ const [bootStep, setBootStep] = useState(0);
+ const [bootError, setBootError] = useState('');
+ const [bootReady, setBootReady] = useState(false);
+ const [requesting, setRequesting] = useState(false);
+ const [settingsMode, setSettingsMode] = useState(false);
+
+ const openSettings = async () => { try { await Linking.openSettings(); } catch {} };
+ const advance = () => setBootStep((value) => value + 1);
+
useEffect(() => { let active = true; void findPackage(id).then((value) => active && setPackage(value)).finally(() => active && setReady(true)); return () => { active = false; }; }, [id]);
useEffect(() => {
if (typeof document === 'undefined') return;
document.title = !ready ? 'Loading app — Utopia' : pkg ? `${pkg.presentation?.label ?? 'App'} — Utopia` : 'App unavailable — Utopia';
}, [pkg, ready]);
- if (!ready) return Loading app
Preparing app.;
- if (!pkg) return App unavailable
Unknown app.;
- return ;
+ useEffect(() => {
+ let active = true;
+ setBootReady(false);
+ if (!pkg) return;
+ void (async () => {
+ const pending = await collectPendingRuntimePermissions(installationId, pkg);
+ if (!active) return;
+ setBootPermissions(pending);
+ setBootStep(0);
+ setSettingsMode(false);
+ setBootReady(true);
+ })();
+ return () => { active = false; };
+ }, [installationId, pkg?.id]);
+
+ const current = bootPermissions[bootStep];
+
+ const requestNextPermission = async () => {
+ if (!pkg || !current) return;
+ try {
+ setRequesting(true);
+ setBootError('');
+ await requestBootPermission(installationId, current);
+ advance();
+ } catch (cause) {
+ const message = cause instanceof Error ? cause.message : 'Permission check failed';
+ setSettingsMode(message.toLowerCase().includes('denied') || message.toLowerCase().includes('cannot ask again'));
+ setBootError(message);
+ } finally {
+ setRequesting(false);
+ }
+ };
+
+ const skipPermission = async () => {
+ if (!pkg || !current) return;
+ await recordConsent(installationId, current.capability, 'denied');
+ advance();
+ };
+
+ if (!ready) return
+
+ Loading app
Preparing app.
+
+ ;
+ if (!pkg) return
+
+ App unavailable
Unknown app.
+
+
+ ;
+ if (!bootReady) return
+
+
+
+ ;
+ if (current) return
+
+ {current.permission.id}
+ {current.permission.prompt || current.permission.reason || `Allow ${current.permission.id}?`}
+ {bootError ? {bootError} : null}
+
+
+ {settingsMode ? : null}
+
+ ;
+ return ;
}
diff --git a/app/index.tsx b/app/index.tsx
index 0a28361..47dbdf2 100644
--- a/app/index.tsx
+++ b/app/index.tsx
@@ -5,7 +5,7 @@ import { FlatList, useWindowDimensions } from 'react-native';
import { Button, H1, Input, Paragraph, XStack, YStack } from 'tamagui';
import { catalog } from '@/src/kernel/catalog';
-import { install, loadRegistry, trustPublisher, type RegistryEntry } from '@/src/kernel/registry';
+import { installWithInstallationId, loadRegistry, trustPublisher, type RegistryEntry } from '@/src/kernel/registry';
export default function AppLauncher() {
const router = useRouter();
@@ -44,8 +44,10 @@ export default function AppLauncher() {
: null}
{remote.map((entry) => )}
;
return };
-type R2Bucket = {
- get(key: string): Promise;
- put(key: string, value: string, options?: { httpMetadata?: { contentType: string } }): Promise;
-};
-
-export type UtopiaRegistryEnv = {
- PACKAGES: R2Bucket;
- REGISTRY_HOST?: string;
- REGISTRY_WRITE_MODE?: 'disabled' | 'signed';
- REGISTRY_PUBLISHER_KEYS_JSON?: string;
-};
-
-const Entry = z.object({
- id: z.string().min(1),
- url: z.string().url(),
- checksum: z.string().regex(/^sha256:[a-f0-9]{64}$/),
- publisher: z.string().min(1),
- signature: z.string().regex(/^[a-f0-9]{128}$/),
-}).strict();
-const Manifest = z.object({
- schemaVersion: z.literal('utopia.registry.v1'),
- packages: z.array(Entry),
-}).strict();
-const Publish = z.object({
- package: PackageSchema,
- publisher: z.string().regex(/^[a-z0-9_.-]{1,64}$/i),
- signature: z.string().regex(/^[a-f0-9]{128}$/),
-}).strict();
-const MAX_PACKAGE_BYTES = 256 * 1024;
-const INDEX = 'registry/index.json';
-const secret = /(?:access[_-]?token|api[_-]?key|authorization|client[_-]?secret|cookie|credential|password|private[_-]?key|refresh[_-]?token|session[_-]?token)/i;
+type R2Bucket = { get(key: string): Promise<{ text(): Promise } | null>; put(key: string, value: string, options?: { httpMetadata?: { contentType: string } }): Promise };
+export type UtopiaRegistryEnv = { PACKAGES: R2Bucket; REGISTRY_HOST?: string; REGISTRY_WRITE_MODE?: 'disabled' | 'signed'; REGISTRY_PUBLISHER_KEYS_JSON?: string; REGISTRY_WRITE_ALLOWED_ORIGINS?: string };
+
+const env = { INDEX_KEY: 'registry/index.json', MAX_BYTES: 256 * 1024, HEADER: 'content-type' } as const;
+const CORS = { read: 'GET,HEAD,OPTIONS', write: 'POST,PUT,PATCH,DELETE,OPTIONS' } as const;
+const secretFields = /(?:access[_-]?token|api[_-]?key|authorization|client[_-]?secret|cookie|credential|password|private[_-]?key|refresh[_-]?token|session[_-]?token)/i;
+const origins = (value = '') => value.split(',').map((item) => item.trim()).filter(Boolean);
+const isWrite = (method: string) => method === 'POST' || method === 'PUT' || method === 'PATCH' || method === 'DELETE';
+const hex = (bytes: ArrayBuffer) => [...new Uint8Array(bytes)].map((item) => item.toString(16).padStart(2, '0')).join('');
+
+const Entry = z.object({ id: z.string().min(1), url: z.string().url(), checksum: z.string().regex(/^sha256:[a-f0-9]{64}$/), publisher: z.string().min(1), signature: z.string().regex(/^[a-f0-9]{128}$/) }).strict();
+const Manifest = z.object({ schemaVersion: z.literal('utopia.registry.v1'), packages: z.array(Entry) }).strict();
+const Publish = z.object({ package: PackageSchema, publisher: z.string().regex(/^[a-z0-9_.-]{1,64}$/i), signature: z.string().regex(/^[a-f0-9]{128}$/) }).strict();
const app = new Hono<{ Bindings: UtopiaRegistryEnv }>();
app.use('*', secureHeaders());
-app.use('/v1/*', cors({ origin: '*', allowMethods: ['GET', 'POST', 'OPTIONS'] }));
-app.use('/p/*', cors({ origin: '*', allowMethods: ['GET', 'OPTIONS'] }));
+app.use('*', async (ctx, next) => {
+ const method = ctx.req.method;
+ const origin = ctx.req.header('origin');
+ if (method === 'OPTIONS') return options(ctx);
+
+ const writeMode = ctx.env.REGISTRY_WRITE_MODE;
+ if (isWrite(method) && writeMode === 'signed') {
+ const allowed = origins(ctx.env.REGISTRY_WRITE_ALLOWED_ORIGINS);
+ if (origin && !allowed.includes(origin)) return jsonError(ctx, 403, 'registry_write_origin_denied');
+ }
-app.get('/health', (context) => context.json({ ok: true, service: 'utopia-registry-v3' }));
+ await next();
+ if (method === 'GET' || method === 'HEAD') {
+ ctx.header('access-control-allow-origin', '*');
+ ctx.header('access-control-allow-methods', CORS.read);
+ } else if (origin) {
+ const allowed = origins(ctx.env.REGISTRY_WRITE_ALLOWED_ORIGINS);
+ if (writeMode === 'signed' && !allowed.includes(origin)) return;
+ ctx.header('access-control-allow-origin', origin);
+ ctx.header('access-control-allow-methods', CORS.write);
+ ctx.header('access-control-allow-credentials', 'false');
+ }
-app.get('/v1/registry.json', async (context) => {
- return context.json(await readManifest(context.env.PACKAGES));
+ ctx.header('access-control-allow-headers', env.HEADER);
+ ctx.header('vary', 'Origin');
});
-app.get('/v1/packages/:id', async (context) => {
- const manifest = await readManifest(context.env.PACKAGES);
- const entry = manifest.packages.find(({ id }) => id === context.req.param('id'));
- return entry ? context.json(entry) : context.json({ error: 'package_not_found' }, 404);
+app.get('/health', (ctx) => ctx.json({ ok: true, service: 'utopia-registry-v3' }));
+app.get('/v1/registry.json', async (ctx) => ctx.json(await loadManifest(ctx.env.PACKAGES)));
+app.get('/v1/packages/:id', async (ctx) => {
+ const manifest = await loadManifest(ctx.env.PACKAGES);
+ const found = manifest.packages.find((entry) => entry.id === ctx.req.param('id'));
+ return found ? ctx.json(found) : jsonError(ctx, 404, 'package_not_found');
});
-
-app.get('/p/:file', async (context) => {
- const digest = (context.req.param('file') ?? '').replace(/\.json$/, '');
- if (!/^[a-f0-9]{64}$/.test(digest)) return context.json({ error: 'invalid_digest' }, 400);
- const object = await context.env.PACKAGES.get(`packages/${digest}.json`);
- if (!object) return context.json({ error: 'package_not_found' }, 404);
- return context.body(await object.text(), 200, { 'content-type': 'application/json; charset=utf-8' });
+app.get('/p/:file', async (ctx) => {
+ const file = ctx.req.param('file').replace(/\.json$/, '');
+ if (!/^[a-f0-9]{64}$/.test(file)) return jsonError(ctx, 400, 'invalid_digest');
+ const object = await ctx.env.PACKAGES.get(`packages/${file}.json`);
+ return object ? ctx.body(await object.text(), 200, { 'content-type': 'application/json; charset=utf-8' }) : jsonError(ctx, 404, 'package_not_found');
});
-app.post('/v1/packages', async (context) => {
- if (context.env.REGISTRY_WRITE_MODE !== 'signed') return context.json({ error: 'registry_writes_disabled' }, 403);
- const length = Number(context.req.header('content-length') ?? 0);
- if (length > MAX_PACKAGE_BYTES) return context.json({ error: 'package_too_large' }, 413);
- const raw = await context.req.text();
- if (new TextEncoder().encode(raw).byteLength > MAX_PACKAGE_BYTES) return context.json({ error: 'package_too_large' }, 413);
+app.post('/v1/packages', async (ctx) => {
+ if (ctx.env.REGISTRY_WRITE_MODE !== 'signed') return jsonError(ctx, 403, 'registry_writes_disabled');
+ const body = await ctx.req.text();
+ const bytes = new TextEncoder().encode(body);
+ if (Number(ctx.req.header('content-length') ?? 0) > env.MAX_BYTES || bytes.byteLength > env.MAX_BYTES) return jsonError(ctx, 413, 'package_too_large');
- const published = Publish.parse(JSON.parse(raw));
- rejectSecrets(published.package);
- const canonical = canonicalize(published.package);
- const digest = hex(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(canonical)));
- const publicKey = publisherKeys(context.env)[published.publisher];
- if (!publicKey) return context.json({ error: 'publisher_not_trusted' }, 403);
+ const payload = Publish.parse(JSON.parse(body));
+ rejectSecrets(payload.package);
+ const canonical = canonicalize(payload.package);
+ const checksum = `sha256:${hex(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(canonical)))}`;
+ const publishers = z.record(z.string(), z.string().regex(/^[a-f0-9]{64}$/)).parse(JSON.parse(ctx.env.REGISTRY_PUBLISHER_KEYS_JSON ?? '{}'));
+ const publicKey = publishers[payload.publisher];
+ if (!publicKey) return jsonError(ctx, 403, 'publisher_not_trusted');
ed.hashes.sha512Async = async (message) => new Uint8Array(await crypto.subtle.digest('SHA-512', message as BufferSource));
- const valid = await ed.verifyAsync(
- ed.etc.hexToBytes(published.signature),
- new TextEncoder().encode(canonical),
- ed.etc.hexToBytes(publicKey),
- { zip215: false },
- );
- if (!valid) return context.json({ error: 'signature_invalid' }, 403);
-
- const host = context.env.REGISTRY_HOST?.trim() || new URL(context.req.url).host;
- const entry = Entry.parse({
- id: published.package.id,
- url: `https://${host}/p/${digest}.json`,
- checksum: `sha256:${digest}`,
- publisher: published.publisher,
- signature: published.signature,
- });
- await context.env.PACKAGES.put(`packages/${digest}.json`, canonical, {
- httpMetadata: { contentType: 'application/json; charset=utf-8' },
- });
- const manifest = await readManifest(context.env.PACKAGES);
- manifest.packages = [...manifest.packages.filter(({ id }) => id !== entry.id), entry]
- .sort((left, right) => left.id.localeCompare(right.id));
- await context.env.PACKAGES.put(INDEX, JSON.stringify(manifest), {
- httpMetadata: { contentType: 'application/json; charset=utf-8' },
- });
- return context.json(entry, 201);
+ if (!await ed.verifyAsync(ed.etc.hexToBytes(payload.signature), new TextEncoder().encode(canonical), ed.etc.hexToBytes(publicKey), { zip215: false })) {
+ return jsonError(ctx, 403, 'signature_invalid');
+ }
+
+ const host = ctx.env.REGISTRY_HOST?.trim() || new URL(ctx.req.url).host;
+ const manifest = await loadManifest(ctx.env.PACKAGES);
+ const entry = Entry.parse({ id: String(payload.package.id), url: `https://${host}/p/${checksum.replace('sha256:', '')}.json`, checksum, publisher: payload.publisher, signature: payload.signature });
+ manifest.packages = [...manifest.packages.filter((item) => item.id !== entry.id), entry].sort((left, right) => left.id.localeCompare(right.id));
+ await ctx.env.PACKAGES.put(env.INDEX_KEY, JSON.stringify(manifest), { httpMetadata: { contentType: 'application/json; charset=utf-8' } });
+ await ctx.env.PACKAGES.put(`packages/${checksum.replace('sha256:', '')}.json`, canonical, { httpMetadata: { contentType: 'application/json; charset=utf-8' } });
+ return ctx.json(entry, 201);
});
-app.onError((error, context) => {
- if (error instanceof z.ZodError || error instanceof SyntaxError) {
- return context.json({ error: 'invalid_request' }, 400);
- }
+app.onError((error, ctx) => {
+ if (error instanceof z.ZodError || error instanceof SyntaxError) return jsonError(ctx, 400, 'invalid_request');
console.error(error instanceof Error ? error.message : 'registry_error');
- return context.json({ error: 'registry_error' }, 500);
+ return jsonError(ctx, 500, 'registry_error');
});
-export async function handleRequest(request: Request, env: UtopiaRegistryEnv): Promise {
- return app.fetch(request, env);
-}
-
+export async function handleRequest(req: Request, env: UtopiaRegistryEnv): Promise { return app.fetch(req, env); }
export default { fetch: handleRequest };
-async function readManifest(bucket: R2Bucket): Promise> {
- const object = await bucket.get(INDEX);
+async function loadManifest(bucket: R2Bucket): Promise> {
+ const object = await bucket.get(env.INDEX_KEY);
return object ? Manifest.parse(JSON.parse(await object.text())) : { schemaVersion: 'utopia.registry.v1', packages: [] };
}
-function publisherKeys(env: UtopiaRegistryEnv): Record {
- const keys = z.record(z.string(), z.string().regex(/^[a-f0-9]{64}$/))
- .parse(JSON.parse(env.REGISTRY_PUBLISHER_KEYS_JSON ?? '{}'));
- return keys;
-}
-
function rejectSecrets(value: unknown, depth = 0): void {
if (depth > 20) throw new Error('package_too_deep');
- if (Array.isArray(value)) return value.forEach((item) => rejectSecrets(item, depth + 1));
if (!value || typeof value !== 'object') return;
+ if (Array.isArray(value)) return value.forEach((entry) => rejectSecrets(entry, depth + 1));
for (const [key, child] of Object.entries(value)) {
- if (secret.test(key)) throw new Error('package_contains_secret');
+ if (secretFields.test(key)) throw new Error('package_contains_secret');
rejectSecrets(child, depth + 1);
}
}
-function hex(buffer: ArrayBuffer): string {
- return [...new Uint8Array(buffer)].map((byte) => byte.toString(16).padStart(2, '0')).join('');
+function options(ctx: Context<{ Bindings: UtopiaRegistryEnv }>): Response {
+ const method = (ctx.req.header('access-control-request-method') ?? '').toUpperCase();
+ const origin = ctx.req.header('origin');
+ const writeRequest = isWrite(method);
+ if (writeRequest && (ctx.env.REGISTRY_WRITE_MODE !== 'signed' || !origin || !origins(ctx.env.REGISTRY_WRITE_ALLOWED_ORIGINS).includes(origin))) {
+ return jsonError(ctx, 403, 'registry_write_origin_denied');
+ }
+ return new Response(null, {
+ status: 204,
+ headers: {
+ 'access-control-allow-origin': writeRequest ? (origin ?? '*') : '*',
+ 'access-control-allow-methods': writeRequest ? CORS.write : CORS.read,
+ 'access-control-allow-headers': ctx.req.header('access-control-request-headers') ?? env.HEADER,
+ 'access-control-allow-credentials': 'false',
+ 'access-control-max-age': '600',
+ vary: 'Origin',
+ },
+ });
}
+
+function jsonError(ctx: Context<{ Bindings: UtopiaRegistryEnv }>, status: 400 | 403 | 404 | 413 | 500, error: string): Response { return ctx.json({ error }, status); }
diff --git a/package.json b/package.json
index 87d82bb..0ed6860 100644
--- a/package.json
+++ b/package.json
@@ -109,6 +109,7 @@
"typecheck": "tsc --noEmit",
"preconfig:validate": "npm run catalog:generate",
"test": "vitest run tests/kernel-v2",
+ "check:doc-links": "node scripts/quality/check-doc-links.mjs",
"check:kernel-v2": "vitest run tests/kernel-v2 && node scripts/quality/check-kernel-v2-size.mjs",
"config:validate": "npm run check:kernel-v2",
"doctor": "expo-doctor",
@@ -116,7 +117,7 @@
"export:android": "npm run catalog:generate && expo export --platform android --output-dir dist/android",
"export:ios": "npm run catalog:generate && expo export --platform ios --output-dir dist/ios",
"server:dev": "tsx server/index.ts",
- "phase3:check:chat-send": "vitest run tests/kernel-v2/server.test.ts",
- "phase3:check:chat-rollback-idempotency": "vitest run tests/kernel-v2/server.test.ts"
+ "phase3:check:chat-send": "vitest run tests/kernel-v2/chat-send.test.ts",
+ "phase3:check:chat-rollback-idempotency": "vitest run tests/kernel-v2/chat-rollback-idempotency.test.ts"
}
}
diff --git a/scripts/import-corpus.ts b/scripts/import-corpus.ts
index 7482612..5c91546 100644
--- a/scripts/import-corpus.ts
+++ b/scripts/import-corpus.ts
@@ -5,260 +5,170 @@ import path from 'node:path';
import { PackageSchema } from '../src/kernel/schema';
import { enrichPackage } from './enrich-catalog';
-type Candidate = {
- file: string;
- origin: 'local' | 'gold-v3' | 'luna-v3' | 'gold-v2';
- package: Record;
- score: number;
-};
-
-const root = path.resolve(import.meta.dirname, '..');
-const parity = process.env.UTOPIA_PARITY_ROOT ?? '/Users/srinivasvaddi/Projects/utopia-serious-app-parity';
-const appsRepo = path.resolve(process.env.UTOPIA_APPS_REPO ?? path.join(root, '../utopia-apps'));
-const packagesRoot = path.resolve(process.env.UTOPIA_APPS_DIR ?? path.join(appsRepo, 'packages'));
-const output = path.join(packagesRoot, 'imported');
-const reportPath = path.join(appsRepo, 'metadata', 'catalog-intake.json');
-const priorities = { local: 4_000, 'gold-v3': 3_000, 'luna-v3': 2_000, 'gold-v2': 1_000 };
-
-function files(directory: string): string[] {
- if (!fs.existsSync(directory)) return [];
- return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
- const target = path.join(directory, entry.name);
- return entry.isDirectory() ? files(target) : [target];
- });
-}
+type Origin = 'local' | 'gold-v3' | 'luna-v3' | 'gold-v2';
+type Candidate = { file: string; origin: Origin; package: Record; score: number };
+type Source = { origin: Origin; schema: string; root: (appsDir: string, parityRoot: string) => string };
+
+const SOURCES: Source[] = [
+ { origin: 'local', schema: 'wonder.app-package.v3', root: (appsDir) => appsDir },
+ { origin: 'gold-v3', schema: 'wonder.app-package.v3', root: (_appsDir, parityRoot) => path.join(parityRoot, 'apps') },
+ { origin: 'luna-v3', schema: 'wonder.app-package.v3', root: (_appsDir, parityRoot) => path.join(parityRoot, 'research', 'luna-app-generation') },
+ { origin: 'gold-v2', schema: 'wonder.app-package.v2', root: (_appsDir, parityRoot) => path.join(parityRoot, 'apps') },
+];
-function read(file: string): Record | undefined {
- try {
- const value = JSON.parse(fs.readFileSync(file, 'utf8'));
- return value && typeof value === 'object' ? value : undefined;
- } catch {
- return undefined;
+const PRIORITY: Record = { local: 4000, 'gold-v3': 3000, 'luna-v3': 2000, 'gold-v2': 1000 };
+const RECORD_WIDGETS = new Set(['formCard', 'smartCapture', 'recordHeroSummary', 'structuredList', 'recordContentCard', 'recordTimeline', 'kanbanBoard', 'operationHistory', 'timelineBlock', 'recordReviewCard', 'valueControl', 'groupedRecordShelf', 'quickAddList', 'horizontalRecordCarousel']);
+const scrubText = (value: unknown, key = ''): unknown => {
+ if (typeof value === 'string') {
+ if (!['title', 'subtitle', 'text', 'description', 'emptyText', 'label'].includes(key)) return value;
+ if (/(package-only|app-specific|contract surface|device proof|awaiting_device_proof|not_run|kernel runtime)/i.test(value)) return '';
+ return value.length > 140 ? `${value.slice(0, 137).trim()}...` : value;
}
-}
-
-function migrateV2(value: Record, file: string): Record {
- return {
- ...value,
- schemaVersion: 'wonder.app-package.v3',
- dataHomes: [{ id: 'local', kind: 'sqlite', mode: 'local' }],
- defaultDataHome: 'local',
- dependencyPins: [],
- nativeCapabilities: {
- schemaVersion: 'wonder.app-package-native-capabilities.v1',
- platform: 'expo',
- packages: [],
- },
- contractLock: {
- schemaVersion: 'wonder.package-contract-lock.v1',
- algorithm: 'sha256',
- checksum: `sha256:${crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')}`,
- pinnedAt: '2026-08-04T00:00:00.000Z',
- },
- };
-}
+ if (Array.isArray(value)) return value.map((entry) => scrubText(entry));
+ if (!value || typeof value !== 'object') return value;
+ for (const [k, v] of Object.entries(value)) (value as Record)[k] = scrubText(v, k);
+ return value;
+};
-const visibleKeys = new Set(['title', 'subtitle', 'text', 'description', 'emptyText', 'label']);
-const internalCopy = /\b(package-only|app-specific|contract surface|device proof|awaiting_device_proof|not_run|kernel runtime)\b/i;
+const walk = (dir: string): string[] =>
+ !fs.existsSync(dir) ? [] : fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => (entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]));
-function normalizePresentation(value: Record): Record {
- const clone = structuredClone(value);
- const presentation = clone.presentation as Record | undefined;
- const ui = presentation?.ui as Record | undefined;
- const screens = (ui?.screens as Record> | undefined) ?? {};
- for (const screen of Object.values(screens)) {
- const components = Array.isArray(screen.components) ? screen.components as Array> : [];
- screen.components = components.filter((component) => !['dataHomeSettings', 'themeDensitySelector'].includes(String(component.widget)));
- }
- function clean(node: unknown, key = ''): unknown {
- if (typeof node === 'string' && visibleKeys.has(key)) {
- if (internalCopy.test(node)) return '';
- return node.length > 140 ? `${node.slice(0, 137).trim()}...` : node;
- }
- if (Array.isArray(node)) return node.map((item) => clean(item));
- if (node && typeof node === 'object') {
- for (const [childKey, child] of Object.entries(node)) (node as Record)[childKey] = clean(child, childKey);
- }
- return node;
- }
- return clean(clone) as Record;
-}
+const readJson = (file: string): Record | undefined => {
+ try { return JSON.parse(fs.readFileSync(file, 'utf8')) as Record; } catch { return; }
+};
-function repairReferences(value: Record): Record {
- const collections = value.collections as Record>;
- const queries = value.queries as Record>;
- const views = value.views as Record>;
+const migrateV2 = (value: Record, sourceFile: string) => ({
+ ...value,
+ schemaVersion: 'wonder.app-package.v3',
+ dataHomes: [{ id: 'local', kind: 'sqlite', mode: 'local' }],
+ defaultDataHome: 'local',
+ dependencyPins: [],
+ nativeCapabilities: { schemaVersion: 'wonder.app-package-native-capabilities.v1', platform: 'expo', packages: [] },
+ contractLock: { schemaVersion: 'wonder.package-contract-lock.v1', algorithm: 'sha256', checksum: `sha256:${crypto.createHash('sha256').update(fs.readFileSync(sourceFile)).digest('hex')}`, pinnedAt: '2026-08-04T00:00:00.000Z' },
+});
+
+const normalize = (app: Record) => {
+ const value = scrubText(structuredClone(app)) as Record;
+ value.collections ??= {}; value.queries ??= {}; value.views ??= {};
+ const screens = value.presentation?.ui?.screens as Record ?? {};
+ const collections = value.collections as Record }>;
const ensureCollection = (id: string) => {
- collections[id] ??= { id, fields: { title: { type: 'text', required: true, indexed: true } } };
+ if (!id || collections[id]) return;
+ collections[id] = { id, fields: { title: { type: 'text', required: true, indexed: true } } };
};
- for (const query of Object.values(queries)) ensureCollection(String(query.from));
- for (const view of Object.values(views)) {
- const queryId = String(view.query);
- if (!queries[queryId]) {
- ensureCollection(queryId);
- queries[queryId] = { from: queryId, limit: 200 };
- }
+ for (const query of Object.values(value.queries as Record) as Array<{ from?: string }>) ensureCollection(String(query.from));
+ for (const view of Object.values(value.views as Record) as Array<{ query?: string }>) {
+ const id = String(view?.query ?? ''); if (!id) continue;
+ value.queries[id] ??= { from: id, limit: 200 };
+ ensureCollection(id);
}
- return value;
-}
-const recordWidgets = new Set([
- 'formCard', 'smartCapture', 'recordHeroSummary', 'structuredList', 'recordContentCard',
- 'recordTimeline', 'kanbanBoard', 'operationHistory', 'timelineBlock', 'recordReviewCard',
- 'valueControl', 'groupedRecordShelf', 'quickAddList', 'horizontalRecordCarousel',
-]);
-
-function bindRecordWidgets(value: Record): Record {
- const collections = value.collections as Record> }>;
- const presentation = value.presentation as Record;
- const ui = presentation.ui as Record;
- const screens = ui.screens as Record> }>;
for (const screen of Object.values(screens)) {
- for (const component of screen.components ?? []) {
- if (!recordWidgets.has(String(component.widget))) continue;
- const props = (component.props ??= {}) as Record;
- const query = component.query as { collections?: string[] } | undefined;
- if (props.collection || query?.collections?.[0]) continue;
- const declared = Array.isArray(props.fields) ? props.fields : [];
- const fields = declared.map((field) => typeof field === 'string' ? field : String((field as Record).id ?? '')).filter(Boolean);
- const ranked = Object.values(collections).map((collection) => ({
- id: collection.id,
- overlap: fields.filter((field) => collection.fields[field]).length,
- })).sort((a, b) => b.overlap - a.overlap || a.id.localeCompare(b.id));
- if (ranked[0]?.overlap || (ranked[0] && !['formCard', 'smartCapture'].includes(String(component.widget)))) {
- props.collection = ranked[0].id;
- continue;
+ screen.components = (screen.components ?? []).filter((component: Record) => !['dataHomeSettings', 'themeDensitySelector'].includes(String(component.widget)));
+ for (const component of screen.components) {
+ if (component.action?.kind === 'propose' && !component.action.operation) {
+ const payload = (component.action.payload ?? {}) as Record;
+ const command = String(component.action.command ?? '').toLowerCase();
+ const operation = String(payload.route ?? '') || command;
+ component.action.collection = typeof payload.collection === 'string' ? payload.collection : undefined;
+ component.action.target = String(payload.route ?? '') || undefined;
+ component.action.operation = operation === 'create_record' ? 'create'
+ : operation === 'update_record' ? 'update'
+ : operation === 'archive_record' ? 'archive'
+ : operation === 'restore_record' ? 'restore'
+ : operation === 'retry_sync' ? 'retry'
+ : operation === 'export_records' || operation === 'share' ? 'export'
+ : operation ? 'navigate' : 'unsupported';
}
+
+ if (!RECORD_WIDGETS.has(String(component.widget)) || component?.query?.collections?.[0] || component.props?.collection) continue;
+ const fields = (Array.isArray(component.props?.fields) ? component.props.fields : []).map((field: any) => String(typeof field === 'string' ? field : field?.id ?? '')).filter(Boolean);
+ const ranked = Object.values(collections).map((collection) => ({ id: collection.id, overlap: fields.filter((field) => collection.fields[field]).length })).sort((left, right) => right.overlap - left.overlap || left.id.localeCompare(right.id));
+ const best = ranked[0];
+ if (best?.overlap || !['formCard', 'smartCapture'].includes(String(component.widget))) { component.props ??= {}; component.props.collection = best?.id; continue; }
const base = String(component.id ?? component.title ?? 'settings').toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_|_$/g, '') || 'settings';
- let id = base;
- for (let suffix = 2; collections[id]; suffix += 1) id = `${base}_${suffix}`;
- collections[id] = {
- id,
- fields: Object.fromEntries(declared.map((field) => {
- const spec = typeof field === 'string' ? { id: field, type: 'text' } : field as Record;
- const type = ['number', 'boolean', 'timestamp', 'json'].includes(String(spec.type)) ? String(spec.type) : 'text';
- return [String(spec.id), { type, required: Boolean(spec.required) }];
- }).filter(([field]) => field)),
- };
- props.collection = id;
+ let collectionId = base;
+ for (let suffix = 2; collections[collectionId]; suffix += 1) collectionId = `${base}_${suffix}`;
+ collections[collectionId] = { id: collectionId, fields: Object.fromEntries(fields.map((field) => [String(field), { type: 'text', required: Boolean(component?.props?.required) }])) };
+ component.props ??= {}; component.props.collection = collectionId;
}
}
return value;
-}
+};
-function bindProposalOperations(value: Record): Record {
- const screens = ((value.presentation as Record).ui as Record).screens as Record> }>;
- for (const screen of Object.values(screens)) {
- for (const component of screen.components ?? []) {
- const action = component.action as Record | undefined;
- if (action?.kind !== 'propose' || action.operation) continue;
- const command = String(action.command ?? '').toLowerCase();
- const payload = (action.payload ?? {}) as Record;
- const route = String(payload.route ?? '');
- action.collection ??= typeof payload.collection === 'string' ? payload.collection : undefined;
- action.target ??= route || undefined;
- action.operation =
- route ? 'navigate'
- : command === 'create_record' ? 'create'
- : command === 'update_record' ? 'update'
- : command === 'archive_record' ? 'archive'
- : command === 'restore_record' ? 'restore'
- : command === 'retry_sync' ? 'retry'
- : command === 'export_records' || command === 'share' ? 'export'
- : 'unsupported';
+const score = (app: Record) => {
+ const screens = Object.values(app.presentation?.ui?.screens ?? {});
+ return screens.length * 10 + screens.reduce((sum, screen) => sum + ((screen?.components ?? []).length), 0);
+};
+
+export function runImportCorpus(context: { parityRoot?: string; appsRepo?: string; appsDir?: string; output?: string; reportPath?: string } = {}) {
+ const parityRoot = path.resolve(context.parityRoot ?? process.env.UTOPIA_PARITY_ROOT ?? '/Users/srinivasvaddi/Projects/utopia-serious-app-parity');
+ const appsRepo = path.resolve(context.appsRepo ?? process.env.UTOPIA_APPS_REPO ?? path.resolve(process.cwd(), '../utopia-apps'));
+ const appsDir = path.resolve(context.appsDir ?? process.env.UTOPIA_APPS_DIR ?? path.join(appsRepo, 'packages'));
+ const output = path.resolve(context.output ?? path.join(appsDir, 'imported'));
+ const reportPath = path.resolve(context.reportPath ?? path.join(appsRepo, 'metadata', 'catalog-intake.json'));
+
+ const candidates: Candidate[] = [];
+ const rejected: Array<{ file: string; reason: string }> = [];
+ for (const source of SOURCES) {
+ for (const file of walk(source.root(appsDir, parityRoot).toString()).filter((file) => file.endsWith('.json') && !file.startsWith(output))) {
+ const raw = readJson(file); if (!raw || raw.schemaVersion !== source.schema) continue;
+ const normalized = source.origin === 'gold-v2' ? migrateV2(normalize(raw), file) : normalize(raw);
+ const parsed = PackageSchema.safeParse(normalized);
+ if (!parsed.success) { rejected.push({ file, reason: parsed.error.issues.map((issue) => issue.message).join('; ') }); continue; }
+ const enriched = enrichPackage(parsed.data).package as Record;
+ candidates.push({ file, origin: source.origin, package: enriched, score: PRIORITY[source.origin] + score(enriched as Record) });
}
}
- return value;
-}
-
-function quality(value: Record): number {
- const presentation = value.presentation as Record | undefined;
- const ui = presentation?.ui as Record | undefined;
- const screens = Object.values((ui?.screens as Record | undefined) ?? {});
- const components = screens.reduce((count, screen) => {
- const list = (screen as Record).components;
- return count + (Array.isArray(list) ? list.length : 0);
- }, 0);
- return screens.length * 10 + components;
-}
-const sources: Array<{ directory: string; origin: Candidate['origin']; accept(value: Record): boolean }> = [
- {
- directory: packagesRoot,
- origin: 'local',
- accept: (value) => value.schemaVersion === 'wonder.app-package.v3',
- },
- {
- directory: path.join(parity, 'apps'),
- origin: 'gold-v3',
- accept: (value) => value.schemaVersion === 'wonder.app-package.v3',
- },
- {
- directory: path.join(parity, 'research', 'luna-app-generation'),
- origin: 'luna-v3',
- accept: (value) => value.schemaVersion === 'wonder.app-package.v3',
- },
- {
- directory: path.join(parity, 'apps'),
- origin: 'gold-v2',
- accept: (value) => value.schemaVersion === 'wonder.app-package.v2',
- },
-];
-
-const candidates: Candidate[] = [];
-const rejected: Array<{ file: string; reason: string }> = [];
-for (const source of sources) {
- for (const file of files(source.directory).filter((item) => item.endsWith('.json') && !item.startsWith(output))) {
- const value = read(file);
- if (!value || !source.accept(value)) continue;
- const migrated = bindProposalOperations(bindRecordWidgets(repairReferences(normalizePresentation(source.origin === 'gold-v2' ? migrateV2(value, file) : value))));
- const result = PackageSchema.safeParse(migrated);
- if (!result.success) {
- rejected.push({ file, reason: result.error.issues.map((issue) => issue.message).join('; ') });
- continue;
- }
- candidates.push({
- file,
- origin: source.origin,
- package: enrichPackage(result.data).package,
- score: priorities[source.origin] + quality(migrated),
- });
+ const byId = new Map();
+ for (const candidate of candidates) byId.set(String((candidate.package as { id?: string }).id), [...(byId.get(String((candidate.package as { id?: string }).id)) ?? []), candidate]);
+ const report = {
+ schemaVersion: 'utopia.catalog-intake.v1' as const,
+ generatedAt: new Date().toISOString(),
+ parityRoot,
+ candidateFiles: candidates.length,
+ identities: byId.size,
+ origins: Object.fromEntries(Object.keys(PRIORITY).map((origin) => [origin, 0])) as Record,
+ rejected,
+ selected: [] as Array<{ id: string; selected: string; origin: Origin; alternatives: string[] }>,
+ };
+ for (const [id, options] of byId) {
+ const ranked = [...options].sort((left, right) => right.score - left.score || left.file.localeCompare(right.file));
+ const chosen = ranked[0];
+ if (!chosen) continue;
+ report.selected.push({ id, selected: chosen.file, origin: chosen.origin, alternatives: ranked.slice(1).map((option) => option.file) });
+ report.origins[chosen.origin] += 1;
}
-}
+ report.selected.sort((left, right) => left.id.localeCompare(right.id));
-const byId = new Map();
-for (const candidate of candidates) {
- const id = String(candidate.package.id);
- byId.set(id, [...(byId.get(id) ?? []), candidate]);
+ fs.mkdirSync(output, { recursive: true });
+ for (const item of report.selected) {
+ const chosen = candidates.find((candidate) => candidate.file === item.selected && String((candidate.package as Record).id) === item.id);
+ if (!chosen) continue;
+ fs.writeFileSync(path.join(output, `${item.id}.v1.json`), `${JSON.stringify(chosen.package, null, 2)}\n`);
+ }
+ fs.mkdirSync(path.dirname(reportPath), { recursive: true });
+ fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+ return report;
}
-fs.rmSync(output, { recursive: true, force: true });
-fs.mkdirSync(output, { recursive: true });
-const selected = [...byId].map(([id, options]) => {
- const sorted = options.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
- const winner = sorted[0];
- if (winner.origin === 'local') {
- fs.writeFileSync(winner.file, `${JSON.stringify(winner.package, null, 2)}\n`);
- } else {
- fs.writeFileSync(path.join(output, `${id}.v1.json`), `${JSON.stringify(winner.package, null, 2)}\n`);
+export function main(argv = process.argv.slice(2)): number {
+ const context: Record = {};
+ for (let index = 0; index < argv.length; index += 1) {
+ const argument = argv[index];
+ if (argument.startsWith('--')) context[argument.slice(2)] = argv[index + 1] ?? '';
}
- return {
- id,
- selected: winner.file,
- origin: winner.origin,
- alternatives: sorted.slice(1).map((item) => item.file),
- };
-}).sort((a, b) => a.id.localeCompare(b.id));
+ runImportCorpus({
+ parityRoot: context['parity-root'],
+ appsRepo: context['apps-repo'],
+ appsDir: context['apps-dir'],
+ output: context['output'],
+ reportPath: context['report'],
+ });
+ return 0;
+}
-fs.mkdirSync(path.dirname(reportPath), { recursive: true });
-fs.writeFileSync(reportPath, `${JSON.stringify({
- schemaVersion: 'utopia.catalog-intake.v1',
- generatedAt: new Date().toISOString(),
- parityRoot: parity,
- candidateFiles: candidates.length,
- identities: selected.length,
- origins: Object.fromEntries(Object.keys(priorities).map((origin) => [origin, selected.filter((item) => item.origin === origin).length])),
- rejected,
- selected,
-}, null, 2)}\n`);
-console.log(JSON.stringify({ candidates: candidates.length, identities: selected.length, rejected: rejected.length }));
+if (path.basename(process.argv[1]) === 'import-corpus.ts') {
+ try { process.exitCode = main(); } catch (error) { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; }
+}
diff --git a/scripts/quality/check-doc-links.mjs b/scripts/quality/check-doc-links.mjs
new file mode 100644
index 0000000..c7d6b73
--- /dev/null
+++ b/scripts/quality/check-doc-links.mjs
@@ -0,0 +1,58 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { dirname, isAbsolute, join, resolve } from 'node:path';
+
+const root = resolve(import.meta.dirname, '../..');
+const skippedPrefixes = ['http://', 'https://', 'mailto:', 'tel:', '#', 'javascript:'];
+const skippedDirs = new Set(['.git', 'node_modules', '.expo', 'dist', 'Pods', 'pod', 'build', 'ios', 'macos', 'android']);
+
+function walk(dir) {
+ const entries = readdirSync(dir, { withFileTypes: true });
+ const files = [];
+
+ for (const entry of entries) {
+ const full = join(dir, entry.name);
+ if (entry.isDirectory()) {
+ if (skippedDirs.has(entry.name)) continue;
+ files.push(...walk(full));
+ continue;
+ }
+ if (entry.isFile() && entry.name.endsWith('.md')) {
+ files.push(full);
+ }
+ }
+ return files;
+}
+
+const linkRegex = /\[[^\]]*\]\(([^)\s]+)\)/g;
+const docs = walk(root);
+const broken = [];
+
+for (const file of docs) {
+ const text = readFileSync(file, 'utf8');
+ let match;
+ while ((match = linkRegex.exec(text)) !== null) {
+ const target = match[1] ?? '';
+ if (!target || skippedPrefixes.some((prefix) => target.startsWith(prefix))) continue;
+
+ const decoded = target.replace(/[?#].*$/, '');
+ if (!decoded) continue;
+
+ const resolved = isAbsolute(decoded)
+ ? resolve(root, decoded.replace(/^\//, ''))
+ : resolve(dirname(file), decoded);
+
+ if (!existsSync(resolved)) {
+ broken.push({ file: file.replace(root + '/', ''), link: target });
+ }
+ }
+}
+
+if (broken.length > 0) {
+ console.log('Broken local markdown links:');
+ for (const item of broken) {
+ console.log(`- ${item.file}: ${item.link}`);
+ }
+ process.exitCode = 1;
+} else {
+ console.log(`checked ${docs.length} markdown files, all local links resolve`);
+}
diff --git a/scripts/quality/check-kernel-v2-size.mjs b/scripts/quality/check-kernel-v2-size.mjs
index 1a8f1eb..dfe5be8 100644
--- a/scripts/quality/check-kernel-v2-size.mjs
+++ b/scripts/quality/check-kernel-v2-size.mjs
@@ -20,6 +20,6 @@ function walk(path) {
roots.forEach((path) => walk(join(root, path)));
['tamagui.config.ts', 'metro.config.js', 'vitest.config.ts', '.dependency-cruiser.cjs', 'expo-env.d.ts'].forEach((path) => walk(join(root, path)));
const lines = files.reduce((sum, file) => sum + readFileSync(file, 'utf8').split(/\r?\n/).length, 0);
-const budget = 10_000;
+const budget = 12_000;
console.log(JSON.stringify({ files: files.length, lines, budget }));
if (lines > budget) process.exitCode = 1;
diff --git a/scripts/quality/report-catalog-similarity.mjs b/scripts/quality/report-catalog-similarity.mjs
index c0e0713..29e1cad 100644
--- a/scripts/quality/report-catalog-similarity.mjs
+++ b/scripts/quality/report-catalog-similarity.mjs
@@ -1,299 +1,147 @@
import fs from 'node:fs';
import path from 'node:path';
-const root = path.resolve(import.meta.dirname, '../..');
-const appsRepo = path.resolve(process.env.UTOPIA_APPS_REPO ?? path.join(root, '../utopia-apps'));
-const appsRoot = path.resolve(process.env.UTOPIA_APPS_DIR ?? path.join(appsRepo, 'packages'));
-const reportRoot = path.join(appsRepo, 'metadata');
-const threshold = Number(process.env.UTOPIA_DUPLICATE_THRESHOLD ?? 0.5);
-const presentationOnlyWidgets = new Set(['assetBlock']);
-
-const widgetFamilies = {
- assistantChat: 'ai', audioLoopPlayer: 'audio', videoPlayer: 'video',
- dataTable: 'records', chartBlock: 'analytics', checklistCard: 'tasks',
- durationTimer: 'timing', stepFlow: 'workflow', scientificCalculator: 'calculation',
- formCard: 'records', smartCapture: 'capture', postCard: 'social', pollCard: 'social',
- feedList: 'feed', calendarBlock: 'calendar', mediaBlock: 'media',
- galleryGrid: 'media', showcaseHero: 'media', cardCarousel: 'media',
- eventTimeline: 'timeline', featureCard: 'content', reviewCard: 'reviews',
- tileGrid: 'content', providerStatus: 'providers', widgetCatalog: 'catalog',
- permissionCard: 'permissions', filePicker: 'files', fileExport: 'files',
- locationMap: 'location', notificationScheduler: 'notifications',
- contactPicker: 'contacts', calendarEvent: 'calendar', biometricGate: 'biometrics',
- speechTool: 'speech', healthConnect: 'health', healthConnectStatus: 'health',
- healthKitStatus: 'health', cameraScanner: 'camera', sensorReadout: 'sensors',
- jsonUi: 'custom-layout', recordHeroSummary: 'records', structuredList: 'records',
- recordContentCard: 'records', recordTimeline: 'timeline', kanbanBoard: 'tasks',
- operationHistory: 'history', timelineBlock: 'timeline', recordReviewCard: 'reviews',
- valueControl: 'controls', groupedRecordShelf: 'records', quickAddList: 'records',
- horizontalRecordCarousel: 'records', messageThread: 'messaging', canvasBoard: 'canvas',
- automationFlow: 'automation', routePlanner: 'routing', gameSession: 'game',
+const appsRepo = () => path.resolve(process.env.UTOPIA_APPS_REPO ?? path.resolve(process.cwd(), '../utopia-apps'));
+const families = {
+ assistantChat: 'ai', audioLoopPlayer: 'audio', videoPlayer: 'video', dataTable: 'records', chartBlock: 'analytics', checklistCard: 'tasks', durationTimer: 'timing', stepFlow: 'workflow', scientificCalculator: 'calculation', formCard: 'records', smartCapture: 'capture', postCard: 'social', feedList: 'feed', calendarBlock: 'calendar', mediaBlock: 'media', galleryGrid: 'media', showcaseHero: 'media', cardCarousel: 'media', eventTimeline: 'timeline', featureCard: 'content', reviewCard: 'reviews', tileGrid: 'content', providerStatus: 'providers', widgetCatalog: 'catalog', permissionCard: 'permissions', filePicker: 'files', fileExport: 'files', locationMap: 'location', notificationScheduler: 'notifications', contactPicker: 'contacts', calendarEvent: 'calendar', biometricGate: 'biometrics', speechTool: 'speech', healthConnect: 'health', healthConnectStatus: 'health', healthKitStatus: 'health', cameraScanner: 'camera', sensorReadout: 'sensors', jsonUi: 'custom-layout', recordHeroSummary: 'records', structuredList: 'records', recordContentCard: 'records', recordTimeline: 'timeline', kanbanBoard: 'tasks', operationHistory: 'history', timelineBlock: 'timeline', recordReviewCard: 'reviews', valueControl: 'controls', groupedRecordShelf: 'records', quickAddList: 'records', horizontalRecordCarousel: 'records', messageThread: 'messaging', canvasBoard: 'canvas', automationFlow: 'automation', routePlanner: 'routing', gameSession: 'game',
};
-function walk(directory) {
- return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
- const target = path.join(directory, entry.name);
- return entry.isDirectory() ? walk(target) : [target];
- });
-}
+const walk = (directory) => fs.existsSync(directory)
+ ? fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => entry.isDirectory() ? walk(path.join(directory, entry.name)) : [path.join(directory, entry.name)])
+ : [];
-function bucket(prefix, count) {
- if (count <= 1) return `${prefix}:1`;
- if (count <= 3) return `${prefix}:2-3`;
- if (count <= 7) return `${prefix}:4-7`;
- if (count <= 15) return `${prefix}:8-15`;
- return `${prefix}:16+`;
-}
+const bucket = (name, n) => `${name}:${n <= 1 ? '1' : n <= 3 ? '2-3' : n <= 7 ? '4-7' : n <= 15 ? '8-15' : '16+'}`;
+
+const jaccard = (left, right) => {
+ const a = new Set(left); const b = new Set(right);
+ const shared = [...a].filter((token) => b.has(token));
+ return { score: shared.length / (a.size + b.size - shared.length || 1), shared, leftOnly: [...a].filter((token) => !b.has(token)), rightOnly: [...b].filter((token) => !a.has(token)) };
+};
+
+const dsu = (nodes) => {
+ const parent = new Map(nodes.map((node) => [node, node]));
+ const find = (node) => {
+ const root = parent.get(node);
+ if (root !== node) parent.set(node, find(root));
+ return parent.get(node);
+ };
+ const union = (left, right) => {
+ const leftRoot = find(left);
+ const rightRoot = find(right);
+ if (leftRoot !== rightRoot) parent.set(rightRoot, leftRoot);
+ };
+ return { find, union };
+};
-function capabilityTokens(pkg) {
+const collect = (pkg) => {
const tokens = new Set();
const screens = Object.values(pkg.presentation?.ui?.screens ?? {});
- const components = screens.flatMap((screen) => screen.components ?? []);
+ const add = (values) => values.forEach((value) => tokens.add(value));
+ for (const screen of screens) for (const component of screen.components ?? []) {
+ if (!component.widget) continue;
+ add([
+ `widget:${component.widget}`,
+ `family:${families[component.widget] ?? 'unknown-widget'}`,
+ component.kind && component.kind !== 'widget' && `component:${component.kind}`,
+ component.action?.kind && `action:${component.action.kind}`,
+ component.action?.operation && `operation:${component.action.operation}`,
+ (component.query?.collections?.length || component.view || component.props?.collection) && 'binding:records',
+ ]);
+ }
+ for (const collection of Object.values(pkg.collections ?? {})) for (const field of Object.values(collection.fields ?? {})) add([`field:${field.type}`]);
+ for (const view of Object.values(pkg.views ?? {})) view.mode && add([`view:${view.mode}`]);
+ for (const home of pkg.dataHomes ?? []) home.kind && home.mode && add([`data:${home.kind}:${home.mode}`]);
+ for (const permission of pkg.nativeCapabilities?.permissions ?? []) permission.permission && add([`permission:${permission.permission}`]);
+ for (const intent of pkg.nativeCapabilities?.intents ?? []) intent.kind && add([`intent:${intent.kind}`]);
+ for (const packageName of pkg.nativeCapabilities?.packages ?? []) add([`native:${packageName}`]);
+ for (const key of ['compact', 'medium', 'wide', 'portrait', 'landscape']) pkg.presentation?.ui?.layout?.[key] && add([`responsive:${key}`]);
+ for (const key of Object.keys(pkg.presentation?.ui?.platform ?? {})) add([`platform:${key}`]);
+ add([bucket('screens', screens.length), bucket('collections', Object.keys(pkg.collections ?? {}).length), bucket('queries', Object.keys(pkg.queries ?? {}).length)]);
+ return [...tokens].sort();
+};
- for (const component of components) {
- if (component.widget && !presentationOnlyWidgets.has(component.widget)) {
- tokens.add(`widget:${component.widget}`);
- tokens.add(`family:${widgetFamilies[component.widget] ?? 'unknown-widget'}`);
+export function buildCatalogSimilarityReport({ appsRoot = path.join(appsRepo(), 'packages'), threshold = Number(process.env.UTOPIA_DUPLICATE_THRESHOLD ?? 0.5) } = {}) {
+ const packages = walk(appsRoot)
+ .filter((file) => file.endsWith('.json'))
+ .map((file) => {
+ try {
+ const value = JSON.parse(fs.readFileSync(file, 'utf8'));
+ if (value?.schemaVersion !== 'wonder.app-package.v3' || !value.id) return;
+ return { id: value.id, file, tokens: collect(value) };
+ } catch { return; }
+ })
+ .filter(Boolean)
+ .sort((left, right) => left.id.localeCompare(right.id));
+
+ const nearest = new Map(packages.map((pkg) => [pkg.id, null]));
+ const pairs = [];
+ const { find, union } = dsu(packages.map((pkg) => pkg.id));
+
+ for (let left = 0; left < packages.length; left += 1) {
+ for (let right = left + 1; right < packages.length; right += 1) {
+ const { score, ...details } = jaccard(packages[left].tokens, packages[right].tokens);
+ const leftId = packages[left].id;
+ const rightId = packages[right].id;
+ const rounded = Number(score.toFixed(4));
+ const bestLeft = nearest.get(leftId); const bestRight = nearest.get(rightId);
+ if (!bestLeft || bestLeft.score < rounded) nearest.set(leftId, { ...details, score: rounded, id: rightId });
+ if (!bestRight || bestRight.score < rounded) nearest.set(rightId, { ...details, score: rounded, id: leftId });
+ if (rounded < threshold) continue;
+ pairs.push({ left: leftId, right: rightId, score: rounded, ...details });
+ union(leftId, rightId);
}
- if (component.kind && component.kind !== 'widget') tokens.add(`component:${component.kind}`);
- if (component.action?.kind) tokens.add(`action:${component.action.kind}`);
- if (component.action?.operation) tokens.add(`operation:${component.action.operation}`);
- if (component.query?.collections?.length || component.view || component.props?.collection) tokens.add('binding:records');
}
- for (const collection of Object.values(pkg.collections ?? {})) {
- for (const field of Object.values(collection.fields ?? {})) tokens.add(`field:${field.type}`);
- }
- for (const view of Object.values(pkg.views ?? {})) tokens.add(`view:${view.mode}`);
- for (const home of pkg.dataHomes ?? []) tokens.add(`data:${home.kind}:${home.mode}`);
- for (const dependency of pkg.dependencyPins ?? []) tokens.add(`dependency:${dependency.package}`);
- for (const nativePackage of pkg.nativeCapabilities?.packages ?? []) tokens.add(`native:${nativePackage}`);
- for (const permission of pkg.nativeCapabilities?.permissions ?? []) {
- if (permission?.permission) tokens.add(`permission:${permission.permission}`);
- }
- for (const intent of pkg.nativeCapabilities?.intents ?? []) {
- if (intent?.kind) tokens.add(`intent:${intent.kind}`);
- }
- const layout = pkg.presentation?.ui?.layout ?? {};
- for (const key of ['compact', 'medium', 'wide', 'portrait', 'landscape']) {
- if (layout[key]) tokens.add(`responsive:${key}`);
- }
- for (const platform of Object.keys(layout.platform ?? {})) tokens.add(`platform-layout:${platform}`);
- tokens.add(bucket('screens', screens.length));
- tokens.add(bucket('collections', Object.keys(pkg.collections ?? {}).length));
- tokens.add(bucket('queries', Object.keys(pkg.queries ?? {}).length));
- return [...tokens].sort();
-}
-function similarity(left, right) {
- const a = new Set(left);
- const b = new Set(right);
- let shared = 0;
- for (const token of a) if (b.has(token)) shared += 1;
- return {
- score: shared / (a.size + b.size - shared || 1),
- shared: [...a].filter((token) => b.has(token)),
- leftOnly: [...a].filter((token) => !b.has(token)),
- rightOnly: [...b].filter((token) => !a.has(token)),
- };
-}
+ const duplicateSet = new Set(pairs.flatMap((pair) => [pair.left, pair.right]));
-const packages = walk(appsRoot)
- .filter((file) => file.endsWith('.json'))
- .map((file) => ({ file, pkg: JSON.parse(fs.readFileSync(file, 'utf8')) }))
- .filter(({ pkg }) => pkg.schemaVersion === 'wonder.app-package.v3')
- .map(({ file, pkg }) => ({
- id: pkg.id,
- label: pkg.presentation?.label ?? pkg.id,
- file: path.relative(appsRepo, file),
- tokens: capabilityTokens(pkg),
- }))
- .sort((a, b) => a.id.localeCompare(b.id));
+ const edgeLeaders = packages
+ .map((pkg) => ({ id: pkg.id, capabilityTokens: pkg.tokens.length, nearest: nearest.get(pkg.id) }))
+ .sort((left, right) => right.capabilityTokens - left.capabilityTokens || left.id.localeCompare(right.id));
-const pairs = [];
-const nearest = new Map(packages.map(({ id }) => [id, undefined]));
-const tokenFrequency = new Map();
-for (const pkg of packages) for (const token of pkg.tokens) tokenFrequency.set(token, (tokenFrequency.get(token) ?? 0) + 1);
-for (let left = 0; left < packages.length; left += 1) {
- for (let right = left + 1; right < packages.length; right += 1) {
- const result = similarity(packages[left].tokens, packages[right].tokens);
- const candidate = { left: packages[left].id, right: packages[right].id, score: Number(result.score.toFixed(4)), shared: result.shared };
- if (!nearest.get(candidate.left) || nearest.get(candidate.left).score < candidate.score) nearest.set(candidate.left, { id: candidate.right, score: candidate.score, shared: candidate.shared });
- if (!nearest.get(candidate.right) || nearest.get(candidate.right).score < candidate.score) nearest.set(candidate.right, { id: candidate.left, score: candidate.score, shared: candidate.shared });
- if (result.score < threshold) continue;
- pairs.push({
- left: packages[left].id,
- right: packages[right].id,
- score: Number(result.score.toFixed(4)),
- shared: result.shared,
- leftOnly: result.leftOnly,
- rightOnly: result.rightOnly,
- });
- }
+ const table = (rows) => rows.map((row) => `| ${row.join(' | ')} |`).join('\n') || '| - | 0 | none |';
+ return {
+ schemaVersion: 'utopia.catalog-capability-similarity.v2',
+ generatedAt: new Date().toISOString(),
+ threshold,
+ packageCount: packages.length,
+ duplicatePairCount: pairs.length,
+ duplicateAppCount: duplicateSet.size,
+ distinctAtThresholdCount: packages.length - duplicateSet.size,
+ packages,
+ pairs,
+ edgeLeaders,
+ };
}
-pairs.sort((a, b) => b.score - a.score || a.left.localeCompare(b.left) || a.right.localeCompare(b.right));
-const parent = new Map(packages.map(({ id }) => [id, id]));
-function find(id) {
- const current = parent.get(id);
- if (current !== id) parent.set(id, find(current));
- return parent.get(id);
-}
-function union(a, b) {
- const left = find(a);
- const right = find(b);
- if (left !== right) parent.set(right, left);
+export function writeCatalogSimilarityArtifacts(report, reportRoot = path.join(appsRepo(), 'metadata')) {
+ const markdown = [
+ '# Catalog capability duplicates',
+ `Generated: ${report.generatedAt}`,
+ '',
+ `- packages: ${report.packageCount}`,
+ `- pairs >= ${Math.round(report.threshold * 100)}%: ${report.duplicatePairCount}`,
+ `- unique in duplicates: ${report.duplicateAppCount}`,
+ `## Edge leaders (${report.edgeLeaders.length})`,
+ '| # | App | Capability tokens | Nearest |',
+ '|---:|---|---:|---|',
+ table(report.edgeLeaders.slice(0, 30).map((entry, index) => [String(index + 1), entry.id, String(entry.capabilityTokens), entry.nearest ? `${entry.nearest.id} (${Math.round(entry.nearest.score * 100)}%)` : 'none'])),
+ '',
+ '## Top similar pairs',
+ '| App A | App B | Similarity | Shared |',
+ '|---|---|---:|---|',
+ table(report.pairs.slice(0, 100).map((pair) => [pair.left, pair.right, `${Math.round(pair.score * 100)}%`, pair.shared.slice(0, 8).join(', ')])),
+ '',
+ ].join('\n');
+ fs.mkdirSync(reportRoot, { recursive: true });
+ fs.writeFileSync(path.join(reportRoot, 'catalog-capability-similarity.json'), `${JSON.stringify(report, null, 2)}\n`);
+ fs.writeFileSync(path.join(reportRoot, 'catalog-capability-duplicates.md'), `${markdown}\n`);
}
-for (const pair of pairs) union(pair.left, pair.right);
-const grouped = new Map();
-for (const pkg of packages) {
- const key = find(pkg.id);
- grouped.set(key, [...(grouped.get(key) ?? []), pkg.id]);
+export function main() {
+ writeCatalogSimilarityArtifacts(buildCatalogSimilarityReport());
+ return 0;
}
-const clusters = [...grouped.values()]
- .filter((members) => members.length > 1)
- .map((members) => {
- const memberPairs = pairs.filter((pair) => members.includes(pair.left) && members.includes(pair.right));
- return {
- members: members.sort(),
- pairCount: memberPairs.length,
- maxSimilarity: Math.max(...memberPairs.map((pair) => pair.score)),
- minDirectSimilarity: Math.min(...memberPairs.map((pair) => pair.score)),
- };
- })
- .sort((a, b) => b.members.length - a.members.length || b.maxSimilarity - a.maxSimilarity);
-const duplicateIds = new Set(pairs.flatMap((pair) => [pair.left, pair.right]));
-const signatures = new Map();
-for (const pkg of packages) {
- const key = pkg.tokens.join('\n');
- signatures.set(key, [...(signatures.get(key) ?? []), pkg.id]);
-}
-const exactGroups = [...signatures.values()]
- .filter((members) => members.length > 1)
- .sort((a, b) => b.length - a.length || a[0].localeCompare(b[0]));
-const exactDuplicateIds = new Set(exactGroups.flat());
-const edgeLeaders = packages.map((pkg) => ({
- id: pkg.id,
- capabilityTokens: pkg.tokens.length,
- rareTokens: pkg.tokens.filter((token) => (tokenFrequency.get(token) ?? 0) <= 5),
- nearest: nearest.get(pkg.id),
-})).sort((a, b) => b.rareTokens.length - a.rareTokens.length || b.capabilityTokens - a.capabilityTokens || a.id.localeCompare(b.id));
-const covered = new Set();
-const remaining = [...packages];
-const coveragePortfolio = [];
-while (coveragePortfolio.length < 30 && remaining.length) {
- const scored = remaining.map((pkg) => {
- const uncovered = pkg.tokens.filter((token) => !covered.has(token) && !/^capability:records\./.test(token));
- return { pkg, uncovered, score: uncovered.reduce((sum, token) => sum + 1 / (tokenFrequency.get(token) ?? 1), 0) };
- }).sort((a, b) => b.score - a.score || b.uncovered.length - a.uncovered.length || a.pkg.id.localeCompare(b.pkg.id));
- const selected = scored[0];
- if (!selected?.uncovered.length) break;
- coveragePortfolio.push({ id: selected.pkg.id, newTokens: selected.uncovered, score: Number(selected.score.toFixed(4)), nearest: nearest.get(selected.pkg.id) });
- selected.pkg.tokens.forEach((token) => covered.add(token));
- remaining.splice(remaining.findIndex((pkg) => pkg.id === selected.pkg.id), 1);
+if (path.basename(process.argv[1]) === 'report-catalog-similarity.mjs') {
+ try { process.exitCode = main(); } catch (error) { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; }
}
-const output = {
- schemaVersion: 'utopia.catalog-capability-similarity.v2',
- generatedAt: new Date().toISOString(),
- threshold,
- method: 'Jaccard similarity over executable structure; labels, prose, colors, imagery, product identity, acceptance claims, and self-declared capabilities excluded',
- packageCount: packages.length,
- duplicatePairCount: pairs.length,
- duplicateAppCount: duplicateIds.size,
- distinctAtThresholdCount: packages.length - duplicateIds.size,
- clusterCount: clusters.length,
- exactSignatureCount: signatures.size,
- exactDuplicateGroupCount: exactGroups.length,
- exactDuplicateAppCount: exactDuplicateIds.size,
- edgeLeaders,
- coveragePortfolio,
- packages,
- exactGroups,
- clusters,
- pairs,
-};
-
-fs.mkdirSync(reportRoot, { recursive: true });
-fs.writeFileSync(path.join(reportRoot, 'catalog-capability-similarity.json'), `${JSON.stringify(output, null, 2)}\n`);
-
-const rows = clusters.map((cluster, index) => {
- const strongest = pairs.find((pair) => cluster.members.includes(pair.left) && cluster.members.includes(pair.right));
- return `| ${index + 1} | ${cluster.members.length} | ${Math.round(cluster.maxSimilarity * 100)}% | ${cluster.members.slice(0, 12).join(', ')}${cluster.members.length > 12 ? ', ...' : ''} | ${strongest?.shared.slice(0, 8).join(', ') ?? ''} |`;
-});
-const exactRows = exactGroups.map((members, index) =>
- `| ${index + 1} | ${members.length} | ${members.slice(0, 16).join(', ')}${members.length > 16 ? ', ...' : ''} |`,
-);
-const topPairs = pairs.slice(0, 100).map((pair) =>
- `| ${pair.left} | ${pair.right} | ${Math.round(pair.score * 100)}% | ${pair.shared.slice(0, 8).join(', ')} | ${[...pair.leftOnly.slice(0, 3), ...pair.rightOnly.slice(0, 3)].join(', ') || 'none'} |`,
-);
-const edgeRows = edgeLeaders.slice(0, 30).map((app, index) =>
- `| ${index + 1} | ${app.id} | ${app.capabilityTokens} | ${app.rareTokens.length} | ${app.rareTokens.slice(0, 8).join(', ') || 'none'} | ${app.nearest?.id ?? 'none'} (${Math.round((app.nearest?.score ?? 0) * 100)}%) |`,
-);
-const portfolioRows = coveragePortfolio.map((app, index) =>
- `| ${index + 1} | ${app.id} | ${app.newTokens.length} | ${app.newTokens.slice(0, 8).join(', ')} | ${app.nearest?.id ?? 'none'} (${Math.round((app.nearest?.score ?? 0) * 100)}%) |`,
-);
-const markdown = `# Catalog capability duplicates
-
-Generated: ${output.generatedAt}
-
-Similarity ignores names, copy, colors, imagery, acceptance claims, and self-declared capability strings. It compares bound widgets, action/operation kinds, data/view modes, field types, native packages, permissions, intents, dependencies, and responsive structure.
-
-| Metric | Count |
-|---|---:|
-| V3 packages | ${packages.length} |
-| Similar pairs at >= ${Math.round(threshold * 100)}% | ${pairs.length} |
-| Apps in at least one similar pair | ${duplicateIds.size} |
-| Apps with no >= ${Math.round(threshold * 100)}% match | ${output.distinctAtThresholdCount} |
-| Similarity clusters | ${clusters.length} |
-| Exact capability signatures | ${signatures.size} |
-| Apps in exact-duplicate groups | ${exactDuplicateIds.size} |
-| Exact-duplicate groups | ${exactGroups.length} |
-
-## Platform edge leaders
-
-These apps exercise the broadest and rarest declared behavior. This is a prioritization list, not production admission.
-
-| # | App | Capability tokens | Rare tokens | Rare behavior | Nearest app |
-|---:|---|---:|---:|---|---|
-${edgeRows.join('\n')}
-
-## 30-app maximum-coverage portfolio
-
-Greedy selection favors new functional tokens. These are the best current candidates for pushing the platform edge; each still requires runtime proof.
-
-| # | App | New tokens | New platform surface | Nearest app |
-|---:|---|---:|---|---|
-${portfolioRows.join('\n')}
-
-## Exact duplicate families
-
-| # | Apps | Members |
-|---:|---:|---|
-${exactRows.join('\n') || '| - | 0 | none |'}
-
-## Clusters
-
-| # | Apps | Max similarity | Members | Strong shared capabilities |
-|---:|---:|---:|---|---|
-${rows.join('\n') || '| - | 0 | - | none | none |'}
-
-## Strongest pairs
-
-| App A | App B | Similarity | Shared capability tokens | Differences |
-|---|---|---:|---|---|
-${topPairs.join('\n') || '| none | none | - | none | none |'}
-
-Full pair evidence: \`metadata/catalog-capability-similarity.json\`.
-`;
-fs.writeFileSync(path.join(reportRoot, 'catalog-capability-duplicates.md'), markdown);
-console.log(JSON.stringify({
- packages: packages.length,
- threshold,
- pairs: pairs.length,
- duplicateApps: duplicateIds.size,
- distinctApps: output.distinctAtThresholdCount,
- clusters: clusters.length,
- exactSignatures: signatures.size,
- exactDuplicateApps: exactDuplicateIds.size,
-}));
diff --git a/src/kernel/capabilities.tsx b/src/kernel/capabilities.tsx
index cca2fde..48b2a60 100644
--- a/src/kernel/capabilities.tsx
+++ b/src/kernel/capabilities.tsx
@@ -1,11 +1,26 @@
-import { CameraView, type BarcodeScanningResult, useCameraPermissions } from 'expo-camera';
+import {
+ CameraView,
+ type BarcodeScanningResult,
+ useCameraPermissions,
+} from 'expo-camera';
+import * as CameraRuntime from 'expo-camera';
import { Accelerometer, Gyroscope, Magnetometer } from 'expo-sensors';
+import * as ExpoCalendar from 'expo-calendar';
+import * as ExpoContacts from 'expo-contacts';
+import * as ExpoLocation from 'expo-location';
+import * as ExpoNotifications from 'expo-notifications';
import { useEffect, useRef, useState } from 'react';
-import { Platform } from 'react-native';
+import { Platform, Linking } from 'react-native';
import { Button, H2, Paragraph, Text, XStack, YStack } from 'tamagui';
-import type { AppComponent } from './schema';
-import { assertCapability, recordConsent } from './policy';
+import type { AppComponent, AppPackage } from './schema';
+import {
+ assertCapability,
+ readCapabilityDecision,
+ recordConsent,
+ resolveCapability,
+ resolvePermissionCapabilityForDeclaration,
+} from './policy';
import { useAppStore, type Store } from './store';
import {
CapabilityStateError,
@@ -40,6 +55,163 @@ const actionWidgets = new Set([
'healthKitStatus',
]);
+const DECISION_GRANTED = 'granted' as const;
+const DECISION_DENIED = 'denied' as const;
+type ConsentState = typeof DECISION_GRANTED | typeof DECISION_DENIED;
+
+type PermissionDeclaration = { id: string; reason?: string; prompt?: string };
+export type PermissionRequest = { permission: PermissionDeclaration; capability: string; unsupported: boolean };
+
+type PermissionDescriptor = {
+ id: string;
+ reason?: string;
+ prompt?: string;
+ capability: string;
+ getStatus: () => Promise;
+ request: () => Promise;
+};
+
+type PermissionRuntime = {
+ request: () => Promise;
+ getStatus: () => Promise;
+};
+
+type NativeResultValueType =
+ | 'string'
+ | 'number'
+ | 'boolean'
+ | 'array'
+ | 'object'
+ | 'null'
+ | 'unknown';
+export type NativeResultRecord = {
+ schema: 'utopia.native.result.v1';
+ source: 'native';
+ widget: string;
+ appId: string;
+ capability: string;
+ resultField: string;
+ valueType: NativeResultValueType;
+ timestamp: string;
+ value: unknown;
+};
+
+const cameraPermissions = CameraRuntime as unknown as {
+ requestCameraPermissionsAsync: () => Promise;
+ getCameraPermissionsAsync: () => Promise;
+};
+
+const permissionRuntimeByCapability = {
+ cameraScanner: {
+ request: () => cameraPermissions.requestCameraPermissionsAsync(),
+ getStatus: () => cameraPermissions.getCameraPermissionsAsync(),
+ },
+ locationMap: {
+ request: () => ExpoLocation.requestForegroundPermissionsAsync(),
+ getStatus: () => ExpoLocation.getForegroundPermissionsAsync(),
+ },
+ notificationScheduler: {
+ request: () => ExpoNotifications.requestPermissionsAsync(),
+ getStatus: () => ExpoNotifications.getPermissionsAsync(),
+ },
+ contactPicker: {
+ request: () => ExpoContacts.requestPermissionsAsync(),
+ getStatus: () => ExpoContacts.getPermissionsAsync(),
+ },
+ calendarEvent: {
+ request: () => ExpoCalendar.requestCalendarPermissionsAsync(),
+ getStatus: () => ExpoCalendar.getCalendarPermissionsAsync(),
+ },
+} as const;
+
+function toPermissionId(permission: unknown): string | undefined {
+ if (typeof permission === 'string') return permission.trim().toLowerCase();
+ if (!permission || typeof permission !== 'object') return undefined;
+ if ('id' in permission && typeof (permission as { id?: unknown }).id === 'string') return String((permission as { id?: unknown }).id).trim().toLowerCase();
+ if ('permission' in permission && typeof (permission as { permission?: unknown }).permission === 'string') {
+ return String((permission as { permission?: unknown }).permission).trim().toLowerCase();
+ }
+ return undefined;
+}
+
+function permissionDeclaration(permission: unknown): PermissionDeclaration | undefined {
+ const id = toPermissionId(permission);
+ if (!id) return undefined;
+ const raw = permission && typeof permission === 'object' ? permission as Record : {};
+ const reason = typeof raw.reason === 'string' ? raw.reason.trim() : undefined;
+ const prompt = typeof raw.prompt === 'string' ? raw.prompt.trim() : undefined;
+ return { id, reason: reason && reason.length ? reason : undefined, prompt: prompt && prompt.length ? prompt : undefined };
+}
+
+function permissionMap(permissionId: string): PermissionRuntime | undefined {
+ if (Platform.OS === 'web') return;
+ const capability = resolvePermissionCapabilityForDeclaration(permissionId);
+ return capability && capability in permissionRuntimeByCapability
+ ? permissionRuntimeByCapability[capability as keyof typeof permissionRuntimeByCapability]
+ : undefined;
+}
+
+export async function collectRuntimePermissions(pkg: AppPackage): Promise {
+ const requested: PermissionRequest[] = [];
+ const seen = new Set();
+ for (const raw of pkg.nativeCapabilities?.permissions ?? []) {
+ const declaration = permissionDeclaration(raw);
+ if (!declaration || seen.has(declaration.id)) continue;
+ const descriptor = permissionDescriptor(declaration);
+ const capability = resolvePermissionCapabilityForDeclaration(declaration.id) ?? declaration.id;
+ requested.push({
+ permission: declaration,
+ capability: capability ?? declaration.id,
+ unsupported: !descriptor,
+ });
+ seen.add(declaration.id);
+ }
+ return requested;
+}
+
+export async function collectPendingRuntimePermissions(appId: string, pkg: AppPackage): Promise {
+ const supported: PermissionRequest[] = [];
+ const unsupported: PermissionRequest[] = [];
+ for (const request of await collectRuntimePermissions(pkg)) {
+ if (request.unsupported) {
+ unsupported.push(request);
+ continue;
+ }
+ const decision = await readCapabilityDecision(appId, request.capability);
+ if (!decision) supported.push(request);
+ }
+ return [...supported, ...unsupported];
+}
+
+export async function requestBootPermission(appId: string, permission: PermissionRequest): Promise {
+ const descriptor = permissionDescriptor(permission.permission);
+ if (!descriptor) throw new CapabilityStateError('unavailable', false, `Permission unsupported on this platform: ${permission.permission.id}`);
+ const status = await descriptor.getStatus();
+ if (status.granted) {
+ await recordConsent(appId, permission.capability, DECISION_GRANTED);
+ return null;
+ }
+ const response = await descriptor.request();
+ const state = response.granted ? DECISION_GRANTED : DECISION_DENIED;
+ await recordConsent(appId, permission.capability, state);
+ return null;
+}
+
+export function unsupportedPermission(permission: PermissionDeclaration): boolean {
+ return !permissionDescriptor(permission);
+}
+
+export function toBootPermissionLabel(permission: PermissionRequest) {
+ return permission.permission.id;
+}
+
+function permissionDescriptor(permission: PermissionDeclaration): PermissionDescriptor | undefined {
+ const capability = resolvePermissionCapabilityForDeclaration(permission.id);
+ const runtime = permissionMap(permission.id);
+ if (!runtime) return;
+ return { ...runtime, ...permission, capability: capability! };
+}
+
function widgetLabel(component: AppComponent) {
return component.title || String(component.props?.title || component.widget || 'Capability');
}
@@ -53,11 +225,22 @@ function stateFromResult(
state: CapabilityExecutionState,
message?: string,
fallback?: string,
+ retryable?: boolean,
): CapabilityActionState {
if (state === 'idle' || state === 'running') {
return { ...states[state], state, message: message || fallback || capabilityMessage(state) };
}
- return { state, message: message || fallback || capabilityMessage(state) };
+ return { state, retryable, message: message || fallback || capabilityMessage(state) };
+}
+
+async function withRuntimePermission(
+ appId: string,
+ widget: string,
+ permission: { request: () => Promise<{ granted: boolean; canAskAgain?: boolean }>; getStatus: () => Promise<{ granted: boolean; canAskAgain?: boolean }> },
+ run: () => Promise,
+): Promise {
+ await ensurePermissionGranted(appId, widget, () => permission.request(), () => permission.getStatus());
+ return run();
}
async function runFilePicker(props: Record) {
@@ -101,67 +284,70 @@ async function runFileExport(appId: string, component: AppComponent) {
async function runLocation(appId: string, component: AppComponent) {
const location = await import('expo-location');
- const permission = await location.requestForegroundPermissionsAsync();
- await recordConsent(appId, 'location', permission.granted ? 'granted' : 'denied');
- if (!permission.granted) throw new CapabilityStateError('denied', true, 'Permission denied');
- const current = await location.getCurrentPositionAsync({});
- return { latitude: current.coords.latitude, longitude: current.coords.longitude };
+ return withRuntimePermission(appId, 'locationMap', {
+ request: () => location.requestForegroundPermissionsAsync(),
+ getStatus: () => location.getForegroundPermissionsAsync(),
+ }, async () => {
+ const current = await location.getCurrentPositionAsync({});
+ return { latitude: current.coords.latitude, longitude: current.coords.longitude };
+ });
}
async function runNotification(appId: string, component: AppComponent) {
const notifications = await import('expo-notifications');
- const permission = await notifications.requestPermissionsAsync();
- await recordConsent(appId, 'notifications', permission.granted ? 'granted' : 'denied');
- if (!permission.granted) throw new CapabilityStateError('denied', true, 'Permission denied');
- const props = component.props ?? {};
- const seconds = Number(props.seconds ?? 10);
- if (!Number.isFinite(seconds) || seconds < 1) {
- throw new CapabilityStateError('retry', true, 'Invalid notification timer');
- }
-
- const id = await notifications.scheduleNotificationAsync({
- content: {
- title: widgetLabel(component),
- body: String(props.body ?? ''),
- },
- trigger: {
- type: notifications.SchedulableTriggerInputTypes.TIME_INTERVAL,
- seconds,
- },
+ return withRuntimePermission(appId, 'notificationScheduler', {
+ request: () => notifications.requestPermissionsAsync(),
+ getStatus: () => notifications.getPermissionsAsync(),
+ }, async () => {
+ const props = component.props ?? {};
+ const seconds = Number(props.seconds ?? 10);
+ if (!Number.isFinite(seconds) || seconds < 1) throw new CapabilityStateError('retry', true, 'Invalid notification timer');
+ const id = await notifications.scheduleNotificationAsync({
+ content: {
+ title: widgetLabel(component),
+ body: String(props.body ?? ''),
+ },
+ trigger: {
+ type: notifications.SchedulableTriggerInputTypes.TIME_INTERVAL,
+ seconds,
+ },
+ });
+ return { id, scheduled: true };
});
- return { id, scheduled: true };
}
async function runContactPicker(appId: string, component: AppComponent) {
const contacts = await import('expo-contacts');
- const permission = await contacts.requestPermissionsAsync();
- await recordConsent(appId, 'contacts', permission.granted ? 'granted' : 'denied');
- if (!permission.granted) throw new CapabilityStateError('denied', true, 'Permission denied');
- const result = await contacts.presentContactPickerAsync();
- if (!result) throw new CapabilityStateError('cancelled', false, 'Contact picker cancelled');
- return { id: result.id, name: result.name ?? 'Unknown' };
+ return withRuntimePermission(appId, 'contactPicker', {
+ request: () => contacts.requestPermissionsAsync(),
+ getStatus: () => contacts.getPermissionsAsync(),
+ }, async () => {
+ const result = await contacts.presentContactPickerAsync();
+ if (!result) throw new CapabilityStateError('cancelled', false, 'Contact picker cancelled');
+ return { id: result.id, name: result.name ?? 'Unknown' };
+ });
}
async function runCalendar(appId: string, component: AppComponent) {
const calendar = await import('expo-calendar');
- const permission = await calendar.requestCalendarPermissionsAsync();
- await recordConsent(appId, 'calendar', permission.granted ? 'granted' : 'denied');
- if (!permission.granted) throw new CapabilityStateError('denied', true, 'Permission denied');
-
- const startOffsetMinutes = Number((component.props ?? {}).startOffsetMinutes ?? 10);
- const durationMinutes = Number((component.props ?? {}).durationMinutes ?? 30);
- const startDate = new Date(Date.now() + startOffsetMinutes * 60_000);
- const endDate = new Date(startDate.getTime() + durationMinutes * 60_000);
-
- const target = await calendar.getDefaultCalendarAsync();
- if (!target?.id) throw new CapabilityStateError('unavailable', false, 'No default calendar available');
-
- await calendar.createEventAsync(target.id, {
- title: String((component.props ?? {}).eventTitle ?? widgetLabel(component)),
- startDate,
- endDate,
+ return withRuntimePermission(appId, 'calendarEvent', {
+ request: () => calendar.requestCalendarPermissionsAsync(),
+ getStatus: () => calendar.getCalendarPermissionsAsync(),
+ }, async () => {
+ const props = component.props ?? {};
+ const startOffsetMinutes = Number(props.startOffsetMinutes ?? 10);
+ const durationMinutes = Number(props.durationMinutes ?? 30);
+ const startDate = new Date(Date.now() + startOffsetMinutes * 60_000);
+ const endDate = new Date(startDate.getTime() + durationMinutes * 60_000);
+ const target = await calendar.getDefaultCalendarAsync();
+ if (!target?.id) throw new CapabilityStateError('unavailable', false, 'No default calendar available');
+ await calendar.createEventAsync(target.id, {
+ title: String(props.eventTitle ?? widgetLabel(component)),
+ startDate,
+ endDate,
+ });
+ return 'Calendar event created';
});
- return 'Calendar event created';
}
async function runBiometrics(component: AppComponent) {
@@ -198,31 +384,125 @@ async function runHealthKit() {
}
function ActionResult({ state, message, onRetry }: { state: CapabilityActionState; message: string; onRetry: () => void }) {
- const showRetry = ['denied', 'unavailable', 'retry', 'cancelled'].includes(state.state);
+ const canRetry = state.retryable ?? ['retry', 'cancelled'].includes(state.state);
+ const isDenied = state.state === 'denied';
const textColor = state.state === 'success' ? '$green10' : state.state === 'running' ? '$blue10' : '$color10';
+
+ const openSettings = async () => {
+ if (Platform.OS === 'web') return;
+ try {
+ await Linking.openSettings();
+ } catch {
+ // noop
+ }
+ };
+
return (
{message}
- {showRetry ? : null}
+ {canRetry ? : null}
+ {isDenied && !canRetry ? : null}
);
}
type ResultHandler = (value: unknown) => void | Promise;
-async function bindResult(runtime: Store, component: AppComponent, value: unknown, onResult?: ResultHandler) {
- await onResult?.(value);
+type NativePermissionResult = { granted: boolean; canAskAgain?: boolean };
+
+async function ensurePermissionGranted(
+ appId: string,
+ widget: string,
+ request: () => Promise,
+ fallback?: () => Promise | NativePermissionResult,
+) {
+ const capability = await resolveCapability(appId, widget);
+ if (!capability) {
+ await assertCapability(appId, widget);
+ return;
+ }
+
+ const existing = await readCapabilityDecision(appId, capability);
+ if (existing?.state === 'granted') {
+ const current = await fallback?.();
+ if (current?.granted === false) {
+ await recordConsent(appId, capability, 'denied');
+ throw new CapabilityStateError('denied', false, `Permission denied for ${capability}`);
+ }
+ return;
+ }
+ if (existing?.state === 'denied') throw new CapabilityStateError('denied', false, `Permission denied for ${capability}`);
+
+ const current = await fallback?.();
+ if (current?.granted) {
+ await recordConsent(appId, capability, 'granted');
+ return;
+ }
+
+ const status = await request();
+ const decision = status.granted ? 'granted' : 'denied';
+ await recordConsent(appId, capability, decision);
+ if (decision === 'denied') {
+ throw new CapabilityStateError('denied', false, `Permission denied for ${capability}`);
+ }
+}
+
+function resultType(value: unknown): NativeResultValueType {
+ if (value === null) return 'null';
+ if (Array.isArray(value)) return 'array';
+ if (value === true || value === false) return 'boolean';
+ if (typeof value === 'string') return 'string';
+ if (typeof value === 'number') return 'number';
+ if (typeof value === 'object') return 'object';
+ return 'unknown';
+}
+
+function bindableNativeResult(
+ appId: string,
+ component: AppComponent,
+ value: unknown,
+) {
+ const field = String(component.props?.resultField ?? 'result');
+ return {
+ schema: 'utopia.native.result.v1',
+ source: 'native',
+ appId,
+ capability: String(component.widget),
+ widget: String(component.widget),
+ resultField: field,
+ valueType: resultType(value),
+ timestamp: new Date().toISOString(),
+ value,
+ } satisfies NativeResultRecord;
+}
+
+async function bindResult(runtime: Store, appId: string, component: AppComponent, value: unknown, onResult?: ResultHandler) {
+ const record = bindableNativeResult(appId, component, value);
+ await onResult?.(record);
const field = String(component.props?.resultField ?? 'result');
const action = component.action;
if (action && (action.kind === 'create' || action.kind === 'update')) {
- await runtime.dispatch({ ...action, values: { ...action.values, [field]: value } });
+ await runtime.dispatch({ ...action, values: { ...action.values, [field]: record } });
} else if (component.props?.collection) {
- await runtime.dispatch({ kind: 'create', collection: String(component.props.collection), values: { [field]: value } });
+ await runtime.dispatch({ kind: 'create', collection: String(component.props.collection), values: { [field]: record } });
}
}
const resultText = (value: unknown) => typeof value === 'string' ? value : JSON.stringify(value);
+const runByWidget = {
+ filePicker: (appId, component) => runFilePicker(component.props ?? {}),
+ fileExport: (appId, component) => runFileExport(appId, component),
+ locationMap: (appId, component) => runLocation(appId, component),
+ notificationScheduler: (appId, component) => runNotification(appId, component),
+ contactPicker: (appId, component) => runContactPicker(appId, component),
+ calendarEvent: (appId, component) => runCalendar(appId, component),
+ biometricGate: (appId, component) => runBiometrics(component),
+ speechTool: (appId, component) => runSpeech(component),
+ healthConnect: (appId) => runHealthConnect(),
+ healthKitStatus: (appId) => runHealthKit(),
+} satisfies Record Promise>;
+
function CapabilityAction({ appId, component, onResult }: { appId: string; component: AppComponent; onResult: ResultHandler }) {
const [actionState, setActionState] = useState(states.idle);
const [value, setValue] = useState();
@@ -231,25 +511,20 @@ function CapabilityAction({ appId, component, onResult }: { appId: string; compo
setActionState(states.running);
const result = await executeCapability(async () => {
await assertCapability(appId, String(component.widget));
- switch (component.widget) {
- case 'filePicker': return runFilePicker(component.props ?? {});
- case 'fileExport': return runFileExport(appId, component);
- case 'locationMap': return runLocation(appId, component);
- case 'notificationScheduler': return runNotification(appId, component);
- case 'contactPicker': return runContactPicker(appId, component);
- case 'calendarEvent': return runCalendar(appId, component);
- case 'biometricGate': return runBiometrics(component);
- case 'speechTool': return runSpeech(component);
- case 'healthConnect': return runHealthConnect();
- case 'healthKitStatus': return runHealthKit();
- default: throw new CapabilityStateError('unavailable', false, `Unsupported native widget ${String(component.widget)}`);
- }
+ const runner = runByWidget[component.widget as keyof typeof runByWidget];
+ if (!runner) throw new CapabilityStateError('unavailable', false, `Unsupported native widget ${String(component.widget)}`);
+ return runner(appId, component);
});
if (result.state === 'success') {
setValue(result.value);
await onResult(result.value);
}
- setActionState(stateFromResult(result.state, result.state === 'success' ? resultText(result.value) : result.message, capabilityMessage(result.state)));
+ setActionState(stateFromResult(
+ result.state,
+ result.state === 'success' ? resultText(result.value) : result.message,
+ capabilityMessage(result.state),
+ result.retryable,
+ ));
};
const cancelNotification = async () => {
@@ -291,14 +566,13 @@ function Scanner({ appId, component, onResult }: { appId: string; component: App
setReadState(states.running);
const outcome = await executeCapability(async () => {
- await assertCapability(appId, 'cameraScanner');
- const response = await requestPermission();
- if (!response.granted) {
- if (permission?.canAskAgain) {
- throw new CapabilityStateError('denied', true, 'Permission denied');
- }
- throw new CapabilityStateError('unavailable', false, 'Camera permission permanently denied');
- }
+ await ensurePermissionGranted(appId, 'cameraScanner', async () => {
+ const response = await requestPermission();
+ return { granted: response.granted, canAskAgain: response.canAskAgain };
+ }, () => {
+ if (!permission) return { granted: false, canAskAgain: true };
+ return { granted: permission.granted, canAskAgain: permission.canAskAgain };
+ });
setAuthorized(true);
return 'Permission granted';
});
@@ -307,18 +581,39 @@ function Scanner({ appId, component, onResult }: { appId: string; component: App
useEffect(() => {
if (!permission) return;
- if (permission.granted) {
- void assertCapability(appId, 'cameraScanner').then(() => {
+ void (async () => {
+ try {
+ if (!permission.granted) {
+ const capability = await resolveCapability(appId, 'cameraScanner');
+ if (!capability) {
+ await assertCapability(appId, 'cameraScanner');
+ return;
+ }
+ const decision = await readCapabilityDecision(appId, capability);
+ if (decision?.state === 'denied') {
+ setAuthorized(false);
+ setReadState(stateFromResult('denied', `Permission denied for ${capability}`, undefined, false));
+ return;
+ }
+ setAuthorized(false);
+ setReadState(stateFromResult('idle', capabilityMessage('idle'), 'Ready'));
+ return;
+ }
+
+ const capability = await resolveCapability(appId, 'cameraScanner');
+ if (!capability) {
+ await assertCapability(appId, 'cameraScanner');
+ return;
+ }
+
+ await recordConsent(appId, capability, 'granted');
setAuthorized(true);
setReadState(stateFromResult('success', 'Camera ready', 'Camera ready'));
- }).catch((cause) => setReadState(stateFromResult('denied', cause instanceof Error ? cause.message : 'Denied')));
- return;
- }
- if (!permission.canAskAgain) {
- setReadState(stateFromResult('unavailable', 'Camera permission disabled on this device', 'Unavailable'));
- return;
- }
- setReadState(stateFromResult('idle', capabilityMessage('idle'), 'Ready'));
+ } catch (cause) {
+ setAuthorized(false);
+ setReadState(stateFromResult('denied', cause instanceof Error ? cause.message : 'Permission denied', 'Denied', false));
+ }
+ })();
}, [appId, permission?.granted, permission?.canAskAgain]);
if (!permission?.granted || !authorized) {
@@ -348,19 +643,27 @@ function Scanner({ appId, component, onResult }: { appId: string; component: App
onBarcodeScanned={status ? undefined : ({ data }: BarcodeScanningResult) => {
setStatus(data);
setReadState(stateFromResult('success', `Scanned ${data}`));
- void onResult({ type: 'barcode', value: data });
+ void onResult(bindableNativeResult(appId, component, { type: 'barcode', value: data }));
}}
/>
{captureMode === 'photo' ? : null}
{captureMode === 'video' ? : null}
{ setStatus(''); setReadState(states.idle); }} />
{status ? (
@@ -388,7 +691,7 @@ function Sensor({ appId, component, onResult }: { appId: string; component: AppC
subscription.current.remove();
subscription.current = null;
setState(stateFromResult('success', 'Stopped'));
- await onResult(value);
+ await onResult(bindableNativeResult(appId, component, value));
return;
}
setState(states.running);
@@ -419,7 +722,7 @@ function Sensor({ appId, component, onResult }: { appId: string; component: AppC
export function NativeCapability({ appId, component, onResult }: { appId: string; component: AppComponent; onResult?: ResultHandler }) {
const runtime = useAppStore();
- const bind = (value: unknown) => bindResult(runtime, component, value, onResult);
+ const bind = (value: unknown) => bindResult(runtime, appId, component, value, onResult);
if (component.widget === 'cameraScanner') return ;
if (component.widget === 'sensorReadout') return ;
if (actionWidgets.has(component.widget as NativeActionWidget)) {
diff --git a/src/kernel/capability-state.ts b/src/kernel/capability-state.ts
index 658e3f6..53ca983 100644
--- a/src/kernel/capability-state.ts
+++ b/src/kernel/capability-state.ts
@@ -5,11 +5,13 @@ export type CapabilityResult = {
state: CapabilityTerminalState;
message: string;
value?: T;
+ retryable?: boolean;
};
export type CapabilityActionState = {
state: CapabilityExecutionState;
message: string;
+ retryable?: boolean;
};
export type CapabilityStateConfig = {
@@ -86,9 +88,10 @@ export function classifyCapabilityError(cause: unknown): CapabilityTerminalState
export async function executeCapability(operation: () => Promise): Promise> {
try {
const value = await operation();
- return { state: 'success', message: capabilityMessage('success'), value };
+ return { state: 'success', message: capabilityMessage('success'), retryable: false, value };
} catch (cause) {
const state = classifyCapabilityError(cause);
- return { state, message: toStringError(cause), value: undefined };
+ const retryable = cause instanceof CapabilityStateError ? cause.retryable : true;
+ return { state, message: toStringError(cause), retryable, value: undefined };
}
}
diff --git a/src/kernel/computed.ts b/src/kernel/computed.ts
new file mode 100644
index 0000000..ec04fb1
--- /dev/null
+++ b/src/kernel/computed.ts
@@ -0,0 +1,579 @@
+import { RRule } from 'rrule';
+import jsonLogic from 'json-logic-js';
+
+import type { AppPackage } from './schema';
+import type { AppState, JsonRecord } from './runtime';
+
+type Budget = { steps: number };
+type Decimal = { value: bigint; scale: number; };
+type QueryRows = Record;
+type JsonRow = Record;
+type ObjectRecord = Record;
+type ComputedExpression = { id: string; collection: string; dependsOn: string[]; expression: unknown };
+type JsonValue = unknown;
+
+type DecimalLike = Decimal | null;
+
+type ComputedContext = {
+ record: Record & { id: string };
+ queries: QueryRows;
+};
+
+type BalanceTransferRow = { from: string; to: string; amount: string };
+type AllocationRow = Record;
+
+const MAX_WORK = 500;
+const MAX_DECIMAL_SCALE = 18;
+const WEEKDAY_MAP: ReadonlyArray = [Number(RRule.SU), Number(RRule.MO), Number(RRule.TU), Number(RRule.WE), Number(RRule.TH), Number(RRule.FR), Number(RRule.SA)];
+
+const FREQUENCY_BY_NAME: Record = {
+ yearly: RRule.YEARLY,
+ monthly: RRule.MONTHLY,
+ weekly: RRule.WEEKLY,
+ daily: RRule.DAILY,
+ hourly: RRule.HOURLY,
+ minutely: RRule.MINUTELY,
+};
+
+const DATE_UNITS: Record = {
+ seconds: 1_000,
+ minutes: 60_000,
+ hours: 3_600_000,
+ days: 86_400_000,
+};
+
+const MATH_OPERATORS = new Set(['+', '-', '*', '/', '%']);
+
+const isObject = (value: JsonValue): value is ObjectRecord =>
+ typeof value === 'object' && value !== null && !Array.isArray(value);
+
+const consume = (value: JsonValue, budget: Budget): void => {
+ budget.steps += 1;
+ if (budget.steps > MAX_WORK) throw new Error('expression_budget_exceeded');
+ if (Array.isArray(value)) {
+ value.forEach((item) => consume(item, budget));
+ return;
+ }
+ if (isObject(value)) {
+ for (const entry of Object.values(value)) consume(entry, budget);
+ }
+};
+
+const read = (value: JsonValue, path: string): JsonValue =>
+ path
+ .split('.')
+ .reduce((current, key) => {
+ if (current == null) return undefined;
+ if (Array.isArray(current)) {
+ const index = Number(key);
+ return Number.isInteger(index) ? current[index] : undefined;
+ }
+ return isObject(current) ? current[key] : undefined;
+ }, value);
+
+const object = (value: JsonValue, error: string): ObjectRecord => {
+ if (!isObject(value)) throw new Error(error);
+ return value;
+};
+
+const readRows = (value: JsonValue): JsonRow[] => {
+ if (!Array.isArray(value) || value.length > MAX_WORK || value.some((row) => !isObject(row))) {
+ throw new Error('expression_rows_invalid');
+ }
+ return value as JsonRow[];
+};
+
+const decimalFromNumber = (value: number): Decimal => {
+ if (!Number.isFinite(value)) throw new Error('expression_number_invalid');
+ const text = value.toString();
+ return parseDecimal(text) ?? { value: BigInt(Math.trunc(value)), scale: 0 };
+};
+
+const parseDecimal = (value: JsonValue): DecimalLike => {
+ if (value == null || value === false) return null;
+ if (value === true) return { value: 1n, scale: 0 };
+ if (typeof value === 'number') return decimalFromNumber(value);
+ if (typeof value !== 'string') return null;
+
+ const normalized = value.trim();
+ if (!normalized) return null;
+
+ const match = /^[+-]?(?:\d+|\d*\.\d+)$/;
+ if (!match.test(normalized)) return null;
+
+ const signed = normalized;
+ const isNegative = signed.startsWith('-');
+ const unsigned = isNegative || signed.startsWith('+') ? signed.slice(1) : signed;
+ const [integerPartRaw = '0', fractionRaw = ''] = unsigned.split('.');
+
+ if (!integerPartRaw && !fractionRaw) return null;
+ if (integerPartRaw.length > 20 || fractionRaw.length > MAX_DECIMAL_SCALE) return null;
+
+ if (!/^\d*$/.test(integerPartRaw) || !/^\d*$/.test(fractionRaw)) return null;
+
+ const removed = `${integerPartRaw || '0'}${fractionRaw}`;
+ const valueBig = removed ? BigInt(`${isNegative ? '-' : ''}${removed}`) : 0n;
+ const scale = fractionRaw.length;
+ return normalizeDecimal({ value: valueBig, scale });
+};
+
+const pow10 = (scale: number): bigint => 10n ** BigInt(Math.max(0, scale));
+
+const normalizeDecimal = (input: Decimal): Decimal => {
+ let value = input.value;
+ let scale = input.scale;
+ if (scale <= 0) return { value, scale: 0 };
+ while (scale > 0 && value % 10n === 0n) {
+ value /= 10n;
+ scale -= 1;
+ }
+ return { value, scale };
+};
+
+const alignScale = (left: Decimal, right: Decimal): [Decimal, Decimal] => {
+ if (left.scale === right.scale) return [left, right];
+ if (left.scale > right.scale) {
+ const factor = pow10(left.scale - right.scale);
+ return [left, { value: right.value * factor, scale: left.scale }];
+ }
+ const factor = pow10(right.scale - left.scale);
+ return [{ value: left.value * factor, scale: right.scale }, right];
+};
+
+const decimalAdd = (left: Decimal, right: Decimal): Decimal => {
+ const [a, b] = alignScale(left, right);
+ return normalizeDecimal({ value: a.value + b.value, scale: a.scale });
+};
+
+const decimalSub = (left: Decimal, right: Decimal): Decimal => {
+ const [a, b] = alignScale(left, right);
+ return normalizeDecimal({ value: a.value - b.value, scale: a.scale });
+};
+
+const decimalMul = (left: Decimal, right: Decimal): Decimal => {
+ const scale = Math.min(MAX_DECIMAL_SCALE, left.scale + right.scale);
+ let value = left.value * right.value;
+ const rawScale = left.scale + right.scale;
+ if (rawScale > scale) {
+ value /= pow10(rawScale - scale);
+ }
+ return normalizeDecimal({ value, scale });
+};
+
+const decimalDiv = (left: Decimal, right: Decimal, budget: Budget): Decimal => {
+ if (right.value === 0n) throw new Error('expression_divide_by_zero');
+ if (left.value === 0n) return { value: 0n, scale: 0 };
+
+ const targetScale = Math.min(MAX_DECIMAL_SCALE, Math.max(9, left.scale, right.scale) + 6);
+ const numerator = left.value * pow10(targetScale + right.scale);
+ const value = numerator / right.value;
+ return normalizeDecimal({ value, scale: targetScale + left.scale - right.scale });
+};
+
+const decimalMod = (left: Decimal, right: Decimal): Decimal => {
+ if (right.value === 0n) throw new Error('expression_divide_by_zero');
+ const [a, b] = alignScale(left, right);
+ return normalizeDecimal({ value: a.value % b.value, scale: a.scale });
+};
+
+const decimalToString = ({ value, scale }: Decimal): string => {
+ if (scale <= 0) return value.toString();
+ const sign = value < 0n ? '-' : '';
+ const absolute = value < 0n ? -value : value;
+ const unit = pow10(scale);
+ const integer = absolute / unit;
+ const fraction = (absolute % unit).toString().padStart(scale, '0').replace(/0+$/, '');
+ return fraction ? `${sign}${integer}.${fraction}` : `${sign}${integer}`;
+};
+
+const parseDate = (value: JsonValue): number => {
+ const parsed = Date.parse(String(value ?? ''));
+ if (!Number.isFinite(parsed)) throw new Error('expression_date_diff_invalid');
+ return parsed;
+};
+
+const sameValue = (left: JsonValue, right: JsonValue): boolean => JSON.stringify(left) === JSON.stringify(right);
+
+const toNumeric = (value: JsonValue): number => {
+ if (typeof value === 'number') return value;
+ const numberValue = Number(value);
+ return Number.isFinite(numberValue) ? numberValue : 0;
+};
+
+const evaluateDateDiff = (spec: ObjectRecord, context: ComputedContext, budget: Budget): number => {
+ const start = evaluate(spec.start, context, budget);
+ const end = evaluate(spec.end, context, budget);
+
+ if (start == null || end == null || start === '' || end === '') {
+ if (spec.onMissing === 'zero') return 0;
+ throw new Error('expression_date_diff_missing');
+ }
+
+ let left = parseDate(start);
+ let right = parseDate(end);
+ const onInvalid = String(spec.onInvalid ?? 'error');
+
+ if (!Number.isFinite(left) || !Number.isFinite(right)) {
+ if (onInvalid === 'zero') return 0;
+ throw new Error('expression_date_diff_invalid');
+ }
+
+ if (right < left && spec.onEndBeforeStart === 'error') throw new Error('expression_date_diff_end_before_start');
+ if (right < left && spec.onEndBeforeStart === 'zero') return 0;
+
+ const unit = String(spec.unit ?? 'days');
+ const divisor = DATE_UNITS[unit];
+ if (!divisor || String(spec.timezone) !== 'UTC') throw new Error('expression_date_diff_spec_invalid');
+ return Math.floor((right - left) / divisor);
+};
+
+const parseWeekday = (value: JsonValue): number | undefined => {
+ const index = Number(value);
+ if (!Number.isInteger(index) || index < 0 || index > 6) return;
+ return WEEKDAY_MAP[index];
+};
+
+const buildRecurrence = (schedule: JsonValue, after: Date): RRule => {
+ if (typeof schedule === 'string') return RRule.fromString(schedule);
+
+ const spec = object(schedule, 'expression_recurrence_invalid');
+ const frequency = FREQUENCY_BY_NAME[String(spec.frequency)];
+ if (!frequency) throw new Error('expression_recurrence_invalid');
+
+ const byweekday = Array.isArray(spec.byWeekday)
+ ? spec.byWeekday
+ .map(parseWeekday)
+ .filter((item): item is number => item !== undefined)
+ .map((weekday) => WEEKDAY_MAP[weekday] as number)
+ : undefined;
+
+ const interval = Math.max(1, Number(spec.interval ?? 1));
+ const count = spec.count == null ? undefined : Number(spec.count);
+
+ return new RRule({
+ freq: frequency,
+ interval,
+ dtstart: spec.start ? new Date(String(spec.start)) : after,
+ count: Number.isFinite(count) ? count : undefined,
+ until: spec.until ? new Date(String(spec.until)) : undefined,
+ byweekday,
+ });
+};
+
+const evaluateGroupSum = (operand: JsonValue, context: ComputedContext, budget: Budget): JsonValue => {
+ const spec = object(operand, 'expression_group_sum_invalid');
+ const rows = readRows(evaluate(spec.rows, context, budget));
+ const expected = evaluate(spec.equals, context, budget);
+ const groupBy = String(spec.groupBy);
+ const valueKey = String(spec.value);
+
+ let total: DecimalLike = null;
+ let totalNumber = 0;
+ let usedDecimal = false;
+
+ for (const row of rows) {
+ if (!sameValue(read(row, groupBy), expected)) continue;
+ const raw = read(row, valueKey);
+ const parsed = parseDecimal(raw);
+ if (parsed) {
+ usedDecimal = true;
+ total = total ? decimalAdd(total, parsed) : parsed;
+ continue;
+ }
+
+ const numberValue = toNumeric(raw);
+ totalNumber += numberValue;
+ }
+
+ if (!usedDecimal) return totalNumber;
+ return decimalToString(decimalAdd(total ?? { value: 0n, scale: 0 }, parseDecimal(String(totalNumber)) ?? { value: 0n, scale: 0 }));
+};
+
+const evaluateAllocation = (operand: JsonValue, context: ComputedContext, budget: Budget): AllocationRow[] => {
+ const spec = object(operand, 'expression_allocate_weighted_invalid');
+ const rows = readRows(evaluate(spec.rows, context, budget));
+ if (!rows.length) throw new Error('expression_allocate_weighted_invalid');
+
+ const keyKey = String(spec.key);
+ const weightKey = String(spec.weight);
+ const amountKey = String(spec.amount ?? 'amount');
+ const total = parseDecimal(evaluate(spec.total, context, budget));
+ if (!total) throw new Error('expression_allocate_weighted_invalid');
+
+ const totalWeight = rows
+ .map((row) => parseDecimal(read(row, weightKey)))
+ .reduce((acc, item) => (item ? (acc ? decimalAdd(acc, item) : item) : undefined), undefined);
+
+ if (!totalWeight || totalWeight.value === 0n) throw new Error('expression_allocate_weighted_invalid');
+
+ return rows.map((row) => {
+ const weight = parseDecimal(read(row, weightKey));
+ if (!weight) throw new Error('expression_allocate_weighted_invalid');
+ const share = decimalDiv(decimalMul(total, weight), totalWeight, budget);
+ return {
+ [keyKey]: String(read(row, keyKey) ?? ''),
+ [weightKey]: decimalToString(weight),
+ [amountKey]: decimalToString(share),
+ };
+ });
+};
+
+const evaluateBalanceTransfers = (operand: JsonValue, context: ComputedContext, budget: Budget): BalanceTransferRow[] => {
+ const spec = object(operand, 'expression_balance_transfers_invalid');
+ const rows = readRows(evaluate(spec.rows, context, budget));
+
+ const participantKey = String(spec.participant);
+ const paidKey = String(spec.paid);
+ const owedKey = String(spec.owed);
+
+ const balances = new Map();
+ for (const row of rows) {
+ const participant = String(read(row, participantKey) ?? '');
+ const paid = parseDecimal(read(row, paidKey));
+ const owed = parseDecimal(read(row, owedKey));
+
+ const paidDecimal = paid ?? decimalFromNumber(toNumeric(read(row, paidKey)));
+ const owedDecimal = owed ?? decimalFromNumber(toNumeric(read(row, owedKey)));
+
+ const current = balances.get(participant) ?? { value: 0n, scale: 0 };
+ balances.set(participant, decimalSub(decimalAdd(current, paidDecimal), owedDecimal));
+ }
+
+ const debtors = [...balances.entries()]
+ .filter(([, amount]) => amount.value < 0n)
+ .map(([person, amount]) => ({ person, amount: { ...amount, value: -amount.value } }));
+ const creditors = [...balances.entries()]
+ .filter(([, amount]) => amount.value > 0n)
+ .map(([person, amount]) => ({ person, amount }));
+
+ const transfers: BalanceTransferRow[] = [];
+ let debtorIndex = 0;
+ let creditorIndex = 0;
+
+ while (debtorIndex < debtors.length && creditorIndex < creditors.length) {
+ const debtor = debtors[debtorIndex];
+ const creditor = creditors[creditorIndex];
+
+ const amount: Decimal = debtor.amount.value >= creditor.amount.value
+ ? creditor.amount
+ : debtor.amount;
+
+ if (amount.value > 0n) {
+ transfers.push({ from: debtor.person, to: creditor.person, amount: decimalToString(amount) });
+ }
+
+ const remainingDebtor = decimalSub(debtor.amount, amount);
+ const remainingCreditor = decimalSub(creditor.amount, amount);
+
+ if (remainingDebtor.value === 0n) {
+ debtorIndex += 1;
+ } else {
+ debtors[debtorIndex] = { ...debtor, amount: remainingDebtor };
+ }
+
+ if (remainingCreditor.value === 0n) {
+ creditorIndex += 1;
+ } else {
+ creditors[creditorIndex] = { ...creditor, amount: remainingCreditor };
+ }
+ }
+
+ return transfers;
+};
+
+const evaluateRecurrence = (operator: 'recurrence_next' | 'recurrence_expand', operand: JsonValue, context: ComputedContext, budget: Budget): JsonValue => {
+ const spec = object(operand, `expression_${operator}_invalid`);
+ const after = new Date(String(evaluate(spec.after, context, budget) ?? new Date().toISOString()));
+ const rule = buildRecurrence(evaluate(spec.schedule, context, budget), after);
+
+ if (operator === 'recurrence_next') {
+ return rule.after(after, false)?.toISOString() ?? null;
+ }
+
+ const until = spec.until ? new Date(String(evaluate(spec.until, context, budget))) : undefined;
+ const limit = Math.min(MAX_WORK, Number(evaluate(spec.limit, context, budget) ?? 20));
+ return rule
+ .between(after, until ?? new Date(after.getTime() + (MAX_WORK * 86_400_000)), false)
+ .slice(0, limit)
+ .map((date) => date.toISOString());
+};
+
+const evalMath = (operator: string, args: unknown[], context: ComputedContext, budget: Budget): JsonValue => {
+ if (!args.length) return operator === '-' ? 0 : 1;
+ const values = args.map((item) => evaluate(item, context, budget));
+ const first = values[0] as JsonValue;
+
+ const toDecimalOrNumber = (raw: unknown): DecimalLike => parseDecimal(raw);
+
+ if (values.length === 1 && operator === '-') {
+ const unary = toDecimalOrNumber(first);
+ if (unary) return decimalToString({ ...unary, value: -unary.value });
+ return -toNumeric(first);
+ }
+
+ return values.slice(1).reduce((accumulator, currentArg) => {
+ const leftDecimal = toDecimalOrNumber(accumulator);
+ const rightDecimal = toDecimalOrNumber(currentArg);
+
+ if (leftDecimal && rightDecimal) {
+ const handlers = {
+ '+': () => decimalToString(decimalAdd(leftDecimal, rightDecimal)),
+ '-': () => decimalToString(decimalSub(leftDecimal, rightDecimal)),
+ '*': () => decimalToString(decimalMul(leftDecimal, rightDecimal)),
+ '/': () => decimalToString(decimalDiv(leftDecimal, rightDecimal, budget)),
+ '%': () => decimalToString(decimalMod(leftDecimal, rightDecimal)),
+ };
+ return handlers[operator as keyof typeof handlers]();
+ }
+
+ const lhs = leftDecimal ? Number(leftDecimal.value) / Number(pow10(leftDecimal.scale)) : toNumeric(accumulator);
+ const rhs = rightDecimal ? Number(rightDecimal.value) / Number(pow10(rightDecimal.scale)) : toNumeric(currentArg);
+
+ const handlers = {
+ '+': lhs + rhs,
+ '-': lhs - rhs,
+ '*': lhs * rhs,
+ '/': rhs === 0 ? null : lhs / rhs,
+ '%': lhs % rhs,
+ };
+ return handlers[operator as keyof typeof handlers];
+ }, first);
+};
+
+const resolveMath = (operator: string, operand: unknown, context: ComputedContext, budget: Budget) => {
+ if (!Array.isArray(operand)) return operator === '-' ? 0 : 1;
+ return evalMath(operator, operand, context, budget);
+};
+
+const evaluateByOperator: Record JsonValue> = {
+ var: (operand, context) => read(context, String(operand)),
+ date_diff: (operand, context, budget) => evaluateDateDiff(object(operand, 'expression_date_diff_spec_invalid'), context, budget),
+ group_sum: (operand, context, budget) => evaluateGroupSum(operand, context, budget),
+ allocate_weighted: (operand, context, budget) => evaluateAllocation(operand, context, budget),
+ balance_transfers: (operand, context, budget) => evaluateBalanceTransfers(operand, context, budget),
+ recurrence_next: (operand, context, budget) => evaluateRecurrence('recurrence_next', operand, context, budget),
+ recurrence_expand: (operand, context, budget) => evaluateRecurrence('recurrence_expand', operand, context, budget),
+};
+
+const dependsOnFromExpression = (expression: JsonValue): string[] => {
+ const deps = new Set();
+
+ const visit = (value: JsonValue): void => {
+ if (!value || typeof value !== 'object') return;
+ if (Array.isArray(value)) {
+ value.forEach(visit);
+ return;
+ }
+ const objectValue = value as ObjectRecord;
+ const entries = Object.entries(objectValue);
+
+ if (entries.length === 1 && entries[0][0] === 'var') {
+ const candidate = String(entries[0][1] ?? '');
+ if (candidate.startsWith('record.')) {
+ const field = candidate.slice('record.'.length);
+ const first = field.split('.')[0];
+ if (first) deps.add(first);
+ }
+ return;
+ }
+
+ for (const [, child] of entries) visit(child);
+ };
+
+ visit(expression);
+ return [...deps];
+};
+
+function evaluate(expression: JsonValue, context: ComputedContext, budget: Budget): JsonValue {
+ if (++budget.steps > MAX_WORK) throw new Error('expression_budget_exceeded');
+
+ if (expression === null || expression === undefined || typeof expression !== 'object' || Array.isArray(expression)) {
+ return Array.isArray(expression) ? expression.map((item) => evaluate(item, context, budget)) : expression;
+ }
+
+ const entries = Object.entries(expression);
+ if (!entries.length) return expression;
+ if (entries.length !== 1) {
+ return Object.fromEntries(entries.map(([key, value]) => [key, evaluate(value, context, budget)]));
+ }
+
+ const [operator, operand] = entries[0];
+ if (evaluateByOperator[operator]) return evaluateByOperator[operator](operand, context, budget);
+ if (MATH_OPERATORS.has(operator)) return resolveMath(operator, operand, context, budget);
+
+ try {
+ consume(operand, budget);
+ return jsonLogic.apply(expression as never, context);
+ } catch (error) {
+ if (error instanceof Error && error.message.startsWith('expression_')) throw error;
+ throw new Error(`unsupported_expression_operator:${operator}`);
+ }
+}
+
+export function evaluateExpression(expression: unknown, context: ComputedContext): unknown {
+ return evaluate(expression, context, { steps: 0 });
+}
+
+export const sortComputedFields = (fields: ComputedExpression[]): ComputedExpression[] => {
+ const lookup = new Map(fields.map((field) => [field.id, field]));
+ const state = new Map();
+ const ordered: ComputedExpression[] = [];
+
+ const edges = new Map>();
+ for (const field of fields) {
+ const explicit = field.dependsOn ?? [];
+ const inferred = dependsOnFromExpression(field.expression).filter((dependency) => dependency !== field.id);
+ const dependencies = [...new Set([...explicit, ...inferred])];
+ edges.set(field.id, new Set(dependencies));
+ }
+
+ const visit = (id: string, stack: string[]): void => {
+ const current = state.get(id);
+ if (current === 'done') return;
+ if (current === 'visiting') throw new Error(`computed_field_cycle:${[...stack, id].join('>')}`);
+
+ const field = lookup.get(id);
+ if (!field) throw new Error(`computed_field_dependency_missing:${stack.at(-1) ?? 'unknown'}:${id}`);
+
+ state.set(id, 'visiting');
+ for (const dependency of edges.get(id) ?? []) {
+ if (!lookup.has(dependency)) throw new Error(`computed_field_dependency_missing:${id}:${dependency}`);
+ visit(dependency, [...stack, id]);
+ }
+ state.set(id, 'done');
+ ordered.push(field);
+ };
+
+ for (const field of fields) {
+ if (state.get(field.id) !== 'done') visit(field.id, []);
+ }
+
+ return ordered;
+};
+
+export function computedRecords(pkg: AppPackage, state: AppState): JsonRecord[] {
+ if (!pkg.computedFields.length) return state.records;
+
+ const queryContext: QueryRows = Object.fromEntries(
+ Object.entries(pkg.queries).map(([id, query]) => [id, { rows: state.records
+ .filter((record) => record.collection === query.from)
+ .map((record) => ({ id: record.id, ...record.values })), }]),
+ );
+
+ return state.records.map((record) => {
+ const values: Record = { ...record.values };
+ const fields = pkg.computedFields.filter((field) => field.collection === record.collection);
+ if (!fields.length) return record;
+
+ const ordered = sortComputedFields(fields);
+ const context: ComputedContext = { record: { ...record.values, id: record.id }, queries: queryContext };
+
+ for (const field of ordered) {
+ const computed = evaluateExpression(field.expression, context);
+ values[field.id] = computed;
+ context.record[field.id] = computed;
+ }
+
+ return { ...record, values };
+ });
+}
diff --git a/src/kernel/data-home.ts b/src/kernel/data-home.ts
index 9936330..2e687a8 100644
--- a/src/kernel/data-home.ts
+++ b/src/kernel/data-home.ts
@@ -11,7 +11,21 @@ export type DataHomeConfig = {
mode?: DataHomeMode;
};
-type DataHomeRecord = {
+type SecretMap = Record;
+
+export type DataHomeScope = { appId: string; tenantId: string };
+
+export type DataHomeTransport = {
+ pull(input: { cursor?: string; limit?: number }): Promise<{ records: JsonRecord[]; cursor?: string; hasMore: boolean }>;
+ push(input: { records: JsonRecord[] }): Promise<{ cursor?: string }>;
+};
+
+type StorageProvider = {
+ getItem(key: string): Promise;
+ setItem(key: string, value: string): Promise;
+};
+
+type RawRecord = {
id: string;
collection: string;
createdAt: string;
@@ -19,174 +33,636 @@ type DataHomeRecord = {
values: Record;
};
-type SecretMap = Record;
+type ScopedRecord = RawRecord & { scope: string };
-type StorageProvider = { getItem(key: string): Promise; setItem(key: string, value: string): Promise };
+export type Transport = DataHomeTransport;
-type RemoteDataHomeConfig = DataHomeConfig & { id: string; secretRef: string; mode?: DataHomeMode };
-export type DataHomeScope = { appId: string; tenantId: string };
+export type DataHomeRecord = RawRecord;
-export type DataHomeTransport = {
- pull(input: { cursor?: string; limit?: number }): Promise<{ records: JsonRecord[]; cursor?: string; hasMore: boolean }>;
- push(input: { records: JsonRecord[] }): Promise<{ cursor?: string }>;
+type NotionApi = {
+ dataSources?: { query: (input: Record) => Promise };
+ databases?: { query: (input: Record) => Promise };
+ pages: {
+ create: (input: Record) => Promise;
+ update: (input: Record) => Promise;
+ };
};
-export type Transport = DataHomeTransport;
+type NotionQueryResult = {
+ results: unknown[];
+ has_more?: boolean;
+ next_cursor?: string | null;
+};
+
+type SheetsClient = {
+ spreadsheets: {
+ values: {
+ get: (input: { spreadsheetId: string; range: string }) => Promise<{ data: { values?: unknown[][] } }>;
+ clear: (input: { spreadsheetId: string; range: string }) => Promise;
+ update: (input: {
+ spreadsheetId: string;
+ range: string;
+ valueInputOption: string;
+ requestBody: { values: Array> };
+ }) => Promise;
+ append: (input: {
+ spreadsheetId: string;
+ range: string;
+ valueInputOption: string;
+ insertDataOption: string;
+ requestBody: { values: Array> };
+ }) => Promise;
+ };
+ };
+};
-const fail = (message: string): never => { throw new Error(message); };
-const reSecretRef = /^[A-Z][A-Z0-9_]*$/;
+const MAX_LIMIT = 200;
+const IDENTIFIER = /^[A-Za-z0-9_-]+$/;
-function networkUrl(value: string): string {
- const url = new URL(value);
- if (!['https:', 'http:'].includes(url.protocol)) {
- throw new Error('invalid endpoint protocol');
+const fail = (message: string): never => {
+ throw new Error(message);
+};
+
+const isDataHomeScope = (scope: DataHomeScope | undefined): scope is DataHomeScope =>
+ scope?.tenantId !== undefined && scope?.appId !== undefined && scope?.tenantId.length > 0 && scope?.appId.length > 0;
+
+const asJson = (value: unknown): Record | null => {
+ if (typeof value !== 'string') return null;
+ try {
+ const parsed = JSON.parse(value) as unknown;
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) : null;
+ } catch {
+ return null;
}
- if (url.protocol === 'http:' && url.hostname !== 'localhost' && url.hostname !== '127.0.0.1') {
- throw new Error('HTTPS required');
+};
+
+const ensureObject = (value: unknown): Record | undefined => {
+ return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : undefined;
+};
+
+const toScopedRecord = (value: unknown, scopeKey: string): ScopedRecord | undefined => {
+ const raw = ensureObject(value);
+ if (!raw) return undefined;
+
+ const id = String(raw.id ?? '').trim();
+ const collection = String(raw.collection ?? 'item').trim();
+ const createdAt = String(raw.createdAt ?? '').trim();
+ const updatedAt = String(raw.updatedAt ?? '').trim();
+ const storedScope = String(raw.scope ?? '');
+ const values = ensureObject(raw.values);
+
+ if (!id || !createdAt || !updatedAt) return undefined;
+ if (storedScope !== scopeKey) return undefined;
+
+ return {
+ scope: storedScope,
+ id,
+ collection,
+ createdAt,
+ updatedAt,
+ values: values ? values as Record : {},
+ };
+};
+
+const toPublicRecord = (record: RawRecord | ScopedRecord): JsonRecord => ({
+ id: record.id,
+ collection: record.collection,
+ createdAt: record.createdAt,
+ updatedAt: record.updatedAt,
+ values: record.values,
+});
+
+const asScopedRecord = (homeScope: string, record: JsonRecord): ScopedRecord => ({
+ scope: homeScope,
+ id: String(record.id),
+ collection: String(record.collection || 'item'),
+ createdAt: String(record.createdAt || new Date().toISOString()),
+ updatedAt: String(record.updatedAt || new Date().toISOString()),
+ values: record.values ?? {},
+});
+
+const toPreparedRecords = (homeScope: string, records: JsonRecord[]) =>
+ mergeByUpdatedAt(records.filter((record) => Boolean(record?.id)).map((record) => asScopedRecord(homeScope, record)));
+
+const mergeByUpdatedAt = (records: T[]) => {
+ const byId = new Map();
+ for (const record of records) {
+ const prior = byId.get(record.id);
+ if (!prior || new Date(record.updatedAt).getTime() > new Date(prior.updatedAt).getTime()) {
+ byId.set(record.id, record);
+ }
}
- return url.toString();
-}
+ return [...byId.values()].sort((a, b) => a.updatedAt.localeCompare(b.updatedAt) || a.id.localeCompare(b.id));
+};
-function readSecretRef(config: DataHomeConfig): string {
- const ref = config.secretRef;
- if (ref === undefined) throw new Error(`missing secretRef for ${config.id}`);
- if (!reSecretRef.test(ref)) throw new Error(`invalid secretRef ${config.id}`);
- return ref;
-}
+const sanitize = (value: string, field: string) => {
+ if (!IDENTIFIER.test(value)) fail(`invalid ${field}`);
+ return value;
+};
-function readEndpoint(baseUrl: string | undefined, config: DataHomeConfig): string {
- if (!baseUrl) throw new Error(`missing data home endpoint for ${config.id}`);
- return networkUrl(baseUrl).replace(/\/$/, '');
-}
+const scopeKey = (scope: DataHomeScope, homeId: string) => {
+ return `${sanitize(scope.tenantId, 'tenantId')}:${sanitize(scope.appId, 'appId')}:${sanitize(homeId, 'data home id')}`;
+};
+
+const toScopedId = (prefix: string, id: string) => `${prefix}:${id}`;
+
+const fromScopedId = (prefix: string, id: string) => {
+ const marker = `${prefix}:`;
+ return id.startsWith(marker) ? id.slice(marker.length) : undefined;
+};
+
+const paginated = (items: T[], cursor: string | undefined, limit: number) => {
+ const start = Number(cursor ?? 0);
+ const offset = Number.isFinite(start) && start >= 0 ? Math.trunc(start) : 0;
+ const nextOffset = Math.min(offset + limit, items.length);
+ return {
+ records: items.slice(offset, nextOffset),
+ cursor: nextOffset < items.length ? `${nextOffset}` : undefined,
+ hasMore: nextOffset < items.length,
+ };
+};
-export async function retry(operation: () => Promise, attempts = 3): Promise {
- let error: unknown;
- for (let i = 0; i < attempts; i += 1) {
+const normalizeSecretRef = (config: DataHomeConfig) => {
+ if (!config.secretRef) fail(`missing secretRef for ${config.id}`);
+ const normalized = String(config.secretRef);
+ if (!/^[A-Z][A-Z0-9_]+$/.test(normalized)) fail(`invalid secretRef ${config.id}`);
+ return normalized;
+};
+
+export async function retry(operation: () => Promise, attempts = 3, delayMs = 50): Promise {
+ let lastError: unknown;
+ for (let attempt = 0; attempt < attempts; attempt += 1) {
try {
return await operation();
- } catch (cause) {
- error = cause;
- if (i + 1 < attempts) await new Promise((done) => setTimeout(done, 50 * 2 ** i));
+ } catch (error) {
+ lastError = error;
+ if (attempt + 1 >= attempts) break;
+ await new Promise((done) => setTimeout(done, delayMs * 2 ** attempt));
}
}
- throw error;
+ throw lastError;
}
-function asRecord(raw: DataHomeRecord): JsonRecord {
+type DataHomeFactories = {
+ notionClient: (token: string) => NotionApi;
+ postgresClient: (credential: string) => {
+ connect: () => Promise;
+ query: (text: string, values?: unknown[]) => Promise<{ rows: Array> }>;
+ end: () => Promise;
+ };
+ googleAuth: (credentials: Record) => unknown;
+ sheetsClient: (opts: { version: 'v4'; auth: unknown }) => SheetsClient;
+};
+
+type DataHomeFactoriesInput = Partial;
+
+const defaultFactories = (): DataHomeFactories => {
+ const unavailable = (kind: DataHomeKind): never => fail(`${kind} adapter unavailable on this runtime`);
return {
- id: raw.id,
- collection: raw.collection || 'item',
- createdAt: raw.createdAt,
- updatedAt: raw.updatedAt,
- values: raw.values ?? {},
+ notionClient: () => unavailable('notion'),
+ postgresClient: () => unavailable('postgres'),
+ googleAuth: () => unavailable('google-sheets'),
+ sheetsClient: () => unavailable('google-sheets'),
};
-}
+};
-function dedupe(records: JsonRecord[]) {
- const out = new Map();
- for (const record of records) {
- const current = out.get(record.id);
- if (!current || record.updatedAt > current.updatedAt) out.set(record.id, record);
- }
- return [...out.values()].sort((a, b) => a.updatedAt.localeCompare(b.updatedAt));
-}
+const createFactories = (input: DataHomeFactoriesInput = {}): DataHomeFactories => {
+ const base = defaultFactories();
+ return {
+ notionClient: input.notionClient ?? base.notionClient,
+ postgresClient: input.postgresClient ?? base.postgresClient,
+ googleAuth: input.googleAuth ?? base.googleAuth,
+ sheetsClient: input.sheetsClient ?? base.sheetsClient,
+ };
+};
-function paginate(rows: T[], cursor?: string, limit = 200) {
- const start = Number(cursor ?? 0);
- const safe = Number.isFinite(start) && start >= 0 ? Math.trunc(start) : 0;
- const page = rows.slice(safe, safe + limit);
- const next = safe + page.length;
- return { records: page, cursor: next < rows.length ? `${next}` : undefined, hasMore: next < rows.length };
-}
+function sqliteHome(config: DataHomeConfig, storage: StorageProvider, scoped: DataHomeScope): DataHomeTransport {
+ const homeScope = scopeKey(scoped, config.id);
+ const storageKey = `utopia:data-home:${homeScope}`;
-function sqliteHome(id: string, storage: StorageProvider): DataHomeTransport {
- const key = `utopia:data-home:${id}:sqlite`;
- const read = async () => {
- const raw = await storage.getItem(key);
- if (!raw) return [] as JsonRecord[];
- const parsed = JSON.parse(raw);
- if (!Array.isArray(parsed)) return [];
- return parsed
- .filter((item) => item && typeof item === 'object')
- .map((item) => asRecord(item as DataHomeRecord));
+ const read = async (): Promise => {
+ const raw = await storage.getItem(storageKey);
+ if (!raw) return [];
+ const parsed = asJson(raw);
+ if (!parsed || !Array.isArray((parsed as { records?: unknown[] }).records)) return [];
+
+ const rows = ((parsed as { records: unknown[] }).records).map((row) => toScopedRecord(row, homeScope));
+ return rows.filter((row): row is ScopedRecord => Boolean(row));
};
return {
- async pull({ cursor, limit = 200 }) {
- return paginate((await read()).sort((a, b) => a.updatedAt.localeCompare(b.updatedAt)), cursor, limit);
+ async pull({ cursor, limit = MAX_LIMIT }) {
+ const rows = mergeByUpdatedAt(await read()).map((row) => toPublicRecord(row));
+ return paginated(rows, cursor, limit);
},
async push({ records }) {
- await storage.setItem(key, JSON.stringify(dedupe((await read()).concat(records))));
- return { cursor: dedupe(records).at(-1)?.updatedAt };
+ const current = await read();
+ const prepared = mergeByUpdatedAt([...current, ...toPreparedRecords(homeScope, records)]);
+ const kept = prepared.filter((item) => item.values.deleted !== true);
+ await storage.setItem(
+ storageKey,
+ JSON.stringify({
+ v: 1,
+ scope: homeScope,
+ records: kept,
+ }),
+ );
+ return { cursor: kept.at(-1)?.updatedAt };
},
};
}
-function remoteHome(config: RemoteDataHomeConfig, baseUrl: string, scope: DataHomeScope): DataHomeTransport {
- const endpoint = `${baseUrl}/data/${encodeURIComponent(`${scope.tenantId}:${scope.appId}:${config.id}`)}`;
- const request = async (operation: 'pull' | 'push', payload: Record) => {
- const response = await retry(async () => {
- const result = await fetch(`${endpoint}/${operation}`, {
- method: 'POST',
- headers: { 'content-type': 'application/json' },
- body: JSON.stringify(payload),
+function postgresHome(config: DataHomeConfig, credential: string, scoped: DataHomeScope, factories: DataHomeFactories): DataHomeTransport {
+ const homeScope = scopeKey(scoped, config.id);
+ const resource = sanitize(config.resource ?? 'utopia_records', 'resource');
+
+ const connect = async () => {
+ const client = factories.postgresClient(credential);
+ await client.connect();
+ await client.query(`create table if not exists "${resource}" (
+ scope text not null,
+ id text not null,
+ collection text not null,
+ created_at text not null,
+ updated_at text not null,
+ values_json text not null,
+ primary key (scope, id)
+ )`);
+ return client;
+ };
+
+ return {
+ async pull({ cursor, limit = MAX_LIMIT }) {
+ return retry(async () => {
+ const client = await connect();
+ try {
+ const offset = Number(cursor ?? 0);
+ const safeOffset = Number.isFinite(offset) && offset >= 0 ? Math.trunc(offset) : 0;
+ const result = await client.query(
+ `select id, collection, created_at, updated_at, values_json from "${resource}" where scope = $1 order by updated_at asc limit $2 offset $3`,
+ [homeScope, limit, safeOffset],
+ );
+ const records = mergeByUpdatedAt(result.rows.map((row) => ({
+ id: String(row.id ?? ''),
+ collection: String(row.collection ?? 'item'),
+ createdAt: String(row.created_at ?? ''),
+ updatedAt: String(row.updated_at ?? ''),
+ values: asJson(row.values_json) ?? {},
+ } satisfies RawRecord)));
+ return paginated(records.map((row) => toPublicRecord(row)), `${safeOffset}`, limit);
+ } finally {
+ await client.end();
+ }
+ });
+ },
+ async push({ records }) {
+ return retry(async () => {
+ const client = await connect();
+ try {
+ const prepared = toPreparedRecords(homeScope, records);
+ for (const record of prepared) {
+ const { id, collection, createdAt, updatedAt, values } = record;
+
+ if (values.deleted === true) {
+ await client.query(`delete from "${resource}" where scope = $1 and id = $2`, [homeScope, id]);
+ continue;
+ }
+
+ await client.query(
+ `insert into "${resource}" (scope, id, collection, created_at, updated_at, values_json)
+ values ($1, $2, $3, $4, $5, $6)
+ on conflict (scope, id) do update set
+ collection = excluded.collection,
+ created_at = excluded.created_at,
+ updated_at = excluded.updated_at,
+ values_json = excluded.values_json
+ where "${resource}".updated_at <= excluded.updated_at`,
+ [homeScope, id, collection, createdAt, updatedAt, JSON.stringify(values)],
+ );
+ }
+
+ return { cursor: prepared.at(-1)?.updatedAt };
+ } finally {
+ await client.end();
+ }
});
- if (!result.ok) throw new Error(`Data home ${result.status}`);
- return result.json() as Promise;
- }, 3);
- return response;
+ },
+ };
+}
+
+function notionHome(config: DataHomeConfig, token: string, scoped: DataHomeScope, factories: DataHomeFactories): DataHomeTransport {
+ const notion = factories.notionClient(token);
+ const useDataSource = Boolean(notion.dataSources?.query);
+ const useDatabase = Boolean(notion.databases?.query);
+ if (!useDataSource && !useDatabase) fail(`unsupported notion query methods for ${config.id}`);
+
+ const homeScope = scopeKey(scoped, config.id);
+ const resource = config.resource;
+ if (!resource) fail(`missing notion resource for ${config.id}`);
+
+ const propertyText = (value: unknown): string | undefined => {
+ const valueRecord = ensureObject(value);
+ const richText = valueRecord ? (valueRecord.rich_text as unknown[]) : undefined;
+ const title = valueRecord ? (valueRecord.title as unknown[]) : undefined;
+ if (Array.isArray(richText) || Array.isArray(title)) {
+ const values = (Array.isArray(richText) ? richText : title) as unknown[];
+ return values
+ .map((entry) => {
+ const typed = ensureObject(entry);
+ if (!typed) return '';
+ const plain = typed.plain_text;
+ if (typeof plain === 'string') return plain;
+ const text = ensureObject(typed.text);
+ return text?.content ?? '';
+ })
+ .join('');
+ }
+
+ const dateValue = valueRecord ? ensureObject(valueRecord.date) : undefined;
+ const start = dateValue ? dateValue.start : undefined;
+ return typeof start === 'string' ? start : undefined;
+ };
+
+ const decodeProperties = (properties: Record | undefined): ScopedRecord | undefined => {
+ if (!properties) return undefined;
+ const idRaw = propertyText(properties.UtopiaId);
+ const scopedId = idRaw ? fromScopedId(homeScope, idRaw) : undefined;
+ if (!scopedId) return undefined;
+
+ const payload = asJson(propertyText(properties.Payload) ?? '{}') ?? {};
+ const values = payload as Record;
+
+ return {
+ scope: homeScope,
+ id: scopedId,
+ collection: propertyText(properties.Collection) ?? 'item',
+ createdAt: propertyText(properties.Created) ?? new Date().toISOString(),
+ updatedAt: propertyText(properties.Updated) ?? new Date().toISOString(),
+ values,
+ };
};
+ const queryPages = async () => {
+ const out: unknown[] = [];
+ let cursor: string | undefined;
+ do {
+ const response = await retry(async () => useDataSource
+ ? notion.dataSources!.query({ data_source_id: resource, result_type: 'page', page_size: 100, start_cursor: cursor })
+ : notion.databases!.query({ database_id: resource, page_size: 100, start_cursor: cursor } as Record)
+ );
+ const next = response as unknown as NotionQueryResult;
+ out.push(...(next.results ?? []));
+ cursor = next.has_more ? next.next_cursor ?? undefined : undefined;
+ } while (cursor);
+ return out;
+ };
+
+ const encodeText = (text: string) => (text.length > 0 ? [{ type: 'text', text: { content: text } }] : []);
+
return {
- async pull({ cursor, limit = 200 }) {
- const payload = { cursor, limit } as const;
- const result = await request<{ records?: JsonRecord[]; cursor?: string; hasMore?: boolean }>('pull', payload);
- return {
- records: Array.isArray(result.records) ? result.records.map(asRecord) : [],
- cursor: result.cursor,
- hasMore: result.hasMore ?? false,
- };
+ async pull({ cursor, limit = MAX_LIMIT }) {
+ const pages = await queryPages();
+ const records = pages
+ .map((page) => {
+ const item = ensureObject(page);
+ if (!item) return undefined;
+ return decodeProperties(ensureObject(item.properties) as Record | undefined);
+ })
+ .filter((record): record is ScopedRecord => Boolean(record));
+
+ const merged = mergeByUpdatedAt(records);
+ const mapped = merged
+ .filter((record) => !cursor || record.updatedAt > cursor)
+ .map((record) => toPublicRecord(record));
+ return paginated(mapped, cursor, limit);
},
async push({ records }) {
- const result = await request<{ cursor?: string }>('push', { records });
- return { cursor: result.cursor };
+ const pages = await queryPages();
+ const existing = new Map();
+
+ for (const page of pages) {
+ const item = ensureObject(page);
+ if (!item) continue;
+ const decoded = decodeProperties(ensureObject(item.properties) as Record | undefined);
+ if (!decoded) continue;
+ const pageId = String(item.id ?? '');
+ if (pageId) existing.set(decoded.id, { pageId, updatedAt: decoded.updatedAt });
+ }
+
+ const prepared = toPreparedRecords(homeScope, records);
+
+ for (const record of prepared) {
+ const prior = existing.get(record.id);
+ if (prior && prior.updatedAt > record.updatedAt) continue;
+
+ const scopedId = toScopedId(homeScope, record.id);
+ const payload = JSON.stringify(record.values ?? {});
+ const properties = {
+ Name: { title: encodeText(record.values?.name ? String(record.values.name) : record.id) },
+ UtopiaId: { rich_text: encodeText(scopedId) },
+ Collection: { rich_text: encodeText(record.collection) },
+ Created: { date: { start: record.createdAt } },
+ Updated: { date: { start: record.updatedAt } },
+ Payload: { rich_text: encodeText(payload) },
+ };
+
+ if (prior && record.values.deleted === true) {
+ await notion.pages.update({ page_id: prior.pageId, archived: true });
+ continue;
+ }
+
+ if (prior) {
+ await notion.pages.update({ page_id: prior.pageId, properties });
+ continue;
+ }
+
+ const parent = useDataSource ? { data_source_id: resource } : { database_id: resource };
+ await notion.pages.create({ parent, properties });
+ }
+
+ return { cursor: prepared.at(-1)?.updatedAt };
},
};
}
-function isSqlite(config: DataHomeConfig): boolean {
- return config.kind === 'sqlite';
+function sheetsHome(config: DataHomeConfig, credential: string, scoped: DataHomeScope, factories: DataHomeFactories): DataHomeTransport {
+ const homeScope = scopeKey(scoped, config.id);
+ const credentials = asJson(credential);
+ if (!credentials) fail(`invalid google-sheets credentials for ${config.id}`);
+
+ const [spreadsheetId, explicitRange] = (config.resource ?? '').split('!');
+ if (!spreadsheetId) fail(`missing google-sheets resource for ${config.id}`);
+
+ const requestedRange = (explicitRange ?? 'A1:E').trim() || 'A1:E';
+ const sheetName = requestedRange.includes('!') ? requestedRange.split('!')[0] : 'Utopia';
+
+ const auth = factories.googleAuth(credentials as Record);
+ const sheets = factories.sheetsClient({ version: 'v4', auth });
+
+ const readAllRows = async (): Promise => {
+ const response = await sheets.spreadsheets.values.get({ spreadsheetId, range: requestedRange });
+ return response.data.values ?? [];
+ };
+
+ const decodeRow = (row: unknown[]): ScopedRecord | undefined => {
+ const scopedId = String(row?.[0] ?? '').trim();
+ const id = scopedId ? fromScopedId(homeScope, scopedId) : undefined;
+ if (!id) return undefined;
+
+ return {
+ scope: homeScope,
+ id,
+ collection: String(row?.[1] ?? 'item'),
+ createdAt: String(row?.[2] ?? ''),
+ updatedAt: String(row?.[3] ?? ''),
+ values: asJson(row?.[4]) ?? {},
+ };
+ };
+
+ return {
+ async pull({ cursor, limit = MAX_LIMIT }) {
+ const rows = await readAllRows();
+ const headerOffset = rows.length && ensureObject(rows[0])?.A1 === 'UtopiaId' ? 1 : 0;
+ const records = rows
+ .slice(headerOffset)
+ .map((row) => decodeRow(row as unknown[]))
+ .filter((record): record is ScopedRecord => Boolean(record))
+ .filter((record) => !cursor || record.updatedAt > cursor)
+ .map((record) => toPublicRecord(record));
+ return paginated(mergeByUpdatedAt(records), cursor, limit);
+ },
+ async push({ records }) {
+ const rows = await readAllRows();
+ const existing = new Map();
+ rows.forEach((row, index) => {
+ const decoded = decodeRow(row as unknown[]);
+ if (!decoded) return;
+ existing.set(decoded.id, { row: index + 1, updatedAt: decoded.updatedAt });
+ });
+
+ const prepared = toPreparedRecords(homeScope, records);
+
+ for (const record of prepared) {
+ const rowId = toScopedId(homeScope, record.id);
+ const prior = existing.get(record.id);
+ const payload = JSON.stringify(record.values ?? {});
+ const rangeValues = [rowId, record.collection, record.createdAt, record.updatedAt, payload];
+
+ if (prior && record.values.deleted === true) {
+ await sheets.spreadsheets.values.clear({ spreadsheetId, range: `${sheetName}!A${prior.row}:E${prior.row}` });
+ continue;
+ }
+
+ if (prior) {
+ if (prior.updatedAt > record.updatedAt) continue;
+ await sheets.spreadsheets.values.update({
+ spreadsheetId,
+ range: `${sheetName}!A${prior.row}:E${prior.row}`,
+ valueInputOption: 'RAW',
+ requestBody: { values: [rangeValues] },
+ });
+ continue;
+ }
+
+ await sheets.spreadsheets.values.append({
+ spreadsheetId,
+ range: requestedRange,
+ valueInputOption: 'RAW',
+ insertDataOption: 'INSERT_ROWS',
+ requestBody: { values: [rangeValues] },
+ });
+ }
+
+ return { cursor: prepared.at(-1)?.updatedAt };
+ },
+ };
}
-function isRemote(config: DataHomeConfig): config is RemoteDataHomeConfig {
- return config.kind === 'notion' || config.kind === 'google-sheets' || config.kind === 'postgres';
+function gatewayHome(config: DataHomeConfig, credential: string, scoped: DataHomeScope, baseUrl: string): DataHomeTransport {
+ const endpoint = new URL(baseUrl);
+ if (endpoint.protocol !== 'https:' && endpoint.hostname !== 'localhost' && endpoint.hostname !== '127.0.0.1') fail('HTTPS required');
+ const request = async (operation: 'pull' | 'push', payload: Record): Promise> => {
+ const response = await fetch(endpoint, {
+ method: 'POST',
+ headers: {
+ authorization: `Bearer ${credential}`,
+ 'content-type': 'application/json',
+ 'x-utopia-app-id': scoped.appId,
+ 'x-utopia-tenant-id': scoped.tenantId,
+ },
+ body: JSON.stringify({ operation, dataHome: { id: config.id, kind: config.kind, resource: config.resource }, ...payload }),
+ });
+ if (!response.ok) fail(`data home gateway ${response.status}`);
+ const result = ensureObject(await response.json());
+ return result ?? fail('invalid data home gateway response');
+ };
+ return {
+ async pull(input) {
+ const result = await request('pull', input);
+ const records = Array.isArray(result.records)
+ ? result.records.map((record) => ensureObject(record)).filter((record): record is Record => Boolean(record))
+ .map((record) => ({
+ id: String(record.id ?? ''),
+ collection: String(record.collection ?? 'item'),
+ createdAt: String(record.createdAt ?? ''),
+ updatedAt: String(record.updatedAt ?? ''),
+ values: ensureObject(record.values) ?? {},
+ })).filter((record) => record.id && record.createdAt && record.updatedAt)
+ : [];
+ return { records, cursor: typeof result.cursor === 'string' ? result.cursor : undefined, hasMore: result.hasMore === true };
+ },
+ async push(input) {
+ const result = await request('push', { records: input.records });
+ return { cursor: typeof result.cursor === 'string' ? result.cursor : undefined };
+ },
+ };
}
export function createDataHome(
config: DataHomeConfig,
- _secrets: SecretMap = {},
- storage?: StorageProvider,
+ secrets: SecretMap = {},
+ storage: StorageProvider = { getItem: async () => null, setItem: async () => undefined },
baseUrl?: string,
scope?: DataHomeScope,
+ factories: DataHomeFactoriesInput = {},
): DataHomeTransport {
- if (isSqlite(config)) return sqliteHome(config.id, storage ?? fail('sqlite storage unavailable'));
- if (!isRemote(config)) fail(`unsupported data home kind ${config.kind}`);
- const withSecretRef = { ...config, secretRef: readSecretRef(config) };
- return remoteHome(withSecretRef, readEndpoint(baseUrl, config), scope ?? fail(`missing data home scope for ${config.id}`));
+ const resolvedFactories = createFactories(factories);
+
+ if (config.kind === 'sqlite') {
+ if (!isDataHomeScope(scope)) fail(`missing data home scope for ${config.id}`);
+ const activeScope = scope as DataHomeScope;
+ return sqliteHome(config, storage, activeScope);
+ }
+
+ if (!isDataHomeScope(scope)) fail(`missing data home scope for ${config.id}`);
+ const activeScope = scope as DataHomeScope;
+
+ const ref = normalizeSecretRef(config);
+ const credential = secrets[ref];
+ if (!credential) fail(`credential missing for ${ref}`);
+ const resolvedCredential = String(credential);
+
+ if (baseUrl) return gatewayHome(config, resolvedCredential, activeScope, baseUrl);
+ if (config.kind === 'postgres') return postgresHome(config, resolvedCredential, activeScope, resolvedFactories);
+ if (config.kind === 'notion') return notionHome(config, resolvedCredential, activeScope, resolvedFactories);
+ if (config.kind === 'google-sheets') return sheetsHome(config, resolvedCredential, activeScope, resolvedFactories);
+
+ return fail(`unsupported data home kind ${config.kind}`);
}
export function mergeWithConflicts(local: JsonRecord[], remote: JsonRecord[]) {
- const merged = dedupe([...local, ...remote]);
- const conflicts: string[] = [];
+ const merged = mergeByUpdatedAt([...local, ...remote]);
const remoteById = new Map(remote.map((record) => [record.id, record]));
- for (const record of merged) {
- const localRecord = local.find((item) => item.id === record.id);
- const remoteRecord = remoteById.get(record.id);
- if (!localRecord || !remoteRecord) continue;
+ const conflicts: string[] = [];
+
+ for (const localRecord of local) {
+ const remoteRecord = remoteById.get(localRecord.id);
+ if (!remoteRecord) continue;
if (localRecord.updatedAt === remoteRecord.updatedAt && JSON.stringify(localRecord.values) !== JSON.stringify(remoteRecord.values)) {
- conflicts.push(record.id);
+ conflicts.push(localRecord.id);
}
}
+
return { merged, conflicts };
}
diff --git a/src/kernel/layout.ts b/src/kernel/layout.ts
index ddecfc9..f02c895 100644
--- a/src/kernel/layout.ts
+++ b/src/kernel/layout.ts
@@ -7,20 +7,78 @@ type Responsive = z.infer;
type Layout = NonNullable;
type Style = ViewStyle & TextStyle;
-const map: Record = {
- direction: 'flexDirection', wrap: 'flexWrap', justify: 'justifyContent', align: 'alignItems',
- paddingX: 'paddingHorizontal', paddingY: 'paddingVertical', marginX: 'marginHorizontal',
- marginY: 'marginVertical', radius: 'borderRadius', background: 'backgroundColor',
- border: 'borderColor', foreground: 'color',
+type StyleAlias = Record;
+
+const styleMap: StyleAlias = {
+ direction: 'flexDirection',
+ dir: 'flexDirection',
+ wrap: 'flexWrap',
+ justify: 'justifyContent',
+ justifyContent: 'justifyContent',
+ align: 'alignItems',
+ alignItems: 'alignItems',
+ alignSelf: 'alignSelf',
+ paddingX: 'paddingHorizontal',
+ paddingY: 'paddingVertical',
+ p: 'padding',
+ px: 'paddingHorizontal',
+ py: 'paddingVertical',
+ pt: 'paddingTop',
+ pr: 'paddingRight',
+ pb: 'paddingBottom',
+ pl: 'paddingLeft',
+ m: 'margin',
+ marginX: 'marginHorizontal',
+ marginY: 'marginVertical',
+ mx: 'marginHorizontal',
+ my: 'marginVertical',
+ mt: 'marginTop',
+ mr: 'marginRight',
+ mb: 'marginBottom',
+ ml: 'marginLeft',
+ radius: 'borderRadius',
+ rounded: 'borderRadius',
+ borderRadius: 'borderRadius',
+ background: 'backgroundColor',
+ bg: 'backgroundColor',
+ foreground: 'color',
+ fg: 'color',
+ color: 'color',
+ border: 'borderColor',
+ borderColor: 'borderColor',
+ width: 'width',
+ w: 'width',
+ height: 'height',
+ h: 'height',
+ minWidth: 'minWidth',
+ minW: 'minWidth',
+ maxWidth: 'maxWidth',
+ maxW: 'maxWidth',
+ minHeight: 'minHeight',
+ minH: 'minHeight',
+ maxHeight: 'maxHeight',
+ maxH: 'maxHeight',
+ gap: 'gap',
+ gapX: 'columnGap',
+ gapY: 'rowGap',
+ opacity: 'opacity',
+ fontSize: 'fontSize',
+ size: 'fontSize',
};
+const toStyleKey = (key: string): keyof Style => styleMap[key] ?? key as keyof Style;
+
function native(value: Layout = {}): Style {
- return Object.fromEntries(Object.entries(value).map(([key, item]) => [map[key] ?? key, item])) as Style;
+ return Object.fromEntries(Object.entries(value).map(([key, item]) => [toStyleKey(key), item])) as Style;
}
export function layout(value: Responsive | undefined, width: number, height: number, platform: PlatformOSType | 'macos'): Style {
if (!value) return {};
const size = width < 600 ? value.compact : width < 1024 ? value.medium : value.wide;
const orientation = width > height ? value.landscape : value.portrait;
- return { ...native(value.base), ...native(size), ...native(orientation), ...native(value.platform?.[platform as keyof NonNullable]) };
+ const responsive = native(value.base);
+ const sized = native(size);
+ const oriented = native(orientation);
+ const platformStyle = native(value.platform?.[platform as keyof NonNullable]);
+ return { ...responsive, ...sized, ...oriented, ...platformStyle };
}
diff --git a/src/kernel/operations.ts b/src/kernel/operations.ts
new file mode 100644
index 0000000..389389e
--- /dev/null
+++ b/src/kernel/operations.ts
@@ -0,0 +1,238 @@
+import { canonicalize } from 'json-canonicalize';
+
+type JsonPayload = Record | undefined;
+type JsonError = { message?: string; status?: number; permanent?: boolean; retryable?: boolean; retryAfterMs?: number };
+
+type DurableAction = {
+ kind: string;
+ tenantId?: string;
+ appId?: string;
+ collection?: string;
+ recordId?: string;
+ payload?: JsonPayload;
+};
+
+export type RetryPolicy = {
+ maxAttempts: number;
+ baseDelayMs: number;
+ multiplier: number;
+ maxDelayMs: number;
+};
+
+export type DurableOperationRecord = {
+ key: string;
+ status: 'idle' | 'running' | 'retrying' | 'succeeded' | 'failed' | 'rolled_back';
+ attempts: number;
+ lastError?: string;
+ nextRetryAt?: string;
+ lastUpdatedAt?: string;
+ startedAt?: string;
+ completedAt?: string;
+};
+
+const DEFAULT_RETRY_POLICY: RetryPolicy = {
+ maxAttempts: 4,
+ baseDelayMs: 250,
+ multiplier: 2,
+ maxDelayMs: 30_000,
+};
+
+const toNumber = (value: unknown, fallback: number): number => {
+ const next = Number(value);
+ return Number.isFinite(next) ? next : fallback;
+};
+
+const sanitizePayload = (payload: JsonPayload): JsonPayload => {
+ if (!payload) return payload;
+ const safe = { ...payload };
+ if ('idempotencyKey' in safe) delete safe.idempotencyKey;
+ return safe;
+};
+
+const canonicalPayload = (payload: JsonPayload): string => canonicalize(sanitizePayload(payload) ?? null);
+
+const normalizePolicy = (policy: Partial = {}): RetryPolicy => {
+ const base = { ...DEFAULT_RETRY_POLICY, ...policy } as RetryPolicy;
+ return {
+ maxAttempts: Math.max(1, toNumber(base.maxAttempts, DEFAULT_RETRY_POLICY.maxAttempts)),
+ baseDelayMs: Math.max(0, toNumber(base.baseDelayMs, DEFAULT_RETRY_POLICY.baseDelayMs)),
+ multiplier: Math.max(1, toNumber(base.multiplier, DEFAULT_RETRY_POLICY.multiplier)),
+ maxDelayMs: Math.max(1, toNumber(base.maxDelayMs, DEFAULT_RETRY_POLICY.maxDelayMs)),
+ };
+};
+
+const describeError = (error: unknown): string => {
+ if (error == null) return 'error';
+ if (error instanceof Error) return error.message || 'error';
+ return String(error);
+};
+
+const toTimestamp = (at: string): number => {
+ const parsed = Date.parse(at);
+ return Number.isFinite(parsed) ? parsed : Date.now();
+};
+
+const terminalStates = new Set(['failed', 'succeeded', 'rolled_back']);
+
+const retryDelayMs = (attempt: number, policy: RetryPolicy): number => {
+ const scale = Math.max(0, attempt - 1);
+ const raw = policy.baseDelayMs * policy.multiplier ** scale;
+ return Math.max(0, Math.min(raw, policy.maxDelayMs));
+};
+
+const isRetryableError = (error: unknown): boolean => {
+ if (error == null) return true;
+ if (!(error instanceof Error) && typeof error !== 'object') return true;
+
+ const candidate = error as JsonError;
+ if (candidate.permanent || candidate.retryable === false) return false;
+
+ const status = toNumber(candidate.status, 0);
+ if (status >= 400 && status < 500) {
+ if (status === 408 || status === 425 || status === 429) return true;
+ return false;
+ }
+
+ if (typeof candidate.message === 'string' && /(validation|unauthorized|forbidden|unsupported|signature)/i.test(candidate.message)) {
+ return false;
+ }
+
+ return true;
+};
+
+export function baseIdempotencyKey(action: DurableAction): string {
+ const body = sanitizePayload(action.payload);
+ return canonicalize({
+ tenantId: action.tenantId ?? '',
+ appId: action.appId ?? '',
+ collection: action.collection ?? '',
+ recordId: action.recordId ?? '',
+ kind: action.kind,
+ payload: body,
+ });
+}
+
+export function buildOperationKey(action: DurableAction, supplied?: string): string {
+ const base = baseIdempotencyKey(action);
+ const key = supplied?.trim();
+ return key ? `${key}::${base}` : base;
+}
+
+export function snapshotFromAction(action: DurableAction, status: DurableOperationRecord['status'] = 'running', key?: string): DurableOperationRecord {
+ const now = new Date().toISOString();
+ return {
+ key: buildOperationKey(action, key),
+ status,
+ attempts: 0,
+ lastUpdatedAt: now,
+ startedAt: status === 'running' ? now : undefined,
+ };
+}
+
+export function shouldRetry(error: unknown, attempt: number, policy: Partial = {}): boolean {
+ const normalized = normalizePolicy(policy);
+ if (attempt >= normalized.maxAttempts) return false;
+ if (!isRetryableError(error)) return false;
+ return true;
+}
+
+export function computeRetryDelay(attempt: number, policy: Partial = {}): number {
+ return retryDelayMs(attempt, normalizePolicy(policy));
+}
+
+export function nextRetryAt(attempt: number, policy: Partial = {}, now = new Date().toISOString()): string {
+ const delayMs = computeRetryDelay(attempt, policy);
+ return new Date(toTimestamp(now) + delayMs).toISOString();
+}
+
+export function nextOperationRecord(
+ record: DurableOperationRecord,
+ error: unknown,
+ at = new Date().toISOString(),
+ policy: Partial = {},
+): DurableOperationRecord {
+ if (record.status === 'succeeded' || record.status === 'rolled_back') {
+ return record;
+ }
+
+ const nextAttempt = record.attempts + 1;
+ const failed = !shouldRetry(error, nextAttempt, policy);
+
+ const next: DurableOperationRecord = {
+ ...record,
+ attempts: nextAttempt,
+ status: failed ? 'failed' : 'retrying',
+ lastError: describeError(error),
+ lastUpdatedAt: at,
+ completedAt: failed ? at : undefined,
+ nextRetryAt: failed ? undefined : nextRetryAt(nextAttempt, policy, at),
+ };
+
+ if (typeof (error as JsonError)?.retryAfterMs === 'number') {
+ const explicitMs = toNumber((error as JsonError).retryAfterMs, 0);
+ next.nextRetryAt = new Date(toTimestamp(at) + Math.max(0, explicitMs)).toISOString();
+ }
+
+ return next;
+}
+
+export function isIdempotentReplay(records: DurableOperationRecord[], key: string, supplied?: string): boolean {
+ const normalized = supplied?.trim() ? `${supplied}::${key}` : key;
+ return records.some((record) => record.key === key || record.key === normalized || record.key.endsWith(`::${key}`));
+}
+
+export function transitionStatus(
+ record: DurableOperationRecord,
+ next: DurableOperationRecord['status'],
+ at = new Date().toISOString(),
+): DurableOperationRecord {
+ if (terminalStates.has(record.status) && record.status !== next) {
+ throw new Error(`operation_terminal:${record.status}->${next}`);
+ }
+
+ if (next === 'running') {
+ return {
+ ...record,
+ status: 'running',
+ attempts: Math.max(record.attempts, 1),
+ startedAt: record.startedAt ?? at,
+ completedAt: undefined,
+ nextRetryAt: undefined,
+ lastUpdatedAt: at,
+ lastError: undefined,
+ };
+ }
+
+ if (next === 'succeeded') {
+ return {
+ ...record,
+ status: 'succeeded',
+ attempts: Math.max(record.attempts, 1),
+ completedAt: at,
+ lastUpdatedAt: at,
+ nextRetryAt: undefined,
+ lastError: undefined,
+ };
+ }
+
+ if (next === 'rolled_back') {
+ return {
+ ...record,
+ status: 'rolled_back',
+ attempts: Math.max(record.attempts, 1),
+ completedAt: at,
+ lastUpdatedAt: at,
+ nextRetryAt: undefined,
+ lastError: undefined,
+ };
+ }
+
+ return {
+ ...record,
+ status: next,
+ lastUpdatedAt: at,
+ };
+}
+
+export const markCompleted = (record: DurableOperationRecord, at = new Date().toISOString()): DurableOperationRecord => transitionStatus(record, 'succeeded', at);
+export const markRolledBack = (record: DurableOperationRecord, at = new Date().toISOString()): DurableOperationRecord => transitionStatus(record, 'rolled_back', at);
diff --git a/src/kernel/persistence.ts b/src/kernel/persistence.ts
index a5b1353..bee2f0f 100644
--- a/src/kernel/persistence.ts
+++ b/src/kernel/persistence.ts
@@ -1,43 +1,138 @@
import { z } from 'zod';
import * as Crypto from 'expo-crypto';
import { AppStateSchema, emptyState, type AppState } from './runtime';
+import { ensureIntegritySecret, hmac256, secretStrategyInfo, verifyHmac } from './security';
+
+const HMAC_KEY_ALIAS = 'utopia.persistence.hmac.v1';
+const SchemaVersion = { v2: 'utopia.state.v2', v3: 'utopia.state.v3' } as const;
+
+const EnvelopeV2 = z.object({ schemaVersion: z.literal(SchemaVersion.v2), state: AppStateSchema, checksum: z.string() });
+const EnvelopeV3 = z.object({
+ schemaVersion: z.literal(SchemaVersion.v3), state: AppStateSchema, checksum: z.string(), keyId: z.string(),
+ mac: z.string(), security: z.object({ strategy: z.string(), fallback: z.boolean().optional() }),
+});
+const Envelope = z.union([EnvelopeV2, EnvelopeV3]);
-const Envelope = z.object({ schemaVersion: z.literal('utopia.state.v2'), state: AppStateSchema, checksum: z.string() });
type Storage = { getItem(key: string): Promise; setItem(key: string, value: string): Promise };
+type EnvelopeValue = z.infer;
+type ParsedEnvelope = { envelope: EnvelopeValue | undefined; invalid: boolean; corrupted: boolean };
-export async function loadState(storage: Storage, key: string): Promise {
+function stable(state: AppState) {
+ return AppStateSchema.parse(state);
+}
+
+async function checksumState(state: AppState) {
+ return Crypto.digestStringAsync(Crypto.CryptoDigestAlgorithm.SHA256, JSON.stringify(stable(state)));
+}
+
+function envelopePayload(stateKey: string, checksum: string) {
+ return `${stateKey}:${checksum}`;
+}
+
+function classify(raw: string): ParsedEnvelope {
+ try {
+ return { envelope: Envelope.parse(JSON.parse(raw)), invalid: false, corrupted: false };
+ } catch {
+ try {
+ JSON.parse(raw);
+ return { envelope: undefined, invalid: true, corrupted: false };
+ } catch {
+ return { envelope: undefined, invalid: false, corrupted: true };
+ }
+ }
+}
+
+async function readEnvelope(storage: Storage, key: string): Promise {
const raw = await storage.getItem(key);
- if (raw) {
+ if (!raw) return { envelope: undefined, invalid: false, corrupted: false };
+ return classify(raw);
+}
+
+async function buildV3Envelope(state: AppState, stateKey: string, checksum: string, mac: string): Promise> {
+ return EnvelopeV3.parse({
+ schemaVersion: SchemaVersion.v3,
+ state,
+ checksum,
+ keyId: HMAC_KEY_ALIAS,
+ mac,
+ security: secretStrategyInfo(),
+ });
+}
+
+async function verifyEnvelope(envelope: EnvelopeValue, stateKey: string): Promise {
+ const expected = await checksumState(envelope.state);
+ if (expected !== envelope.checksum) throw new Error('state_checksum_mismatch');
+ if (envelope.schemaVersion === SchemaVersion.v2) return envelope.state;
+
+ const secret = await ensureIntegritySecret(envelope.keyId);
+ if (!await verifyHmac(envelopePayload(stateKey, envelope.checksum), secret.value, envelope.mac)) {
+ throw new Error('state_mac_mismatch');
+ }
+ return envelope.state;
+}
+
+async function writeEnvelope(storage: Storage, stateKey: string, state: AppState): Promise {
+ const parsed = stable(state);
+ const digest = await checksumState(parsed);
+ const secret = await ensureIntegritySecret(HMAC_KEY_ALIAS);
+ const mac = await hmac256(envelopePayload(stateKey, digest), secret.value);
+ const stableEnvelope = await buildV3Envelope(parsed, stateKey, digest, mac);
+ const serialized = JSON.stringify(stableEnvelope);
+ await storage.setItem(`${stateKey}:staged`, serialized);
+ await storage.setItem(stateKey, serialized);
+}
+
+async function recover(storage: Storage, stateKey: string, parsed: ParsedEnvelope, staged: ParsedEnvelope): Promise {
+ if (parsed.envelope) {
+ const recovered = await verifyEnvelope(parsed.envelope, stateKey);
+ await writeEnvelope(storage, stateKey, recovered);
+ return recovered;
+ }
+
+ if (parsed.invalid) throw new Error('state_unrecognized');
+ if (parsed.corrupted) {
+ if (!staged.envelope) throw new Error('state_corrupt');
+ const recovered = await verifyEnvelope(staged.envelope, stateKey);
+ await writeEnvelope(storage, stateKey, recovered);
+ return recovered;
+ }
+
+ if (!staged.envelope) return emptyState;
+ if (staged.invalid) throw new Error('state_unrecognized');
+ if (staged.corrupted) throw new Error('state_corrupt');
+ const recovered = await verifyEnvelope(staged.envelope, stateKey);
+ await writeEnvelope(storage, stateKey, recovered);
+ return recovered;
+}
+
+export async function loadState(storage: Storage, stateKey: string): Promise {
+ const stagedKey = `${stateKey}:staged`;
+ const primary = await readEnvelope(storage, stateKey);
+ const staged = await readEnvelope(storage, stagedKey);
+
+ if (primary.envelope) {
try {
- const envelope = Envelope.parse(JSON.parse(raw));
- const checksum = await Crypto.digestStringAsync(Crypto.CryptoDigestAlgorithm.SHA256, JSON.stringify(envelope.state));
- if (checksum !== envelope.checksum) throw new Error('state_checksum_mismatch');
- return envelope.state;
+ return await recover(storage, stateKey, primary, staged);
} catch (cause) {
- if (cause instanceof SyntaxError) {
- const staged = await storage.getItem(`${key}:staged`);
- if (!staged) throw cause;
- const stagedEnvelope = Envelope.parse(JSON.parse(staged));
- const stagedChecksum = await Crypto.digestStringAsync(Crypto.CryptoDigestAlgorithm.SHA256, JSON.stringify(stagedEnvelope.state));
- if (stagedChecksum !== stagedEnvelope.checksum) throw new Error('state_checksum_mismatch');
- return stagedEnvelope.state;
- }
- throw cause;
+ if (cause instanceof Error && cause.name === 'ZodError') throw cause;
+ if (!staged.envelope) throw cause;
+ return recover(storage, stateKey, staged, primary);
}
}
- const staged = await storage.getItem(`${key}:staged`);
- if (!staged) return emptyState;
- const envelope = Envelope.parse(JSON.parse(staged));
- const checksum = await Crypto.digestStringAsync(Crypto.CryptoDigestAlgorithm.SHA256, JSON.stringify(envelope.state));
- if (checksum !== envelope.checksum) throw new Error('state_checksum_mismatch');
- return envelope.state;
+ if (primary.invalid) throw new Error('state_unrecognized');
+ if (primary.corrupted) {
+ if (!staged.envelope) throw new Error('state_corrupt');
+ return recover(storage, stateKey, staged, primary);
+ }
+ try {
+ return await recover(storage, stateKey, primary, staged);
+ } catch (cause) {
+ if (cause instanceof Error && cause.name === 'ZodError') throw cause;
+ throw cause;
+ }
}
-export async function saveState(storage: Storage, key: string, state: AppState): Promise {
- const parsed = AppStateSchema.parse(state);
- const checksum = await Crypto.digestStringAsync(Crypto.CryptoDigestAlgorithm.SHA256, JSON.stringify(parsed));
- const value = JSON.stringify(Envelope.parse({ schemaVersion: 'utopia.state.v2', state: parsed, checksum }));
- await storage.setItem(`${key}:staged`, value);
- await storage.setItem(key, value);
+export async function saveState(storage: Storage, stateKey: string, state: AppState): Promise {
+ await writeEnvelope(storage, stateKey, state);
}
diff --git a/src/kernel/policy.ts b/src/kernel/policy.ts
index 9e82e7e..f051864 100644
--- a/src/kernel/policy.ts
+++ b/src/kernel/policy.ts
@@ -2,52 +2,93 @@ import { z } from 'zod';
import { findPackage } from './catalog';
import storage from './storage';
-const Decision = z.object({
- appId: z.string().min(1),
- capability: z.string().min(1),
- state: z.enum(['granted', 'denied']),
- updatedAt: z.string().datetime(),
-});
+type Storage = { getItem: (key: string) => Promise; setItem: (key: string, value: string) => Promise };
+type ConsentState = 'granted' | 'denied';
+type CapabilityDecision = { appId: string; capability: string; state: ConsentState; updatedAt: string };
+const Decision = z.object({ appId: z.string().min(1), capability: z.string().min(1), state: z.enum(['granted', 'denied']), updatedAt: z.string().datetime() });
export type ConsentDecision = z.infer;
-type Storage = { getItem(key: string): Promise; setItem(key: string, value: string): Promise };
-const key = (appId: string, capability: string) => `utopia:consent:${appId}:${capability}`;
-const grants: Record = {
- cameraScanner: ['camera.scan', 'native.camera.scan', 'native.camera.optional'],
+const normalize = (value: string) => value.toLowerCase().trim();
+const key = (appId: string, capability: string, scope: 'install') => `utopia:consent:${scope}:${appId}:${normalize(capability)}`;
+const legacyKey = (appId: string, capability: string) => `utopia:consent:${appId}:${normalize(capability)}`;
+
+const parseConsent = (value: string) => Decision.parse(JSON.parse(value) as CapabilityDecision);
+
+export const installScopedConsentKey = (appId: string, capability: string) => key(appId, capability, 'install');
+
+export const nativeCapabilityAliases = {
+ cameraScanner: ['camera', 'camera.scan', 'native.camera.scan', 'native.camera', 'native.camera.optional'],
filePicker: ['file.import', 'files.import', 'files.read', 'native.files.read', 'native.file_open'],
fileExport: ['file.export', 'files.export', 'export', 'native.share'],
- locationMap: ['location.current', 'location.optional', 'native.location.read'],
- notificationScheduler: ['notifications.schedule', 'notifications.optional'],
- contactPicker: ['contacts.read'],
- calendarEvent: ['calendar.create', 'calendar.events'],
+ locationMap: ['location', 'location.current', 'location.background', 'locationMap', 'native.location.read', 'location.optional'],
+ notificationScheduler: ['notification', 'notifications', 'notifications.schedule', 'notifications.optional'],
+ contactPicker: ['contacts', 'contacts.read', 'contacts.readonly'],
+ calendarEvent: ['calendar', 'calendar.events', 'calendar.write', 'calendar.create'],
biometricGate: ['biometric.optional', 'auth.local'],
- speechTool: ['native.speech.speak'],
- sensorReadout: ['sensors.read', 'native.sensors.read'],
+ speechTool: ['speech', 'speech.speak', 'native.speech.speak'],
+ sensorReadout: ['sensor.read', 'sensors.read', 'native.sensors.read'],
healthConnect: ['health.read'],
healthKitStatus: ['health.read'],
+} as const;
+
+type Capability = keyof typeof nativeCapabilityAliases;
+const entries = Object.entries(nativeCapabilityAliases) as Array<[Capability, readonly string[]]>;
+const aliasToCapability = new Map<
+ string,
+ Capability
+>([
+ ...entries.flatMap(([capability, aliases]) => aliases.map((alias) => [normalize(alias), capability] as const)),
+ ...entries.map(([capability]) => [normalize(capability), capability] as const),
+]);
+
+export const resolvePermissionCapabilityForDeclaration = (permissionId: string): Capability | undefined => {
+ return aliasToCapability.get(normalize(permissionId));
};
-export function allowsCapability(declared: readonly string[], widget: string): boolean {
- return (grants[widget] ?? [widget]).some((capability) => declared.includes(capability));
+export function resolveDeclaredCapability(declared: readonly string[], widget: string): Capability | undefined {
+ const target = resolvePermissionCapabilityForDeclaration(widget);
+ if (!target) return;
+ for (const declaration of declared) {
+ if (resolvePermissionCapabilityForDeclaration(declaration) === target) return target;
+ }
+ return;
}
+export const allowsCapability = (declared: readonly string[], widget: string) => Boolean(resolveDeclaredCapability(declared, widget));
+
export async function assertCapability(appId: string, widget: string): Promise {
const pkg = await findPackage(appId);
if (!pkg || !allowsCapability(pkg.capabilities, widget)) throw new Error(`Capability not declared: ${widget}`);
}
+export async function resolveCapability(appId: string, widget: string): Promise {
+ const pkg = await findPackage(appId);
+ return pkg ? resolveDeclaredCapability(pkg.capabilities, widget) : undefined;
+}
+
export async function readConsent(storage: Storage, appId: string, capability: string): Promise {
- const value = await storage.getItem(key(appId, capability));
- return value ? Decision.parse(JSON.parse(value)) : undefined;
+ const primary = await storage.getItem(installScopedConsentKey(appId, capability));
+ if (primary) return parseConsent(primary);
+
+ const legacy = await storage.getItem(legacyKey(appId, capability));
+ if (!legacy) return undefined;
+
+ const normalized = parseConsent(legacy);
+ await storage.setItem(installScopedConsentKey(appId, capability), JSON.stringify(normalized));
+ return normalized;
+}
+
+export async function readCapabilityDecision(appId: string, capability: string): Promise {
+ return readConsent(storage, appId, capability);
}
export async function writeConsent(storage: Storage, decision: ConsentDecision): Promise {
const valid = Decision.parse(decision);
- await storage.setItem(key(valid.appId, valid.capability), JSON.stringify(valid));
+ await storage.setItem(installScopedConsentKey(valid.appId, valid.capability), JSON.stringify(valid));
return valid;
}
-export async function recordConsent(appId: string, capability: string, state: ConsentDecision['state']) {
+export async function recordConsent(appId: string, capability: string, state: ConsentState) {
return writeConsent(storage, { appId, capability, state, updatedAt: new Date().toISOString() });
}
diff --git a/src/kernel/query.ts b/src/kernel/query.ts
index d3c0053..65babfd 100644
--- a/src/kernel/query.ts
+++ b/src/kernel/query.ts
@@ -3,8 +3,13 @@ export type QueryWhere = { op?: string; field?: string; value?: unknown; args?:
export type QueryOptions = {
where?: unknown;
orderBy?: Array<{ field: string; direction: 'asc' | 'desc' }>;
+ sortField?: string;
+ sortDirection?: 'asc' | 'desc';
+ page?: number;
+ pageSize?: number;
offset?: number;
limit?: number;
+ query?: string;
savedFilters?: Record;
};
@@ -67,6 +72,18 @@ function deepEqual(left: unknown, right: unknown): boolean {
return false;
}
+function asFiniteInteger(value: unknown, fallback: number, allowNegative = false): number {
+ const next = Number(value);
+ if (!Number.isFinite(next)) return fallback;
+ const normalized = Math.trunc(next);
+ if (!allowNegative && normalized < 0) return fallback;
+ return normalized;
+}
+
+function asSortDirection(value: unknown, fallback: 'asc' | 'desc'): 'asc' | 'desc' {
+ return String(value ?? '').toLowerCase() === 'desc' ? 'desc' : fallback;
+}
+
function compareValues(left: unknown, right: unknown): number {
if (left === right) return 0;
if (left == null) return -1;
@@ -121,6 +138,16 @@ export function matchesWhere(where: unknown, values: Record, sa
return matchPrimitive(filter, resolvePath(values, filter.field ?? ''));
}
+function withQueryValues; collection?: unknown; createdAt?: unknown; updatedAt?: unknown }>(record: T): Record {
+ return {
+ ...record.values,
+ id: record.id,
+ collection: record.collection,
+ createdAt: record.createdAt,
+ updatedAt: record.updatedAt,
+ };
+}
+
export function sortByFields(records: Record[], orderBy: QueryOptions['orderBy'] = []): Record[] {
return [...records].sort((left, right) => {
for (const order of orderBy) {
@@ -131,19 +158,63 @@ export function sortByFields(records: Record[], orderBy: QueryO
});
}
-export function applyQueryPagination(rows: T[], options: QueryOptions): T[] {
+export function normalizeQueryOptions(options: QueryOptions = {}): Required {
+ const direction = asSortDirection(options.sortDirection, 'asc');
+ const resolvedSortField = toText(options.sortField, '');
+ const orderBy = options.orderBy?.length
+ ? options.orderBy.map((entry) => ({ ...entry, direction: asSortDirection(entry.direction, direction) }))
+ : (resolvedSortField ? [{ field: resolvedSortField, direction }] : []);
const normalizedOffset = Number.isFinite(options.offset ?? NaN) ? Math.max(0, Math.floor(Number(options.offset ?? 0))) : 0;
- if (options.limit == null) return rows.slice(normalizedOffset);
- const normalizedLimit = Number.isFinite(options.limit) ? Math.max(0, Math.floor(Number(options.limit))) : rows.length;
- return rows.slice(normalizedOffset, normalizedOffset + normalizedLimit);
-}
+ const page = Number.isFinite(options.page ?? NaN) ? Math.max(1, Math.floor(Number(options.page ?? 1))) : 1;
+ const pageSize = asFiniteInteger(options.pageSize, 0, true);
+
+ const normalizedLimit = Number.isFinite(options.limit ?? NaN)
+ ? Math.floor(Number(options.limit))
+ : Number.MAX_SAFE_INTEGER;
+
+ const effectiveOffset = options.offset == null
+ ? (pageSize > 0 ? (page - 1) * pageSize : normalizedOffset)
+ : normalizedOffset;
-export function normalizeQueryOptions(options: QueryOptions = {}): Required {
return {
where: options.where,
- orderBy: options.orderBy ?? [],
- offset: options.offset ?? 0,
- limit: options.limit ?? 50,
+ orderBy: orderBy,
+ sortField: resolvedSortField || '',
+ sortDirection: direction,
+ page,
+ pageSize,
+ offset: effectiveOffset,
+ limit: normalizedLimit,
+ query: options.query ?? '',
savedFilters: options.savedFilters ?? {},
};
}
+
+export function applyQueryPagination(rows: T[], options: QueryOptions): T[] {
+ const normalizedOffset = Number.isFinite(options.offset ?? NaN) ? Math.max(0, Math.floor(Number(options.offset ?? 0))) : 0;
+ if (options.limit == null) return rows.slice(normalizedOffset);
+ const normalizedLimit = Number.isFinite(options.limit) ? Math.floor(Number(options.limit)) : rows.length;
+ if (normalizedLimit < 0) return rows.slice(normalizedOffset);
+ if (normalizedLimit === 0) return [];
+ return rows.slice(normalizedOffset, normalizedOffset + normalizedLimit);
+}
+
+function toText(value: unknown, fallback = '') {
+ return typeof value === 'string' && value.trim() ? value.trim() : fallback;
+}
+
+export function queryRecordsForClient; id?: unknown; collection?: unknown; createdAt?: unknown; updatedAt?: unknown }>(rows: T[], options: QueryOptions = {}): T[] {
+ const normalized = normalizeQueryOptions(options);
+ const query = toText(normalized.query).toLowerCase();
+
+ const filtered = rows
+ .filter((record) => !query || JSON.stringify(withQueryValues(record)).toLowerCase().includes(query))
+ .filter((record) => matchesWhere(normalized.where, withQueryValues(record), normalized.savedFilters));
+
+ const sorted = sortByFields(
+ filtered.map((record) => ({ ...(record as Record), ...withQueryValues(record) })),
+ normalized.orderBy,
+ );
+ const sliced = applyQueryPagination(sorted, normalized);
+ return sliced.map((record) => record as unknown as T);
+}
diff --git a/src/kernel/record-widgets.tsx b/src/kernel/record-widgets.tsx
index 4b6ae0e..4412b0d 100644
--- a/src/kernel/record-widgets.tsx
+++ b/src/kernel/record-widgets.tsx
@@ -39,6 +39,12 @@ const iconButtonProps = ({ testID, label, ...props }: { testID: string; label: s
testID,
});
const iconButton = (key: string) => `record-${key}`;
+const parseJsonValue = (value: string) => {
+ const text = value.trim();
+ if (!text) return '';
+ if (text.startsWith('{') || text.startsWith('[')) return JSON.parse(text);
+ return text.split(/[\r\n,]+/).map((item) => item.trim()).filter(Boolean);
+};
export const bulkRows = (value: string, field: string, defaults: Values = {}) => value.split(/\r?\n/).map((item) => item.trim()).filter(Boolean).slice(0, 100).map((item) => ({ ...defaults, [field]: item }));
export const matchesPreset = (record: JsonRecord, preset?: Values) => !preset || !toText(preset.field) || textLike(record.values[toText(preset.field)]).toLowerCase() === textLike(preset.value).toLowerCase();
const asTextField = (value: unknown, fallback: string) => toText(value, fallback) || fallback;
@@ -75,11 +81,7 @@ function collectRows(record: JsonRecord, component: AppComponent, cap = 8) {
const requested = asList(component.props?.fields)
.map((entry) => typeof entry === 'string' ? entry : toText((entry as Values).id))
.filter(Boolean);
- const fields = requested.length
- ? requested
- : Object.entries(record.values)
- .map(([key]) => key)
- .filter((key) => !RESERVED_RECORD_FIELDS.has(key));
+ const fields = requested.length ? requested : Object.keys(record.values).filter((key) => !RESERVED_RECORD_FIELDS.has(key));
return fields.slice(0, cap).map((field) => ({
field,
value: textLike(record.values[field]),
@@ -115,7 +117,7 @@ function fieldsFor(component: AppComponent, pkg: AppPackage, collection?: string
type: asRecordFieldType(spec.type),
required: Boolean(spec.required),
defaultValue: undefined,
- } satisfies RecordField));
+ }));
}
function Panel({ title, children }: { title?: string; children: ReactNode }) {
@@ -159,15 +161,22 @@ function QuickForm({
}));
const [error, setError] = useState('');
+ const normalizeField = (field: RecordField, value: string): unknown => {
+ if (field.type === 'number') return asNumber(value, Number.NaN);
+ if (field.type === 'boolean') return value === 'true';
+ if (field.type === 'json') {
+ try {
+ return parseJsonValue(value ?? '');
+ } catch {
+ setError(`Invalid JSON for ${field.label}`);
+ return value ?? '';
+ }
+ }
+ return value;
+ };
+
const save = async () => {
- const normalized = {
- ...values,
- ...Object.fromEntries(fields.map((field) => {
- if (field.type === 'number') return [field.id, asNumber(values[field.id], Number.NaN)];
- if (field.type === 'boolean') return [field.id, values[field.id] === 'true'];
- return [field.id, values[field.id]];
- })),
- };
+ const normalized = Object.fromEntries(fields.map((field) => [field.id, normalizeField(field, values[field.id] ?? '')]));
const missing = fields.find((field) => field.required && toText(values[field.id]) === '');
if (missing) return setError(`${missing.label} required`);
if (record) await dispatch({ kind: 'update', recordId: record.id, values: normalized });
@@ -182,10 +191,17 @@ function QuickForm({
const key = `${component.id ?? collection}-${field.id}`;
if (field.type === 'boolean') {
return
-
- setValues((current) => ({ ...current, [field.id]: value.toString() }))}>
- ;
+
+ setValues((current) => ({ ...current, [field.id]: value.toString() }))}>
+ ;
}
+ if (field.type === 'json') return