From cb81923a9e16119cd3d5f4eb4bf924b60ebeec3f Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Wed, 9 Sep 2026 14:44:26 +0200 Subject: [PATCH 1/3] Run LocalThought integrations entirely in the browser --- .dagger/src/index.ts | 20 +- Cargo.lock | 6 +- Cargo.toml | 2 +- TESTING_COVERAGE.md | 19 +- .../chunks/PluginRuns/ConnectLocalThought.tsx | 69 ++- .../chunks/PluginRuns/LocalThoughtCatalog.tsx | 24 +- .../PluginRuns/installationResources.test.ts | 34 ++ .../PluginRuns/installationResources.ts | 67 ++- .../chunks/PluginRuns/localImportVerdict.ts | 49 ++ .../src/chunks/PluginRuns/localThought.ts | 82 ++- browser/data-browser/src/locales/de.po | 8 + browser/data-browser/src/locales/en.po | 8 + browser/data-browser/src/locales/es.po | 8 + browser/data-browser/src/locales/fr.po | 8 + browser/e2e/tests/plugins.spec.ts | 25 +- integrations/localthought/README.md | 132 ++--- integrations/localthought/browser-smoke.mjs | 63 +++ integrations/localthought/browser.test.ts | 126 +++++ integrations/localthought/browser.ts | 278 ++++++++++ integrations/localthought/mock-proxy.mjs | 6 + .../localthought/syncables/Cargo.toml | 47 ++ integrations/localthought/syncables/LICENSE | 201 +++++++ integrations/localthought/syncables/README.md | 209 +++++++ .../localthought/syncables/UPSTREAM.md | 1 + .../syncables/src/client/client.rs | 369 +++++++++++++ .../localthought/syncables/src/client/mod.rs | 5 + .../syncables/src/client/storage.rs | 84 +++ .../localthought/syncables/src/error.rs | 95 ++++ .../syncables/src/fake_data/generate.rs | 83 +++ .../syncables/src/fake_data/mod.rs | 3 + .../localthought/syncables/src/lib.rs | 81 +++ .../syncables/src/mock_server/mod.rs | 6 + .../syncables/src/mock_server/server.rs | 91 +++ .../syncables/src/mock_server/store.rs | 89 +++ .../syncables/src/openapi/load.rs | 91 +++ .../localthought/syncables/src/openapi/mod.rs | 12 + .../syncables/src/openapi/overlay.rs | 209 +++++++ .../syncables/src/openapi/resolve_refs.rs | 92 ++++ .../syncables/src/openapi/types.rs | 250 +++++++++ .../syncables/src/pagination/autodetect.rs | 169 ++++++ .../syncables/src/pagination/items.rs | 91 +++ .../syncables/src/pagination/mod.rs | 20 + .../src/pagination/request_builder.rs | 144 +++++ .../src/pagination/response_parser.rs | 237 ++++++++ .../syncables/src/pagination/types.rs | 255 +++++++++ .../syncables/src/pagination/validate.rs | 112 ++++ .../syncables/src/resources/discover.rs | 106 ++++ .../syncables/src/resources/mod.rs | 3 + .../localthought/syncables/src/routing/mod.rs | 3 + .../syncables/src/routing/router.rs | 59 ++ .../localthought/syncables/src/sync/client.rs | 520 ++++++++++++++++++ .../syncables/src/sync/constants.rs | 129 +++++ .../syncables/src/sync/credentials.rs | 70 +++ .../localthought/syncables/src/sync/mod.rs | 24 + .../syncables/src/sync/ontology.rs | 273 +++++++++ .../syncables/src/sync/resource_model.rs | 426 ++++++++++++++ .../syncables/src/sync/storage.rs | 221 ++++++++ integrations/localthought/tsconfig.json | 2 +- integrations/localthought/wasm-smoke.mjs | 72 +++ planning/api-plugins.md | 9 + server/Cargo.toml | 1 - server/src/handlers/integration_proxy.rs | 316 ----------- server/src/handlers/integration_proxy_sync.rs | 515 ----------------- server/src/handlers/mod.rs | 4 - server/src/routes.rs | 20 - wasm/Cargo.toml | 4 + wasm/src/integrations.rs | 260 +++++++++ wasm/src/lib.rs | 2 + 68 files changed, 6097 insertions(+), 1022 deletions(-) create mode 100644 browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts create mode 100644 integrations/localthought/browser-smoke.mjs create mode 100644 integrations/localthought/browser.test.ts create mode 100644 integrations/localthought/browser.ts create mode 100644 integrations/localthought/syncables/Cargo.toml create mode 100644 integrations/localthought/syncables/LICENSE create mode 100644 integrations/localthought/syncables/README.md create mode 100644 integrations/localthought/syncables/UPSTREAM.md create mode 100644 integrations/localthought/syncables/src/client/client.rs create mode 100644 integrations/localthought/syncables/src/client/mod.rs create mode 100644 integrations/localthought/syncables/src/client/storage.rs create mode 100644 integrations/localthought/syncables/src/error.rs create mode 100644 integrations/localthought/syncables/src/fake_data/generate.rs create mode 100644 integrations/localthought/syncables/src/fake_data/mod.rs create mode 100644 integrations/localthought/syncables/src/lib.rs create mode 100644 integrations/localthought/syncables/src/mock_server/mod.rs create mode 100644 integrations/localthought/syncables/src/mock_server/server.rs create mode 100644 integrations/localthought/syncables/src/mock_server/store.rs create mode 100644 integrations/localthought/syncables/src/openapi/load.rs create mode 100644 integrations/localthought/syncables/src/openapi/mod.rs create mode 100644 integrations/localthought/syncables/src/openapi/overlay.rs create mode 100644 integrations/localthought/syncables/src/openapi/resolve_refs.rs create mode 100644 integrations/localthought/syncables/src/openapi/types.rs create mode 100644 integrations/localthought/syncables/src/pagination/autodetect.rs create mode 100644 integrations/localthought/syncables/src/pagination/items.rs create mode 100644 integrations/localthought/syncables/src/pagination/mod.rs create mode 100644 integrations/localthought/syncables/src/pagination/request_builder.rs create mode 100644 integrations/localthought/syncables/src/pagination/response_parser.rs create mode 100644 integrations/localthought/syncables/src/pagination/types.rs create mode 100644 integrations/localthought/syncables/src/pagination/validate.rs create mode 100644 integrations/localthought/syncables/src/resources/discover.rs create mode 100644 integrations/localthought/syncables/src/resources/mod.rs create mode 100644 integrations/localthought/syncables/src/routing/mod.rs create mode 100644 integrations/localthought/syncables/src/routing/router.rs create mode 100644 integrations/localthought/syncables/src/sync/client.rs create mode 100644 integrations/localthought/syncables/src/sync/constants.rs create mode 100644 integrations/localthought/syncables/src/sync/credentials.rs create mode 100644 integrations/localthought/syncables/src/sync/mod.rs create mode 100644 integrations/localthought/syncables/src/sync/ontology.rs create mode 100644 integrations/localthought/syncables/src/sync/resource_model.rs create mode 100644 integrations/localthought/syncables/src/sync/storage.rs create mode 100644 integrations/localthought/wasm-smoke.mjs delete mode 100644 server/src/handlers/integration_proxy.rs delete mode 100644 server/src/handlers/integration_proxy_sync.rs create mode 100644 wasm/src/integrations.rs diff --git a/.dagger/src/index.ts b/.dagger/src/index.ts index c177cf9aef..dc4e0a4b36 100644 --- a/.dagger/src/index.ts +++ b/.dagger/src/index.ts @@ -636,6 +636,10 @@ export class AtomicServer { this.source.directory('plugin-runtime'), ) .withDirectory('/code/wasm', this.source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + this.source.directory('integrations/localthought/syncables'), + ) .withDirectory('/code/server', this.source.directory('server')) .withDirectory('/code/cli', this.source.directory('cli')) .withDirectory('/code/desktop', this.source.directory('desktop')) @@ -713,6 +717,10 @@ export class AtomicServer { .withDirectory('/code/cli', this.source.directory('cli')) .withDirectory('/code/desktop', this.source.directory('desktop')) .withDirectory('/code/wasm', this.source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + this.source.directory('integrations/localthought/syncables'), + ) .withDirectory( '/code/plugin-examples', this.source.directory('plugin-examples'), @@ -1127,7 +1135,9 @@ export class AtomicServer { // Surfaces /app/dev-drive and /app/prunetests in the production // build the e2e tests run against. See `devRoutesEnabled()` in // data-browser/src/config.ts. - buildContainer = buildContainer.withEnvVariable('VITE_E2E', 'true'); + buildContainer = buildContainer + .withEnvVariable('VITE_E2E', 'true') + .withEnvVariable('VITE_INTEGRATION_PROXY_URL', 'http://127.0.0.1:19090'); } return buildContainer.withExec(['pnpm', 'run', 'build']); @@ -1182,6 +1192,10 @@ export class AtomicServer { .withDirectory('/code/cli', source.directory('cli')) .withDirectory('/code/desktop', source.directory('desktop')) .withDirectory('/code/wasm', source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + source.directory('integrations/localthought/syncables'), + ) .withDirectory( '/code/plugin-examples', source.directory('plugin-examples'), @@ -1380,6 +1394,10 @@ export class AtomicServer { .withDirectory('/code/cli', source.directory('cli')) .withDirectory('/code/desktop', source.directory('desktop')) .withDirectory('/code/wasm', source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + source.directory('integrations/localthought/syncables'), + ) .withDirectory( '/code/plugin-examples', source.directory('plugin-examples'), diff --git a/Cargo.lock b/Cargo.lock index 7f5b7ea106..e675c1e34f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,7 +1282,6 @@ dependencies = [ "sha1 0.11.0", "simple-server-timing-header", "static-files 0.3.1", - "syncables", "tempfile", "text-splitter", "tokio", @@ -1345,14 +1344,17 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" name = "atomic-wasm" version = "0.1.0" dependencies = [ + "async-trait", "atomic_lib", "blake3", + "chrono", "console_error_panic_hook", "getrandom 0.3.4", "js-sys", "serde", "serde-wasm-bindgen", "serde_json", + "syncables", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", @@ -13442,7 +13444,6 @@ dependencies = [ [[package]] name = "syncables" version = "0.1.0" -source = "git+https://github.com/localthought/syncables-rs?rev=d48e4d9bad3ed9ec826d1c2040e989171701965d#d48e4d9bad3ed9ec826d1c2040e989171701965d" dependencies = [ "async-trait", "httpdate", @@ -13453,7 +13454,6 @@ dependencies = [ "serde_yaml_ng", "thiserror 2.0.18", "tokio", - "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a575b5bc79..740f704af9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ "plugin-runtime", "tools/cargo-bin", ] -exclude = ["flutter/rust"] +exclude = ["flutter/rust", "integrations/localthought/syncables"] # Debuginfo dominates target/ size: with ~1460 deps (tauri, actix, iroh) the # default `debug = true` produces a multi-GB tree per build flavor, and this diff --git a/TESTING_COVERAGE.md b/TESTING_COVERAGE.md index 6e3c8f1b5a..dbae936d47 100644 --- a/TESTING_COVERAGE.md +++ b/TESTING_COVERAGE.md @@ -1,5 +1,18 @@ # Testing coverage map +LocalThought browser migration: `integrations/localthought/browser.test.ts` +covers tenant HMAC, actor/drive ownership, rotation before dispatch, pagination, +uncertain-response refusal and cross-origin pagination refusal. The real generated +WASM bundle is exercised by `wasm-smoke.mjs` for pagination, typed ontology, +timestamps and provider failures. `browser-smoke.mjs` exercises the complete +mock consent/import/review/OPFS/reload journey with AtomicServer unavailable +(verified locally). Local installation/schema lookup tests reject missing or +incomplete local databases rather than inferring permission to create duplicates. +The companion Syncables branch has 142 passing native tests and a wasm32 build; +the companion proxy branch has 39 passing tests including CORS preflight and +exposed headers. Live OAuth on the browser path still requires deployment of +the companion proxy CORS change and is not yet verified. + What is tested, at which layer, and — the part that matters — **what is not**. This exists because the protocol is far better tested than the glue around it, @@ -65,12 +78,12 @@ return to the same drive, rotating connection codes, two-page Syncables fetch, review/apply, and five displayed records with integer/boolean/float/timestamp properties. Dagger starts the mock for E2E; local runs opt in with `ATOMIC_MOCK_INTEGRATION_PROXY=1` and the README configuration. -`integration_proxy` Rust tests cover actor/drive binding, tenant HMAC, +`browser.test.ts` and the real WASM smoke cover actor/drive binding, tenant HMAC, Syncables pagination/ontology and duplicate-page refusal. The mock's Node test covers invalid tenant proofs and replayed/rotated codes. The mapping tests cover typed proposals, missing identities, repeat imports, local edits and duplicates. -Live catalog OAD endpoints currently return 404 (integration-proxy #25), so live -OAuth and GitHub data fetching are not yet certified. +The historical server path was live-verified for GitHub and Google Calendar. +The new browser path awaits deployment of the companion proxy CORS change. Run it against a production build to catch missing translation catalog entries: Vite dev extracts them automatically and can hide blank production labels. The GitHub setup flow also covers opting into assistant-led automation creation: diff --git a/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx b/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx index bb33f0ea6e..4be4f39df2 100644 --- a/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx +++ b/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx @@ -3,7 +3,7 @@ import { core, dataBrowser, ensureSchema, - executeServerPlugin, + pluginSchema, useStore, type Resource, } from '@tomic/react'; @@ -12,10 +12,13 @@ import { Column } from '@components/Row'; import Field from '@components/forms/Field'; import { Input, ErrMessage } from '@components/forms/InputStyles'; import { AtomicLink } from '@components/AtomicLink'; -import { pluginClassesFor } from './runScript'; -import { ensureInstallationResource } from './installationResources'; +import { + localSchemaStore, + ensureLocalInstallationResource as ensureInstallationResource, +} from './installationResources'; import { RunPluginDialog } from './RunPluginDialog'; import { + browserIntegrations, connectionKey, platformName, proxyRequest, @@ -27,6 +30,7 @@ import { type FetchedPlatform, } from '../../../../../integrations/localthought/schema'; import type { Config } from '../../../../../integrations/localthought/plugin'; +import { localImportVerdict } from './localImportVerdict'; import source from '../../../../../integrations/localthought/plugin.js?raw'; export function ConnectLocalThought({ @@ -55,6 +59,7 @@ export function ConnectLocalThought({ start: new Date().toISOString().slice(0, 10), end: new Date(Date.now() + 30 * 86400000).toISOString().slice(0, 10), })); + const [tenantSecret, setTenantSecret] = useState(''); const [busy, setBusy] = useState(false); const [error, setError] = useState(''); const [preview, setPreview] = useState<{ @@ -65,13 +70,10 @@ export function ConnectLocalThought({ const [tables, setTables] = useState([]); useEffect(() => { const controller = new AbortController(); - fetch( - `${store.getServerUrl()}/integration-proxy/platform?platform=${encodeURIComponent(platform)}`, - { signal: controller.signal }, - ) - .then(async response => { - if (!response.ok) throw new Error(await response.text()); - const data = await response.json(); + browserIntegrations() + .describe(platform) + .then(data => { + if (controller.signal.aborted) return; setParameters(data.parameters); setCollections(data.collections); setConstants( @@ -104,7 +106,12 @@ export function ConnectLocalThought({ const result = await proxyRequest<{ url: string; state: string }>( store, 'start', - { drive, platform, returnUrl: `${location.origin}/app/integrations` }, + { + drive, + platform, + tenantSecret, + returnUrl: `${location.origin}/app/integrations`, + }, ); sessionStorage.setItem( 'localthought-pending', @@ -131,7 +138,8 @@ export function ConnectLocalThought({ }); if (fetched.platform !== platform) throw new Error('Imported platform did not match this connection'); - const terms = await pluginClassesFor(store, drive); + const schemaStore = localSchemaStore(store); + const terms = await ensureSchema(schemaStore, drive, pluginSchema()); const name = platformName(platform); const identity = `localthought:${connection.connection}:${JSON.stringify(Object.entries(constants).sort())}`; const resource = await ensureInstallationResource(store, drive, { @@ -145,7 +153,7 @@ export function ConnectLocalThought({ }, }); const schema = await ensureSchema( - store, + schemaStore, drive, platformSchema(platform, fetched.ontology.terms), ); @@ -203,24 +211,13 @@ export function ConnectLocalThought({ localthought: { ...config, connection: connection.connection }, }); await resource.save(); - const result = await executeServerPlugin(store, { - drive, - plugin: resource.subject, - source, - input: { - config: { ...config, records: fetched.records }, - trigger: { - kind: 'manual', - at: Date.now(), - subject: resource.subject, - }, - }, + const verdict = await localImportVerdict(store, drive, { + ...config, + records: fetched.records, }); - if (result.error || !result.verdict) - throw new Error(result.error ?? 'Import returned no preview'); setPreview({ resource, - verdict: result.verdict, + verdict, tables: Object.values(destinations).map(d => d.table), }); } catch (reason) { @@ -236,7 +233,21 @@ export function ConnectLocalThought({ Connect your personal account through LocalThought, then return here to preview an import.

- {connection && ( diff --git a/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx b/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx index 7e10caf095..733f9a6170 100644 --- a/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx +++ b/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx @@ -6,7 +6,12 @@ import { Button } from '@components/Button'; import { Dialog, useDialog } from '@components/Dialog'; import { ErrMessage } from '@components/forms/InputStyles'; import { ConnectLocalThought } from './ConnectLocalThought'; -import { connectionKey, platformName, proxyRequest } from './localThought'; +import { + browserIntegrations, + connectionKey, + platformName, + proxyRequest, +} from './localThought'; const DirectGitHub = lazy(() => import('./ConnectGitHub').then(m => ({ default: m.ConnectGitHub })), @@ -26,20 +31,9 @@ export function LocalThoughtCatalog({ const completing = useRef(false); useEffect(() => { const controller = new AbortController(); - fetch(`${store.getServerUrl()}/integration-proxy/catalog`, { - signal: controller.signal, - }) - .then(async response => { - if (!response.ok) - throw new Error('Could not load the LocalThought catalog'); - const data = await response.json(); - if ( - !Array.isArray(data.platforms) || - data.platforms.some((id: unknown) => typeof id !== 'string') - ) - throw new Error('Invalid integration catalog'); - setPlatforms(data.platforms); - }) + browserIntegrations() + .catalog(controller.signal) + .then(setPlatforms) .catch(reason => { if (!controller.signal.aborted) setError(String(reason)); }); diff --git a/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts b/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts index 81dc6fc793..97ead3f8d7 100644 --- a/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts +++ b/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts @@ -47,3 +47,37 @@ it('never interprets a failed identity query as permission to create', async () ).rejects.toThrow('offline'); expect(store.newResource).not.toHaveBeenCalled(); }); + +it('uses complete local identities without a server and refuses a missing local DB', async () => { + const { ensureLocalInstallationResource } = + await import('./installationResources'); + const resource = { + get: (p: string) => ({ parent: 'parent', isA: ['class'] })[p], + }; + const queryLocalDb = vi + .fn() + .mockResolvedValue({ subjects: ['saved'], count: 1 }); + const store = { + queryLocalDb, + getResource: async () => resource, + newResource: vi.fn(), + }; + const options = { + parent: 'parent', + localId: 'step', + isA: ['class'], + propVals: {}, + }; + expect( + await ensureLocalInstallationResource(store as never, 'drive', options), + ).toBe(resource); + queryLocalDb.mockResolvedValue(null); + await expect( + ensureLocalInstallationResource(store as never, 'drive', options), + ).rejects.toThrow('Local installation query'); + queryLocalDb.mockResolvedValue({ subjects: [], count: 1 }); + await expect( + ensureLocalInstallationResource(store as never, 'drive', options), + ).rejects.toThrow('Local installation query'); + expect(store.newResource).not.toHaveBeenCalled(); +}); diff --git a/browser/data-browser/src/chunks/PluginRuns/installationResources.ts b/browser/data-browser/src/chunks/PluginRuns/installationResources.ts index e4a7be3b9e..62c808ce40 100644 --- a/browser/data-browser/src/chunks/PluginRuns/installationResources.ts +++ b/browser/data-browser/src/chunks/PluginRuns/installationResources.ts @@ -16,14 +16,12 @@ export async function ensureInstallationResource( isA: string[]; propVals: Record; }, + local = false, ) { const find = async () => { - const ids = await readConnectionSubjects( - store, - drive, - core.properties.localId, - options.localId, - ); + const ids = await ( + local ? readLocalInstallationSubjects : readConnectionSubjects + )(store, drive, core.properties.localId, options.localId); const resources = await Promise.all(ids.map(id => store.getResource(id))); const matches = resources.filter( r => @@ -71,3 +69,60 @@ export async function ensureInstallationResource( throw error; } } + +async function readLocalInstallationSubjects( + store: Store, + drive: string, + property: string, + value: string, +) { + const result = await store.queryLocalDb({ + drive, + property, + value, + limit: 10001, + }); + if (!result || result.count !== result.subjects.length) + throw new Error( + 'Local installation query failed or is incomplete; refusing to create duplicates', + ); + return result.subjects; +} + +/** LocalThought installations use the same identity checks against local OPFS. */ +export function ensureLocalInstallationResource( + store: Store, + drive: string, + options: Parameters[2], +) { + return ensureInstallationResource(store, drive, options, true); +} + +/** Schema recovery must use the same local identity authority as installation. */ +export function localSchemaStore(store: Store) { + return { + getResource: store.getResource.bind(store), + newResource: store.newResource.bind(store), + findByLocalId: async (drive: string, parent: string, localId: string) => { + const subjects = await readLocalInstallationSubjects( + store, + drive, + core.properties.localId, + localId, + ); + const resources = await Promise.all( + subjects.map(s => store.getResource(s)), + ); + const matches = resources.filter( + r => + String(r.get(core.properties.parent)).split('?')[0] === + parent.split('?')[0], + ); + if (matches.length > 1) + throw new Error( + 'Duplicate local schema identity; resolve before importing', + ); + return matches[0]; + }, + }; +} diff --git a/browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts b/browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts new file mode 100644 index 0000000000..7f373d547c --- /dev/null +++ b/browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts @@ -0,0 +1,49 @@ +// @wc-ignore-file +import { core, type Store } from '@tomic/react'; +import { + run, + type Config, +} from '../../../../../integrations/localthought/plugin'; + +/** The shipped pure mapper gets a local read snapshot, never network or credentials. */ +export async function localImportVerdict( + store: Store, + drive: string, + config: Config, +) { + const rows = new Map>(); + for (const { table } of Object.values(config.destinations)) { + let offset = 0; + for (;;) { + const result = await store.queryLocalDb({ + drive, + property: core.properties.parent, + value: table, + offset, + limit: 1000, + }); + if (!result) + throw new Error('Local database must be available before importing'); + for (const subject of result.subjects) + rows.set(subject, (await store.getResource(subject)).getPropVals()); + offset += result.subjects.length; + if (offset >= result.count) break; + if (!result.subjects.length) + throw new Error('Local import snapshot is incomplete'); + } + } + return JSON.stringify( + run({ + config, + query: (property, value) => + [...rows] + .filter(([, row]) => row[property] === value) + .map(([subject]) => subject), + read: subject => { + const row = rows.get(subject); + if (!row) throw new Error('Missing local import record'); + return row; + }, + }), + ); +} diff --git a/browser/data-browser/src/chunks/PluginRuns/localThought.ts b/browser/data-browser/src/chunks/PluginRuns/localThought.ts index 6dcf43024e..8751989c66 100644 --- a/browser/data-browser/src/chunks/PluginRuns/localThought.ts +++ b/browser/data-browser/src/chunks/PluginRuns/localThought.ts @@ -1,5 +1,5 @@ // @wc-ignore-file -import { signRequest, type Store } from '@tomic/react'; +import { type Store } from '@tomic/react'; export const platformName = (id: string) => ({ @@ -7,25 +7,73 @@ export const platformName = (id: string) => 'google-calendar': 'Google Calendar', pets: 'Pets', })[id] ?? id; +import { + BrowserIntegrations, + type Engine, +} from '../../../../../integrations/localthought/browser'; +import { wasmJsUrl, wasmBinaryUrl } from '../../helpers/wasmUrls'; +let loaded: Promise | undefined; +async function engine(): Promise { + return (loaded ??= (async () => { + const url = wasmJsUrl(); + const module = await import(/* @vite-ignore */ url); + await module.default({ module_or_path: wasmBinaryUrl() }); + if (typeof module.fetchIntegration !== 'function') + throw new Error('Rebuild the WASM bundle and reload Atomic'); + return module; + })().catch(error => { + loaded = undefined; + throw error; + })); +} +export const browserIntegrations = () => + new BrowserIntegrations( + localStorage, + engine, + import.meta.env.VITE_INTEGRATION_PROXY_URL || undefined, + ); export async function proxyRequest( store: Store, action: string, - body: object, + body: { + drive: string; + platform?: string; + returnUrl?: string; + tenantSecret?: string; + state?: string; + connectionCode?: string; + connection?: string; + constants?: Record; + calendarRange?: { start: string; end: string }; + }, ): Promise { - const agent = store.getAgent(); - if (!agent) throw new Error('Sign in before connecting an account'); - const url = `${store.getServerUrl()}/integration-proxy/${action}`; - const response = await fetch(url, { - method: 'POST', - headers: { - ...(await signRequest(url, agent, {})), - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }); - if (!response.ok) throw new Error(await response.text()); - - return response.json(); + const actor = store.getAgent()?.subject; + if (!actor) throw new Error('Sign in before connecting an account'); + const client = browserIntegrations(); + if (action === 'start') + return (await client.start( + body.drive, + actor, + body.platform!, + body.returnUrl!, + body.tenantSecret!, + )) as T; + if (action === 'finish') + return client.finish( + body.drive, + actor, + body.state!, + body.connectionCode!, + ) as T; + if (action === 'fetch') + return client.fetchRecords( + body.drive, + actor, + body.connection!, + body.constants ?? {}, + body.calendarRange, + ); + throw new Error('Unknown browser integration action'); } export interface SavedConnection { connection: string; @@ -34,4 +82,4 @@ export interface SavedConnection { actor: string; } export const connectionKey = (drive: string, actor: string, platform: string) => - `localthought:${JSON.stringify([drive, actor, platform])}`; + `localthought-browser:${JSON.stringify([drive, actor, platform])}`; diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index ee9bc15a69..e3ca21bfa9 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -9659,3 +9659,11 @@ msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "LocalThought tenant secret" +msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index 2cbaf4b6de..3d23fe2d7d 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -9698,3 +9698,11 @@ msgstr "Events from (UTC)" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "Events before (UTC)" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "LocalThought tenant secret" +msgstr "LocalThought tenant secret" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "The tenant secret is used in this tab. Connection credentials stay in this browser." diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index 73252fe2f3..0e1c244ed0 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -9659,3 +9659,11 @@ msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "LocalThought tenant secret" +msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index 7dcbb3f208..b74aceab3e 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -9659,3 +9659,11 @@ msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "LocalThought tenant secret" +msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "" diff --git a/browser/e2e/tests/plugins.spec.ts b/browser/e2e/tests/plugins.spec.ts index 1d4e1f0a6f..1c0f9b0392 100644 --- a/browser/e2e/tests/plugins.spec.ts +++ b/browser/e2e/tests/plugins.spec.ts @@ -28,10 +28,9 @@ test.describe('plugins', () => { !process.env.ATOMIC_MOCK_INTEGRATION_PROXY, 'Run with the documented mock integration-proxy server configuration', ); - await page.getByRole('link', { name: 'Integrations', exact: true }).click(); // CI's browser and server are in different containers. Forward the mock's - // loopback address to the server container; the real connect/rotation logic runs there. + // loopback address to the server container before catalog loading starts. if (process.env.ATOMIC_SERVICE_URL) await page.route('http://127.0.0.1:19090/**', async route => { const target = new URL(route.request().url()); @@ -42,6 +41,7 @@ test.describe('plugins', () => { }); await route.fulfill({ response }); }); + await page.getByRole('link', { name: 'Integrations', exact: true }).click(); const pets = page.locator('[data-integration=pets]'); await expect( pets.getByRole('heading', { name: 'Pets', exact: true }), @@ -52,6 +52,9 @@ test.describe('plugins', () => { await expect( setup.getByRole('button', { name: 'Install and connect', exact: true }), ).toBeVisible(); + await setup + .getByLabel('LocalThought tenant secret') + .fill('bW9jay10ZW5hbnQ.mock-signature'); await setup .getByRole('button', { name: 'Install and connect', exact: true }) .click(); @@ -61,23 +64,11 @@ test.describe('plugins', () => { ).toBeVisible(); await page.getByRole('button', { name: 'Connect test account' }).click(); await expect(page).not.toHaveURL(/connection_code=/); - const [preview] = await Promise.all([ - page.waitForResponse( - response => - response.url().endsWith('/plugin-run') && - response.request().method() === 'POST', - { timeout: 45_000 }, - ), - page.getByRole('button', { name: 'Fetch and preview' }).click(), - ]); - expect(preview.ok(), await preview.text()).toBe(true); + await page.getByRole('button', { name: 'Fetch and preview' }).click(); const review = page.locator('dialog[open]'); - // Installing walks pluginClassesFor, two ensureSchema calls, three - // ensureInstallationResource calls and the actual sandboxed plugin run — - // measured ~24s locally even on a fresh, otherwise-idle server. The - // default 10s expect timeout is tuned for interaction latency, not this - // one-time setup cost. + // The browser creates the local ontology, tables and reviewed proposal. + // Allow the one-time installation more than the interaction timeout. await expect( review.getByRole('button', { name: 'Apply 5 changes', exact: true }), ).toBeEnabled({ timeout: 45_000 }); diff --git a/integrations/localthought/README.md b/integrations/localthought/README.md index d6f4504702..9ea97f7efc 100644 --- a/integrations/localthought/README.md +++ b/integrations/localthought/README.md @@ -1,40 +1,67 @@ -# LocalThought API plugins +# LocalThought browser integrations + +The LocalThought flow runs entirely in the browser: catalog discovery, tenant +challenge signing, OAuth return handling, paginated Syncables reads, ontology +creation, proposal review and local Store/OPFS writes. No AtomicServer HTTP +instance is needed. LocalThought remains the remote OAuth and API proxy. + +Open Integrations, select a platform and paste your `TENANT_SECRET`. It is used +in tab memory to sign the handoff and is not persisted. OAuth returns to the +same frontend `/app/integrations` page. The short-lived return is bound to the +agent, drive and proxy; its code is removed from the address bar immediately. +Connection codes are stored in this browser's localStorage, outside the synced +graph, and may be read by code running on this frontend origin. Clearing site +data requires reconnecting. Existing server-held connections require reconnecting. +Web Locks serialize rotating codes across tabs; a request consumes its code +before dispatch and saves the replacement before processing data. Uncertain +requests cannot silently replay credentials. + +Syncables is vendored temporarily under `syncables/` with upstream provenance in +`UPSTREAM.md`; the matching upstream branch is `codex/browser-integrations`. +`wasm/src/integrations.rs` exposes its in-memory engine through wasm-bindgen. +The shipped pure import mapper reads a local snapshot and produces the existing +reviewed intents; user-edited plugin source is not executed on this path. +Local edits and repeated imports retain the existing reconciliation behavior. + +## Build and proxy requirements + +- Build `atomic-wasm` using `cd browser/data-browser && pnpm build:wasm`. +- Set `VITE_INTEGRATION_PROXY_URL` at frontend build/dev time to override the + default `https://localthought.io`. HTTPS or loopback HTTP origins only. +- Deploy the companion integration-proxy CORS change. It handles preflights for + explicit Authorization headers and exposes `X-Connection-Code`, `Link`, + pagination/count headers, `ETag` and `Retry-After`. Cookie credentials are not + enabled; login and consent use top-level navigation. +- Native AtomicServer's `TENANT_SECRET`, `ATOMIC_INTEGRATION_PROXY_URL` and + `ATOMIC_INTEGRATION_FRONTEND_ORIGIN` no longer configure this flow. Its + `/integration-proxy/*` handlers and Syncables dependency have been removed. + +Limits remain 200 requests, 5,000 records, 10 MB per page/document and 120 +seconds of network work. Calendar imports require explicit UTC date bounds. +Imports are manual; closed tabs do not run schedules. Other legacy integrations, +server plugin execution, actions and schedules are outside this migration. + +## Checks -AtomicServer reads `/catalog` from `https://localthought.io` and offers one -connection card per advertised platform. It loads `/catalog/{platform}.yaml` -for collection discovery, required scope parameters, pagination and ontology. - -`syncables-rs` is pinned in `server/Cargo.toml`. Its `SyncClient` receives the -catalog's overlaid OpenAPI document and a host-owned transport that forwards -GET requests through `/proxy/{platform}/{path}`, preserving query parameters -and pagination headers. Rotating connection codes are held in AtomicServer's -secret store, scoped to a connection, drive and authenticated agent. The -browser carries a one-time OAuth return code to signed completion and removes -it from the address bar. Tenant secrets and provider tokens never enter graph -resources, plugin source or browser storage. +```sh +cargo check -p atomic-wasm --target wasm32-unknown-unknown +browser/node_modules/.bin/vitest run --config integrations/localthought/vitest.config.ts +node integrations/localthought/wasm-smoke.mjs # after building wasm/pkg +``` -Each platform gets drive-local classes and properties derived by Syncables. -The platform ID prefixes every generated schema identity. Scalars retain their -datatypes; RFC3339 timestamps become milliseconds; arrays and nested objects -use Atomic JSON properties. API-required fields are recommended in Atomic -because nullable/partial API representations may omit them. Each resource type -gets a table and view. The shared sandbox importer proposes changes for review, -uses provider/resource/namespace/ID for reconciliation, and preserves local edits. -A failed collection aborts the preview instead of applying a partial snapshot. +For the browser-only mock journey (no AtomicServer on port 19999): -This is a manual read/import flow, with limits of 200 requests, 5,000 records, -10 MB per page/document and 120 seconds per import. No provider writes or -background syncing are enabled. The existing direct GitHub token workflow -remains available explicitly in the GitHub connection dialog. +```sh +MOCK_PROXY_PORT=19091 MOCK_FRONTEND_ORIGIN=http://localhost:6748 node integrations/localthought/mock-proxy.mjs +# Separate terminal, browser/data-browser: +VITE_INTEGRATION_PROXY_URL=http://127.0.0.1:19091 VITE_ATOMIC_SERVER_URL=http://127.0.0.1:19999 pnpm exec vite --host 127.0.0.1 --port 6748 +# Repository root: +node integrations/localthought/browser-smoke.mjs +``` -## Server configuration +The mock is test-only. It uses synthetic credentials and data; never deploy it. -- `TENANT_SECRET`: the LocalThought tenant secret, stored only on AtomicServer. -- `ATOMIC_INTEGRATION_FRONTEND_ORIGIN`: exact frontend origin, e.g. - `https://your-atomic-app.example` (no trailing slash). Returns must use its - `/app/integrations` path. -- `ATOMIC_INTEGRATION_PROXY_URL`: optional; defaults to `https://localthought.io`. - Loopback HTTP is accepted for development. +## Historical server-flow verification Live verification on 2026-09-09 succeeded against proxy Heroku release v38 (`5960ae43`): OAuth returned to AtomicServer, Syncables fetched 29 issue/PR @@ -55,44 +82,3 @@ An unbounded fetch successfully traversed multiple pages but exceeded the 5,000-record preview limit; the UI now defaults to the next 30 days. Date bounds and recurrence expansion are passed to Syncables as collection query settings. The importer remains a manual snapshot, not a background sync. - -## Local mock and tests - -The mock is test-only, with a synthetic tenant, consent screen, expiring-by-use -challenges, rotating single-use codes, and an OpenAPI document serving five -Pets over two pages. Never deploy it as a real credential service. - -```sh -node integrations/localthought/mock-proxy.mjs -``` - -Start AtomicServer with: - -```sh -TENANT_SECRET=bW9jay10ZW5hbnQ.mock-signature \ -ATOMIC_INTEGRATION_PROXY_URL=http://127.0.0.1:19090 \ -ATOMIC_INTEGRATION_FRONTEND_ORIGIN=http://localhost:6747 \ -ATOMICSERVER_SKIP_JS_BUILD=true \ -cargo run -p atomic-server --features light,wasm-plugins -- --port 9883 -``` - -Run one Vite server with `VITE_ATOMIC_SERVER_URL=http://localhost:9883`, then: - -```sh -cd browser/e2e -ATOMIC_MOCK_INTEGRATION_PROXY=1 pnpm exec playwright test tests/plugins.spec.ts \ - --project chromium --grep 'Pets imports' -``` - -Additional checks from the repository root: - -```sh -node --test integrations/localthought/mock-proxy.test.mjs -browser/node_modules/.bin/vitest run --config integrations/localthought/vitest.config.ts -browser/node_modules/.bin/tsc -p integrations/localthought/tsconfig.json -ATOMICSERVER_SKIP_JS_BUILD=true cargo test -p atomic-server --lib \ - --features light,wasm-plugins integration_proxy -``` - -Dagger's E2E server starts the mock alongside AtomicServer, and the focused -GitHub Actions workflow runs the typed Pets journey on the API-plugin branches. diff --git a/integrations/localthought/browser-smoke.mjs b/integrations/localthought/browser-smoke.mjs new file mode 100644 index 0000000000..87ce370fcf --- /dev/null +++ b/integrations/localthought/browser-smoke.mjs @@ -0,0 +1,63 @@ +/** Run against Vite + the mock proxy, with VITE_ATOMIC_SERVER_URL on a closed port. */ +import { chromium } from '../../browser/e2e/node_modules/@playwright/test/index.mjs'; +const browser = await chromium.launch({ headless: true }); +const page = await browser.newPage(); +page.setDefaultTimeout(45000); +await page.route('http://127.0.0.1:19999/**', route => route.abort()); +const errors = []; +page.on('pageerror', e => errors.push(e.message)); +try { + console.log('Opening offline dev drive'); + await page.goto('http://localhost:6748/app/dev-drive'); + await page.waitForURL(/app\/show\?subject=/, { timeout: 120000 }); + console.log('Opening integrations'); + await page.getByRole('link', { name: 'Integrations', exact: true }).click(); + console.log('Opening Pets'); + const pets = page.locator('[data-integration=pets]'); + await pets.getByRole('button', { name: 'Set up connection' }).click(); + await page + .getByLabel('LocalThought tenant secret') + .fill('bW9jay10ZW5hbnQ.mock-signature'); + await page + .getByRole('button', { name: 'Install and connect', exact: true }) + .click(); + await page.getByRole('button', { name: 'Connect test account' }).click(); + console.log('Fetching preview'); + await page.getByRole('button', { name: 'Fetch and preview' }).click(); + console.log('Applying preview'); + await page + .getByRole('button', { name: 'Apply 5 changes', exact: true }) + .click(); + await page + .getByRole('link', { name: 'Open imported records', exact: true }) + .click(); + for (const name of ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles']) + await page + .getByRole('main') + .getByText(name, { exact: true }) + .first() + .waitFor(); + await page.reload(); + for (const name of ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles']) + await page + .getByRole('main') + .getByText(name, { exact: true }) + .first() + .waitFor(); + console.log( + 'Browser-only OAuth, WASM import, review, OPFS apply and reload passed', + ); +} catch (error) { + console.error( + error.message, + '\nURL:', + page.url(), + '\nUI:', + (await page.locator('body').innerText()).slice(-7000), + '\nErrors:', + errors, + ); + process.exitCode = 1; +} finally { + await browser.close(); +} diff --git a/integrations/localthought/browser.test.ts b/integrations/localthought/browser.test.ts new file mode 100644 index 0000000000..d7194196bf --- /dev/null +++ b/integrations/localthought/browser.test.ts @@ -0,0 +1,126 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { BrowserIntegrations, proxyOrigin, sign, type Engine } from './browser'; +const secret = 'bW9jay10ZW5hbnQ.mock-signature'; +const origin = 'https://proxy.example'; +function setup() { + const values = new Map(); + const storage = { + getItem: (k: string) => values.get(k) ?? null, + setItem: (k: string, v: string) => { + values.set(k, v); + }, + } as Storage; + vi.stubGlobal('location', { origin: 'https://atomic.example' }); + vi.stubGlobal('navigator', { + locks: { request: (_: string, f: () => unknown) => f() }, + }); + const http = vi.fn(async (url: string, init?: RequestInit) => { + expect(init?.credentials).toBe('omit'); + expect(init?.redirect).toBe('error'); + if (url.endsWith('/catalog')) return new Response('["pets"]'); + if (url.endsWith('/session')) + return Response.json({ ts: 1, nonce: 'nonce', challenge: 'challenge' }); + return new Response('{}'); + }); + const engine: Engine = { + describeIntegration: async () => + JSON.stringify({ upstream: 'https://pets.example' }), + fetchIntegration: async (_text, _platform, _constants, _range, fetch) => { + const first = JSON.parse(await fetch('https://pets.example/pets')); + expect(first.headers['x-connection-code']).toBeUndefined(); + await fetch('https://pets.example/pets?page=2'); + return '{"records":[]}'; + }, + }; + const client = new BrowserIntegrations( + storage, + async () => engine, + origin, + http as typeof fetch, + ); + const start = () => + client.start( + 'drive', + 'actor', + 'pets', + 'https://atomic.example/app/integrations', + secret, + ); + return { values, storage, http, engine, client, start }; +} +afterEach(() => vi.unstubAllGlobals()); +it('matches the tenant HMAC protocol and rejects non-origin proxy URLs', async () => { + expect(await sign('key', 'The quick brown fox jumps over the lazy dog')).toBe( + '97yD9DBThCSxMpjmqm-xQ-9NWaFJRhdZl0edvC0aPNg', + ); + expect(() => proxyOrigin('https://proxy.example/path')).toThrow(); + expect(() => proxyOrigin('http://proxy.example')).toThrow(); +}); +it('binds returns to actor, drive and expiry without saving tenant secrets', async () => { + const { client, start, values } = setup(); + const { state, url } = await start(); + expect(url).not.toContain(secret); + expect([...values.values()].join()).not.toContain(secret); + expect(() => client.finish('other', 'actor', state, 'code')).toThrow(); + expect(() => client.finish('drive', 'other', state, 'code')).toThrow(); + expect(client.finish('drive', 'actor', state, 'code').platform).toBe('pets'); + expect(() => client.finish('drive', 'actor', state, 'code')).toThrow(); +}); +it('consumes before dispatch and preserves rotation and pagination', async () => { + const { client, start, http, values } = setup(); + const { state } = await start(); + client.finish('drive', 'actor', state, 'first'); + const codes: string[] = []; + http.mockImplementation(async (url, init) => { + if (url.includes('/catalog/')) return new Response('{}'); + expect(JSON.parse([...values.values()][0]).code).toBeUndefined(); + codes.push((init?.headers as Record).Authorization); + return new Response('[]', { + headers: { + 'x-connection-code': 'second', + link: '; rel=next', + }, + }); + }); + await client.fetchRecords('drive', 'actor', state, {}); + expect(codes).toEqual(['Bearer first', 'Bearer second']); + expect(http.mock.calls.at(-1)?.[0]).toBe(`${origin}/proxy/pets/pets?page=2`); +}); +it('never retries an uncertain consumed credential', async () => { + const { client, start, http } = setup(); + const { state } = await start(); + client.finish('drive', 'actor', state, 'first'); + http.mockImplementation(async url => { + if (url.includes('/catalog/')) return new Response('{}'); + throw new Error('connection lost'); + }); + await expect( + client.fetchRecords('drive', 'actor', state, {}), + ).rejects.toThrow('lost'); + const calls = http.mock.calls.length; + await expect( + client.fetchRecords('drive', 'actor', state, {}), + ).rejects.toThrow('Reconnect'); + expect(http.mock.calls).toHaveLength(calls); +}); +it('rejects pagination to a different provider before spending a credential', async () => { + const { client, start, engine, values } = setup(); + const { state } = await start(); + client.finish('drive', 'actor', state, 'first'); + engine.fetchIntegration = async (_t, _p, _c, _r, fetch) => + fetch('https://evil.example/pets'); + await expect( + client.fetchRecords('drive', 'actor', state, {}), + ).rejects.toThrow('origin'); + expect(JSON.parse([...values.values()][0]).code).toBe('first'); +}); + +it('calls the browser fetch function without binding it to the client', async () => { + const { storage, engine } = setup(); + vi.stubGlobal('fetch', function (this: unknown) { + expect(this).not.toBeInstanceOf(BrowserIntegrations); + return Promise.resolve(new Response('["pets"]')); + }); + const client = new BrowserIntegrations(storage, async () => engine, origin); + expect(await client.catalog()).toEqual(['pets']); +}); diff --git a/integrations/localthought/browser.ts b/integrations/localthought/browser.ts new file mode 100644 index 0000000000..9ed1248fbc --- /dev/null +++ b/integrations/localthought/browser.ts @@ -0,0 +1,278 @@ +/** Browser-owned credentials. Never store these in Atomic graph resources. */ +export const DEFAULT_PROXY = 'https://localthought.io'; +const key = 'localthought-browser-v1:'; +export interface Connection { + drive: string; + actor: string; + platform: string; + origin: string; + expires: number; + code?: string; + ready: boolean; +} +export interface Engine { + describeIntegration(text: string): Promise; + fetchIntegration( + text: string, + platform: string, + constants: string, + range: string | undefined, + fetch: (url: string) => Promise, + ): Promise; +} +export function proxyOrigin(value = DEFAULT_PROXY): string { + const u = new URL(value); + if ( + u.origin !== value || + (u.protocol !== 'https:' && + !( + u.protocol === 'http:' && + ['localhost', '127.0.0.1'].includes(u.hostname) + )) + ) + throw new Error('Proxy must be an HTTPS origin or localhost'); + return value; +} +const base64url = (bytes: Uint8Array) => + btoa(String.fromCharCode(...bytes)) + .replaceAll('+', '-') + .replaceAll('/', '_') + .replaceAll('=', ''); +export async function sign(secret: string, text: string) { + const encoder = new TextEncoder(); + const k = await crypto.subtle.importKey( + 'raw', + encoder.encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + return base64url( + new Uint8Array(await crypto.subtle.sign('HMAC', k, encoder.encode(text))), + ); +} +async function limitedText(response: Response): Promise { + if (!response.body) throw new Error('Empty proxy response'); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let size = 0, + text = ''; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > 10 * 1024 * 1024) + throw new Error('Proxy response exceeds 10 MB'); + text += decoder.decode(value, { stream: true }); + } + return text + decoder.decode(); + } finally { + await reader.cancel(); + } +} +export class BrowserIntegrations { + constructor( + private storage: Storage, + private engine: () => Promise, + readonly origin = DEFAULT_PROXY, + private http: typeof fetch = (...args) => fetch(...args), + ) { + proxyOrigin(origin); + } + private async get(path: string, signal?: AbortSignal) { + const response = await this.http(`${this.origin}${path}`, { + credentials: 'omit', + redirect: 'error', + signal: signal ?? AbortSignal.timeout(30000), + }); + if (!response.ok) + throw new Error(`LocalThought returned HTTP ${response.status}`); + return limitedText(response); + } + async catalog(signal?: AbortSignal): Promise { + const names = JSON.parse(await this.get('/catalog', signal)); + if ( + !Array.isArray(names) || + names.length > 200 || + names.some(s => typeof s !== 'string' || !/^[a-z0-9-]{1,80}$/.test(s)) + ) + throw new Error('Invalid platform catalog'); + return names; + } + private async document(platform: string) { + if (!/^[a-z0-9-]{1,80}$/.test(platform)) + throw new Error('Invalid platform'); + return this.get(`/catalog/${platform}.yaml`); + } + async describe(platform: string) { + return JSON.parse( + await ( + await this.engine() + ).describeIntegration(await this.document(platform)), + ) as { + parameters: string[]; + collections: string[]; + upstream: string; + }; + } + async start( + drive: string, + actor: string, + platform: string, + returnUrl: string, + secret: string, + ) { + if (!(await this.catalog()).includes(platform)) + throw new Error('Unknown platform'); + const encoded = secret.split('.')[0]; + let tenant: string; + try { + tenant = new TextDecoder('utf-8', { fatal: true }).decode( + Uint8Array.from( + atob(encoded.replaceAll('-', '+').replaceAll('_', '/')), + c => c.charCodeAt(0), + ), + ); + } catch { + throw new Error('Invalid tenant secret'); + } + if (!tenant || !secret.includes('.')) + throw new Error('Invalid tenant secret'); + const callback = new URL(returnUrl); + if ( + callback.origin !== location.origin || + callback.pathname !== '/app/integrations' || + callback.search || + callback.hash + ) + throw new Error('Invalid integration return URL'); + const state = base64url(crypto.getRandomValues(new Uint8Array(32))); + callback.searchParams.set('integration_state', state); + callback.searchParams.set('platform', platform); + const challenge = JSON.parse(await this.get('/session')); + const url = new URL(`${this.origin}/connect`); + for (const [k, v] of Object.entries({ + redirect_uri: callback.href, + platform, + ts: String(challenge.ts), + nonce: challenge.nonce, + challenge: challenge.challenge, + tenant_id: tenant, + user_id: actor, + user_id_sig: await sign(secret, actor), + response: await sign(secret, challenge.challenge), + })) + url.searchParams.set(k, v as string); + // Only the short-lived handoff survives navigation, never the tenant secret. + this.storage.setItem( + key + state, + JSON.stringify({ + drive, + actor, + platform, + origin: this.origin, + expires: Date.now() + 600000, + ready: false, + } satisfies Connection), + ); + return { state, url: url.href }; + } + private connection(id: string, drive: string, actor: string) { + const raw = this.storage.getItem(key + id); + if (!raw) throw new Error('Reconnect your account'); + const c: Connection = JSON.parse(raw); + if (c.drive !== drive || c.actor !== actor || c.origin !== this.origin) + throw new Error('Connection belongs to another drive, agent or proxy'); + return c; + } + finish(drive: string, actor: string, state: string, code: string) { + const c = this.connection(state, drive, actor); + if (c.ready || c.expires < Date.now() || !code || code.length > 4096) + throw new Error('Invalid, expired or completed connection'); + this.storage.setItem( + key + state, + JSON.stringify({ ...c, ready: true, code }), + ); + return { connection: state, platform: c.platform }; + } + async fetchRecords( + drive: string, + actor: string, + id: string, + constants: Record, + range?: { start: string; end: string }, + ) { + // Web Locks serialize rotating credentials across tabs as well as UI actions. + if (!navigator.locks) + throw new Error('This browser needs Web Locks for integrations'); + return navigator.locks.request(key + id, async () => { + const c = this.connection(id, drive, actor); + if (!c.ready || !c.code) throw new Error('Reconnect your account'); + const text = await this.document(c.platform); + const engine = await this.engine(); + const { upstream } = JSON.parse(await engine.describeIntegration(text)); + const base = new URL(upstream); + let requests = 0; + const signal = AbortSignal.timeout(120000); + const transport = async (raw: string) => { + signal.throwIfAborted(); + const target = new URL(raw); + if ( + target.origin !== base.origin || + target.username || + target.password || + target.hash + ) + throw new Error('Pagination left the catalog API origin'); + if (++requests > 200) + throw new Error('Import exceeds 200 requests; narrow its scope'); + const current = this.connection(id, drive, actor); + if (!current.code) + throw new Error('Reconnect before retrying an uncertain request'); + const code = current.code; + delete current.code; + // Consume durably BEFORE dispatch; a lost response must never replay a code. + this.storage.setItem(key + id, JSON.stringify(current)); + const response = await this.http( + `${this.origin}/proxy/${c.platform}${target.pathname}${target.search}`, + { + headers: { Authorization: `Bearer ${code}` }, + credentials: 'omit', + redirect: 'error', + signal, + }, + ); + const next = response.headers.get('x-connection-code'); + if (next) + this.storage.setItem( + key + id, + JSON.stringify({ ...current, code: next }), + ); + else + throw new Error( + 'Proxy did not expose a rotated code; reconnect and check CORS', + ); + const headers = Object.fromEntries( + [...response.headers].filter( + ([name]) => name !== 'x-connection-code', + ), + ); + return JSON.stringify({ + status: response.status, + headers, + body: await limitedText(response), + }); + }; + const result = await engine.fetchIntegration( + text, + c.platform, + JSON.stringify(constants), + range ? JSON.stringify(range) : undefined, + transport, + ); + signal.throwIfAborted(); + return JSON.parse(result); + }); + } +} diff --git a/integrations/localthought/mock-proxy.mjs b/integrations/localthought/mock-proxy.mjs index 8560e618b7..650c9aee31 100644 --- a/integrations/localthought/mock-proxy.mjs +++ b/integrations/localthought/mock-proxy.mjs @@ -12,6 +12,12 @@ export function mockProxy() { const challenges = new Set(); const issueCode = platform => { const code = randomBytes(32).toString('base64url'); codes.set(code, platform); return code; }; return createServer((req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); + res.setHeader('Access-Control-Allow-Headers', 'Authorization, Content-Type'); + res.setHeader('Access-Control-Expose-Headers', 'X-Connection-Code, Link'); + if (req.method === 'OPTIONS') { res.writeHead(204); return res.end(); } + const url = new URL(req.url, 'http://localhost'); const json = (status, value, headers = {}) => { res.writeHead(status, { 'Content-Type': 'application/json', ...headers }); res.end(JSON.stringify(value)); }; if (url.pathname === '/catalog') return json(200, ['github-issues', 'google-calendar', 'pets']); diff --git a/integrations/localthought/syncables/Cargo.toml b/integrations/localthought/syncables/Cargo.toml new file mode 100644 index 0000000000..d162e2ac8d --- /dev/null +++ b/integrations/localthought/syncables/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "syncables" +version = "0.1.0" +edition = "2021" +rust-version = "1.82" +description = "Reads an OpenAPI document to run a mock API server and a client that syncs a local copy of the API's dataset." +authors = ["Michiel de Jong "] +license = "Apache-2.0" +repository = "https://github.com/localthought/syncables-rs" +readme = "README.md" +keywords = ["openapi", "pagination", "sync", "mock-server", "local-first"] +categories = ["web-programming", "api-bindings"] + +[dependencies] +async-trait = "0.1" +httpdate = "1" +indexmap = { version = "2", features = ["serde"] } +percent-encoding = "2" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +serde_yaml_ng = "0.10" +thiserror = "2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { version = "1", features = ["fs"] } + +[dev-dependencies] +uuid = { version = "1", features = ["v4"] } +tokio = { version = "1", features = ["full"] } + +[lints.rust] +missing_docs = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +module_name_repetitions = "allow" +# The module tree deliberately mirrors the TypeScript original's `src/` +# layout one-to-one, which puts `client/client.rs` inside `client/`. +module_inception = "allow" +# "OpenAPI", "JSONPath" and friends are proper nouns, not code. +doc_markdown = "allow" +# `schema` vs. `scheme` are the spec's own two words, not a typo. +similar_names = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" diff --git a/integrations/localthought/syncables/LICENSE b/integrations/localthought/syncables/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/integrations/localthought/syncables/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/integrations/localthought/syncables/README.md b/integrations/localthought/syncables/README.md new file mode 100644 index 0000000000..f48da88994 --- /dev/null +++ b/integrations/localthought/syncables/README.md @@ -0,0 +1,209 @@ +# syncables-rs + +A Rust port of [localthought/syncables](https://github.com/localthought/syncables). + +Reads an OpenAPI document and gives you: + +- a **mock API server** that implements it, backed by a real (in-memory) + CRUD store per resource, seeded with fake data generated from the + document's schemas; +- an **API client** that talks to any server implementing that OpenAPI + document and keeps a local copy of each resource collection in sync. + +Alongside that port, the crate also carries a second, unrelated surface: +a **sync engine** (`SyncClient`) that reads a document's +[CRUD Causality Extension](https://github.com/pondersource/openapi-extensions/tree/main/spec/crud-causality) +(`components.crudResources`) and syncs records — including nested +collections, walked once per parent record — into a host-provided +`Storage` implementation, deriving a neutral, Atomic-Data-shaped ontology +along the way without the crate itself depending on `atomic_lib`. This is +new scope, not part of the original TypeScript port; see +[Sync engine](#sync-engine) below. + +## Port status + +This section covers the original TypeScript port only — see +[Sync engine](#sync-engine) below for that surface's status. + +This crate is **scaffolding**. The module tree mirrors the TypeScript +original's `src/` one-to-one, and the following are ported and tested: + +| Area | Module | Status | +| --- | --- | --- | +| Document loading | `openapi::load`, `openapi::resolve_refs` | ported | +| Overlays | `openapi::overlay` | ported | +| OpenAPI type surface | `openapi::types` | ported | +| Resource discovery | `resources::discover` | ported | +| Path routing | `routing::router` | ported | +| Fake data | `fake_data::generate` | ported | +| Pagination | `pagination::{types, validate, autodetect, items, request_builder, response_parser}` | ported | +| Mock server store | `mock_server::store` | ported | +| Client storage | `client::storage` | ported | +| Mock server handler | `mock_server::server` | **scaffolded** — public surface only | +| Client sync/writes | `client::client` | **scaffolded** — public surface only | + +The two scaffolded modules carry their full ported public API, doc +comments and constants; their function bodies are `todo!()`, each naming +the TypeScript source file and function it is to be ported from. Nothing +in this crate silently returns a wrong answer — the parts that exist are +covered by tests, including acceptance tests against unmodified real-world +documents. + +## Usage + +```sh +cargo build +cargo test +cargo clippy --all-targets -- -D warnings +cargo fmt --all --check +``` + +Once the client and mock server are ported, the shape will be: + +```rust,ignore +use syncables::{create_api_client, create_mock_server, load_open_api_document, ApiClientOptions}; + +let document = load_open_api_document("./petstore.yaml").await?; + +let server = create_mock_server(document.clone()); +let address = server.listen(None).await?; + +let client = create_api_client(document, ApiClientOptions::new(address.url)); +client.sync().await?; // pulls every discovered resource collection into local storage + +let pets = client.list("/pets").await?; +``` + +A "resource" is any pair of an OpenAPI collection path and its matching +item path, e.g. `/pets` and `/pets/{petId}`. Paths without that pairing +(health checks, one-off actions, etc.) are served from their documented +examples/schemas but aren't treated as syncable resources. + +## Sync engine + +New scope, not part of the original TypeScript port; tracked by +[issues #1–#9](https://github.com/localthought/syncables-rs/issues/1). +Where the port above discovers resources from plain collection/item path +pairing, the sync engine derives a richer resource model from a +document's [CRUD Causality Extension](https://github.com/pondersource/openapi-extensions/tree/main/spec/crud-causality) +(`components.crudResources`) — including nested collections, like a +repository's issues and each issue's comments — and drives a full, +paginated read of every collection into a host-provided `Storage`, +deriving a neutral ontology (Classes and Properties, Atomic-Data-shaped +but not Atomic-Data-typed) along the way. + +| Area | Module | Status | +| --- | --- | --- | +| Resource model (`crudResources`, `x-crud`) | `sync::resource_model` | ported and tested | +| Binding configured constants into path templates | `sync::constants` | ported and tested | +| Credentials, API base URL | `sync::credentials` | ported and tested | +| Ontology derivation | `sync::ontology` | ported and tested | +| `Storage` trait, `InMemoryStorage` | `sync::storage` | ported and tested | +| `SyncClient::sync()` — full read | `sync::client` | ported and tested | +| `SyncClient` — local-first write-back | `sync::client` | **not implemented** — [#9](https://github.com/localthought/syncables-rs/issues/9) | + +```rust,ignore +use std::sync::Arc; +use syncables::{ClientConfig, Credentials, InMemoryStorage, SyncClient}; + +let config = ClientConfig { + document: "./github-issues.openapi.yaml".into(), + overlays: vec!["./auth-overlay.yaml".into(), "./crud-causality-overlay.yaml".into()], + credentials: Credentials::Bearer(std::env::var("GITHUB_TOKEN")?), + constants: [("owner".to_string(), "localthought".to_string()), + ("repo".to_string(), "test-repo-1".to_string())].into(), + ontology_base_url: "https://my-ontologies.com".to_string(), +}; + +let client = SyncClient::new(config, Arc::new(my_fetch_impl))?; +let storage = InMemoryStorage::new(); +let report = client.sync(&storage).await?; // walks issues, then each issue's comments +``` + +`SyncClient::new` takes an `Arc` (the same injectable-transport +trait `ApiClient` above uses) alongside `ClientConfig` — the crate has no +HTTP client dependency of its own, so a host supplies one. This is the one +deliberate divergence from the `ClientConfig`/`SyncClient` contract +[`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +is already written against in its `src/syncables.rs`, which otherwise this +module matches field-for-field; that module is meant to be deleted once +reflector-rs points its `use`s here instead. + +## Differences from the TypeScript original + +The port keeps the original's names, comments and behaviour wherever it +can. Where Rust forced a decision, it went like this: + +- **`Record` → `serde_json::Map`**, and + `unknown` → `serde_json::Value`. Maps that need document order + (schema properties, paths, collections) use `IndexMap`, and + `serde_json` is built with `preserve_order`: `locate_items_field` picks + the *first* array-typed property, so order is load-bearing. +- **The injectable `fetch` becomes a `Fetch` trait** + (`client::client::Fetch`). As in the original, it is the only extension + point — there is no built-in notion of auth, so authenticating means + supplying an implementation that adds the right header to every request. +- **`StorageAdapter` is an `async_trait`**, with `InMemoryStorageAdapter` + as the default, matching the original's pluggable adapter. +- **Pagination roles stay strings** rather than becoming closed enums: the + spec allows `x-` extension roles, and `validate` is what decides + validity, so an unknown role has to survive parsing to be reported. + `PaginationSchemeObject::type` likewise deserializes leniently, so one + malformed scheme (e.g. Giphy's former `type: offset`) does not fail the + whole document. +- **Errors are a `thiserror` enum** (`syncables::Error`) instead of thrown + strings; every fallible function returns `syncables::Result`. +- **Numbers are `f64`** in `PaginationResponseState`, mirroring JavaScript + number semantics for values read out of arbitrary JSON bodies. + +## Tests + +`tests/unit/` mirrors the original's `__tests__/unit/` layout module for +module. Cargo builds it as one test binary (`tests/unit/main.rs`), so the +module tree is declared there rather than discovered per file. + +`tests/fixtures/real-world/` holds real OpenAPI documents and pagination +overlays vendored unmodified from apis.guru and localthought/overlays (see +the header comment in each file for provenance). The acceptance tests run +the ported pipeline against these and deliberately document real quirks +rather than working around them. When extending them, keep that spirit: +assert what actually happens against the unmodified real document, not an +idealized result. + +## NLnet milestone 1 + +The TypeScript original is the reference implementation for +[milestone 1](https://github.com/tubsproject/syncables/blob/main/nlnet-milestones.md#1-syncables) +of the project's NLnet grant. This port tracks it. + +## Generative AI use + +syncables-rs is developed collaboratively with **Claude Code** +(Anthropic), an agentic coding assistant: a human directs the design and +reviews, edits, and tests the changes it proposes before they're +committed. + +As an NLnet-funded project, this follows +[NLnet's Generative AI policy](https://nlnet.nl/foundation/policies/generativeAI/): + +- Commits produced with AI assistance carry a `Claude-Session: ` + trailer identifying the session that produced them. +- [`docs/ai-logs/`](docs/ai-logs) holds prompt/output disclosure logs, + redacted for secrets and personal information. +- AI-drafted content is reviewed and edited by a human before being + committed; it is not represented as unassisted human work. + +## License + +Apache-2.0, matching the original. + +## Browser / WASM + +`cargo check --target wasm32-unknown-unknown --lib` builds the engine without +Tokio filesystem or native runtime dependencies. Load catalog text with +`openapi::load::parse_yaml`, pass the resulting value to +`load_open_api_document`, then call `SyncClient::sync_document(&doc, &storage)`. +This path never opens `ClientConfig.document` or `overlays`; apply overlays in +memory before calling it. File sources return an explicit unsupported error +in WASM. Implement browser `Fetch` with `#[async_trait(?Send)]`; it may hold +a JS callback. Native `Fetch` keeps its Send + Sync contract. diff --git a/integrations/localthought/syncables/UPSTREAM.md b/integrations/localthought/syncables/UPSTREAM.md new file mode 100644 index 0000000000..1cd8d62970 --- /dev/null +++ b/integrations/localthought/syncables/UPSTREAM.md @@ -0,0 +1 @@ +Vendored from localthought/syncables-rs commit 0ab3521 (codex/browser-integrations), based on d48e4d9bad3ed9ec826d1c2040e989171701965d. This snapshot adds sync_document and target-specific transport/filesystem support. Replace with a pinned upstream dependency after that branch is merged. Apache-2.0; see LICENSE. diff --git a/integrations/localthought/syncables/src/client/client.rs b/integrations/localthought/syncables/src/client/client.rs new file mode 100644 index 0000000000..152700df4c --- /dev/null +++ b/integrations/localthought/syncables/src/client/client.rs @@ -0,0 +1,369 @@ +//! The API client: talks to any server implementing the document, and +//! keeps a local copy of each resource collection in sync. +//! +//! Reads are served from local storage ([`StorageAdapter`]; +//! [`InMemoryStorageAdapter`] is the default). `sync()` and the standalone +//! `paginate()` both walk every page of a paginated GET operation before +//! returning. +//! +//! `create`/`update`/`remove` are local-first: each writes to storage +//! immediately and returns without waiting on the network, then applies +//! itself against the server in the background via a per-record write +//! queue (keyed by `{resource}:{id}`, one write in flight at a time so +//! writes to the same record land in server order), retrying failures with +//! exponential backoff. +//! +//! Nothing in this crate reads an OpenAPI document's +//! `security`/`securitySchemes` — there is no built-in notion of auth. +//! [`ApiClientOptions::fetch`] is the only extension point, so +//! authenticating (a bearer token, an API key from an env var, etc.) means +//! passing a [`Fetch`] implementation that adds the right header to every +//! request. +//! +//! # Port status +//! +//! Scaffolding. The public surface, options and types below are ported; +//! the sync/write/pagination machinery is not yet implemented. + +// The `async` on the stubs below is part of the ported API surface, +// not an accident of the current `todo!()` bodies. +#![allow(clippy::unused_async)] + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use crate::error::Result; +use crate::openapi::types::OpenApiDocument; +use crate::resources::discover::{discover_resources, ResourceRoute}; + +use super::storage::{InMemoryStorageAdapter, StorageAdapter}; + +/// Hard ceiling on pages walked in one traversal, mirroring the original's +/// `MAX_PAGES`. +pub const MAX_PAGES: usize = 50; + +/// One outgoing HTTP request. +#[derive(Debug, Clone)] +pub struct HttpRequest { + /// HTTP method. + pub method: String, + /// Absolute URL. + pub url: String, + /// Request headers. + pub headers: IndexMap, + /// Request body, if any. + pub body: Option>, +} + +/// One HTTP response. +#[derive(Debug, Clone)] +pub struct HttpResponse { + /// HTTP status code. + pub status: u16, + /// Response headers. + pub headers: IndexMap, + /// Response body. + pub body: Vec, +} + +/// The client's only extension point for how requests reach the network. +/// +/// This is the Rust equivalent of the original's injectable `fetch`. +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait Fetch: FetchBounds { + /// Sends one request and returns the response. + async fn fetch(&self, request: HttpRequest) -> Result; +} + +/// How a failed background write is retried. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RetryOptions { + /// Delay before the first retry of a failed write. Doubles on each + /// subsequent attempt. Default 200ms. + pub base_delay: Duration, + /// Ceiling for the exponential backoff between retries. Default 30s. + pub max_delay: Duration, + /// Stop auto-retrying a write after this many attempts. Default is + /// unlimited (keep retrying until it succeeds). + pub max_attempts: Option, +} + +impl Default for RetryOptions { + fn default() -> Self { + Self { + base_delay: Duration::from_millis(200), + max_delay: Duration::from_secs(30), + max_attempts: None, + } + } +} + +/// How to build an [`ApiClient`]. +pub struct ApiClientOptions { + /// Base URL of the server to talk to. + pub base_url: String, + /// Where the local copy lives. Defaults to [`InMemoryStorageAdapter`]. + pub storage: Option>, + /// How requests reach the network. + pub fetch: Option>, + /// How failed background writes are retried. + pub retry: RetryOptions, + /// Record property that holds a resource's identity — the value used + /// as the local storage key, read back from a create response to + /// reconcile the server-assigned id, and substituted into the item + /// URL's path variable. + /// + /// Defaults to `id`. Set this when the API addresses a resource by a + /// different field (e.g. GitHub issues are keyed by `number`, not the + /// global `id` the payload also carries). + pub identity_field: String, +} + +impl ApiClientOptions { + /// Options pointing at `base_url`, with everything else defaulted. + pub fn new(base_url: impl Into) -> Self { + Self { + base_url: base_url.into(), + storage: None, + fetch: None, + retry: RetryOptions::default(), + identity_field: "id".to_string(), + } + } +} + +/// Options for one [`ApiClient::paginate`] traversal. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PaginateOptions { + /// Page size to request. Falls back to the server's own default when + /// omitted. + pub page_size: Option, +} + +/// What one [`ApiClient::sync`] changed. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SyncResult { + /// Collection paths whose local copy was actually added to, updated, + /// or pruned by this sync. + pub changed: Vec, +} + +/// Called after each sync while polling. +pub type SyncCallback = Box; +/// Called when a sync fails while polling. +pub type ErrorCallback = Box; + +/// How [`ApiClient::start_polling`] should behave. +pub struct PollOptions { + /// How often to call `sync()`. An initial sync runs immediately. + pub interval: Duration, + /// Called after every sync while polling, including ones where nothing + /// changed. + pub on_sync: Option, + /// Called when a sync fails while polling; polling continues on the + /// next interval. + pub on_error: Option, +} + +/// Cancels a running poll loop. +#[derive(Debug)] +pub struct PollingHandle { + _private: (), +} + +impl PollingHandle { + /// Stops future polling. Does not cancel a sync already in flight. + pub fn stop(self) { + todo!("port src/client/client.ts: startPolling") + } +} + +/// Which kind of write is still pending. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PendingWriteType { + /// A `create` not yet confirmed by the server. + Create, + /// An `update` not yet confirmed by the server. + Update, + /// A `remove` not yet confirmed by the server. + Delete, +} + +/// A write local storage already reflects, but the server has not yet +/// confirmed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PendingWriteInfo { + /// Collection path the write belongs to. + pub resource: String, + /// The id the write is filed under locally. For an unsettled `create`, + /// this is the client-generated id, not (yet) whatever the server + /// assigns. + pub id: String, + /// Which kind of write this is. + pub write_type: PendingWriteType, + /// How many attempts to reach the server have failed so far. + /// + /// If [`RetryOptions::max_attempts`] is set and reached, this stops + /// growing and the write stops auto-retrying — it stays listed until + /// `create`/`update`/`remove` is called again for the same record. + pub attempts: u32, + /// The most recent failure, if at least one attempt has failed. + pub last_error: Option, +} + +/// A local-first client for a server implementing the document. +pub struct ApiClient { + document: Arc, + routes: Arc>, + storage: Arc, + fetch: Option>, + options: Arc, +} + +impl ApiClient { + /// Collection paths this client knows how to sync. + pub fn resources(&self) -> Vec { + self.routes + .iter() + .map(|route| route.collection_path.clone()) + .collect() + } + + /// The document this client was built from. + pub fn document(&self) -> &OpenApiDocument { + &self.document + } + + /// Where the local copy lives. + pub fn storage(&self) -> &Arc { + &self.storage + } + + /// Pulls every discovered resource collection into local storage. + /// + /// Safe to call repeatedly: for a non-paginated collection it + /// conditionally re-fetches, keyed by exact request URL, sending + /// `If-None-Match`/`If-Modified-Since` from the prior response's + /// `ETag`/`Last-Modified` and treating a `304` as "nothing to do". + /// Even on a fresh `200`, or for a paginated collection, it only + /// touches storage for items that actually changed. + pub async fn sync(&self) -> Result { + let _ = &self.fetch; + let _ = &self.options; + todo!("port src/client/client.ts: sync") + } + + /// Calls [`Self::sync`] on `options.interval`, skipping a tick if the + /// previous sync is still running. + // `options` is consumed by the poll loop once this is implemented. + #[allow(clippy::needless_pass_by_value)] + pub fn start_polling(&self, options: PollOptions) -> PollingHandle { + let _ = options; + todo!("port src/client/client.ts: startPolling") + } + + /// Every locally held record of `resource`. + pub async fn list(&self, resource: &str) -> Result>> { + self.storage.list(resource).await + } + + /// One locally held record, by id. + pub async fn get(&self, resource: &str, id: &str) -> Result>> { + self.storage.get(resource, id).await + } + + /// Writes `data` to local storage immediately, under a client-generated + /// id (or `data[identity_field]`, if already set) and returns without + /// waiting on the network. + /// + /// The write to the server happens in the background and is retried on + /// failure — see [`Self::pending_writes`] for its outcome so far. If + /// the server assigns a different id than the one used locally, the + /// record is moved to it once the write settles. + pub async fn create( + &self, + resource: &str, + data: Map, + ) -> Result> { + let _ = (resource, data); + todo!("port src/client/client.ts: create") + } + + /// Merges `data` into the local copy of `id` immediately and returns + /// without waiting on the network; the corresponding `PUT` (or `PATCH`, + /// per [`ResourceRoute::update_method`]) is sent, and retried on + /// failure, in the background. + pub async fn update( + &self, + resource: &str, + id: &str, + data: Map, + ) -> Result> { + let _ = (resource, id, data); + todo!("port src/client/client.ts: update") + } + + /// Removes `id` from local storage immediately and returns without + /// waiting on the network; the corresponding `DELETE` is sent, and + /// retried on failure, in the background. + pub async fn remove(&self, resource: &str, id: &str) -> Result<()> { + let _ = (resource, id); + todo!("port src/client/client.ts: remove") + } + + /// Writes not yet confirmed by the server, across every resource (or + /// just `resource`, if given). + pub fn pending_writes(&self, resource: Option<&str>) -> Vec { + let _ = resource; + todo!("port src/client/client.ts: pendingWrites") + } + + /// Fetches every item from a GET list operation at `path`, walking + /// every page per its resolved pagination scheme (explicit + /// `x-pagination` or auto-detected from `components.paginationSchemes`). + /// + /// `path` need not be a discovered resource — any GET operation in the + /// document works, e.g. a search/listing endpoint with no paired item + /// route. + pub async fn paginate( + &self, + path: &str, + options: PaginateOptions, + ) -> Result>> { + let _ = (path, options); + todo!("port src/client/client.ts: paginate") + } +} + +/// Builds a client for `document` against the server in `options`. +pub fn create_api_client(document: OpenApiDocument, options: ApiClientOptions) -> ApiClient { + let routes = discover_resources(&document.paths); + let storage = options + .storage + .clone() + .unwrap_or_else(|| Arc::new(InMemoryStorageAdapter::new())); + let fetch = options.fetch.clone(); + ApiClient { + document: Arc::new(document), + routes: Arc::new(routes), + storage, + fetch, + options: Arc::new(options), + } +} + +/// Native transports must be thread safe; browser transports run on one JS thread. +#[cfg(not(target_arch = "wasm32"))] +pub trait FetchBounds: Send + Sync {} +#[cfg(not(target_arch = "wasm32"))] +impl FetchBounds for T {} +/// Browser transports may own JavaScript callbacks. +#[cfg(target_arch = "wasm32")] +pub trait FetchBounds {} +#[cfg(target_arch = "wasm32")] +impl FetchBounds for T {} diff --git a/integrations/localthought/syncables/src/client/mod.rs b/integrations/localthought/syncables/src/client/mod.rs new file mode 100644 index 0000000000..27ea15e9c5 --- /dev/null +++ b/integrations/localthought/syncables/src/client/mod.rs @@ -0,0 +1,5 @@ +//! An API client that talks to any server implementing the OpenAPI +//! document and keeps a local copy of each resource collection in sync. + +pub mod client; +pub mod storage; diff --git a/integrations/localthought/syncables/src/client/storage.rs b/integrations/localthought/syncables/src/client/storage.rs new file mode 100644 index 0000000000..65b2a20bef --- /dev/null +++ b/integrations/localthought/syncables/src/client/storage.rs @@ -0,0 +1,84 @@ +//! Where the client keeps its local copy of each resource collection. + +use std::collections::HashMap; +use std::sync::Mutex; + +use async_trait::async_trait; +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use crate::error::Result; + +/// Pluggable local storage for the client's copy of a collection. +/// +/// Implement this to persist somewhere other than memory; +/// [`InMemoryStorageAdapter`] is the default. +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait StorageAdapter: Send + Sync { + /// Every record held for `resource`. + async fn list(&self, resource: &str) -> Result>>; + /// One record, by id. + async fn get(&self, resource: &str, id: &str) -> Result>>; + /// Inserts or replaces a record. + async fn put(&self, resource: &str, id: &str, value: Map) -> Result<()>; + /// Removes a record. + async fn delete(&self, resource: &str, id: &str) -> Result<()>; +} + +/// One collection per resource path, each keyed by record id. +type Collections = HashMap>>; + +/// The default [`StorageAdapter`]: everything in process memory. +#[derive(Debug, Default)] +pub struct InMemoryStorageAdapter { + collections: Mutex, +} + +impl InMemoryStorageAdapter { + /// An adapter with no collections. + pub fn new() -> Self { + Self::default() + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl StorageAdapter for InMemoryStorageAdapter { + async fn list(&self, resource: &str) -> Result>> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + Ok(collections + .entry(resource.to_string()) + .or_default() + .values() + .cloned() + .collect()) + } + + async fn get(&self, resource: &str, id: &str) -> Result>> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + Ok(collections + .entry(resource.to_string()) + .or_default() + .get(id) + .cloned()) + } + + async fn put(&self, resource: &str, id: &str, value: Map) -> Result<()> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + collections + .entry(resource.to_string()) + .or_default() + .insert(id.to_string(), value); + Ok(()) + } + + async fn delete(&self, resource: &str, id: &str) -> Result<()> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + collections + .entry(resource.to_string()) + .or_default() + .shift_remove(id); + Ok(()) + } +} diff --git a/integrations/localthought/syncables/src/error.rs b/integrations/localthought/syncables/src/error.rs new file mode 100644 index 0000000000..ebddbd6c05 --- /dev/null +++ b/integrations/localthought/syncables/src/error.rs @@ -0,0 +1,95 @@ +//! The crate's error type. + +use std::path::PathBuf; + +use thiserror::Error; + +/// Anything that can go wrong loading a document or talking to a server. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum Error { + /// Reading a document from disk failed. + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), + + /// A document could not be parsed as YAML or JSON. + #[error("could not parse document: {0}")] + Yaml(#[from] serde_yaml_ng::Error), + + /// A document did not fit the OpenAPI type surface this crate expects. + #[error("could not read document: {0}")] + Json(#[from] serde_json::Error), + + /// A document or overlay file could not be read or parsed. Wraps the + /// underlying [`Error::Io`] or [`Error::Yaml`] with the path that caused + /// it, since a bare i/o or parse error doesn't otherwise name the file. + #[error("could not load \"{path}\": {source}")] + FileLoad { + /// The file that could not be loaded. + path: PathBuf, + /// The underlying read or parse failure. + #[source] + source: Box, + }, + + /// An overlay used a JSONPath target outside the supported subset. + #[error( + "unsupported overlay target \"{0}\": only \"$\", simple dot-paths like \ + \"$.components...\", and quoted bracket segments like \ + \"$.paths['/pets'].get\" are supported" + )] + UnsupportedOverlayTarget(String), + + /// An overlay target segment resolved to something that isn't an object. + #[error("overlay target segment \"{0}\" does not resolve to an object")] + OverlayTargetNotAnObject(String), + + /// An overlay tried to `remove` the document root. + #[error("overlay cannot remove the document root")] + OverlayRemovesRoot, + + /// The client could not reach the server, or the server rejected the request. + #[error("http error: {0}")] + Http(String), + + /// A resource path was asked for that the document does not declare. + #[error("unknown resource \"{0}\"")] + UnknownResource(String), + + /// The document declares no `components.crudResources`; the + /// CRUD-causality overlay hasn't been applied. + #[error("document declares no crudResources; apply the CRUD-causality overlay")] + NoCrudResources, + + /// A constant named a parameter the document does not declare (path or + /// query, on any operation). A typo'd key here would otherwise sync + /// nothing, or scope the sync far wider than intended. + #[error("constant \"{0}\" does not name a parameter this document declares")] + UnknownConstant(String), + + /// A path template variable is bound by neither a constant, a parent + /// record's context provider, nor the resource's own identity binding. + #[error( + "path variable \"{0}\" is not bound by a constant, a parent record, \ + or an identity binding" + )] + UnboundContextParam(String), + + /// Two differently-named resources or fields normalized to the same + /// ontology shortname — the ontology would then mint one term for two + /// distinct things. + #[error( + "\"{first}\" and \"{second}\" both normalize to the ontology shortname \"{shortname}\"" + )] + ShortnameCollision { + /// The shortname both names normalized to. + shortname: String, + /// The name that claimed the shortname first. + first: String, + /// The name that collided with it. + second: String, + }, +} + +/// `Result` specialized to this crate's [`Error`]. +pub type Result = std::result::Result; diff --git a/integrations/localthought/syncables/src/fake_data/generate.rs b/integrations/localthought/syncables/src/fake_data/generate.rs new file mode 100644 index 0000000000..9602c95912 --- /dev/null +++ b/integrations/localthought/syncables/src/fake_data/generate.rs @@ -0,0 +1,83 @@ +//! Schema-to-value synthesis, shared by the mock server for seeding and +//! for example responses. +//! +//! A schema's own fixed `example` is reused verbatim on every call, so +//! callers generating more than one item from the same schema (resource +//! seeding, paginated list generation) must inject their own unique `id` +//! afterward rather than relying on the generated value to differ per item. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde_json::{Map, Number, Value}; + +use crate::openapi::types::SchemaObject; + +static STRING_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Synthesizes a value matching a JSON Schema, preferring an `example` on +/// the schema itself when present. +pub fn generate_from_schema(schema: Option<&SchemaObject>) -> Value { + let Some(schema) = schema else { + return Value::Null; + }; + if let Some(example) = &schema.example { + return example.clone(); + } + if let Some(enum_values) = &schema.enum_values { + if let Some(first) = enum_values.first() { + return first.clone(); + } + } + if let Some(all_of) = &schema.all_of { + let mut merged = Map::new(); + for sub in all_of { + if let Value::Object(object) = generate_from_schema(Some(sub)) { + merged.extend(object); + } + } + return Value::Object(merged); + } + if let Some(one_of) = &schema.one_of { + if let Some(first) = one_of.first() { + return generate_from_schema(Some(first)); + } + } + if let Some(any_of) = &schema.any_of { + if let Some(first) = any_of.first() { + return generate_from_schema(Some(first)); + } + } + + match schema.schema_type.as_deref() { + Some("string") => Value::String(generate_string(schema)), + Some("integer" | "number") => { + Number::from_f64(schema.minimum.unwrap_or(1.0)).map_or(Value::Null, Value::Number) + } + Some("boolean") => Value::Bool(true), + Some("array") => Value::Array(vec![generate_from_schema(schema.items.as_deref())]), + Some("object") | None => Value::Object(generate_object(schema)), + Some(_) => Value::Null, + } +} + +fn generate_object(schema: &SchemaObject) -> Map { + let mut result = Map::new(); + for (key, property_schema) in schema.properties.iter().flatten() { + result.insert(key.clone(), generate_from_schema(Some(property_schema))); + } + result +} + +fn generate_string(schema: &SchemaObject) -> String { + match schema.format.as_deref() { + Some("date-time") => "1970-01-01T00:00:00.000Z".to_string(), + Some("date") => "1970-01-01".to_string(), + Some("uuid") => "00000000-0000-4000-8000-000000000000".to_string(), + Some("email") => "user@example.com".to_string(), + Some("uri" | "url") => "https://example.com".to_string(), + _ => { + let next = STRING_COUNTER.fetch_add(1, Ordering::Relaxed) + 1; + format!("string-{next}") + } + } +} diff --git a/integrations/localthought/syncables/src/fake_data/mod.rs b/integrations/localthought/syncables/src/fake_data/mod.rs new file mode 100644 index 0000000000..8c6a87ef33 --- /dev/null +++ b/integrations/localthought/syncables/src/fake_data/mod.rs @@ -0,0 +1,3 @@ +//! Synthesizing values from OpenAPI schemas. + +pub mod generate; diff --git a/integrations/localthought/syncables/src/lib.rs b/integrations/localthought/syncables/src/lib.rs new file mode 100644 index 0000000000..0b9e3fc9f3 --- /dev/null +++ b/integrations/localthought/syncables/src/lib.rs @@ -0,0 +1,81 @@ +//! Reads an OpenAPI document and gives you: +//! +//! - a **mock API server** ([`create_mock_server`]) that implements it, +//! backed by a real (in-memory) CRUD store per resource, seeded with +//! fake data generated from the document's schemas; +//! - an **API client** ([`create_api_client`]) that talks to any server +//! implementing that OpenAPI document and keeps a local copy of each +//! resource collection in sync. +//! +//! Both understand the [OpenAPI Pagination Schemes Extension](https://github.com/pondersource/openapi-pagination-schemes-extension) +//! when a document declares `components.paginationSchemes`. +//! +//! A "resource" is any pair of an OpenAPI collection path and its matching +//! item path, e.g. `/pets` and `/pets/{petId}`. Paths without that pairing +//! (health checks, one-off actions, etc.) are served from their documented +//! examples/schemas but aren't treated as syncable resources. +//! +//! # Port status +//! +//! This crate is a port of [localthought/syncables](https://github.com/localthought/syncables) +//! (TypeScript) and is **scaffolding**: the module layout mirrors the +//! original's `src/` one-to-one, and the document, resource, pagination +//! and storage layers are ported. The mock server's request handler +//! ([`mock_server::server`]) and the client's sync/write machinery +//! ([`client::client`]) carry their full public surface but are not +//! implemented yet — those functions `todo!()`, each naming the +//! TypeScript source it is to be ported from. + +pub mod client; +pub mod error; +pub mod fake_data; +pub mod mock_server; +pub mod openapi; +pub mod pagination; +pub mod resources; +pub mod routing; +pub mod sync; + +pub use crate::error::{Error, Result}; + +pub use crate::openapi::load::{load_open_api_document, OpenApiSource}; +pub use crate::openapi::overlay::{ + apply_overlay, load_open_api_document_with_overlays, load_overlay, OverlayAction, + OverlayDocument, +}; +pub use crate::openapi::resolve_refs::resolve_refs; +pub use crate::openapi::types::{ + OpenApiDocument, OperationObject, ParameterObject, SchemaObject, ServerObject, +}; + +pub use crate::resources::discover::{discover_resources, ResourceRoute}; + +pub use crate::fake_data::generate::generate_from_schema; + +pub use crate::mock_server::server::{create_mock_server, MockServer}; + +pub use crate::client::client::{ + create_api_client, ApiClient, ApiClientOptions, PaginateOptions, PollOptions, PollingHandle, + SyncResult, +}; +pub use crate::client::storage::{InMemoryStorageAdapter, StorageAdapter}; + +pub use crate::pagination::autodetect::{resolve_effective_scheme, EffectiveScheme}; +pub use crate::pagination::types::{ + AutoDetectObject, PaginationApplicationObject, PaginationResponseState, PaginationSchemeObject, + PaginationSchemesMap, RequestRole, ResponseRole, SchemeType, +}; +pub use crate::pagination::validate::validate_pagination_scheme; + +pub use crate::sync::client::{ClientConfig, SyncClient, SyncError, SyncReport}; +pub use crate::sync::constants::{bind_url, validate_constants}; +pub use crate::sync::credentials::{base_url, Credentials}; +pub use crate::sync::ontology::{ + derive_ontology, ontology_shortname, Ontology, OntologyTerm, TermKind, +}; +pub use crate::sync::resource_model::{ + crud_operation, discover_resource_model, AddedField, CollectionMembership, ContextProvider, + CrudAction, CrudOperation, CrudResourceObject, IdentityBindingObject, ManagedCollection, + ResourceCollectionObject, ResourceIdentityObject, ResourceModel, +}; +pub use crate::sync::storage::{InMemoryStorage, Record, Storage, StorageError}; diff --git a/integrations/localthought/syncables/src/mock_server/mod.rs b/integrations/localthought/syncables/src/mock_server/mod.rs new file mode 100644 index 0000000000..68e080ed5e --- /dev/null +++ b/integrations/localthought/syncables/src/mock_server/mod.rs @@ -0,0 +1,6 @@ +//! A mock API server that implements an OpenAPI document, backed by a real +//! (in-memory) CRUD store per resource, seeded with fake data generated +//! from the document's schemas. + +pub mod server; +pub mod store; diff --git a/integrations/localthought/syncables/src/mock_server/server.rs b/integrations/localthought/syncables/src/mock_server/server.rs new file mode 100644 index 0000000000..2dd754b1b5 --- /dev/null +++ b/integrations/localthought/syncables/src/mock_server/server.rs @@ -0,0 +1,91 @@ +//! The mock server: an HTTP request handler generated from an OpenAPI +//! document. +//! +//! For a `GET` operation it first checks whether a pagination scheme +//! applies (see [`crate::pagination::autodetect`]) and, if so, serves it +//! as a paginated list; otherwise it treats the match as a resource +//! ([`ResourceStore`], CRUD semantics based on collection vs. item path +//! and HTTP method) or falls back to serving the operation's documented +//! example/generated schema response verbatim. Resource collections are +//! lazily seeded with [`SEED_COUNT`] fake records on first `GET`. +//! +//! # Port status +//! +//! Scaffolding. The types, constants and the public surface below are +//! ported; the request handler itself is not yet implemented. + +// The `async` on the stubs below is part of the ported API surface, +// not an accident of the current `todo!()` bodies. +#![allow(clippy::unused_async)] + +use std::sync::{Arc, Mutex}; + +use crate::error::Result; +use crate::openapi::types::OpenApiDocument; +use crate::resources::discover::{discover_resources, ResourceRoute}; + +use super::store::ResourceStore; + +/// Fake records seeded into a resource collection on its first `GET`. +pub const SEED_COUNT: usize = 3; +/// Total fake items generated for a paginated list endpoint, once per path template. +pub const PAGINATED_TOTAL_COUNT: usize = 7; +/// Page size used when the request doesn't specify one. +pub const DEFAULT_PAGE_SIZE: usize = 3; + +/// Where a started mock server is listening. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerAddress { + /// The bound TCP port. + pub port: u16, + /// The base URL clients should use, e.g. `http://127.0.0.1:8080`. + pub url: String, +} + +/// A mock server built from an OpenAPI document. +#[derive(Clone)] +pub struct MockServer { + document: Arc, + resources: Arc>, + store: Arc>, +} + +impl MockServer { + /// The document this server implements. + pub fn document(&self) -> &OpenApiDocument { + &self.document + } + + /// The resources discovered in the document. + pub fn resources(&self) -> &[ResourceRoute] { + &self.resources + } + + /// The backing CRUD store. + pub fn store(&self) -> &Mutex { + &self.store + } + + /// Binds a port and starts serving. + /// + /// Pass `None` for `port` to bind an ephemeral one. + pub async fn listen(&self, port: Option) -> Result { + let _ = port; + todo!("port src/mock-server/server.ts: handleRequest and the HTTP listener") + } + + /// Stops serving. + pub async fn close(&self) -> Result<()> { + todo!("port src/mock-server/server.ts: server shutdown") + } +} + +/// Builds a mock server for `document`. +pub fn create_mock_server(document: OpenApiDocument) -> MockServer { + let resources = discover_resources(&document.paths); + MockServer { + document: Arc::new(document), + resources: Arc::new(resources), + store: Arc::new(Mutex::new(ResourceStore::new())), + } +} diff --git a/integrations/localthought/syncables/src/mock_server/store.rs b/integrations/localthought/syncables/src/mock_server/store.rs new file mode 100644 index 0000000000..003c1c3919 --- /dev/null +++ b/integrations/localthought/syncables/src/mock_server/store.rs @@ -0,0 +1,89 @@ +//! The mock server's in-memory CRUD store, one collection per resource path. + +use std::collections::HashMap; +use std::time::SystemTime; + +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +#[derive(Debug, Clone)] +struct CollectionMeta { + version: u64, + last_modified: String, +} + +/// One in-memory `Map` per collection path, with CRUD semantics based on +/// collection vs. item path and HTTP method. +#[derive(Debug, Default)] +pub struct ResourceStore { + collections: HashMap>>, + meta: HashMap, +} + +impl ResourceStore { + /// A store with no collections. + pub fn new() -> Self { + Self::default() + } + + /// Whether `resource`'s collection has been created (seeded) yet. + pub fn has(&self, resource: &str) -> bool { + self.collections.contains_key(resource) + } + + /// Every record in `resource`'s collection, in insertion order. + pub fn list(&mut self, resource: &str) -> Vec> { + self.collection(resource).values().cloned().collect() + } + + /// One record, by id. + pub fn get(&mut self, resource: &str, id: &str) -> Option> { + self.collection(resource).get(id).cloned() + } + + /// Inserts or replaces a record. + pub fn put(&mut self, resource: &str, id: &str, value: Map) { + self.collection(resource).insert(id.to_string(), value); + self.touch(resource); + } + + /// Removes a record, reporting whether it existed. + pub fn delete(&mut self, resource: &str, id: &str) -> bool { + let deleted = self.collection(resource).shift_remove(id).is_some(); + if deleted { + self.touch(resource); + } + deleted + } + + /// Weak ETag for the current state of `resource`'s collection, once it + /// has been populated at least once. + pub fn etag(&self, resource: &str) -> Option { + self.meta + .get(resource) + .map(|meta| format!("W/\"{}\"", meta.version)) + } + + /// RFC 7231 HTTP-date of the last mutation to `resource`'s collection, + /// if any. + pub fn last_modified(&self, resource: &str) -> Option { + self.meta + .get(resource) + .map(|meta| meta.last_modified.clone()) + } + + fn touch(&mut self, resource: &str) { + let version = self.meta.get(resource).map_or(0, |meta| meta.version) + 1; + self.meta.insert( + resource.to_string(), + CollectionMeta { + version, + last_modified: httpdate::fmt_http_date(SystemTime::now()), + }, + ); + } + + fn collection(&mut self, resource: &str) -> &mut IndexMap> { + self.collections.entry(resource.to_string()).or_default() + } +} diff --git a/integrations/localthought/syncables/src/openapi/load.rs b/integrations/localthought/syncables/src/openapi/load.rs new file mode 100644 index 0000000000..7df664c9a3 --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/load.rs @@ -0,0 +1,91 @@ +//! Loading an OpenAPI document from a file path or an in-memory value. + +use std::path::Path; + +use serde_json::Value; + +use super::resolve_refs::resolve_refs; +use super::types::OpenApiDocument; +use crate::error::{Error, Result}; + +/// Where a document comes from: a file path, or an already-parsed value. +/// +/// The TypeScript original takes `string | Record`; +/// this is the same union, made explicit. +#[derive(Debug, Clone)] +pub enum OpenApiSource<'a> { + /// A path to a YAML or JSON document on disk. + Path(&'a Path), + /// An in-memory document. + Value(Value), +} + +impl<'a> From<&'a str> for OpenApiSource<'a> { + fn from(path: &'a str) -> Self { + Self::Path(Path::new(path)) + } +} + +impl<'a> From<&'a Path> for OpenApiSource<'a> { + fn from(path: &'a Path) -> Self { + Self::Path(path) + } +} + +impl From for OpenApiSource<'_> { + fn from(value: Value) -> Self { + Self::Value(value) + } +} + +/// Loads an OpenAPI document from a JSON/YAML file path or an in-memory +/// value, and resolves all local `$ref`s. +/// +/// YAML is a superset of JSON, so the file format does not need to be +/// detected separately. +pub async fn load_open_api_document<'a>( + source: impl Into> + Send, +) -> Result { + let raw = match source.into() { + OpenApiSource::Path(path) => load_yaml_file(path).await?, + OpenApiSource::Value(value) => value, + }; + let resolved = resolve_refs(&raw); + serde_json::from_value(resolved).map_err(Error::from) +} + +/// Parses YAML (or JSON) text into a [`Value`]. +pub fn parse_yaml(text: &str) -> Result { + serde_yaml_ng::from_str(text).map_err(Error::from) +} + +/// Reads and parses a YAML/JSON file, wrapping any i/o or parse failure in +/// [`Error::FileLoad`] so it names the offending path — a bare +/// [`std::io::Error`] from a failed read doesn't otherwise mention which +/// file was missing or unreadable. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) async fn load_yaml_file(path: &Path) -> Result { + async { + let text = tokio::fs::read_to_string(path).await?; + parse_yaml(&text) + } + .await + .map_err(|source| Error::FileLoad { + path: path.to_path_buf(), + source: Box::new(source), + }) +} + +#[cfg(target_arch = "wasm32")] +pub(crate) async fn load_yaml_file(path: &Path) -> Result { + Err(Error::FileLoad { + path: path.to_path_buf(), + source: Box::new( + std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Use an in-memory OpenAPI document in the browser", + ) + .into(), + ), + }) +} diff --git a/integrations/localthought/syncables/src/openapi/mod.rs b/integrations/localthought/syncables/src/openapi/mod.rs new file mode 100644 index 0000000000..81904227c8 --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/mod.rs @@ -0,0 +1,12 @@ +//! Reading an OpenAPI document and preparing it for everything downstream. +//! +//! [`load`] reads a document from a file path or an in-memory value and +//! passes it through [`resolve_refs`], which inlines all local `#/...` +//! JSON-pointer `$ref`s. Everything downstream assumes refs are already +//! resolved; [`types`] holds the minimal OpenAPI type surface actually +//! used (not a full spec typing). + +pub mod load; +pub mod overlay; +pub mod resolve_refs; +pub mod types; diff --git a/integrations/localthought/syncables/src/openapi/overlay.rs b/integrations/localthought/syncables/src/openapi/overlay.rs new file mode 100644 index 0000000000..20751b120c --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/overlay.rs @@ -0,0 +1,209 @@ +//! An intentionally minimal [OpenAPI Overlay](https://spec.openapis.org/overlay/v1.0.0.html) +//! implementation: `update`/`remove` actions against `$`, plain dot-paths +//! like `$.components`, and quoted bracket segments like +//! `$.paths['/pets/{petId}'].get` — not the full JSONPath grammar (no +//! wildcards, filters, or numeric/array indexing). + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use super::load::{load_open_api_document, load_yaml_file, OpenApiSource}; +use super::types::OpenApiDocument; +use crate::error::{Error, Result}; + +/// A single overlay action. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OverlayAction { + /// JSONPath target — `$`, a dot-path such as `$.components`, or a + /// dot-path with quoted bracket segments such as + /// `$.paths['/pets/{petId}'].get`. + pub target: String, + /// Object to deep-merge onto the target. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub update: Option>, + /// When true, delete the target instead of merging. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remove: Option, +} + +/// Metadata of an overlay document. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OverlayInfo { + /// Overlay title. + pub title: String, + /// Overlay version. + pub version: String, +} + +/// An OpenAPI Overlay document. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OverlayDocument { + /// Overlay specification version. + pub overlay: String, + /// Overlay metadata. + pub info: OverlayInfo, + /// Actions to apply, in order. + pub actions: Vec, +} + +/// Loads an OpenAPI Overlay document from a YAML/JSON file path or value. +pub async fn load_overlay<'a>( + source: impl Into> + Send, +) -> Result { + let raw = match source.into() { + OpenApiSource::Path(path) => load_yaml_file(path).await?, + OpenApiSource::Value(value) => value, + }; + serde_json::from_value(raw).map_err(Error::from) +} + +/// Loads an OpenAPI document and applies a list of Overlays to it, in the +/// order given — a later overlay may refine what an earlier one added. Each +/// overlay is loaded from a file path. +/// +/// Overlays are applied to the document after its own `$ref`s are resolved +/// (mirroring the TypeScript original's `buildDocumentFrom` in +/// `src/sync/document.ts` of `localthought/reflector`), so an overlay's +/// `update` can safely assume there are no refs left to chase. +pub async fn load_open_api_document_with_overlays<'a>( + document: impl Into> + Send, + overlay_paths: &[PathBuf], +) -> Result { + let document = load_open_api_document(document).await?; + let mut value = serde_json::to_value(document).map_err(Error::from)?; + for path in overlay_paths { + let overlay = load_overlay(path.as_path()).await?; + value = apply_overlay(&value, &overlay)?; + } + serde_json::from_value(value).map_err(Error::from) +} + +fn deep_merge_value(existing: Option<&Value>, incoming: &Value) -> Value { + match (existing, incoming) { + (Some(Value::Object(existing)), Value::Object(incoming)) => { + let mut merged = existing.clone(); + for (key, value) in incoming { + merged.insert(key.clone(), deep_merge_value(merged.get(key), value)); + } + Value::Object(merged) + } + _ => incoming.clone(), + } +} + +/// Tokenizes the intentionally small subset of Overlay JSONPath targets this +/// crate supports into property segments: `$` (the document root), a +/// dot-path like `$.components.schemas.Foo`, and quoted bracket segments +/// like `$.paths['/repos/{owner}/{repo}/issues'].get` — needed because a +/// path template contains slashes and braces that a plain `.split('.')` +/// would mangle. No wildcards, filters, or numeric/array indexing. +fn parse_target(target: &str) -> Result> { + let unsupported = || Error::UnsupportedOverlayTarget(target.to_string()); + + if target == "$" { + return Ok(Vec::new()); + } + if !target.starts_with('$') { + return Err(unsupported()); + } + + let mut segments = Vec::new(); + let bytes = target.as_bytes(); + let mut index = 1; + while index < bytes.len() { + match bytes[index] { + b'.' => { + index += 1; + let start = index; + while index < bytes.len() && bytes[index] != b'.' && bytes[index] != b'[' { + index += 1; + } + if index == start { + return Err(unsupported()); + } + segments.push(target[start..index].to_string()); + } + b'[' => { + let quote = *bytes.get(index + 1).ok_or_else(unsupported)?; + if quote != b'\'' && quote != b'"' { + return Err(unsupported()); + } + let key_start = index + 2; + let end = target[key_start..] + .find(quote as char) + .map(|offset| key_start + offset) + .ok_or_else(unsupported)?; + segments.push(target[key_start..end].to_string()); + index = end + 1; + if bytes.get(index) != Some(&b']') { + return Err(unsupported()); + } + index += 1; + } + _ => return Err(unsupported()), + } + } + Ok(segments) +} + +fn navigate<'v>( + root: &'v mut Value, + segments: &[String], + create_missing: bool, +) -> Result> { + let mut node = root; + for segment in segments { + let object = node + .as_object_mut() + .ok_or_else(|| Error::OverlayTargetNotAnObject(segment.clone()))?; + if !object.contains_key(segment) { + if !create_missing { + return Ok(None); + } + object.insert(segment.clone(), Value::Object(Map::new())); + } + let child = object + .get_mut(segment) + .expect("segment inserted or already present"); + if !child.is_object() { + return Err(Error::OverlayTargetNotAnObject(segment.clone())); + } + node = child; + } + Ok(Some(node)) +} + +/// Applies an overlay to a document, returning a new document. +/// +/// Supports `update` (deep-merged onto the target) and `remove` actions. +pub fn apply_overlay(document: &Value, overlay: &OverlayDocument) -> Result { + let mut result = document.clone(); + + for action in &overlay.actions { + let segments = parse_target(&action.target)?; + if action.remove == Some(true) { + let Some((key, parent_segments)) = segments.split_last() else { + return Err(Error::OverlayRemovesRoot); + }; + if let Some(parent) = navigate(&mut result, parent_segments, false)? { + if let Some(object) = parent.as_object_mut() { + object.shift_remove(key); + } + } + } else if let Some(update) = &action.update { + let target = navigate(&mut result, &segments, true)? + .expect("navigate with create_missing always yields a node"); + let object = target + .as_object_mut() + .ok_or_else(|| Error::OverlayTargetNotAnObject(action.target.clone()))?; + for (key, value) in update { + let merged = deep_merge_value(object.get(key), value); + object.insert(key.clone(), merged); + } + } + } + + Ok(result) +} diff --git a/integrations/localthought/syncables/src/openapi/resolve_refs.rs b/integrations/localthought/syncables/src/openapi/resolve_refs.rs new file mode 100644 index 0000000000..7a9c24a36a --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/resolve_refs.rs @@ -0,0 +1,92 @@ +//! Resolves local `#/...` JSON pointer `$ref`s in place. +//! +//! Each `{ "$ref": ... }` node is replaced with the value it points to. +//! Reused (non-cyclic) targets are resolved once and cached so diamond +//! references share a result; a target still being resolved when it is +//! referenced again is a genuine cycle, so that occurrence is left +//! unresolved to avoid recursing forever. Non-local refs (external files, +//! URLs) are left unresolved too, since there is nothing in the document +//! to resolve them against — real-world documents sometimes use these in +//! vendor extensions unrelated to the schemas this crate cares about. + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +/// Resolves every local `$ref` in `document`, returning a new value. +pub fn resolve_refs(document: &Value) -> Value { + let mut resolver = Resolver { + root: document, + resolving: HashSet::new(), + resolved: HashMap::new(), + }; + resolver.walk(document) +} + +/// A JSON pointer identifying a node, used as the cache/cycle key. +/// +/// The TypeScript original keys its `Set`/`Map` on object *identity*; +/// `serde_json` values are cloned rather than shared, so the pointer that +/// reached a node stands in for that identity here. +type NodeKey = String; + +struct Resolver<'a> { + root: &'a Value, + resolving: HashSet, + resolved: HashMap, +} + +impl Resolver<'_> { + fn resolve_pointer(&self, reference: &str) -> Option<&Value> { + let trimmed = reference.strip_prefix("#/")?; + let mut node = self.root; + for segment in trimmed.split('/') { + let segment = segment.replace("~1", "/").replace("~0", "~"); + node = node.get(&segment)?; + } + Some(node) + } + + fn walk(&mut self, node: &Value) -> Value { + match node { + Value::Array(items) => Value::Array(items.iter().map(|i| self.walk(i)).collect()), + Value::Object(object) => { + if let Some(Value::String(reference)) = object.get("$ref") { + return self.walk_ref(node, &reference.clone()); + } + let mut result = serde_json::Map::new(); + for (key, value) in object { + result.insert(key.clone(), self.walk(value)); + } + Value::Object(result) + } + other => other.clone(), + } + } + + fn walk_ref(&mut self, node: &Value, reference: &str) -> Value { + if !reference.starts_with("#/") { + return node.clone(); + } + let Some(target) = self.resolve_pointer(reference) else { + return Value::Null; + }; + if !target.is_object() && !target.is_array() { + return target.clone(); + } + let key: NodeKey = reference.to_string(); + if let Some(cached) = self.resolved.get(&key) { + return cached.clone(); + } + if self.resolving.contains(&key) { + return target.clone(); + } + + let target = target.clone(); + self.resolving.insert(key.clone()); + let result = self.walk(&target); + self.resolving.remove(&key); + self.resolved.insert(key, result.clone()); + result + } +} diff --git a/integrations/localthought/syncables/src/openapi/types.rs b/integrations/localthought/syncables/src/openapi/types.rs new file mode 100644 index 0000000000..ebdc5c7305 --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/types.rs @@ -0,0 +1,250 @@ +//! The minimal OpenAPI type surface this crate actually uses — not a full +//! spec typing. Every struct keeps an `extensions` catch-all so a document +//! round-trips through these types without losing vendor extensions +//! (`x-pagination` in particular is read back out of `OperationObject`). + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::pagination::types::PaginationSchemesMap; + +/// A JSON object with insertion order preserved. +/// +/// Order is load-bearing: [`crate::pagination::items::locate_items_field`] +/// picks the *first* array-typed property of a response schema. +pub type JsonMap = IndexMap; + +/// A JSON Schema subset, as it appears inside an OpenAPI document. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct SchemaObject { + /// `type` keyword (`string`, `integer`, `object`, `array`, ...). + #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] + pub schema_type: Option, + /// `format` keyword (`date-time`, `uuid`, `email`, ...). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, + /// Properties of an object schema, in document order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub properties: Option>, + /// Names of required properties. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option>, + /// Item schema of an array schema. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, + /// Allowed values; the first is used when generating fake data. + #[serde(rename = "enum", default, skip_serializing_if = "Option::is_none")] + pub enum_values: Option>, + /// A fixed example, preferred over synthesis when generating fake data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, + /// Lower bound for numeric schemas. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub minimum: Option, + /// `oneOf` branches. + #[serde(rename = "oneOf", default, skip_serializing_if = "Option::is_none")] + pub one_of: Option>, + /// `anyOf` branches. + #[serde(rename = "anyOf", default, skip_serializing_if = "Option::is_none")] + pub any_of: Option>, + /// `allOf` branches, merged when generating data or listing properties. + #[serde(rename = "allOf", default, skip_serializing_if = "Option::is_none")] + pub all_of: Option>, + /// Any other keyword present on the schema. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One entry of a `content` map, keyed by media type. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct MediaTypeObject { + /// Schema of the payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// A fixed example payload, preferred over the schema when serving mocks. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// A documented response for one status code. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ResponseObject { + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Response payloads, keyed by media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option>, +} + +/// A documented request body. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RequestBodyObject { + /// Whether the body is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option, + /// Request payloads, keyed by media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option>, +} + +/// Where a parameter is carried. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ParameterLocation { + /// Query string parameter. + Query, + /// Path template variable. + Path, + /// Request header. + Header, + /// Cookie. + Cookie, +} + +/// A single operation parameter. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ParameterObject { + /// Parameter name. + pub name: String, + /// Where the parameter is carried. + #[serde(rename = "in")] + pub location: ParameterLocation, + /// Whether the parameter is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option, + /// Schema of the parameter's value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// Any other key present on the parameter. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One HTTP operation on a path. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OperationObject { + /// `operationId`, when the document declares one. + #[serde( + rename = "operationId", + default, + skip_serializing_if = "Option::is_none" + )] + pub operation_id: Option, + /// Declared parameters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option>, + /// Declared request body. + #[serde( + rename = "requestBody", + default, + skip_serializing_if = "Option::is_none" + )] + pub request_body: Option, + /// Documented responses, keyed by status code. + #[serde(default)] + pub responses: IndexMap, + /// Any other key on the operation — notably `x-pagination`. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// The set of operations declared on one path template. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct PathItem { + /// `GET` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub get: Option, + /// `PUT` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub put: Option, + /// `POST` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub post: Option, + /// `PATCH` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub patch: Option, + /// `DELETE` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete: Option, + /// Any other key on the path item. + #[serde(flatten)] + pub extensions: JsonMap, +} + +impl PathItem { + /// The operation declared for `method` (case-insensitive), if any. + pub fn operation(&self, method: &str) -> Option<&OperationObject> { + match method.to_ascii_uppercase().as_str() { + "GET" => self.get.as_ref(), + "PUT" => self.put.as_ref(), + "POST" => self.post.as_ref(), + "PATCH" => self.patch.as_ref(), + "DELETE" => self.delete.as_ref(), + _ => None, + } + } +} + +/// Document metadata. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct InfoObject { + /// Document title. + pub title: String, + /// Document version. + pub version: String, +} + +/// One entry of the document's top-level `servers` list. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ServerObject { + /// Root URL of the API, e.g. `https://api.github.com`. + pub url: String, + /// Any other key on the server entry. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// The `components` section, narrowed to what this crate reads. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ComponentsObject { + /// Reusable schemas. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schemas: Option>, + /// Pagination schemes, per the OpenAPI Pagination Schemes Extension. + #[serde( + rename = "paginationSchemes", + default, + skip_serializing_if = "Option::is_none" + )] + pub pagination_schemes: Option, + /// Any other key under `components`. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// An OpenAPI document with all local `$ref`s already resolved. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OpenApiDocument { + /// OpenAPI version string. + #[serde(default)] + pub openapi: String, + /// Document metadata. + #[serde(default)] + pub info: InfoObject, + /// Path templates, in document order. + #[serde(default)] + pub paths: IndexMap, + /// The API's base URL(s). The credential layer targets requests at the + /// first entry rather than any URL configured separately, so a document + /// can't be pointed at the wrong host by mistake. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub servers: Option>, + /// Reusable components. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub components: Option, + /// Any other top-level key. + #[serde(flatten)] + pub extensions: JsonMap, +} diff --git a/integrations/localthought/syncables/src/pagination/autodetect.rs b/integrations/localthought/syncables/src/pagination/autodetect.rs new file mode 100644 index 0000000000..c604c0e98f --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/autodetect.rs @@ -0,0 +1,169 @@ +//! Resolving which pagination scheme applies to an operation. + +use std::collections::HashSet; + +use indexmap::IndexMap; +use serde_json::Value; + +use super::types::{ + AutoDetect, AutoDetectObject, PaginationApplicationObject, PaginationSchemeObject, +}; +use super::validate::validate_pagination_scheme; +use crate::openapi::types::{OpenApiDocument, OperationObject, ParameterLocation}; + +/// The scheme that applies to an operation, and the name it is declared under. +#[derive(Debug, Clone, PartialEq)] +pub struct EffectiveScheme { + /// Key in `components.paginationSchemes`. + pub scheme_name: String, + /// The scheme itself, with any per-operation overrides merged in. + pub scheme: PaginationSchemeObject, +} + +/// Schemes that fail validation (spec §9) are excluded here rather than +/// thrown on eagerly — one malformed scheme in a document (see e.g. +/// Giphy's `type: offset`, which isn't a valid scheme type) shouldn't +/// prevent using the rest of the document or its other schemes. +fn valid_schemes(document: &OpenApiDocument) -> IndexMap { + document + .components + .as_ref() + .and_then(|c| c.pagination_schemes.as_ref()) + .into_iter() + .flatten() + .filter(|(name, scheme)| validate_pagination_scheme(name, scheme).is_empty()) + .map(|(name, scheme)| (name.clone(), scheme.clone())) + .collect() +} + +fn query_param_names(operation: &OperationObject) -> HashSet<&str> { + operation + .parameters + .as_deref() + .unwrap_or_default() + .iter() + .filter(|parameter| parameter.location == ParameterLocation::Query) + .map(|parameter| parameter.name.as_str()) + .collect() +} + +fn body_field_names(operation: &OperationObject) -> HashSet<&str> { + operation + .request_body + .as_ref() + .and_then(|body| body.content.as_ref()) + .and_then(|content| content.get("application/json")) + .and_then(|media| media.schema.as_ref()) + .and_then(|schema| schema.properties.as_ref()) + .into_iter() + .flatten() + .map(|(name, _)| name.as_str()) + .collect() +} + +/// Default auto-detection rules (spec §6.2/§6.3): a dimension only +/// contributes to the match when the scheme actually declares fields for +/// it — an empty declaration isn't treated as vacuously satisfied, or +/// every scheme with no query parameters would match every operation. +fn auto_detect_matches(scheme: &PaginationSchemeObject, operation: &OperationObject) -> bool { + let options = match &scheme.auto_detect { + Some(AutoDetect::Enabled(false)) => return false, + Some(AutoDetect::Options(options)) => (**options).clone(), + _ => AutoDetectObject::default(), + }; + let require_all = options.require_all.unwrap_or(true); + let mut results: Vec = Vec::new(); + + let request = scheme.request.as_ref(); + + if options.match_query_params.unwrap_or(true) { + let required: Vec<&String> = request + .and_then(|r| r.query_parameters.as_ref()) + .into_iter() + .flatten() + .map(|(name, _)| name) + .collect(); + if !required.is_empty() { + let declared = query_param_names(operation); + results.push(required.iter().all(|name| declared.contains(name.as_str()))); + } + } + + if options.match_body_fields.unwrap_or(true) { + let required: Vec<&String> = request + .and_then(|r| r.body_fields.as_ref()) + .into_iter() + .flatten() + .map(|(name, _)| name) + .collect(); + if !required.is_empty() { + let declared = body_field_names(operation); + results.push(required.iter().all(|name| declared.contains(name.as_str()))); + } + } + + if results.is_empty() { + return false; + } + if require_all { + results.iter().all(|matched| *matched) + } else { + results.iter().any(|matched| *matched) + } +} + +fn deep_merge(base: &Value, overrides: &Value) -> Value { + match (base, overrides) { + (Value::Object(base), Value::Object(overrides)) => { + let mut result = base.clone(); + for (key, value) in overrides { + let merged = match result.get(key) { + Some(existing) if existing.is_object() && value.is_object() => { + deep_merge(existing, value) + } + _ => value.clone(), + }; + result.insert(key.clone(), merged); + } + Value::Object(result) + } + _ => overrides.clone(), + } +} + +/// Resolves the pagination scheme that applies to an operation: an +/// explicit `x-pagination` application (with overrides merged in) takes +/// priority, falling back to auto-detection against the document's valid +/// `paginationSchemes`. +pub fn resolve_effective_scheme( + document: &OpenApiDocument, + operation: &OperationObject, +) -> Option { + let schemes = valid_schemes(document); + + if let Some(explicit) = operation.extensions.get("x-pagination") { + let applications: Vec = + serde_json::from_value(explicit.clone()).unwrap_or_default(); + let application = applications.into_iter().next()?; + let base = schemes.get(&application.scheme)?; + let scheme = match &application.overrides { + Some(overrides) => { + let merged = deep_merge(&serde_json::to_value(base).ok()?, overrides); + serde_json::from_value(merged).ok()? + } + None => base.clone(), + }; + return Some(EffectiveScheme { + scheme_name: application.scheme, + scheme, + }); + } + + schemes + .into_iter() + .find(|(_, scheme)| auto_detect_matches(scheme, operation)) + .map(|(scheme_name, scheme)| EffectiveScheme { + scheme_name, + scheme, + }) +} diff --git a/integrations/localthought/syncables/src/pagination/items.rs b/integrations/localthought/syncables/src/pagination/items.rs new file mode 100644 index 0000000000..af0f3d14f1 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/items.rs @@ -0,0 +1,91 @@ +//! Locating which response property actually holds the list of items. +//! +//! The extension itself only describes pagination metadata, not where +//! items live, so this excludes whatever fields the scheme claims as +//! metadata, then picks the remaining array-typed property (falling back +//! to common envelope names). This is also what makes real enveloped +//! responses (e.g. `{ data: [...], meta, pagination }`) work at all, +//! pagination or not. + +use std::collections::HashSet; + +use indexmap::IndexMap; + +use super::types::PaginationSchemeObject; +use crate::openapi::types::SchemaObject; + +/// Field names real-world APIs commonly wrap a list response in, tried +/// when the response schema itself doesn't unambiguously point at one +/// array property. +const COMMON_ITEMS_FIELDS: [&str; 5] = ["items", "data", "results", "records", "content"]; + +/// Flattens a schema's own properties together with any `allOf` branches' +/// properties into one map. +/// +/// Real paginated response schemas often compose a shared "paging" base +/// schema with a branch that adds the concrete `items` property (e.g. +/// Spotify's `PagingSimplifiedAlbumObject`). +pub fn effective_properties(schema: Option<&SchemaObject>) -> IndexMap { + let Some(schema) = schema else { + return IndexMap::new(); + }; + if let Some(all_of) = &schema.all_of { + let mut merged = IndexMap::new(); + for branch in all_of { + merged.extend(effective_properties(Some(branch))); + } + return merged; + } + schema.properties.clone().unwrap_or_default() +} + +/// The top-level field names a pagination scheme claims for its own metadata. +fn metadata_field_roots(scheme: Option<&PaginationSchemeObject>) -> HashSet { + scheme + .and_then(|s| s.response.as_ref()) + .and_then(|r| r.body_fields.as_ref()) + .into_iter() + .flatten() + .map(|(key, _)| key.split('.').next().unwrap_or(key).to_string()) + .collect() +} + +/// Finds the property in a response schema that holds the actual list of +/// items: the first array-typed property that isn't claimed by the +/// pagination scheme as a metadata field, falling back to common +/// enveloping field names. +pub fn locate_items_field( + schema: Option<&SchemaObject>, + scheme: Option<&PaginationSchemeObject>, +) -> Option { + let properties = effective_properties(schema); + let excluded = metadata_field_roots(scheme); + + for (name, property_schema) in &properties { + if !excluded.contains(name) && property_schema.schema_type.as_deref() == Some("array") { + return Some(name.clone()); + } + } + + COMMON_ITEMS_FIELDS + .iter() + .find(|name| properties.contains_key(**name) && !excluded.contains(**name)) + .map(|name| (*name).to_string()) +} + +/// The schema of a single item, given the schema of the whole (enveloped) +/// response. +pub fn item_schema_for( + schema: Option<&SchemaObject>, + scheme: Option<&PaginationSchemeObject>, +) -> Option { + if let Some(schema) = schema { + if schema.schema_type.as_deref() == Some("array") { + return schema.items.as_deref().cloned(); + } + } + let field = locate_items_field(schema, scheme)?; + effective_properties(schema) + .get(&field) + .and_then(|property| property.items.as_deref().cloned()) +} diff --git a/integrations/localthought/syncables/src/pagination/mod.rs b/integrations/localthought/syncables/src/pagination/mod.rs new file mode 100644 index 0000000000..cd61a08926 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/mod.rs @@ -0,0 +1,20 @@ +//! The [OpenAPI Pagination Schemes Extension](https://github.com/pondersource/openapi-pagination-schemes-extension) +//! (`components.paginationSchemes`), applied to third-party documents via +//! [OpenAPI Overlays](https://spec.openapis.org/overlay/v1.0.0.html). +//! +//! **Pagination is orthogonal to the collection/item resource model.** In +//! real APIs, the paths that pair into a "resource" (batch-get-by-IDs +//! style, e.g. Giphy's `/gifs`, Spotify's `/albums`) are often *not* the +//! paginated ones — real pagination usually lives on separate search/list +//! endpoints (`/gifs/trending`, `/artists/{id}/albums`) that have no +//! sibling item path and are therefore invisible to +//! [`crate::resources::discover`]. So pagination support in both the mock +//! server and `paginate()` operates on *any* GET operation matched by a +//! scheme, not just discovered resources. + +pub mod autodetect; +pub mod items; +pub mod request_builder; +pub mod response_parser; +pub mod types; +pub mod validate; diff --git a/integrations/localthought/syncables/src/pagination/request_builder.rs b/integrations/localthought/syncables/src/pagination/request_builder.rs new file mode 100644 index 0000000000..318b247d3e --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/request_builder.rs @@ -0,0 +1,144 @@ +//! Building the query for one page request, and computing the next cursor. + +use super::types::{PaginationQuery, PaginationResponseState, PaginationSchemeObject, SchemeType}; + +/// Hard ceiling on pages walked in one paginated traversal, so a +/// misconfigured `Link` header — or any response that always claims +/// another page exists — cannot spin forever. Mirrors +/// [`crate::client::client::MAX_PAGES`], the equivalent constant for the +/// crate's existing `ApiClient` surface. +pub const MAX_PAGES: usize = 50; + +/// Where the client is in a paginated traversal, independent of scheme type. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PageCursor { + /// Item offset, for offset-based schemes. + pub offset: Option, + /// Page number, for page-based schemes. + pub page: Option, + /// Opaque token, for token/cursor-based schemes. + pub page_token: Option, +} + +fn fields_with_role(scheme: &PaginationSchemeObject, role: &str) -> Vec { + scheme + .request + .as_ref() + .and_then(|request| request.query_parameters.as_ref()) + .into_iter() + .flatten() + .filter(|(_, field)| field.role.as_deref() == Some(role)) + .map(|(name, _)| name.clone()) + .collect() +} + +/// Builds the query parameters for one page request from the current cursor. +pub fn build_query( + scheme: &PaginationSchemeObject, + cursor: &PageCursor, + page_size: Option, +) -> PaginationQuery { + let mut query = PaginationQuery::new(); + + if let Some(page_size) = page_size { + for name in fields_with_role(scheme, "pageSize") { + query.insert(name, page_size.to_string()); + } + } + for name in fields_with_role(scheme, "offset") { + query.insert(name, cursor.offset.unwrap_or(0).to_string()); + } + for name in fields_with_role(scheme, "page") { + query.insert(name, cursor.page.unwrap_or(1).to_string()); + } + if let Some(page_token) = &cursor.page_token { + for name in fields_with_role(scheme, "pageToken") + .into_iter() + .chain(fields_with_role(scheme, "cursor")) + { + query.insert(name, page_token.clone()); + } + } + + query +} + +/// Computes the cursor for the next page from the previous cursor and the +/// parsed response state. +/// +/// Returns `None` when there is no next page, or when the scheme type is +/// `nextLink` — for that type the caller should follow +/// [`PaginationResponseState::next_link`] directly rather than rebuilding +/// query parameters. +pub fn next_cursor( + scheme: &PaginationSchemeObject, + cursor: &PageCursor, + state: &PaginationResponseState, + items_returned: u64, +) -> Option { + if !state.has_next_page { + return None; + } + + match scheme.typed()? { + SchemeType::PageToken => state.next_page_token.clone().map(|token| PageCursor { + page_token: Some(token), + ..PageCursor::default() + }), + SchemeType::NextLink => None, + SchemeType::PageNumber => { + if !fields_with_role(scheme, "offset").is_empty() { + return Some(PageCursor { + offset: Some(cursor.offset.unwrap_or(0) + items_returned), + ..PageCursor::default() + }); + } + if !fields_with_role(scheme, "page").is_empty() { + return Some(PageCursor { + page: Some(cursor.page.unwrap_or(1) + 1), + ..PageCursor::default() + }); + } + None + } + } +} + +/// What a paginated traversal should do next, after parsing one page's +/// response. +#[derive(Debug, Clone, PartialEq)] +pub enum PageStep { + /// Follow this absolute URL directly (a `nextLink` scheme) — built from + /// the response's own `Link` header, not rebuilt from the operation's + /// path template. + FollowLink(String), + /// Request the next page with this cursor. + NextPage(PageCursor), + /// No more pages: the response says so, or [`MAX_PAGES`] was reached. + Done, +} + +/// Decides the next step of a paginated traversal, wrapping [`next_cursor`] +/// with the two cases it deliberately doesn't handle: a `nextLink` scheme +/// (whose next page is a URL, not a cursor) and the [`MAX_PAGES`] cap. +/// +/// `pages_fetched` is the number of pages fetched so far, including the one +/// `state` describes (i.e. after fetching the first page, pass `1`). +pub fn next_step( + scheme: &PaginationSchemeObject, + cursor: &PageCursor, + state: &PaginationResponseState, + items_returned: u64, + pages_fetched: usize, +) -> PageStep { + if !state.has_next_page || pages_fetched >= MAX_PAGES { + return PageStep::Done; + } + if scheme.typed() == Some(SchemeType::NextLink) { + return state + .next_link + .clone() + .map_or(PageStep::Done, PageStep::FollowLink); + } + next_cursor(scheme, cursor, state, items_returned).map_or(PageStep::Done, PageStep::NextPage) +} diff --git a/integrations/localthought/syncables/src/pagination/response_parser.rs b/integrations/localthought/syncables/src/pagination/response_parser.rs new file mode 100644 index 0000000000..90e007aa42 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/response_parser.rs @@ -0,0 +1,237 @@ +//! Parsing pagination state back out of a server response. + +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use super::types::{PaginationResponseState, PaginationSchemeObject, ResponseRole, SchemeType}; + +/// Reads a dot-separated path out of a JSON object, e.g. `pagination.total_count`. +pub fn read_nested_field<'v>(body: &'v Value, path: &str) -> Option<&'v Value> { + let mut node = body; + for segment in path.split('.') { + node = node.as_object()?.get(segment)?; + } + Some(node) +} + +/// Writes a value at a dot-separated path, creating intermediate objects +/// as needed. +pub fn set_nested_field(body: &mut Map, path: &str, value: Value) { + let segments: Vec<&str> = path.split('.').collect(); + let Some((last, parents)) = segments.split_last() else { + return; + }; + + let mut node = body; + for segment in parents { + let entry = node + .entry((*segment).to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if !entry.is_object() { + *entry = Value::Object(Map::new()); + } + node = entry.as_object_mut().expect("just ensured object"); + } + node.insert((*last).to_string(), value); +} + +/// Parses an RFC 8288 `Link` header value and extracts the URL with +/// `rel="next"`. +/// +/// Example: `; rel="next", <...>; rel="prev"` +pub fn parse_link_header(header: &str) -> Option { + if header.is_empty() { + return None; + } + for part in split_links(header) { + let part = part.trim_start(); + let Some(rest) = part.strip_prefix('<') else { + continue; + }; + let Some(end) = rest.find('>') else { + continue; + }; + let (url, attrs) = rest.split_at(end); + if rel_is_next(&attrs[1..]) { + return Some(url.to_string()); + } + } + None +} + +/// Splits on commas that begin a new `` link-value, mirroring the +/// original's `/,\s*(?=<)/` lookahead. +fn split_links(header: &str) -> Vec<&str> { + let bytes = header.as_bytes(); + let mut parts = Vec::new(); + let mut start = 0; + for (index, byte) in bytes.iter().enumerate() { + if *byte != b',' { + continue; + } + let after = header[index + 1..].trim_start(); + if after.starts_with('<') { + parts.push(&header[start..index]); + start = index + 1; + } + } + parts.push(&header[start..]); + parts +} + +/// Finds `rel=next` (quoted or bare) among a link-value's attributes. +fn rel_is_next(attrs: &str) -> bool { + let lowered = attrs.to_ascii_lowercase(); + let mut search = lowered.as_str(); + while let Some(index) = search.find("rel") { + let before_is_boundary = index == 0 + || !search.as_bytes()[index - 1].is_ascii_alphanumeric() + && search.as_bytes()[index - 1] != b'_'; + let after = search[index + 3..].trim_start(); + if before_is_boundary { + if let Some(value) = after.strip_prefix('=') { + let value = value.trim_start().trim_start_matches('"'); + let value = value + .split(|c: char| c == '"' || c == ';' || c == ',' || c.is_whitespace()) + .next() + .unwrap_or(""); + if value == "next" { + return true; + } + } + } + search = &search[index + 3..]; + } + false +} + +fn to_string_or_none(value: Option<&Value>) -> Option { + match value? { + Value::Null => None, + Value::String(s) if s.is_empty() => None, + Value::String(s) => Some(s.clone()), + other => Some(other.to_string()), + } +} + +fn to_number_or_none(value: Option<&Value>) -> Option { + match value? { + Value::Number(n) => n.as_f64(), + Value::String(s) => s.trim().parse::().ok(), + Value::Bool(b) => Some(f64::from(u8::from(*b))), + _ => None, + } +} + +fn extract_by_role( + scheme: &PaginationSchemeObject, + body: &Value, + headers: &IndexMap, +) -> IndexMap { + let mut roles = IndexMap::new(); + + let response = scheme.response.as_ref(); + + for (path, field) in response + .and_then(|r| r.body_fields.as_ref()) + .into_iter() + .flatten() + { + let Some(role) = &field.role else { continue }; + if let Some(value) = read_nested_field(body, path) { + roles.insert(role.clone(), value.clone()); + } + } + + for (name, field) in response + .and_then(|r| r.headers.as_ref()) + .into_iter() + .flatten() + { + let Some(role) = &field.role else { continue }; + let raw = headers + .get(name) + .or_else(|| headers.get(&name.to_lowercase())) + .or_else(|| headers.get(&name.to_uppercase())); + let Some(raw) = raw else { continue }; + if role == "nextLink" { + if let Some(parsed) = parse_link_header(raw) { + roles.insert("nextLink".to_string(), Value::String(parsed)); + } + } else { + roles.insert(role.clone(), Value::String(raw.clone())); + } + } + + roles +} + +/// A `nextLink`/`nextPageToken` value is a strong, type-agnostic signal +/// that another page exists — real APIs sometimes include one even on a +/// scheme whose `type` is `pageNumber` (e.g. Spotify's offset-based +/// endpoints all carry a `next` URL). Checking it first, ahead of the +/// type-specific counting rules, means traversal still terminates +/// correctly for those schemes even without a `currentPage`/`totalPages` +/// role declared. +/// +/// `totalCount` (role: `all` per spec §4.5) is checked next against +/// `items_fetched_so_far`, which the *caller* tracks — some real schemes +/// (e.g. Giphy's) report `totalCount` and `pageSize` but no `currentPage` +/// at all, so there's nothing here to compute "current page * pageSize" +/// from; the client already knows exactly how many items it has pulled +/// across all pages so far, which is the more direct signal anyway. +fn derive_has_next_page( + scheme_type: Option, + state: &PaginationResponseState, + items_fetched_so_far: Option, +) -> bool { + if state.next_link.is_some() || state.next_page_token.is_some() { + return true; + } + if let (Some(total_count), Some(fetched)) = (state.total_count, items_fetched_so_far) { + #[allow(clippy::cast_precision_loss)] + return (fetched as f64) < total_count; + } + if scheme_type == Some(SchemeType::PageNumber) { + if let (Some(current_page), Some(total_pages)) = (state.current_page, state.total_pages) { + return current_page < total_pages; + } + if let (Some(current_page), Some(total_count), Some(page_size)) = + (state.current_page, state.total_count, state.page_size) + { + return current_page * page_size < total_count; + } + } + false +} + +/// Parses a server response into pagination state, per the resolved scheme. +/// +/// `items_fetched_so_far` — the cumulative item count across all pages +/// fetched so far, including this one — lets `has_next_page` be derived +/// from a plain `totalCount` field even when no `currentPage` role is +/// declared. +pub fn parse_pagination_state( + scheme: &PaginationSchemeObject, + body: &Value, + headers: &IndexMap, + items_fetched_so_far: Option, +) -> PaginationResponseState { + let roles = extract_by_role(scheme, body, headers); + + let mut state = PaginationResponseState { + next_page_token: to_string_or_none( + roles + .get("nextPageToken") + .or_else(|| roles.get("nextCursor")), + ), + next_link: to_string_or_none(roles.get("nextLink")), + current_page: to_number_or_none(roles.get("currentPage")), + total_count: to_number_or_none(roles.get("totalCount")), + total_pages: to_number_or_none(roles.get("totalPages")), + page_size: to_number_or_none(roles.get("pageSize")), + has_next_page: false, + }; + state.has_next_page = derive_has_next_page(scheme.typed(), &state, items_fetched_so_far); + state +} diff --git a/integrations/localthought/syncables/src/pagination/types.rs b/integrations/localthought/syncables/src/pagination/types.rs new file mode 100644 index 0000000000..be15cfad62 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/types.rs @@ -0,0 +1,255 @@ +//! Type definitions for the [OpenAPI Pagination Schemes Extension](https://github.com/pondersource/openapi-pagination-schemes-extension). +//! +//! Spec version 0.1.0. Field names, optionality, and enum values are taken +//! verbatim from the spec so a [`PaginationSchemeObject`] can be lifted +//! from an OAS document without transformation. +//! +//! Roles are modelled as string newtypes rather than closed enums: the +//! spec allows `x-`-prefixed extension roles alongside the standard ones, +//! and [`crate::pagination::validate`] is what decides validity, so an +//! unknown role has to survive deserialization to be reported. + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Which family of pagination a scheme belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SchemeType { + /// Page- or offset-numbered traversal. + #[serde(rename = "pageNumber")] + PageNumber, + /// Opaque-token traversal. + #[serde(rename = "pageToken")] + PageToken, + /// Follow-the-link traversal. + #[serde(rename = "nextLink")] + NextLink, +} + +/// The role a request field plays. Spec-defined values are `page`, +/// `pageSize`, `offset`, `pageToken` and `cursor`. +pub type RequestRole = String; + +/// The role a response field plays. Spec-defined values are +/// `nextPageToken`, `nextCursor`, `nextLink`, `totalCount`, `totalPages`, +/// `pageSize` and `currentPage`. +pub type ResponseRole = String; + +/// One declared request field. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RequestFieldObject { + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Schema of the field's value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// The field's role in the scheme. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + /// Whether the field is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// Request-side fields a scheme declares, grouped by where they travel. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RequestPaginationFieldsObject { + /// Query parameters. + #[serde( + rename = "queryParameters", + default, + skip_serializing_if = "Option::is_none" + )] + pub query_parameters: Option>, + /// Request body fields. + #[serde( + rename = "bodyFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub body_fields: Option>, + /// Request headers. + /// + /// Part of the type surface (mirroring the spec); nothing reads it yet. + #[serde( + rename = "headerFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub header_fields: Option>, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// One declared response field. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ResponseFieldObject { + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Schema of the field's value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// The field's role in the scheme. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// Response-side fields a scheme declares, grouped by where they travel. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ResponsePaginationFieldsObject { + /// Response body fields; keys may be dotted paths into nested objects. + #[serde( + rename = "bodyFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub body_fields: Option>, + /// Response headers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub headers: Option>, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// Fine-grained auto-detection options. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct AutoDetectObject { + /// Match the scheme's declared query parameters against the operation's. + #[serde( + rename = "matchQueryParams", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_query_params: Option, + /// Match the scheme's declared body fields against the operation's. + #[serde( + rename = "matchBodyFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_body_fields: Option, + /// Part of the type surface (mirroring the spec); nothing reads it yet. + #[serde( + rename = "matchResponseFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_response_fields: Option, + /// Part of the type surface (mirroring the spec); nothing reads it yet. + #[serde( + rename = "matchHeaders", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_headers: Option, + /// Require every considered dimension to match, rather than any. + #[serde( + rename = "requireAll", + default, + skip_serializing_if = "Option::is_none" + )] + pub require_all: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// `autoDetect` is either a plain toggle or a set of options. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum AutoDetect { + /// `autoDetect: true` / `autoDetect: false`. + Enabled(bool), + /// `autoDetect: { ... }`. + Options(Box), +} + +/// One entry of `components.paginationSchemes`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PaginationSchemeObject { + /// Which family of pagination this scheme belongs to. + /// + /// Deserialized leniently: an invalid value (e.g. Giphy's `offset`) + /// has to survive parsing so [`crate::pagination::validate`] can + /// report it rather than failing the whole document. + #[serde(rename = "type")] + pub scheme_type: Value, + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Whether and how the scheme may be auto-detected. + #[serde( + rename = "autoDetect", + default, + skip_serializing_if = "Option::is_none" + )] + pub auto_detect: Option, + /// Request-side declared fields. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request: Option, + /// Response-side declared fields. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub response: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +impl PaginationSchemeObject { + /// The scheme's type, when it is one the spec defines. + pub fn typed(&self) -> Option { + serde_json::from_value(self.scheme_type.clone()).ok() + } +} + +/// One entry of an operation's `x-pagination` array. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PaginationApplicationObject { + /// Name of the scheme in `components.paginationSchemes`. + pub scheme: String, + /// Per-operation overrides, deep-merged onto the named scheme. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overrides: Option, + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// `components.paginationSchemes`, keyed by scheme name. +pub type PaginationSchemesMap = IndexMap; + +/// Everything derivable from a server response about the state of pagination. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PaginationResponseState { + /// Token to request the next page with, if the response carried one. + pub next_page_token: Option, + /// URL of the next page, if the response carried one. + pub next_link: Option, + /// The page number this response represents. + pub current_page: Option, + /// Total number of items across all pages. + pub total_count: Option, + /// Total number of pages. + pub total_pages: Option, + /// Number of items per page. + pub page_size: Option, + /// Whether another page exists after this one. + pub has_next_page: bool, +} + +/// Query parameters to send for a single page request. +pub type PaginationQuery = IndexMap; diff --git a/integrations/localthought/syncables/src/pagination/validate.rs b/integrations/localthought/syncables/src/pagination/validate.rs new file mode 100644 index 0000000000..faa1ae1b18 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/validate.rs @@ -0,0 +1,112 @@ +//! Validating a pagination scheme against the extension's own rules (§9). + +use indexmap::IndexMap; + +use super::types::{PaginationSchemeObject, RequestFieldObject, ResponseFieldObject}; + +const SCHEME_TYPES: [&str; 3] = ["pageNumber", "pageToken", "nextLink"]; +const REQUEST_ROLES: [&str; 5] = ["page", "pageSize", "offset", "pageToken", "cursor"]; +const RESPONSE_ROLES: [&str; 7] = [ + "nextPageToken", + "nextCursor", + "nextLink", + "totalCount", + "totalPages", + "pageSize", + "currentPage", +]; + +fn is_extension_key(key: &str) -> bool { + key.starts_with("x-") +} + +fn check_request_roles( + errors: &mut Vec, + path: &str, + section: &str, + fields: Option<&IndexMap>, +) { + for (field_name, field) in fields.into_iter().flatten() { + if let Some(role) = &field.role { + if !is_extension_key(role) && !REQUEST_ROLES.contains(&role.as_str()) { + errors.push(format!( + "{path}.request.{section}.{field_name}.role is not a valid request role (got \"{role}\")" + )); + } + } + } +} + +fn check_response_roles( + errors: &mut Vec, + path: &str, + section: &str, + fields: Option<&IndexMap>, +) { + for (field_name, field) in fields.into_iter().flatten() { + if let Some(role) = &field.role { + if !is_extension_key(role) && !RESPONSE_ROLES.contains(&role.as_str()) { + errors.push(format!( + "{path}.response.{section}.{field_name}.role is not a valid response role (got \"{role}\")" + )); + } + } + } +} + +/// Validates a single scheme against spec section 9. +/// +/// Returns a list of human-readable errors (each naming the offending +/// location), empty if the scheme is valid. Schemes with errors are +/// excluded from auto-detection rather than thrown on — one malformed +/// scheme in a document shouldn't prevent using the others. +pub fn validate_pagination_scheme(name: &str, scheme: &PaginationSchemeObject) -> Vec { + let mut errors = Vec::new(); + let path = format!("paginationSchemes.{name}"); + + let declared_type = scheme + .scheme_type + .as_str() + .map_or_else(|| scheme.scheme_type.to_string(), str::to_string); + if !SCHEME_TYPES.contains(&declared_type.as_str()) { + errors.push(format!( + "{path}.type must be one of pageNumber, pageToken, or nextLink (got \"{declared_type}\")" + )); + } + + if scheme.request.is_none() && scheme.response.is_none() { + errors.push(format!( + "{path} must define at least one of \"request\" or \"response\"" + )); + } + + let request = scheme.request.as_ref(); + check_request_roles( + &mut errors, + &path, + "queryParameters", + request.and_then(|r| r.query_parameters.as_ref()), + ); + check_request_roles( + &mut errors, + &path, + "bodyFields", + request.and_then(|r| r.body_fields.as_ref()), + ); + + let response = scheme.response.as_ref(); + check_response_roles( + &mut errors, + &path, + "bodyFields", + response.and_then(|r| r.body_fields.as_ref()), + ); + check_response_roles( + &mut errors, + &path, + "headers", + response.and_then(|r| r.headers.as_ref()), + ); + + errors +} diff --git a/integrations/localthought/syncables/src/resources/discover.rs b/integrations/localthought/syncables/src/resources/discover.rs new file mode 100644 index 0000000000..44fb66e66c --- /dev/null +++ b/integrations/localthought/syncables/src/resources/discover.rs @@ -0,0 +1,106 @@ +//! Pairing collection paths with their item paths. +//! +//! This pairing is the core concept the rest of the crate builds on: a +//! "resource" only exists where a collection path (`/pets`) has a direct +//! item-path child (`/pets/{petId}`). Paths without such a pairing +//! (health checks, one-off actions) are not resources and are handled +//! separately as raw request/response passthroughs. + +use indexmap::IndexMap; + +use crate::openapi::types::PathItem; + +/// HTTP method a background `update` sends. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum UpdateMethod { + /// Full replace. + Put, + /// Partial update, for APIs (e.g. GitHub) that expose no `PUT`. + Patch, +} + +impl UpdateMethod { + /// The method name as it goes on the wire. + pub fn as_str(self) -> &'static str { + match self { + Self::Put => "PUT", + Self::Patch => "PATCH", + } + } +} + +/// A collection path paired with its item path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResourceRoute { + /// The collection path, e.g. `/pets`. + pub collection_path: String, + /// The item path, e.g. `/pets/{petId}`. + pub item_path: String, + /// Name of the item path's trailing variable, e.g. `petId`. + pub item_param: String, + /// HTTP method a background `update` sends. + /// + /// Derived from the item path's declared operations: `PUT` when the + /// item path has a `put` operation (a full replace), otherwise + /// `PATCH` when it only offers a partial update. + pub update_method: UpdateMethod, +} + +/// Chooses the update method for an item path from the operations it +/// declares: prefer `PUT` (replace) when present, fall back to `PATCH` +/// when the path only offers a partial update, and default to `PUT` when +/// neither is declared. +fn update_method_for(paths: &IndexMap, item_path: &str) -> UpdateMethod { + match paths.get(item_path) { + Some(item) if item.put.is_none() && item.patch.is_some() => UpdateMethod::Patch, + _ => UpdateMethod::Put, + } +} + +/// Pairs each collection path with its item path so mock server and +/// client can apply CRUD semantics. +pub fn discover_resources(paths: &IndexMap) -> Vec { + let all_paths: Vec<&String> = paths.keys().collect(); + + let mut resources = Vec::new(); + for collection_path in all_paths.iter().filter(|p| !is_item_path(p)) { + let item_path = all_paths + .iter() + .find(|p| is_item_path(p) && is_direct_child(collection_path, p)); + if let Some(item_path) = item_path { + resources.push(ResourceRoute { + collection_path: (*collection_path).clone(), + item_path: (*item_path).clone(), + item_param: extract_param_name(item_path), + update_method: update_method_for(paths, item_path), + }); + } + } + resources +} + +fn is_item_path(path: &str) -> bool { + path.split('/') + .rfind(|s| !s.is_empty()) + .is_some_and(|last| last.starts_with('{') && last.ends_with('}')) +} + +fn extract_param_name(item_path: &str) -> String { + let last = item_path.split('/').rfind(|s| !s.is_empty()).unwrap_or(""); + last.get(1..last.len().saturating_sub(1)) + .unwrap_or("") + .to_string() +} + +fn is_direct_child(collection_path: &str, candidate: &str) -> bool { + let collection: Vec<&str> = collection_path + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + let candidate: Vec<&str> = candidate.split('/').filter(|s| !s.is_empty()).collect(); + candidate.len() == collection.len() + 1 + && collection + .iter() + .enumerate() + .all(|(index, segment)| candidate.get(index) == Some(segment)) +} diff --git a/integrations/localthought/syncables/src/resources/mod.rs b/integrations/localthought/syncables/src/resources/mod.rs new file mode 100644 index 0000000000..03d24596b1 --- /dev/null +++ b/integrations/localthought/syncables/src/resources/mod.rs @@ -0,0 +1,3 @@ +//! Turning `document.paths` into the list of syncable resources. + +pub mod discover; diff --git a/integrations/localthought/syncables/src/routing/mod.rs b/integrations/localthought/syncables/src/routing/mod.rs new file mode 100644 index 0000000000..65ca30eb38 --- /dev/null +++ b/integrations/localthought/syncables/src/routing/mod.rs @@ -0,0 +1,3 @@ +//! Matching incoming request paths against OpenAPI path templates. + +pub mod router; diff --git a/integrations/localthought/syncables/src/routing/router.rs b/integrations/localthought/syncables/src/routing/router.rs new file mode 100644 index 0000000000..8ded223da7 --- /dev/null +++ b/integrations/localthought/syncables/src/routing/router.rs @@ -0,0 +1,59 @@ +//! Matching a concrete request path against OpenAPI path templates. + +use std::borrow::Cow; + +use indexmap::IndexMap; +use percent_encoding::percent_decode_str; + +/// A template that matched, together with the path variables it bound. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RouteMatch { + /// The OpenAPI path template that matched, e.g. `/pets/{petId}`. + pub template: String, + /// Path variables bound by the match, percent-decoded. + pub params: IndexMap, +} + +fn decode(segment: &str) -> String { + match percent_decode_str(segment).decode_utf8() { + Ok(Cow::Borrowed(value)) => value.to_string(), + Ok(Cow::Owned(value)) => value, + Err(_) => segment.to_string(), + } +} + +fn match_template(template: &str, actual: &str) -> Option> { + let template_segments: Vec<&str> = template.split('/').filter(|s| !s.is_empty()).collect(); + let actual_segments: Vec<&str> = actual.split('/').filter(|s| !s.is_empty()).collect(); + if template_segments.len() != actual_segments.len() { + return None; + } + + let mut params = IndexMap::new(); + for (index, template_segment) in template_segments.iter().enumerate() { + let actual_segment = actual_segments.get(index).copied().unwrap_or(""); + if let Some(name) = template_segment + .strip_prefix('{') + .and_then(|s| s.strip_suffix('}')) + { + params.insert(name.to_string(), decode(actual_segment)); + } else if *template_segment != actual_segment { + return None; + } + } + Some(params) +} + +/// Returns the first template in `templates` that matches `actual`. +pub fn find_route>(templates: &[S], actual: &str) -> Option { + for template in templates { + let template = template.as_ref(); + if let Some(params) = match_template(template, actual) { + return Some(RouteMatch { + template: template.to_string(), + params, + }); + } + } + None +} diff --git a/integrations/localthought/syncables/src/sync/client.rs b/integrations/localthought/syncables/src/sync/client.rs new file mode 100644 index 0000000000..237cd80021 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/client.rs @@ -0,0 +1,520 @@ +//! `SyncClient`: the read half of [issue #9](https://github.com/localthought/syncables-rs/issues/9) +//! — "the call reflector-rs makes." Loads the document and its overlays +//! (#2), derives the resource model (#3) and validates the configured +//! constants (#6), derives and stores the ontology (#8) before any record, +//! then walks every managed collection — following pagination to the end +//! (#4) and binding constants (#6) — putting each record into a +//! host-provided [`Storage`] (#7). +//! +//! **Local-first writes are out of scope here.** The issue explicitly +//! allows that: "Steps beyond \[the full read\] can land later." `create`/ +//! `update`/`remove`, the per-record write queue and retry/backoff, and +//! reading back `x-crud`'s `addedFields` from a create response are a +//! separate, later piece of #9. +//! +//! `ClientConfig`/`SyncReport`/`SyncError` are copied field-for-field from +//! the contract [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +//! is already written against, in its `src/syncables.rs`; that module is +//! meant to be deleted once reflector-rs points its `use`s here instead. +//! One deliberate divergence: that stub's `ClientConfig` carries no way to +//! actually reach the network, and `SyncClient::new` takes only a +//! `ClientConfig`. This crate has no HTTP client dependency (mirroring +//! [`crate::client::client`]'s existing design, where +//! [`ApiClientOptions::fetch`](crate::client::client::ApiClientOptions::fetch) +//! is the host's only extension point for making requests), so +//! [`SyncClient::new`] additionally takes a [`Fetch`] implementation. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; + +use indexmap::IndexMap; +use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; +use serde_json::{Map, Value}; + +use crate::client::client::{Fetch, HttpRequest}; +use crate::error::Error; +use crate::openapi::overlay::load_open_api_document_with_overlays; +use crate::openapi::types::{OpenApiDocument, SchemaObject}; +use crate::pagination::autodetect::resolve_effective_scheme; +use crate::pagination::items::locate_items_field; +use crate::pagination::request_builder::{build_query, next_step, PageCursor, PageStep}; +use crate::pagination::response_parser::parse_pagination_state; +use crate::pagination::types::PaginationSchemeObject; + +use super::constants::{bind_url, validate_constants}; +use super::credentials::{base_url, Credentials}; +use super::ontology::derive_ontology; +use super::resource_model::{ + discover_resource_model, ContextProvider, ManagedCollection, ResourceModel, +}; +use super::storage::{Record, Storage, StorageError}; + +/// Everything the engine needs to derive and run a sync. +#[derive(Clone, Debug)] +pub struct ClientConfig { + /// Path to the OpenAPI document describing the API. + pub document: PathBuf, + /// Overlays applied to that document, in order. + pub overlays: Vec, + /// The credential sent to the API. + pub credentials: Credentials, + /// Values bound into the document's path and query parameters. This is + /// what narrows a sync to one issue tracker (`owner`/`repo`) instead of + /// every tracker the credential can reach. + pub constants: BTreeMap, + /// Canonical base URL the derived ontology's terms are minted under, + /// e.g. `https://my-ontologies.com`. No trailing slash. + pub ontology_base_url: String, +} + +/// What one [`SyncClient::sync`] did. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SyncReport { + /// Records read from the API and written to storage, per resource. + pub read: BTreeMap, + /// Ontology terms stored. + pub ontology_terms: usize, + /// Non-fatal problems: one collection failing does not abandon the + /// rest of the sync. + pub errors: Vec, +} + +/// Errors the engine itself raises. +#[derive(Debug)] +pub enum SyncError { + /// The document or its overlays could not be loaded or reconciled. + Document(String), + /// The API rejected or failed a request. + Transport(String), + /// The [`Storage`] the host supplied failed. + Storage(StorageError), + /// This build of the engine has no behaviour behind the given call yet. + NotImplemented(&'static str), +} + +impl std::fmt::Display for SyncError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SyncError::Document(message) => write!(f, "OpenAPI document error: {message}"), + SyncError::Transport(message) => write!(f, "transport error: {message}"), + SyncError::Storage(error) => write!(f, "storage error: {error}"), + SyncError::NotImplemented(what) => write!(f, "not implemented yet: {what}"), + } + } +} + +impl std::error::Error for SyncError {} + +impl From for SyncError { + /// Every failure this crate's document/overlay/resource-model/ontology + /// pipeline raises is, from the engine's point of view, a problem with + /// the document or its configuration. + fn from(error: Error) -> Self { + SyncError::Document(error.to_string()) + } +} + +/// The sync engine. +pub struct SyncClient { + config: ClientConfig, + fetch: Arc, +} + +impl std::fmt::Debug for SyncClient { + /// `Fetch` implementations aren't required to be `Debug`, so this + /// shows the configuration only. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyncClient") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl SyncClient { + /// Builds a client for `config`, reaching the network through `fetch`. + /// + /// Errors if `config.ontology_base_url` is empty — ontology terms need + /// a canonical, resolvable base URL. + pub fn new(config: ClientConfig, fetch: Arc) -> Result { + if config.ontology_base_url.is_empty() { + return Err(SyncError::Document( + "ontology_base_url is required: ontology terms need a canonical, resolvable base URL" + .to_string(), + )); + } + Ok(SyncClient { config, fetch }) + } + + /// The configuration this client was built from. + #[must_use] + pub fn config(&self) -> &ClientConfig { + &self.config + } + + /// Reads everything the document describes — narrowed by + /// [`ClientConfig::constants`] — into `storage`, and stores the + /// ontology derived from the document alongside it, before any record. + /// + /// One collection failing partway through — a 404, an unparseable + /// response, a storage error — is recorded in the returned + /// [`SyncReport::errors`] and does not abandon the rest of the sync. + pub async fn sync(&self, storage: &dyn Storage) -> Result { + let document = load_open_api_document_with_overlays( + self.config.document.as_path(), + &self.config.overlays, + ) + .await?; + self.sync_document(&document, storage).await + } + + /// Sync an already loaded, resolved document without filesystem access. + pub async fn sync_document( + &self, + document: &OpenApiDocument, + storage: &dyn Storage, + ) -> Result { + let model = discover_resource_model(document)?; + validate_constants(document, &model, &self.config.constants)?; + + // Fail before writing anything if there's nowhere to sync from — + // no point minting an ontology for a sync that can't run at all. + let base = base_url(document) + .ok_or_else(|| SyncError::Document("document declares no servers".to_string()))?; + + let ontology = derive_ontology(document)?; + storage + .put_ontology(&ontology) + .await + .map_err(SyncError::Storage)?; + + let mut report = SyncReport { + ontology_terms: ontology.terms.len(), + ..SyncReport::default() + }; + self.walk_all(document, &model, base, storage, &mut report) + .await; + Ok(report) + } + + /// Walks every managed collection in dependency order: a collection + /// whose context parameters are all constants runs once; one with a + /// parent-record-provided parameter (see + /// [`ResourceModel::provider_for`]) runs once per parent record + /// already read, its own records feeding any collection nested under + /// it in turn. + async fn walk_all( + &self, + document: &OpenApiDocument, + model: &ResourceModel, + base: &str, + storage: &dyn Storage, + report: &mut SyncReport, + ) { + let mut records_by_collection: BTreeMap>> = BTreeMap::new(); + let mut pending: Vec<&ManagedCollection> = model.collections.iter().collect(); + + loop { + let mut still_pending = Vec::new(); + let mut progressed = false; + + for collection in pending { + let provider_params: Vec<(&str, &ContextProvider)> = collection + .context_params + .iter() + .filter(|param| !self.config.constants.contains_key(param.as_str())) + .filter_map(|param| { + model + .provider_for(param) + .map(|provider| (param.as_str(), provider)) + }) + .collect(); + + let ready = provider_params + .iter() + .all(|(_, provider)| records_by_collection.contains_key(&provider.collection)); + if !ready { + still_pending.push(collection); + continue; + } + progressed = true; + + let mut collection_records = Vec::new(); + for values in binding_combinations( + &self.config.constants, + &provider_params, + &records_by_collection, + ) { + match self + .walk_collection(document, base, collection, &values) + .await + { + Ok(records) => { + let namespace = collection + .context_params + .iter() + .map(|param| values.get(param).cloned().unwrap_or_default()) + .collect::>() + .join("/"); + for record in &records { + let id = record + .get(&collection.id_field) + .map(json_to_string) + .unwrap_or_default(); + let stored = Record { + namespace: namespace.clone(), + resource: collection.resource.clone(), + id, + value: record.clone(), + }; + if let Err(error) = storage.put(&stored).await { + report.errors.push(format!("{}: {error}", collection.name)); + } + } + collection_records.extend(records); + } + Err(message) => report + .errors + .push(format!("{}: {message}", collection.name)), + } + } + + *report.read.entry(collection.resource.clone()).or_insert(0) += + collection_records.len(); + records_by_collection.insert(collection.name.clone(), collection_records); + } + + if still_pending.is_empty() { + break; + } + if !progressed { + // Unreachable once `validate_constants` has passed against + // the same model, but never spin: report and stop rather + // than loop forever if it somehow is. + for collection in still_pending { + report.errors.push(format!( + "{}: could not resolve its context parameters", + collection.name + )); + } + break; + } + pending = still_pending; + } + } + + /// Fetches every item of one managed collection under `values`, + /// walking every page per its resolved pagination scheme. + async fn walk_collection( + &self, + document: &OpenApiDocument, + base: &str, + collection: &ManagedCollection, + values: &BTreeMap, + ) -> std::result::Result>, String> { + let operation = document + .paths + .get(&collection.collection_url) + .and_then(|item| item.get.as_ref()) + .ok_or_else(|| format!("{} declares no GET operation", collection.collection_url))?; + + let effective = resolve_effective_scheme(document, operation); + let response_schema = operation + .responses + .get("200") + .and_then(|response| response.content.as_ref()) + .and_then(|content| content.get("application/json")) + .and_then(|media| media.schema.as_ref()); + + let path = + bind_url(&collection.collection_url, values).map_err(|error| error.to_string())?; + + let mut items = Vec::new(); + let mut cursor = PageCursor::default(); + let mut pages_fetched = 0usize; + let mut next_url: Option = None; + + loop { + let request_url = if let Some(url) = next_url.take() { + url + } else { + let mut query = collection.list_query.clone(); + if let Some(effective) = &effective { + for (name, value) in build_query(&effective.scheme, &cursor, None) { + query.insert(name, value); + } + } + request_url(base, &path, &query) + }; + + let mut headers = IndexMap::new(); + if let Some(authorization) = self.config.credentials.authorization_header() { + headers.insert("Authorization".to_string(), authorization); + } + let diagnostic_url = redacted_request_url(&request_url); + let response = self + .fetch + .fetch(HttpRequest { + method: "GET".to_string(), + url: request_url, + headers, + body: None, + }) + .await + .map_err(|error| error.to_string())?; + + if !(200..300).contains(&response.status) { + return Err(format!( + "GET {} responded {}", + diagnostic_url, response.status + )); + } + + let body: Value = + serde_json::from_slice(&response.body).map_err(|error| error.to_string())?; + let page_items = response_items( + response_schema, + effective.as_ref().map(|e| &e.scheme), + &body, + )?; + let page_count = u64::try_from(page_items.len()).unwrap_or(u64::MAX); + items.extend(page_items); + pages_fetched += 1; + + let Some(effective) = &effective else { break }; + let items_so_far = u64::try_from(items.len()).unwrap_or(u64::MAX); + let state = parse_pagination_state( + &effective.scheme, + &body, + &response.headers, + Some(items_so_far), + ); + match next_step( + &effective.scheme, + &cursor, + &state, + page_count, + pages_fetched, + ) { + PageStep::Done => break, + PageStep::NextPage(next) => cursor = next, + PageStep::FollowLink(url) => next_url = Some(url), + } + } + + Ok(items) + } +} + +/// Returns a request URL that is useful in an error message without exposing +/// values sent as query parameters. Authentication normally travels in a +/// header, but OpenAPI also permits query-parameter API keys. +fn redacted_request_url(url: &str) -> String { + match url.split_once('?') { + Some((base, _)) => format!("{base}?"), + None => url.to_owned(), + } +} + +/// Every combination of constants plus one parent record's value per +/// `provider_params` entry — the cartesian product across however many +/// ancestor collections `collection` is nested under. Empty +/// `provider_params` yields exactly the constants unchanged (a root +/// collection, walked once); a provider whose collection has no records +/// yet read yields no combinations at all (nothing to nest under). +fn binding_combinations( + constants: &BTreeMap, + provider_params: &[(&str, &ContextProvider)], + records_by_collection: &BTreeMap>>, +) -> Vec> { + let mut combinations = vec![constants.clone()]; + for (param, provider) in provider_params { + let empty = Vec::new(); + let parent_records = records_by_collection + .get(&provider.collection) + .unwrap_or(&empty); + let mut next = Vec::new(); + for combination in &combinations { + for record in parent_records { + let Some(value) = record.get(&provider.field) else { + continue; + }; + let mut extended = combination.clone(); + extended.insert((*param).to_string(), json_to_string(value)); + next.push(extended); + } + } + combinations = next; + } + combinations +} + +/// The array of item objects in a paginated response: the whole body when +/// its schema is itself an array, otherwise the property +/// [`locate_items_field`] finds. +fn response_items( + schema: Option<&SchemaObject>, + scheme: Option<&PaginationSchemeObject>, + body: &Value, +) -> std::result::Result>, String> { + let is_bare_array = schema.and_then(|s| s.schema_type.as_deref()) == Some("array"); + let array = if is_bare_array { + body.as_array() + } else if let Some(field) = locate_items_field(schema, scheme) { + body.get(&field).and_then(Value::as_array) + } else { + body.as_array() + }; + let array = + array.ok_or_else(|| "could not locate the items array in the response".to_string())?; + Ok(array + .iter() + .filter_map(|item| item.as_object().cloned()) + .collect()) +} + +/// Renders a JSON scalar as a plain string, for use as a record id or a +/// namespace segment. +fn json_to_string(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +/// Characters percent-encoded in a query string component — the reserved +/// characters that would otherwise be mistaken for delimiters (`&`, `=`, +/// `#`, `+`), not every non-alphanumeric character: a query parameter name +/// like `per_page` should round-trip as `per_page`, not `per%5Fpage`. +const QUERY_COMPONENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'%') + .add(b'&') + .add(b'\'') + .add(b'+') + .add(b'<') + .add(b'>') + .add(b'=') + .add(b'`'); + +/// Builds the absolute URL for one page request. +fn request_url(base: &str, path: &str, query: &IndexMap) -> String { + let mut url = format!("{base}{path}"); + if !query.is_empty() { + let pairs: Vec = query + .iter() + .map(|(name, value)| { + format!( + "{}={}", + utf8_percent_encode(name, QUERY_COMPONENT), + utf8_percent_encode(value, QUERY_COMPONENT) + ) + }) + .collect(); + url.push('?'); + url.push_str(&pairs.join("&")); + } + url +} diff --git a/integrations/localthought/syncables/src/sync/constants.rs b/integrations/localthought/syncables/src/sync/constants.rs new file mode 100644 index 0000000000..0c13c38994 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/constants.rs @@ -0,0 +1,129 @@ +//! Binding configured constants into a resource model's path templates — +//! what scopes a sync to one issue tracker (`owner`/`repo`) instead of +//! every tracker the credential can reach, per +//! [issue #6](https://github.com/localthought/syncables-rs/issues/6). +//! +//! A GitHub token can read every repository its owner can reach; narrowing +//! that down has to be part of the sync's configuration, not a filter +//! applied after the fact — fetching every tracker and discarding most of +//! it is both slow and a data-handling problem. + +use std::collections::{BTreeMap, HashSet}; + +use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; + +use crate::error::{Error, Result}; +use crate::openapi::types::OpenApiDocument; + +use super::resource_model::{path_variables, ResourceModel}; + +/// Characters percent-encoded when a value is substituted into a URL path +/// segment — everything outside what's safe unescaped in a path segment, +/// mirroring the `url` crate's `PATH_SEGMENT_ENCODE_SET` (not pulled in as +/// a dependency for one constant). +const PATH_SEGMENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'<') + .add(b'>') + .add(b'?') + .add(b'`') + .add(b'{') + .add(b'}') + .add(b'/') + .add(b'%'); + +/// Every parameter name declared by some operation in the document — path +/// or query, on any method — the set a constant is allowed to name. +fn declared_parameter_names(document: &OpenApiDocument) -> HashSet<&str> { + document + .paths + .values() + .flat_map(|item| { + [ + item.get.as_ref(), + item.put.as_ref(), + item.post.as_ref(), + item.patch.as_ref(), + item.delete.as_ref(), + ] + }) + .flatten() + .flat_map(|operation| operation.parameters.iter().flatten()) + .map(|parameter| parameter.name.as_str()) + .collect() +} + +/// A path variable is resolvable if a constant supplies it, or a parent +/// record's context provider does. +fn is_resolvable(model: &ResourceModel, constants: &BTreeMap, param: &str) -> bool { + constants.contains_key(param) || model.provider_for(param).is_some() +} + +/// Validates `constants` against `document` and `model`, before any request +/// is made: +/// +/// - every constant must name a parameter the document actually declares +/// ([`Error::UnknownConstant`]) — a typo'd key would otherwise silently +/// sync nothing, or scope the sync far wider than intended; +/// - every path variable a managed collection needs — in its list URL, or +/// in its own item URL beyond what its identity binding already supplies +/// — must be resolvable by a constant or a parent record's provider +/// ([`Error::UnboundContextParam`]). +pub fn validate_constants( + document: &OpenApiDocument, + model: &ResourceModel, + constants: &BTreeMap, +) -> Result<()> { + let declared = declared_parameter_names(document); + for key in constants.keys() { + if !declared.contains(key.as_str()) { + return Err(Error::UnknownConstant(key.clone())); + } + } + + for collection in &model.collections { + for param in &collection.context_params { + if !is_resolvable(model, constants, param) { + return Err(Error::UnboundContextParam(param.clone())); + } + } + for param in path_variables(&collection.item_url) { + if collection.identity_params.contains(¶m) { + continue; + } + if !is_resolvable(model, constants, ¶m) { + return Err(Error::UnboundContextParam(param)); + } + } + } + Ok(()) +} + +/// Substitutes every `{param}` in `template` with its percent-encoded value +/// from `values`. +/// +/// Errors with [`Error::UnboundContextParam`] if `template` names a +/// variable `values` has no entry for — [`validate_constants`] is meant to +/// have already ruled this out for every template the resource model +/// declares, so this only fires on a template built outside that check. +pub fn bind_url(template: &str, values: &BTreeMap) -> Result { + let mut bound = String::with_capacity(template.len()); + let mut rest = template; + while let Some(start) = rest.find('{') { + bound.push_str(&rest[..start]); + let Some(end) = rest[start..].find('}') else { + bound.push_str(&rest[start..]); + return Ok(bound); + }; + let name = &rest[start + 1..start + end]; + let value = values + .get(name) + .ok_or_else(|| Error::UnboundContextParam(name.to_string()))?; + bound.push_str(&utf8_percent_encode(value, PATH_SEGMENT).to_string()); + rest = &rest[start + end + 1..]; + } + bound.push_str(rest); + Ok(bound) +} diff --git a/integrations/localthought/syncables/src/sync/credentials.rs b/integrations/localthought/syncables/src/sync/credentials.rs new file mode 100644 index 0000000000..9a89c23368 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/credentials.rs @@ -0,0 +1,70 @@ +//! Credentials presented to the API being reflected, and the base URL they +//! are presented against. +//! +//! Ported from `Credentials` in the syncables-rs API contract mirrored at +//! [`localthought/reflector-rs`'s `src/syncables.rs`](https://github.com/localthought/reflector-rs/blob/main/src/syncables.rs), +//! and from `StaticTokenManager` in +//! [`localthought/reflector`](https://github.com/localthought/reflector)'s +//! `src/oauth/static-token.ts`, which is what actually attaches the header +//! and retargets a request at the real API base in the TypeScript +//! original. + +use crate::openapi::types::OpenApiDocument; + +/// Credentials presented to the API being reflected. +/// +/// Only a static bearer token is modelled so far: that is what the GitHub +/// auth overlay's `http`/`bearer` security scheme asks for. An interactive +/// OAuth profile — derived from the document the way the TypeScript +/// Reflector derives Google Calendar's — is tracked separately (see +/// [issue #5](https://github.com/localthought/syncables-rs/issues/5)). +#[derive(Clone, PartialEq, Eq)] +pub enum Credentials { + /// Sent as `Authorization: Bearer `. + Bearer(String), + /// No credential — only useful against a public, unauthenticated API. + Anonymous, +} + +impl std::fmt::Debug for Credentials { + /// Never renders the secret, so `{:?}` on a config that holds one is + /// safe to log. A host logging its resolved configuration at startup is + /// exactly the scenario this guards against. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Credentials::Bearer(_) => f.write_str("Bearer()"), + Credentials::Anonymous => f.write_str("Anonymous"), + } + } +} + +impl Credentials { + /// The `Authorization` header value to send with every request, if any. + /// + /// [`Credentials::Anonymous`] sends no `Authorization` header at all — + /// distinct in principle from an empty bearer token, which would still + /// be presented. + #[must_use] + pub fn authorization_header(&self) -> Option { + match self { + Credentials::Bearer(token) => Some(format!("Bearer {token}")), + Credentials::Anonymous => None, + } + } +} + +/// The API's base URL, read from the document's `servers` — never from +/// separate configuration, so a document can't be pointed at the wrong host +/// by a configuration mismatch. +/// +/// Returns the first declared server, matching the TypeScript original's +/// `StaticTokenManager`/`TokenManager`, which likewise target whatever +/// `servers[0]` names. `None` if the document declares no servers at all. +#[must_use] +pub fn base_url(document: &OpenApiDocument) -> Option<&str> { + document + .servers + .as_ref()? + .first() + .map(|server| server.url.as_str()) +} diff --git a/integrations/localthought/syncables/src/sync/mod.rs b/integrations/localthought/syncables/src/sync/mod.rs new file mode 100644 index 0000000000..8d3819ded1 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/mod.rs @@ -0,0 +1,24 @@ +//! The sync engine surface: `SyncClient`, `ClientConfig`, [`storage::Storage`] +//! and friends. +//! +//! This is a *different* surface from the crate's existing +//! [`crate::client`]/[`crate::mock_server`] pair (a port of the +//! TypeScript `syncables` package). It is new scope, tracked by +//! [localthought/syncables-rs#1](https://github.com/localthought/syncables-rs/issues/1) +//! and the issues under it: a generic engine that reads an OpenAPI +//! document plus a resource model derived from it, and syncs records into +//! a host-provided [`storage::Storage`] implementation. +//! [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +//! is the first intended host. +//! +//! [`storage`] (issue #7), [`credentials`] (issue #5), [`resource_model`] +//! (issue #3), [`constants`] (issue #6), [`ontology`] (issue #8) and +//! [`client`] (issue #9 — the read half; local-first writes are a +//! follow-up) exist so far. + +pub mod client; +pub mod constants; +pub mod credentials; +pub mod ontology; +pub mod resource_model; +pub mod storage; diff --git a/integrations/localthought/syncables/src/sync/ontology.rs b/integrations/localthought/syncables/src/sync/ontology.rs new file mode 100644 index 0000000000..e8094f6a5d --- /dev/null +++ b/integrations/localthought/syncables/src/sync/ontology.rs @@ -0,0 +1,273 @@ +//! Deriving an Atomic Data ontology from the document's `crudResources` +//! extension, per [issue #8](https://github.com/localthought/syncables-rs/issues/8): +//! a Class per resource and a Property per field of that resource's schema. +//! +//! **This crate must not depend on `atomic_lib`.** Terms are handed over as +//! a neutral description — a path, a kind, a shortname, a description, an +//! optional datatype URL, and cross-references by path — not as Atomic Data +//! `Resource`s. Rendering those into `Resource`s at their public/internal +//! subjects is the host's job: see `AtomicStorage` in +//! [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs)'s +//! `src/store.rs`, which is written against exactly this shape. +//! +//! The engine never invents an origin for a term's identity: [`Ontology`] +//! and [`OntologyTerm`] carry only relative paths (no leading slash), which +//! the host mints under its own `ClientConfig::ontology_base_url`. + +use std::collections::HashSet; + +use indexmap::IndexMap; +use serde_json::Value; + +use crate::error::{Error, Result}; +use crate::openapi::types::{OpenApiDocument, SchemaObject}; + +use super::resource_model::{crud_resources, CrudResourceObject}; + +const DATATYPE_STRING: &str = "https://atomicdata.dev/datatypes/string"; +const DATATYPE_INTEGER: &str = "https://atomicdata.dev/datatypes/integer"; +const DATATYPE_FLOAT: &str = "https://atomicdata.dev/datatypes/float"; +const DATATYPE_BOOLEAN: &str = "https://atomicdata.dev/datatypes/boolean"; +const DATATYPE_TIMESTAMP: &str = "https://atomicdata.dev/datatypes/timestamp"; +const DATATYPE_DATE: &str = "https://atomicdata.dev/datatypes/date"; + +/// Whether a term describes a Class or a Property. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TermKind { + /// A resource, e.g. `issue`. + Class, + /// A field of a resource's schema, e.g. `title`. + Property, +} + +/// One term of the ontology derived from an OpenAPI document. +/// +/// Paths are relative to a host-supplied base URL and are the term's +/// identity: `github-issues/property/title` is published (by the host, not +/// this crate) as e.g. `https://my-ontologies.com/github-issues/property/title`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OntologyTerm { + /// Path under the ontology's own path, without a leading slash. + pub path: String, + /// Whether this is a Class or a Property. + pub kind: TermKind, + /// Atomic Data shortname — lowercase, `-`-separated. + pub shortname: String, + /// Human-readable description. + pub description: String, + /// For a Property: the Atomic Data datatype URL its values carry. + /// `None` for a Class, or for a Property whose schema type this crate's + /// mapping can't place — omitted rather than guessed; the host falls + /// back to inferring from the JSON value. + pub datatype: Option, + /// For a Class: the paths (or absolute URLs) of its required + /// properties, from the schema's `required` list. + pub requires: Vec, + /// For a Class: the paths (or absolute URLs) of its recommended + /// (present but not required) properties. + pub recommends: Vec, +} + +/// The ontology derived from one OpenAPI document. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Ontology { + /// Path of the ontology resource itself, e.g. `github-issues`. + pub path: String, + /// Atomic Data shortname for the ontology itself. + pub shortname: String, + /// Human-readable description. + pub description: String, + /// Every term the ontology declares, Classes and Properties mixed, in + /// the order their resources appear in `crudResources`. + pub terms: Vec, +} + +/// Normalizes an OpenAPI name into the Atomic Data shortname used by the +/// generated ontology: lowercase, `-`-separated, with every run of +/// non-alphanumeric characters collapsed to one `-` (`state_reason` → +/// `state-reason`, `updated_at` → `updated-at`). +/// +/// Hosts use this when matching raw API record keys or resource names to the +/// ontology terms returned by [`derive_ontology`]. +pub fn ontology_shortname(name: &str) -> String { + let mut slug = String::with_capacity(name.len()); + let mut pending_dash = false; + for ch in name.chars() { + if ch.is_ascii_alphanumeric() { + if pending_dash && !slug.is_empty() { + slug.push('-'); + } + pending_dash = false; + slug.push(ch.to_ascii_lowercase()); + } else { + pending_dash = true; + } + } + slug +} + +/// Claims `original`'s slug in `claimed`, reusing the existing slug if +/// `original` already claimed one (properties are shared across every +/// resource that has a same-named field) and erroring if a *different* +/// name has already claimed the same slug. +fn claim_shortname(claimed: &mut IndexMap, original: &str) -> Result { + let candidate = ontology_shortname(original); + match claimed.get(&candidate) { + Some(existing) if existing == original => Ok(candidate), + Some(existing) => Err(Error::ShortnameCollision { + shortname: candidate, + first: existing.clone(), + second: original.to_string(), + }), + None => { + claimed.insert(candidate.clone(), original.to_string()); + Ok(candidate) + } + } +} + +/// Maps a property schema's `type`/`format` to an Atomic Data datatype URL. +/// A type this mapping doesn't recognize is `None` rather than guessed. +fn datatype_url(schema: &SchemaObject) -> Option { + match (schema.schema_type.as_deref(), schema.format.as_deref()) { + (Some("string"), Some("date-time")) => Some(DATATYPE_TIMESTAMP.to_string()), + (Some("string"), Some("date")) => Some(DATATYPE_DATE.to_string()), + (Some("string"), _) => Some(DATATYPE_STRING.to_string()), + (Some("integer"), _) => Some(DATATYPE_INTEGER.to_string()), + (Some("number"), _) => Some(DATATYPE_FLOAT.to_string()), + (Some("boolean"), _) => Some(DATATYPE_BOOLEAN.to_string()), + _ => None, + } +} + +/// A property schema's own `description`, if the document gives one — not +/// a typed field on [`SchemaObject`], so it's read from the catch-all. +fn schema_description(schema: &SchemaObject) -> Option { + schema + .extensions + .get("description")? + .as_str() + .map(str::to_string) +} + +/// Resolves a `crudResources..schema` to the [`SchemaObject`] it +/// names. The overlay declares it as a `$ref` (`{ $ref: '#/components/schemas/issue' }`) +/// — added to the document *after* `resolve_refs` already ran (overlays +/// apply on top of an already-resolved document), so it's never inlined +/// automatically and has to be resolved here by name instead. An inline +/// schema (no `$ref`) is also accepted. +fn resource_schema( + document: &OpenApiDocument, + resource: &CrudResourceObject, +) -> Option { + let raw = resource.extensions.get("schema")?; + if let Some(pointer) = raw.get("$ref").and_then(Value::as_str) { + let name = pointer.strip_prefix("#/components/schemas/")?; + document + .components + .as_ref()? + .schemas + .as_ref()? + .get(name) + .cloned() + } else { + serde_json::from_value(raw.clone()).ok() + } +} + +/// Derives the ontology from `document`'s `components.crudResources`: one +/// Class per resource, and one Property per field of that resource's +/// schema — shared across every resource that has a same-named field, +/// rather than minted again per resource. +/// +/// Returns [`Error::NoCrudResources`] if the document declares none, and +/// [`Error::ShortnameCollision`] if two differently-named resources or +/// fields would normalize to the same slug. +pub fn derive_ontology(document: &OpenApiDocument) -> Result { + let resources = crud_resources(document)?; + + let title = document.info.title.trim(); + let ontology_path = if title.is_empty() { + "ontology".to_string() + } else { + ontology_shortname(title) + }; + let description = if title.is_empty() { + "Derived from an OpenAPI document.".to_string() + } else { + format!("Derived from the \"{title}\" OpenAPI document.") + }; + + let mut terms: Vec = Vec::new(); + let mut class_shortnames = IndexMap::new(); + let mut property_shortnames = IndexMap::new(); + // shortname -> index into `terms`, so a field shared across resources + // reuses its one Property term instead of minting a duplicate. + let mut property_terms: IndexMap = IndexMap::new(); + + for (resource_name, resource) in &resources { + let class_shortname = claim_shortname(&mut class_shortnames, resource_name)?; + let schema = resource_schema(document, resource); + + let required: HashSet<&str> = schema + .as_ref() + .and_then(|s| s.required.as_deref()) + .into_iter() + .flatten() + .map(String::as_str) + .collect(); + + let mut requires = Vec::new(); + let mut recommends = Vec::new(); + for (field_name, field_schema) in schema.iter().flat_map(|s| s.properties.iter()).flatten() + { + // Validate the shortname (and any collision) before ever + // consulting `property_terms`, so two different field names + // that happen to normalize the same way can never be silently + // merged into one reused term. + let shortname = claim_shortname(&mut property_shortnames, field_name)?; + let path = if let Some(&index) = property_terms.get(&shortname) { + terms[index].path.clone() + } else { + let path = format!("{ontology_path}/property/{shortname}"); + property_terms.insert(shortname.clone(), terms.len()); + terms.push(OntologyTerm { + path: path.clone(), + kind: TermKind::Property, + shortname, + description: schema_description(field_schema) + .unwrap_or_else(|| format!("`{field_name}` of `{resource_name}`.")), + datatype: datatype_url(field_schema), + requires: Vec::new(), + recommends: Vec::new(), + }); + path + }; + if required.contains(field_name.as_str()) { + requires.push(path); + } else { + recommends.push(path); + } + } + + terms.push(OntologyTerm { + path: format!("{ontology_path}/class/{class_shortname}"), + kind: TermKind::Class, + shortname: class_shortname, + description: resource + .description + .clone() + .unwrap_or_else(|| format!("The `{resource_name}` resource.")), + datatype: None, + requires, + recommends, + }); + } + + Ok(Ontology { + path: ontology_path.clone(), + shortname: ontology_path, + description, + terms, + }) +} diff --git a/integrations/localthought/syncables/src/sync/resource_model.rs b/integrations/localthought/syncables/src/sync/resource_model.rs new file mode 100644 index 0000000000..829619031e --- /dev/null +++ b/integrations/localthought/syncables/src/sync/resource_model.rs @@ -0,0 +1,426 @@ +//! Deriving the resource model from `components.crudResources` — the +//! [CRUD Causality Extension](https://github.com/pondersource/openapi-extensions/tree/main/spec/crud-causality) +//! overlay's contribution to a document, and the `x-crud` block it adds to +//! each operation. +//! +//! Ported from `discoverResourceModel` in +//! [`localthought/reflector`](https://github.com/localthought/reflector)'s +//! `src/sync/resources.ts`, which is the reference for the traversal below +//! (a resource's identity binding, a collection's context parameters, and +//! how a nested collection's parent is resolved). That file also derives a +//! client-generated-id policy for Google Calendar's resources; GitHub's +//! `addedFields` are all server-assigned, so this port doesn't carry that +//! part over — see [issue #3](https://github.com/localthought/syncables-rs/issues/3). + +use indexmap::IndexMap; +use serde::de::Error as _; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; + +use crate::error::{Error, Result}; +use crate::openapi::types::{JsonMap, OpenApiDocument, OperationObject, SchemaObject}; + +// --- The raw `crudResources` shape, as the overlay declares it ------------ + +/// One resource declared under `components.crudResources`, keyed by its +/// resource name (e.g. `issue`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CrudResourceObject { + /// Human-readable description of the resource. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// How a single item of this resource is addressed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identity: Option, + /// The collections that list, and are written through, this resource. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collections: Option>, + /// Any other key on the resource, notably `schema`. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// A resource's single-item URL template and how its path variables map to +/// record fields. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResourceIdentityObject { + /// Single-item URL template, e.g. `/repos/{owner}/{repo}/issues/{issue_number}`. + #[serde(rename = "urlTemplate")] + pub url_template: String, + /// Path variable to record field, e.g. `issue_number` binds to `number` + /// — the resource's URL identity need not be the payload's own `id`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bindings: Option>, + /// Any other key on the identity. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One entry of a [`ResourceIdentityObject::bindings`] map. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct IdentityBindingObject { + /// The record field the path variable is read from (and reconciled + /// against, on write). + pub field: String, + /// Any other key on the binding. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One entry of a [`CrudResourceObject::collections`] map. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResourceCollectionObject { + /// Collection URL template, e.g. `/repos/{owner}/{repo}/issues`. + #[serde(rename = "urlTemplate")] + pub url_template: String, + /// Fixed query parameters added to every list request against this + /// collection — e.g. GitHub's issues list returns only open issues by + /// default, so the overlay declares `{ state: all }` to include closed + /// ones too. + #[serde( + rename = "x-list-query", + default, + skip_serializing_if = "Option::is_none" + )] + pub list_query: Option>, + /// Any other key on the collection. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// Reads `document.components.crudResources`, keyed by resource name. +/// +/// Returns [`Error::NoCrudResources`] if the document declares none — the +/// CRUD-causality overlay hasn't been applied. +pub(super) fn crud_resources( + document: &OpenApiDocument, +) -> Result> { + let raw = document + .components + .as_ref() + .and_then(|components| components.extensions.get("crudResources")) + .ok_or(Error::NoCrudResources)?; + serde_json::from_value(raw.clone()).map_err(Error::from) +} + +// --- The derived resource model -------------------------------------------- + +/// One resource collection the sync engine can walk, derived from one +/// `crudResources..collections.` entry and its resource's +/// `identity`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedCollection { + /// The collection's key, e.g. `issues`. + pub name: String, + /// The resource key in `crudResources`, e.g. `issue`. + pub resource: String, + /// Collection URL template, e.g. `/repos/{owner}/{repo}/issues`. + pub collection_url: String, + /// Single-item URL template (the resource's identity), e.g. + /// `/repos/{owner}/{repo}/issues/{issue_number}`. Empty if the resource + /// declares no `identity`. + pub item_url: String, + /// Record field that carries the item's own id (from the identity + /// binding whose path variable the item URL — not the collection + /// URL — uses), usually `id` but not always: GitHub's issues are + /// addressed by `number`. + pub id_field: String, + /// Path variables in `collection_url` that must be supplied by a + /// parent record or a configured constant (see + /// [issue #6](https://github.com/localthought/syncables-rs/issues/6)). + pub context_params: Vec, + /// Path variables in `item_url` that this resource's own identity + /// binding supplies from the record itself (e.g. `issue_number`, from + /// `number`) — not from a constant or a parent record. + pub identity_params: Vec, + /// Fixed query parameters to add to the list request, from the + /// collection's `x-list-query`. + pub list_query: IndexMap, +} + +/// How a collection's context variable is filled: enumerate `collection` +/// and read `field` off each of its records. E.g. `issue_number` is +/// provided by listing `issues` and reading each record's `number`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContextProvider { + /// The collection to enumerate. + pub collection: String, + /// The record field whose value fills the context variable. + pub field: String, +} + +/// The resource model derived from `components.crudResources`: every +/// managed collection, and how a nested collection's unresolved context +/// variable is resolved from a parent's own records. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ResourceModel { + /// Every managed collection, in document order. + pub collections: Vec, + providers: IndexMap, +} + +impl ResourceModel { + /// The managed collection named `name`, if any. + #[must_use] + pub fn by_name(&self, name: &str) -> Option<&ManagedCollection> { + self.collections + .iter() + .find(|collection| collection.name == name) + } + + /// The collection+field that supplies values for context path variable + /// `param`, if any resource's identity binds it. + #[must_use] + pub fn provider_for(&self, param: &str) -> Option<&ContextProvider> { + self.providers.get(param) + } +} + +/// The path variables (`{...}` segments) in a URL template, in order. +pub(super) fn path_variables(template: &str) -> Vec { + let mut variables = Vec::new(); + let mut rest = template; + while let Some(start) = rest.find('{') { + let Some(end) = rest[start..].find('}') else { + break; + }; + variables.push(rest[start + 1..start + end].to_string()); + rest = &rest[start + end + 1..]; + } + variables +} + +/// Whether any of the resource's own collection URLs contains `{param}` — +/// used to tell an identity binding that names the item's own id apart +/// from one that merely repeats a parent-scoping context variable. +fn collection_url_has(resource: &CrudResourceObject, param: &str) -> bool { + let needle = format!("{{{param}}}"); + resource + .collections + .iter() + .flatten() + .any(|(_, collection)| collection.url_template.contains(&needle)) +} + +/// Renders a JSON scalar the way GitHub's `x-list-query` values are meant +/// to be sent — as the literal query string value. +fn stringify(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + Value::Null => String::new(), + other => other.to_string(), + } +} + +/// Builds the resource model from `components.crudResources`. Each resource +/// may declare an `identity` (its single-item URL and the binding from a +/// path variable to a record field) and one or more `collections` (list +/// URLs). This emits one [`ManagedCollection`] per declared collection, +/// across every resource — nothing about issues, comments, calendars or +/// events is compiled in. +pub fn discover_resource_model(document: &OpenApiDocument) -> Result { + let resources = crud_resources(document)?; + + let mut collections = Vec::new(); + // resource name -> its first managed collection name (used as an + // enumeration source for that resource's own identity bindings). + let mut collection_of_resource: IndexMap = IndexMap::new(); + // path variable -> (resource, field), from every resource's identity bindings. + let mut bindings: Vec<(String, String, String)> = Vec::new(); + + for (resource_name, resource) in &resources { + let item_url = resource + .identity + .as_ref() + .map(|identity| identity.url_template.as_str()) + .unwrap_or_default(); + + let mut id_field = "id".to_string(); + let mut identity_params = Vec::new(); + if let Some(identity_bindings) = resource + .identity + .as_ref() + .and_then(|identity| identity.bindings.as_ref()) + { + for (param, binding) in identity_bindings { + bindings.push((param.clone(), resource_name.clone(), binding.field.clone())); + // The variable bound in the item URL — as opposed to one + // that merely repeats a parent-scoping context variable + // this resource's own collections also carry — is this + // resource's own id field. + let is_own_identity = item_url.contains(&format!("{{{param}}}")) + && !collection_url_has(resource, param); + if is_own_identity { + id_field.clone_from(&binding.field); + identity_params.push(param.clone()); + } + } + } + + let Some(resource_collections) = &resource.collections else { + continue; + }; + for (name, collection) in resource_collections { + collection_of_resource + .entry(resource_name.clone()) + .or_insert_with(|| name.clone()); + + let list_query = collection + .list_query + .iter() + .flatten() + .map(|(key, value)| (key.clone(), stringify(value))) + .collect(); + + collections.push(ManagedCollection { + name: name.clone(), + resource: resource_name.clone(), + collection_url: collection.url_template.clone(), + item_url: item_url.to_string(), + id_field: id_field.clone(), + context_params: path_variables(&collection.url_template), + identity_params: identity_params.clone(), + list_query, + }); + } + } + + let mut providers = IndexMap::new(); + for (param, resource_name, field) in bindings { + // A resource can supply a context value only if it is itself + // enumerable (has a managed collection). + if let Some(collection) = collection_of_resource.get(&resource_name) { + providers.entry(param).or_insert_with(|| ContextProvider { + collection: collection.clone(), + field, + }); + } + } + + Ok(ResourceModel { + collections, + providers, + }) +} + +// --- `x-crud`, the operation-level half of the extension ------------------- + +/// Which CRUD action an operation performs, from its `x-crud.action`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum CrudAction { + /// Lists a collection's items. + List, + /// Reads one item. + Read, + /// Creates an item. + Create, + /// Updates an item. + Update, + /// Deletes an item. + Delete, +} + +/// One field a create response adds beyond what the client sent — read back +/// from the response and merged into the record. GitHub's issue `number` is +/// server-assigned this way; the client never supplies it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AddedField { + /// The field's schema. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// Where the field's value comes from, e.g. `server`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Any other key on the added field. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// Which collections a write affects, from `memberOf`/`removesFrom`: either +/// a fixed list of collection names, or every collection the resource +/// belongs to (`"*"`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CollectionMembership { + /// Only the named collections. + Named(Vec), + /// Every collection the resource belongs to. + All, +} + +impl Serialize for CollectionMembership { + fn serialize(&self, serializer: S) -> std::result::Result { + match self { + CollectionMembership::Named(names) => names.serialize(serializer), + CollectionMembership::All => serializer.serialize_str("*"), + } + } +} + +impl<'de> Deserialize<'de> for CollectionMembership { + fn deserialize>(deserializer: D) -> std::result::Result { + match Value::deserialize(deserializer)? { + Value::String(marker) if marker == "*" => Ok(CollectionMembership::All), + value @ Value::Array(_) => { + let names = Vec::::deserialize(value).map_err(D::Error::custom)?; + Ok(CollectionMembership::Named(names)) + } + other => Err(D::Error::custom(format!( + "expected \"*\" or an array of collection names, got {other}" + ))), + } + } +} + +/// The `x-crud` annotation on one operation, declaring what it does in +/// terms of the resource model rather than of any particular API. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CrudOperation { + /// Which action the operation performs. + pub action: CrudAction, + /// The resource this operation concerns, e.g. `issue`. + pub resource: String, + /// The collection a `list` operation reads, or a `create` adds to via + /// `url.source: template`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collection: Option, + /// How an `update` is sent — GitHub exposes PATCH and no PUT for + /// issues, so this is `patch` rather than the default PUT semantics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mode: Option, + /// How a PATCH body is interpreted, e.g. `merge`. + #[serde( + rename = "patchFormat", + default, + skip_serializing_if = "Option::is_none" + )] + pub patch_format: Option, + /// Fields a `create` response adds beyond what the client sent. + #[serde(rename = "addedFields", default)] + pub added_fields: IndexMap, + /// Collections a `create` adds the new record to. + #[serde(rename = "memberOf", default)] + pub member_of: Vec, + /// Collections a `delete` removes the record from. + #[serde( + rename = "removesFrom", + default, + skip_serializing_if = "Option::is_none" + )] + pub removes_from: Option, + /// Any other key on the annotation. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// Reads the `x-crud` annotation off one operation, if it declares one. +pub fn crud_operation(operation: &OperationObject) -> Result> { + operation + .extensions + .get("x-crud") + .map(|raw| serde_json::from_value(raw.clone()).map_err(Error::from)) + .transpose() +} diff --git a/integrations/localthought/syncables/src/sync/storage.rs b/integrations/localthought/syncables/src/sync/storage.rs new file mode 100644 index 0000000000..193c44407c --- /dev/null +++ b/integrations/localthought/syncables/src/sync/storage.rs @@ -0,0 +1,221 @@ +//! Host-provided persistence for records read by the sync engine. +//! +//! The engine (built in +//! [issue #9](https://github.com/localthought/syncables-rs/issues/9)) never +//! decides where the local-first copy of a synced dataset lives — it talks +//! to whatever implements [`Storage`]. [`InMemoryStorage`] is a reference +//! implementation used by this crate's own tests; +//! [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs)'s +//! `AtomicStorage` (`src/store.rs`) is the intended real-world +//! implementation, storing records in an Atomic Data `Storelike`. + +use std::collections::HashMap; +use std::error::Error as StdError; +use std::sync::Mutex; + +use async_trait::async_trait; +use thiserror::Error; + +use super::ontology::Ontology; + +/// One record read from or written to a host's [`Storage`]. +/// +/// Deliberately plain JSON: `value` is exactly the record's own fields. +/// The trait must not require `atomic_lib`, or any other host-side model, +/// to construct a `Record` — and this crate itself takes no `atomic_lib` +/// dependency. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Record { + /// Separates otherwise-identical `resource`/`id` pairs that belong to + /// different parents — e.g. every issue's comments share the resource + /// name `issueComment`, so without a namespace two issues' comment + /// sets would overwrite each other. The engine derives the namespace + /// from the resource model's context parameters + /// ([issue #3](https://github.com/localthought/syncables-rs/issues/3)) + /// and is consistent about it across a run, since a host indexes by + /// it. + pub namespace: String, + /// The resource name, e.g. `issue` or `issueComment`. + pub resource: String, + /// The record's id within its `namespace`/`resource`. + pub id: String, + /// The record's own fields, as plain JSON. + pub value: serde_json::Map, +} + +/// Something a host's [`Storage`] implementation failed to do. +/// +/// A `StorageError` is fatal for the one call it came from, not for the +/// sync as a whole: the engine +/// ([issue #9](https://github.com/localthought/syncables-rs/issues/9)) +/// collects errors like this into `SyncReport::errors` and continues +/// syncing the rest of the dataset. +#[derive(Debug, Error)] +#[error("{message}")] +pub struct StorageError { + /// What went wrong, in a form suitable for logging or for surfacing + /// in a `SyncReport::errors` list. + pub message: String, + /// The underlying error from the host's storage backend, if any. + #[source] + pub source: Option>, +} + +impl StorageError { + /// Builds a [`StorageError`] with no underlying source error. + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + source: None, + } + } + + /// Builds a [`StorageError`] that wraps an underlying error from the + /// host's storage backend. + pub fn with_source( + message: impl Into, + source: impl StdError + Send + Sync + 'static, + ) -> Self { + Self { + message: message.into(), + source: Some(Box::new(source)), + } + } +} + +/// Host-provided persistence for the records and ontology a sync produces. +/// +/// The engine talks to this trait rather than deciding for itself where +/// the local-first copy of a dataset lives — [`InMemoryStorage`] is a +/// reference implementation used by this crate's own tests, and +/// `AtomicStorage` in +/// [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +/// (`src/store.rs`) is the intended real-world one, storing records in an +/// Atomic Data `Storelike` under `internal:///` +/// subjects. +/// +/// # Contract +/// +/// [`Storage::put_ontology`] is called once per sync, before any +/// [`Storage::put`] — this is documented behavior for callers of this +/// trait, not something enforced by the trait or by any implementation of +/// it. That ordering is the engine's responsibility +/// ([issue #9](https://github.com/localthought/syncables-rs/issues/9)); +/// `reflector-rs`'s `AtomicStorage` is written against it. +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait Storage: Send + Sync { + /// Inserts or replaces a record, keyed by its `namespace`, `resource` + /// and `id`. + async fn put(&self, record: &Record) -> Result<(), StorageError>; + + /// One record, by `namespace`, `resource` and id — `Ok(None)` if no + /// such record has been [`put`](Storage::put). + async fn get( + &self, + namespace: &str, + resource: &str, + id: &str, + ) -> Result, StorageError>; + + /// Every record held for `namespace`/`resource`. + async fn list(&self, namespace: &str, resource: &str) -> Result, StorageError>; + + /// Removes a record, by `namespace`, `resource` and id. Removing a + /// record that isn't present is not an error. + async fn delete(&self, namespace: &str, resource: &str, id: &str) -> Result<(), StorageError>; + + /// Stores the ontology derived from the synced document. Called once + /// per sync, before any [`put`](Storage::put) — see the trait-level + /// "Contract" section. + async fn put_ontology(&self, ontology: &Ontology) -> Result<(), StorageError>; +} + +/// Key a [`Record`] is stored under: its `namespace`, `resource` and `id`. +type RecordKey = (String, String, String); + +/// A reference [`Storage`] implementation that keeps everything in process +/// memory, analogous to [`crate::client::storage::InMemoryStorageAdapter`] +/// but namespace-keyed. Used by this crate's own tests; hosts that need +/// persistence implement [`Storage`] themselves. +#[derive(Debug, Default)] +pub struct InMemoryStorage { + records: Mutex>, + ontologies: Mutex>, +} + +impl InMemoryStorage { + /// A store with no records and no ontology. + pub fn new() -> Self { + Self::default() + } + + /// Every ontology passed to [`Storage::put_ontology`] so far, in call + /// order. Exposed for tests to assert `put_ontology` was called. + pub fn ontologies(&self) -> Vec { + self.ontologies + .lock() + .expect("ontologies mutex poisoned") + .clone() + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl Storage for InMemoryStorage { + async fn put(&self, record: &Record) -> Result<(), StorageError> { + let key = ( + record.namespace.clone(), + record.resource.clone(), + record.id.clone(), + ); + self.records + .lock() + .expect("records mutex poisoned") + .insert(key, record.clone()); + Ok(()) + } + + async fn get( + &self, + namespace: &str, + resource: &str, + id: &str, + ) -> Result, StorageError> { + let key = (namespace.to_string(), resource.to_string(), id.to_string()); + Ok(self + .records + .lock() + .expect("records mutex poisoned") + .get(&key) + .cloned()) + } + + async fn list(&self, namespace: &str, resource: &str) -> Result, StorageError> { + Ok(self + .records + .lock() + .expect("records mutex poisoned") + .values() + .filter(|record| record.namespace == namespace && record.resource == resource) + .cloned() + .collect()) + } + + async fn delete(&self, namespace: &str, resource: &str, id: &str) -> Result<(), StorageError> { + let key = (namespace.to_string(), resource.to_string(), id.to_string()); + self.records + .lock() + .expect("records mutex poisoned") + .remove(&key); + Ok(()) + } + + async fn put_ontology(&self, ontology: &Ontology) -> Result<(), StorageError> { + self.ontologies + .lock() + .expect("ontologies mutex poisoned") + .push(ontology.clone()); + Ok(()) + } +} diff --git a/integrations/localthought/tsconfig.json b/integrations/localthought/tsconfig.json index 7de173b3fc..e412e3c473 100644 --- a/integrations/localthought/tsconfig.json +++ b/integrations/localthought/tsconfig.json @@ -8,5 +8,5 @@ "types": ["node"], "noEmit": true }, - "include": ["data.ts", "schema.ts", "plugin.ts"] + "include": ["data.ts", "schema.ts", "plugin.ts", "browser.ts"] } diff --git a/integrations/localthought/wasm-smoke.mjs b/integrations/localthought/wasm-smoke.mjs new file mode 100644 index 0000000000..4381e77c1a --- /dev/null +++ b/integrations/localthought/wasm-smoke.mjs @@ -0,0 +1,72 @@ +/** Real WASM engine, no AtomicServer or provider network required. */ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import init, { + describeIntegration, + fetchIntegration, +} from '../../wasm/pkg/atomic_wasm.js'; +await init({ + module_or_path: await readFile( + new URL('../../wasm/pkg/atomic_wasm_bg.wasm', import.meta.url), + ), +}); +const document = await readFile( + new URL('./mock-document.json', import.meta.url), + 'utf8', +); +const description = JSON.parse(await describeIntegration(document)); +assert.deepEqual(description.collections, ['pets']); +const urls = []; +const output = JSON.parse( + await fetchIntegration(document, 'pets', '{}', undefined, async url => { + urls.push(url); + const second = url.includes('page=2'); + return JSON.stringify({ + status: 200, + headers: second + ? {} + : { link: '; rel="next"' }, + body: JSON.stringify([ + { + id: second ? 2 : 1, + name: 'Pet', + age: 3, + vaccinated: true, + weight: 2.5, + updated_at: '2026-09-09T00:00:00Z', + }, + ]), + }); + }), +); +assert.deepEqual(urls, [ + 'https://pets.example/pets', + 'https://pets.example/pets?page=2', +]); +assert.equal(output.records.length, 2); +assert.equal(output.records[0].values.age, 3); +assert.equal(output.records[0].values.vaccinated, true); +assert.equal(output.records[0].values['updated-at'], 1788912000000); +assert.equal( + output.ontology.terms.find(t => t.shortname === 'age').datatype, + 'https://atomicdata.dev/datatypes/integer', +); +await assert.rejects( + fetchIntegration(document, 'pets', '{}', undefined, async () => + JSON.stringify({ status: 503, headers: {}, body: '{}' }), + ), + /Import incomplete/, +); +await assert.rejects( + fetchIntegration(document, 'pets', '{}', undefined, async () => + JSON.stringify({ + status: 200, + headers: { link: '; rel="next"' }, + body: JSON.stringify([{ id: 1, name: 'Repeated' }]), + }), + ), + /Import incomplete/, +); +console.log( + 'Real WASM pagination, typed ontology, timestamps and failure refusal passed', +); diff --git a/planning/api-plugins.md b/planning/api-plugins.md index 3068d8e099..22809fb0e5 100644 --- a/planning/api-plugins.md +++ b/planning/api-plugins.md @@ -62,3 +62,12 @@ OAuth and paginated reads work, but an unbounded import exceeds 5,000 records. Main run 34350517612 passed dependency installation with the shared locks, then failed the full-app compiler sweep at its 5-second default (7.8 seconds actual). Give only that bulk test a 30-second budget; retain all compilation assertions. + +## Browser migration + +`codex/browser-integrations` moves the LocalThought flow off AtomicServer. +Catalog parsing and pagination run in the Atomic WASM bundle; the browser owns +the tenant handoff and rotating connection code, then maps fetched records into +locally reviewed proposals. Companion branches in Syncables and integration-proxy +provide WASM compatibility and CORS. See `integrations/localthought/README.md`. +Legacy direct integrations, action infrastructure and scheduling remain separate. diff --git a/server/Cargo.toml b/server/Cargo.toml index 9f7fbdcac8..3230bb8dee 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -33,7 +33,6 @@ static-files = "0.3.1" walkdir = "2" [dependencies] -syncables = { git = "https://github.com/localthought/syncables-rs", rev = "d48e4d9bad3ed9ec826d1c2040e989171701965d" } tempfile = "3" async-trait = "0.1.89" actix = "0.13.5" diff --git a/server/src/handlers/integration_proxy.rs b/server/src/handlers/integration_proxy.rs deleted file mode 100644 index 34f51bc604..0000000000 --- a/server/src/handlers/integration_proxy.rs +++ /dev/null @@ -1,316 +0,0 @@ -//! LocalThought catalog and actor-bound OAuth handoff. Credentials stay in host storage. -use crate::{appstate::AppState, context::RequestContext, errors::AtomicServerResult as Result}; -use actix_web::{web, HttpRequest, HttpResponse}; -use atomic_lib::{ - db::{ - plugin_secret::{PluginSecret, PluginSecretKey}, - trees::Tree, - }, - Db, -}; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use rand::RngCore; -use serde::{Deserialize, Serialize}; -use serde_json::json; - -pub(super) fn client() -> Result { - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|_| "Could not initialize integration proxy client".into()) -} -fn origin() -> Result { - let value = std::env::var("ATOMIC_INTEGRATION_PROXY_URL") - .unwrap_or_else(|_| "https://localthought.io".into()); - let u = url::Url::parse(&value).map_err(|_| "Invalid integration proxy URL")?; - if u.origin().ascii_serialization() != value - || (u.scheme() != "https" - && !(u.scheme() == "http" && matches!(u.host_str(), Some("localhost" | "127.0.0.1")))) - { - return Err("Integration proxy must be an HTTPS origin (or localhost for testing)".into()); - } - Ok(value) -} -fn random_id() -> String { - let mut b = [0u8; 32]; - rand::rngs::OsRng.fill_bytes(&mut b); - URL_SAFE_NO_PAD.encode(b) -} -fn sign(secret: &str, text: &str) -> String { - URL_SAFE_NO_PAD.encode(ring::hmac::sign( - &ring::hmac::Key::new(ring::hmac::HMAC_SHA256, secret.as_bytes()), - text.as_bytes(), - )) -} -async fn platforms(base: &str) -> Result> { - let response = client()? - .get(format!("{base}/catalog")) - .send() - .await - .map_err(|_| "Could not reach integration catalog")?; - if !response.status().is_success() { - return Err("Integration catalog is unavailable".into()); - } - let names: Vec = response - .json() - .await - .map_err(|_| "Invalid integration catalog")?; - if names.len() > 200 - || names.iter().any(|s| { - s.is_empty() - || s.len() > 80 - || !s - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - }) - { - return Err("Invalid catalog platform identifier".into()); - } - Ok(names) -} -pub async fn catalog() -> Result { - let base = origin()?; - Ok(HttpResponse::Ok().json(json!({"platforms": platforms(&base).await?, "origin": base}))) -} -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Start { - drive: String, - platform: String, - return_url: String, -} -#[derive(Clone, Serialize, Deserialize)] -pub(super) struct Connection { - pub(super) drive: String, - pub(super) actor: String, - pub(super) platform: String, - pub(super) origin: String, - pub(super) expires: i64, - pub(super) ready: bool, -} -fn key(id: &str) -> String { - format!("integration-proxy/v1/{id}") -} -fn put(db: &Db, id: &str, value: &Connection) -> Result<()> { - db.kv.insert( - Tree::PluginMeta, - key(id).as_bytes(), - &serde_json::to_vec(value).map_err(|_| "Could not encode connection")?, - )?; - db.flush()?; - Ok(()) -} -fn get(db: &Db, id: &str) -> Result { - let bytes = db - .kv - .get(Tree::PluginMeta, key(id).as_bytes())? - .ok_or("Connection not found")?; - serde_json::from_slice(&bytes).map_err(|_| "Invalid connection".into()) -} -pub(super) fn secret_key(c: &Connection, id: &str) -> PluginSecretKey { - PluginSecretKey::new(&c.drive, &format!("integration-proxy:{id}"), "connection") -} -fn owned(c: &Connection, drive: &str, actor: &str) -> Result<()> { - if c.drive != drive || c.actor != actor { - return Err("This connection belongs to another drive or agent".into()); - } - Ok(()) -} -fn return_url(raw: &str) -> Result { - let u = url::Url::parse(raw).map_err(|_| "Invalid return URL")?; - let allowed = std::env::var("ATOMIC_INTEGRATION_FRONTEND_ORIGIN") - .map_err(|_| "Configure ATOMIC_INTEGRATION_FRONTEND_ORIGIN on this server")?; - if u.origin().ascii_serialization() != allowed - || u.path() != "/app/integrations" - || !u.username().is_empty() - || u.password().is_some() - || u.fragment().is_some() - || u.query().is_some() - { - return Err("Return URL must be the configured frontend integrations page".into()); - } - Ok(u) -} -pub async fn start( - app: web::Data, - body: web::Json, - req: HttpRequest, - ctx: RequestContext, -) -> Result { - let actor = super::plugin_schedule::authorize(&app, &req, &ctx, &body.drive) - .await? - .to_string(); - let base = origin()?; - if !platforms(&base).await?.contains(&body.platform) { - return Err("Platform is not in the integration catalog".into()); - } - let secret = std::env::var("TENANT_SECRET") - .ok() - .filter(|s| !s.is_empty()) - .ok_or("Configure TENANT_SECRET on this server to connect accounts")?; - let tenant = secret - .split_once('.') - .and_then(|(id, _)| URL_SAFE_NO_PAD.decode(id).ok()) - .and_then(|b| String::from_utf8(b).ok()) - .ok_or("Invalid tenant secret configuration")?; - let mut callback = return_url(&body.return_url)?; - let id = random_id(); - callback - .query_pairs_mut() - .append_pair("integration_state", &id) - .append_pair("platform", &body.platform); - #[derive(Deserialize)] - struct Challenge { - ts: u64, - nonce: String, - challenge: String, - } - let response = client()? - .get(format!("{base}/session")) - .send() - .await - .map_err(|_| "Could not obtain integration challenge")?; - if !response.status().is_success() { - return Err("Integration challenge is unavailable".into()); - } - let challenge: Challenge = response - .json() - .await - .map_err(|_| "Invalid integration challenge")?; - let mut connect = - url::Url::parse(&format!("{base}/connect")).map_err(|_| "Invalid proxy URL")?; - connect - .query_pairs_mut() - .append_pair("redirect_uri", callback.as_str()) - .append_pair("platform", &body.platform) - .append_pair("ts", &challenge.ts.to_string()) - .append_pair("nonce", &challenge.nonce) - .append_pair("challenge", &challenge.challenge) - .append_pair("tenant_id", &tenant) - .append_pair("user_id", &actor) - .append_pair("user_id_sig", &sign(&secret, &actor)) - .append_pair("response", &sign(&secret, &challenge.challenge)); - put( - &app.store, - &id, - &Connection { - drive: body.drive.clone(), - actor, - platform: body.platform.clone(), - origin: base, - expires: atomic_lib::utils::now() + 600_000, - ready: false, - }, - )?; - Ok(HttpResponse::Ok().json(json!({"url":connect.as_str(),"state":id}))) -} -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Finish { - drive: String, - state: String, - connection_code: String, -} -pub async fn finish( - app: web::Data, - body: web::Json, - req: HttpRequest, - ctx: RequestContext, -) -> Result { - let actor = super::plugin_schedule::authorize(&app, &req, &ctx, &body.drive) - .await? - .to_string(); - let _lock = app.store.lock_plugin(&key(&body.state)).await; - let mut c = get(&app.store, &body.state)?; - owned(&c, &body.drive, &actor)?; - if c.ready - || c.expires < atomic_lib::utils::now() - || body.connection_code.is_empty() - || body.connection_code.len() > 4096 - { - return Err("Invalid, expired or already completed connection request".into()); - } - app.store.set_plugin_secret( - &secret_key(&c, &body.state), - &PluginSecret::new( - body.connection_code.clone(), - vec![c.origin.clone()], - atomic_lib::utils::now(), - ), - )?; - c.ready = true; - put(&app.store, &body.state, &c)?; - Ok(HttpResponse::Ok().json(json!({"connection":body.state,"platform":c.platform}))) -} -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Fetch { - drive: String, - connection: String, - constants: std::collections::BTreeMap, - calendar_range: Option, -} -pub async fn fetch_records( - app: web::Data, - body: web::Json, - req: HttpRequest, - ctx: RequestContext, -) -> Result { - let actor = super::plugin_schedule::authorize(&app, &req, &ctx, &body.drive) - .await? - .to_string(); - let _lock = app.store.lock_plugin(&key(&body.connection)).await; - let c = get(&app.store, &body.connection)?; - owned(&c, &body.drive, &actor)?; - if !c.ready { - return Err("Finish connecting your account first".into()); - } - let data = super::integration_proxy_sync::sync( - app.store.clone(), - c, - body.connection.clone(), - body.constants.clone(), - body.calendar_range.as_ref(), - ) - .await?; - Ok(HttpResponse::Ok().json(data)) -} -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn connection_is_bound_to_actor_and_drive() { - let c = Connection { - drive: "a".into(), - actor: "b".into(), - platform: "github-issues".into(), - origin: "https://localthought.io".into(), - expires: 0, - ready: true, - }; - assert!(owned(&c, "a", "b").is_ok()); - assert!(owned(&c, "other", "b").is_err()); - assert!(owned(&c, "a", "other").is_err()); - } - #[test] - fn signatures_match_proxy_hmac_contract() { - assert_eq!( - sign("key", "The quick brown fox jumps over the lazy dog"), - "97yD9DBThCSxMpjmqm-xQ-9NWaFJRhdZl0edvC0aPNg" - ); - } -} - -#[derive(Deserialize)] -pub struct PlatformQuery { - platform: String, -} -pub async fn platform(query: web::Query) -> Result { - let base = origin()?; - if !platforms(&base).await?.contains(&query.platform) { - return Err("Platform is not in the catalog".into()); - } - Ok(HttpResponse::Ok() - .json(super::integration_proxy_sync::describe(&base, &query.platform).await?)) -} diff --git a/server/src/handlers/integration_proxy_sync.rs b/server/src/handlers/integration_proxy_sync.rs deleted file mode 100644 index 275e81af12..0000000000 --- a/server/src/handlers/integration_proxy_sync.rs +++ /dev/null @@ -1,515 +0,0 @@ -//! Syncables transport and preview storage. No graph writes happen while fetching. -use super::integration_proxy::{client, secret_key, Connection}; -use crate::errors::AtomicServerResult as Result; -use atomic_lib::{db::plugin_secret::PluginSecret, Db}; -use serde_json::{json, Value}; -use std::{ - collections::{BTreeMap, BTreeSet}, - io::Write, - sync::{Arc, Mutex}, -}; -use syncables::{ - client::client::{Fetch, HttpRequest, HttpResponse}, - ontology_shortname, ClientConfig, Credentials, Ontology, Record, Storage, StorageError, - SyncClient, TermKind, -}; - -struct ProxyTransport { - db: Db, - connection: Connection, - id: String, - upstream: url::Url, - requests: Mutex, -} -#[async_trait::async_trait] -impl Fetch for ProxyTransport { - async fn fetch(&self, request: HttpRequest) -> syncables::Result { - let error = |message: &str| syncables::Error::Http(message.into()); - let target = - url::Url::parse(&request.url).map_err(|_| error("Invalid OpenAPI request URL"))?; - if request.method != "GET" - || target.origin() != self.upstream.origin() - || !target.username().is_empty() - || target.password().is_some() - || target.fragment().is_some() - { - return Err(error( - "Only read requests to the catalog API origin are allowed", - )); - } - { - let mut count = self - .requests - .lock() - .map_err(|_| error("Request counter failed"))?; - *count += 1; - if *count > 200 { - return Err(error("Import exceeds 200 requests; narrow its scope")); - } - } - let c = &self.connection; - let mut proxy = url::Url::parse(&format!( - "{}/proxy/{}{}", - c.origin, - c.platform, - target.path() - )) - .map_err(|_| error("Invalid proxy URL"))?; - proxy.set_query(target.query()); - let key = secret_key(c, &self.id); - let request = client() - .map_err(|_| error("Could not initialize proxy client"))? - .get(proxy); - let request = self - .db - .use_plugin_secret(&key, &c.origin, atomic_lib::utils::now(), |code| { - request.bearer_auth(code) - }) - .map_err(|_| error("Could not read connection credential"))? - .ok_or_else(|| error("Reconnect your account before fetching again"))?; - self.db - .delete_plugin_secret(&key) - .map_err(|_| error("Could not consume connection code"))?; - self.db - .flush() - .map_err(|_| error("Could not persist consumed code"))?; - let mut response = request - .send() - .await - .map_err(|_| error("Proxy request failed; reconnect before retrying"))?; - if let Some(next) = response - .headers() - .get("x-connection-code") - .and_then(|v| v.to_str().ok()) - { - self.db - .set_plugin_secret( - &key, - &PluginSecret::new( - next.to_string(), - vec![c.origin.clone()], - atomic_lib::utils::now(), - ), - ) - .map_err(|_| error("Could not save rotated code; reconnect"))?; - self.db - .flush() - .map_err(|_| error("Could not persist rotated code; reconnect"))?; - } - let status = response.status().as_u16(); - // Preserve pagination headers. Never expose the rotated credential to Syncables or the browser. - let headers = response - .headers() - .iter() - .filter(|(name, _)| name.as_str() != "x-connection-code") - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|v| (name.as_str().to_string(), v.to_string())) - }) - .collect(); - let mut body = Vec::new(); - while let Some(chunk) = response - .chunk() - .await - .map_err(|_| error("Could not read provider response"))? - { - if body.len() + chunk.len() > 10 * 1024 * 1024 { - return Err(error("Provider page exceeds 10 MB")); - } - body.extend_from_slice(&chunk); - } - Ok(HttpResponse { - status, - headers, - body, - }) - } -} -#[derive(Default)] -struct PreviewStorage { - records: Mutex>, - ontology: Mutex>, -} -#[async_trait::async_trait] -impl Storage for PreviewStorage { - async fn put(&self, record: &Record) -> std::result::Result<(), StorageError> { - let mut records = self.records.lock().unwrap(); - if records.len() >= 5000 { - return Err(StorageError::new( - "Import exceeds 5000 records; narrow its scope", - )); - } - if record.id.is_empty() - || records.iter().any(|r| { - r.namespace == record.namespace - && r.resource == record.resource - && r.id == record.id - }) - { - return Err(StorageError::new( - "Missing or repeated record identity; pagination may not be forwarded by the proxy", - )); - } - records.push(record.clone()); - Ok(()) - } - async fn get( - &self, - namespace: &str, - resource: &str, - id: &str, - ) -> std::result::Result, StorageError> { - Ok(self - .records - .lock() - .unwrap() - .iter() - .find(|r| r.namespace == namespace && r.resource == resource && r.id == id) - .cloned()) - } - async fn list( - &self, - namespace: &str, - resource: &str, - ) -> std::result::Result, StorageError> { - Ok(self - .records - .lock() - .unwrap() - .iter() - .filter(|r| r.namespace == namespace && r.resource == resource) - .cloned() - .collect()) - } - async fn delete(&self, _: &str, _: &str, _: &str) -> std::result::Result<(), StorageError> { - Err(StorageError::new("Preview storage cannot delete records")) - } - async fn put_ontology(&self, ontology: &Ontology) -> std::result::Result<(), StorageError> { - *self.ontology.lock().unwrap() = Some(ontology.clone()); - Ok(()) - } -} -fn typed_value(value: &Value, datatype: Option<&str>) -> Result> { - if value.is_null() { - return Ok(None); - } - if datatype == Some("https://atomicdata.dev/datatypes/timestamp") { - let text = value.as_str().ok_or("Expected RFC3339 timestamp")?; - return Ok(Some(json!(chrono::DateTime::parse_from_rfc3339(text) - .map_err(|_| "Invalid provider timestamp")? - .timestamp_millis()))); - } - Ok(Some(value.clone())) -} -fn preview(ontology: &Ontology, records: &[Record], platform: &str) -> Result { - let field_terms: BTreeMap<_, _> = ontology - .terms - .iter() - .filter(|t| t.kind == TermKind::Property) - .map(|t| (t.shortname.as_str(), t)) - .collect(); - let terms: Vec<_> = ontology.terms.iter().map(|t| json!({ "path":t.path, "kind":if t.kind == TermKind::Class {"class"} else {"property"}, "shortname":t.shortname, "description":t.description, "datatype":t.datatype.as_deref().unwrap_or("https://atomicdata.dev/datatypes/json"), "requires":t.requires, "recommends":t.recommends })).collect(); - let mut output = Vec::new(); - for record in records { - let mut values = serde_json::Map::new(); - for (key, value) in &record.value { - let short = ontology_shortname(key); - if let Some(term) = field_terms.get(short.as_str()) { - if let Some(value) = typed_value(value, term.datatype.as_deref())? { - values.insert(short, value); - } - } - } - output.push(json!({"resource":ontology_shortname(&record.resource),"namespace":record.namespace,"id":record.id,"values":values,"name":record.value.get("title").or_else(||record.value.get("summary")).or_else(||record.value.get("name")).cloned().unwrap_or(json!(record.id))})); - } - Ok( - json!({"platform":platform,"ontology":{"description":ontology.description,"terms":terms},"records":output}), - ) -} -/// UTC date boundaries; the end is exclusive, matching Calendar's timeMax. -#[derive(serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct CalendarRange { - start: String, - end: String, -} - -fn scope_calendar(document: &mut Value, range: &CalendarRange) -> Result<()> { - let parse = |value: &str| { - chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") - .map_err(|_| "Calendar dates must use YYYY-MM-DD") - }; - let start = parse(&range.start)?; - let end = parse(&range.end)?; - if start >= end { - return Err("Calendar end date must be after its start date".into()); - } - let collection = document - .pointer_mut("/components/crudResources/event/collections/events") - .and_then(Value::as_object_mut) - .ok_or("Calendar catalog has no events collection")?; - if collection.get("urlTemplate").and_then(Value::as_str) - != Some("/calendars/{calendarId}/events") - { - return Err("Calendar catalog events path has changed".into()); - } - let query = collection - .entry("x-list-query") - .or_insert_with(|| json!({})) - .as_object_mut() - .ok_or("Invalid Calendar collection query")?; - query.insert("timeMin".into(), json!(format!("{start}T00:00:00Z"))); - query.insert("timeMax".into(), json!(format!("{end}T00:00:00Z"))); - query.insert("singleEvents".into(), json!(true)); - Ok(()) -} - -pub(super) async fn sync( - db: Db, - c: Connection, - id: String, - constants: BTreeMap, - calendar_range: Option<&CalendarRange>, -) -> Result { - let file = load_document(&c.origin, &c.platform).await?; - let document = syncables::load_open_api_document(file.path()) - .await - .map_err(|e| e.to_string())?; - let upstream = url::Url::parse( - syncables::base_url(&document).ok_or("OpenAPI document has no API server")?, - ) - .map_err(|_| "Invalid OpenAPI server URL")?; - let mut scoped_file = tempfile::NamedTempFile::new()?; - if let Some(range) = calendar_range { - if c.platform != "google-calendar" { - return Err("Calendar date range only applies to Google Calendar".into()); - } - let mut scoped = serde_json::to_value(&document).map_err(|e| e.to_string())?; - scope_calendar(&mut scoped, range)?; - serde_json::to_writer(&mut scoped_file, &scoped).map_err(|e| e.to_string())?; - } - let platform = c.platform.clone(); - let fetch = ProxyTransport { - db, - connection: c, - id, - upstream, - requests: Mutex::new(0), - }; - let engine = SyncClient::new( - ClientConfig { - document: if calendar_range.is_some() { - scoped_file.path() - } else { - file.path() - } - .into(), - overlays: vec![], - credentials: Credentials::Anonymous, - constants, - ontology_base_url: format!("https://atomicdata.dev/integrations/{platform}"), - }, - Arc::new(fetch), - ) - .map_err(|e| e.to_string())?; - let storage = PreviewStorage::default(); - let report = tokio::time::timeout(std::time::Duration::from_secs(120), engine.sync(&storage)) - .await - .map_err(|_| "Import timed out; no partial data was proposed")? - .map_err(|e| e.to_string())?; - if !report.errors.is_empty() { - return Err(format!( - "Import incomplete; no changes proposed: {}", - report - .errors - .iter() - .map(String::as_str) - .collect::>() - .into_iter() - .collect::>() - .join("; ") - ) - .into()); - } - let ontology = storage.ontology.lock().unwrap(); - let records = storage.records.lock().unwrap(); - preview( - ontology - .as_ref() - .ok_or("Syncables did not generate an ontology")?, - &records, - &platform, - ) -} - -async fn load_document(base: &str, platform: &str) -> Result { - let document_url = format!("{}/catalog/{}.yaml", base, platform); - let mut response = client()? - .get(&document_url) - .send() - .await - .map_err(|_| "Could not load platform OpenAPI document")?; - if !response.status().is_success() { - return Err(format!( - "Platform OpenAPI document returned HTTP {} at {document_url}", - response.status().as_u16() - ) - .into()); - } - let mut bytes = Vec::new(); - while let Some(chunk) = response - .chunk() - .await - .map_err(|_| "Could not read OpenAPI document")? - { - if bytes.len() + chunk.len() > 10 * 1024 * 1024 { - return Err("OpenAPI document exceeds 10 MB".into()); - } - bytes.extend_from_slice(&chunk); - } - let mut file = - tempfile::NamedTempFile::new().map_err(|_| "Could not create temporary OpenAPI file")?; - file.write_all(&bytes) - .map_err(|_| "Could not store temporary OpenAPI document")?; - Ok(file) -} - -pub(super) async fn describe(base: &str, platform: &str) -> Result { - let file = load_document(base, platform).await?; - let doc = syncables::load_open_api_document(file.path()) - .await - .map_err(|e| e.to_string())?; - let model = syncables::discover_resource_model(&doc).map_err(|e| e.to_string())?; - let parameters: std::collections::BTreeSet<_> = model - .collections - .iter() - .flat_map(|c| c.context_params.iter()) - .filter(|p| model.provider_for(p).is_none()) - .cloned() - .collect(); - let collections: Vec<_> = model.collections.iter().map(|c| c.name.clone()).collect(); - Ok(json!({"parameters": parameters, "collections": collections})) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn calendar_range_preserves_catalog_query_and_rejects_invalid_bounds() { - let mut doc = json!({"components":{"crudResources":{"event":{"collections":{"events":{ - "urlTemplate":"/calendars/{calendarId}/events", "x-list-query":{"maxResults":2} - }}}}}}); - scope_calendar( - &mut doc, - &CalendarRange { - start: "2026-09-09".into(), - end: "2026-10-09".into(), - }, - ) - .unwrap(); - let query = - &doc["components"]["crudResources"]["event"]["collections"]["events"]["x-list-query"]; - assert_eq!(query["timeMin"], "2026-09-09T00:00:00Z"); - assert_eq!(query["timeMax"], "2026-10-09T00:00:00Z"); - assert_eq!(query["singleEvents"], true); - assert_eq!(query["maxResults"], 2); - for (start, end) in [ - ("2026-09-09", "2026-09-09"), - ("2026-09-10", "2026-09-09"), - ("2026-02-30", "2026-10-09"), - ] { - assert!(scope_calendar( - &mut doc, - &CalendarRange { - start: start.into(), - end: end.into() - } - ) - .is_err()); - } - } - struct Pages(Mutex>); - #[async_trait::async_trait] - impl Fetch for Pages { - async fn fetch(&self, req: HttpRequest) -> syncables::Result { - self.0.lock().unwrap().push(req.url.clone()); - let second = req.url.ends_with("?page=2"); - let headers = if second { - Default::default() - } else { - [( - "link".into(), - "; rel=\"next\"".into(), - )] - .into() - }; - let body = json!([{ "id": if second {2} else {1}, "name": if second {"Whiskers"} else {"Rex"}, "age":3,"vaccinated":true,"weight":2.5,"updated_at":"2026-09-09T00:00:00Z" }]); - Ok(HttpResponse { - status: 200, - headers, - body: serde_json::to_vec(&body).unwrap(), - }) - } - } - #[tokio::test] - async fn syncables_walks_catalog_pages_and_generates_typed_ontology() { - let mut file = tempfile::NamedTempFile::new().unwrap(); - file.write_all(include_bytes!( - "../../../integrations/localthought/mock-document.json" - )) - .unwrap(); - let transport = Arc::new(Pages(Mutex::new(vec![]))); - let client = SyncClient::new( - ClientConfig { - document: file.path().into(), - overlays: vec![], - credentials: Credentials::Anonymous, - constants: BTreeMap::new(), - ontology_base_url: "https://example.com/ontology".into(), - }, - transport.clone(), - ) - .unwrap(); - let storage = PreviewStorage::default(); - let report = client.sync(&storage).await.unwrap(); - assert!(report.errors.is_empty(), "{:?}", report.errors); - assert_eq!(report.read["pet"], 2); - assert_eq!( - transport.0.lock().unwrap().as_slice(), - &[ - "https://pets.example/pets", - "https://pets.example/pets?page=2" - ] - ); - let ontology = storage.ontology.lock().unwrap(); - let records = storage.records.lock().unwrap(); - let output = preview(ontology.as_ref().unwrap(), &records, "pets").unwrap(); - assert_eq!(output["records"][0]["values"]["age"], 3); - assert_eq!(output["records"][0]["values"]["vaccinated"], true); - assert_eq!( - output["records"][0]["values"]["updated-at"], - 1788912000000i64 - ); - let age = output["ontology"]["terms"] - .as_array() - .unwrap() - .iter() - .find(|t| t["shortname"] == "age") - .unwrap(); - assert_eq!(age["datatype"], "https://atomicdata.dev/datatypes/integer"); - } - #[tokio::test] - async fn duplicate_pages_are_an_error_not_a_successful_partial_import() { - let storage = PreviewStorage::default(); - let row = Record { - namespace: "".into(), - resource: "pet".into(), - id: "1".into(), - value: Default::default(), - }; - storage.put(&row).await.unwrap(); - assert!(storage.put(&row).await.is_err()); - } -} diff --git a/server/src/handlers/mod.rs b/server/src/handlers/mod.rs index dba9e243fc..ea549fdc70 100644 --- a/server/src/handlers/mod.rs +++ b/server/src/handlers/mod.rs @@ -44,7 +44,3 @@ pub mod plugin_sync; pub mod integration_action; pub mod integration_oauth; - -pub mod integration_proxy; - -mod integration_proxy_sync; diff --git a/server/src/routes.rs b/server/src/routes.rs index 4f2ac3d6fe..2d020245ee 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -369,26 +369,6 @@ pub fn config_routes(app: &mut actix_web::web::ServiceConfig) { .route(web::get().to(handlers::plugin_secret::handle_list_secrets)) .route(web::delete().to(handlers::plugin_secret::handle_delete_secret)), ) - .service( - web::resource("/integration-proxy/platform") - .route(web::get().to(handlers::integration_proxy::platform)), - ) - .service( - web::resource("/integration-proxy/catalog") - .route(web::get().to(handlers::integration_proxy::catalog)), - ) - .service( - web::resource("/integration-proxy/start") - .route(web::post().to(handlers::integration_proxy::start)), - ) - .service( - web::resource("/integration-proxy/finish") - .route(web::post().to(handlers::integration_proxy::finish)), - ) - .service( - web::resource("/integration-proxy/fetch") - .route(web::post().to(handlers::integration_proxy::fetch_records)), - ) .service( web::resource("/integration-oauth/notion/list") .route(web::post().to(handlers::integration_oauth::list)), diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 151765060e..16fceebaa6 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -42,6 +42,10 @@ serde-wasm-bindgen = "0.6" js-sys = "0.3" console_error_panic_hook = "0.1" +syncables = { path = "../integrations/localthought/syncables" } +async-trait = "0.1" +chrono = { version = "0.4", default-features = false, features = ["std"] } + [dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/wasm/src/integrations.rs b/wasm/src/integrations.rs new file mode 100644 index 0000000000..6baa3ef249 --- /dev/null +++ b/wasm/src/integrations.rs @@ -0,0 +1,260 @@ +//! Browser Syncables bridge: catalog parsing, typed previews, no filesystem or server. +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{Arc, Mutex}, +}; +use syncables::{ + client::client::{Fetch, HttpRequest, HttpResponse}, + ontology_shortname, ClientConfig, Credentials, Ontology, Record, Storage, StorageError, + SyncClient, TermKind, +}; +use wasm_bindgen::prelude::*; +type Result = std::result::Result>; + +struct BrowserFetch(js_sys::Function); +#[async_trait::async_trait(?Send)] +impl Fetch for BrowserFetch { + async fn fetch(&self, request: HttpRequest) -> syncables::Result { + if request.method != "GET" { + return Err(syncables::Error::Http( + "Browser imports only allow GET".into(), + )); + } + let error = + || syncables::Error::Http("Browser proxy request failed; reconnect if needed".into()); + let result = self + .0 + .call1(&JsValue::NULL, &JsValue::from_str(&request.url)) + .map_err(|_| error())?; + let value = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::resolve(&result)) + .await + .map_err(|_| error())?; + #[derive(serde::Deserialize)] + struct Response { + status: u16, + headers: BTreeMap, + body: String, + } + let value: Response = + serde_json::from_str(&value.as_string().ok_or_else(error)?).map_err(|_| error())?; + Ok(HttpResponse { + status: value.status, + headers: value.headers.into_iter().collect(), + body: value.body.into_bytes(), + }) + } +} +fn js_error(e: impl std::fmt::Display) -> JsValue { + js_sys::Error::new(&e.to_string()).into() +} +async fn document(text: &str) -> std::result::Result { + let value = syncables::openapi::load::parse_yaml(text).map_err(js_error)?; + syncables::load_open_api_document(value) + .await + .map_err(js_error) +} +#[wasm_bindgen(js_name = describeIntegration)] +pub async fn describe_integration(text: String) -> std::result::Result { + let doc = document(&text).await?; + let model = syncables::discover_resource_model(&doc).map_err(js_error)?; + let parameters: BTreeSet<_> = model + .collections + .iter() + .flat_map(|c| c.context_params.iter()) + .filter(|p| model.provider_for(p).is_none()) + .cloned() + .collect(); + Ok(json!({"parameters": parameters, "collections": model.collections.iter().map(|c| &c.name).collect::>(), + "upstream": syncables::base_url(&doc)}).to_string()) +} +#[wasm_bindgen(js_name = fetchIntegration)] +pub async fn fetch_integration( + text: String, + platform: String, + constants: String, + range: Option, + fetch: js_sys::Function, +) -> std::result::Result { + let mut doc = document(&text).await?; + if let Some(range) = range { + if platform != "google-calendar" { + return Err(js_error("Calendar range only applies to Google Calendar")); + } + let range: CalendarRange = serde_json::from_str(&range).map_err(js_error)?; + let mut value = serde_json::to_value(&doc).map_err(js_error)?; + scope_calendar(&mut value, &range).map_err(js_error)?; + doc = serde_json::from_value(value).map_err(js_error)?; + } + let engine = SyncClient::new( + ClientConfig { + document: Default::default(), + overlays: vec![], + credentials: Credentials::Anonymous, + constants: serde_json::from_str(&constants).map_err(js_error)?, + ontology_base_url: format!("https://atomicdata.dev/integrations/{platform}"), + }, + Arc::new(BrowserFetch(fetch)), + ) + .map_err(js_error)?; + let storage = PreviewStorage::default(); + let report = engine + .sync_document(&doc, &storage) + .await + .map_err(js_error)?; + if !report.errors.is_empty() { + return Err(js_error(format!( + "Import incomplete; no changes proposed: {}", + report.errors.join("; ") + ))); + } + let ontology = storage.ontology.lock().unwrap(); + let records = storage.records.lock().unwrap(); + preview( + ontology + .as_ref() + .ok_or_else(|| js_error("Missing ontology"))?, + &records, + &platform, + ) + .map(|v| v.to_string()) + .map_err(js_error) +} +#[derive(Default)] +struct PreviewStorage { + records: Mutex>, + ontology: Mutex>, +} +#[async_trait::async_trait(?Send)] +impl Storage for PreviewStorage { + async fn put(&self, record: &Record) -> std::result::Result<(), StorageError> { + let mut records = self.records.lock().unwrap(); + if records.len() >= 5000 { + return Err(StorageError::new( + "Import exceeds 5000 records; narrow its scope", + )); + } + if record.id.is_empty() + || records.iter().any(|r| { + r.namespace == record.namespace + && r.resource == record.resource + && r.id == record.id + }) + { + return Err(StorageError::new( + "Missing or repeated record identity; pagination may not be forwarded by the proxy", + )); + } + records.push(record.clone()); + Ok(()) + } + async fn get( + &self, + namespace: &str, + resource: &str, + id: &str, + ) -> std::result::Result, StorageError> { + Ok(self + .records + .lock() + .unwrap() + .iter() + .find(|r| r.namespace == namespace && r.resource == resource && r.id == id) + .cloned()) + } + async fn list( + &self, + namespace: &str, + resource: &str, + ) -> std::result::Result, StorageError> { + Ok(self + .records + .lock() + .unwrap() + .iter() + .filter(|r| r.namespace == namespace && r.resource == resource) + .cloned() + .collect()) + } + async fn delete(&self, _: &str, _: &str, _: &str) -> std::result::Result<(), StorageError> { + Err(StorageError::new("Preview storage cannot delete records")) + } + async fn put_ontology(&self, ontology: &Ontology) -> std::result::Result<(), StorageError> { + *self.ontology.lock().unwrap() = Some(ontology.clone()); + Ok(()) + } +} +fn typed_value(value: &Value, datatype: Option<&str>) -> Result> { + if value.is_null() { + return Ok(None); + } + if datatype == Some("https://atomicdata.dev/datatypes/timestamp") { + let text = value.as_str().ok_or("Expected RFC3339 timestamp")?; + return Ok(Some(json!(chrono::DateTime::parse_from_rfc3339(text) + .map_err(|_| "Invalid provider timestamp")? + .timestamp_millis()))); + } + Ok(Some(value.clone())) +} +fn preview(ontology: &Ontology, records: &[Record], platform: &str) -> Result { + let field_terms: BTreeMap<_, _> = ontology + .terms + .iter() + .filter(|t| t.kind == TermKind::Property) + .map(|t| (t.shortname.as_str(), t)) + .collect(); + let terms: Vec<_> = ontology.terms.iter().map(|t| json!({ "path":t.path, "kind":if t.kind == TermKind::Class {"class"} else {"property"}, "shortname":t.shortname, "description":t.description, "datatype":t.datatype.as_deref().unwrap_or("https://atomicdata.dev/datatypes/json"), "requires":t.requires, "recommends":t.recommends })).collect(); + let mut output = Vec::new(); + for record in records { + let mut values = serde_json::Map::new(); + for (key, value) in &record.value { + let short = ontology_shortname(key); + if let Some(term) = field_terms.get(short.as_str()) { + if let Some(value) = typed_value(value, term.datatype.as_deref())? { + values.insert(short, value); + } + } + } + output.push(json!({"resource":ontology_shortname(&record.resource),"namespace":record.namespace,"id":record.id,"values":values,"name":record.value.get("title").or_else(||record.value.get("summary")).or_else(||record.value.get("name")).cloned().unwrap_or(json!(record.id))})); + } + Ok( + json!({"platform":platform,"ontology":{"description":ontology.description,"terms":terms},"records":output}), + ) +} +/// UTC date boundaries; the end is exclusive, matching Calendar's timeMax. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct CalendarRange { + start: String, + end: String, +} + +fn scope_calendar(document: &mut Value, range: &CalendarRange) -> Result<()> { + let parse = |value: &str| { + chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") + .map_err(|_| "Calendar dates must use YYYY-MM-DD") + }; + let start = parse(&range.start)?; + let end = parse(&range.end)?; + if start >= end { + return Err("Calendar end date must be after its start date".into()); + } + let collection = document + .pointer_mut("/components/crudResources/event/collections/events") + .and_then(Value::as_object_mut) + .ok_or("Calendar catalog has no events collection")?; + if collection.get("urlTemplate").and_then(Value::as_str) + != Some("/calendars/{calendarId}/events") + { + return Err("Calendar catalog events path has changed".into()); + } + let query = collection + .entry("x-list-query") + .or_insert_with(|| json!({})) + .as_object_mut() + .ok_or("Invalid Calendar collection query")?; + query.insert("timeMin".into(), json!(format!("{start}T00:00:00Z"))); + query.insert("timeMax".into(), json!(format!("{end}T00:00:00Z"))); + query.insert("singleEvents".into(), json!(true)); + Ok(()) +} diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index cd4cf14b77..04b195c56f 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -1125,3 +1125,5 @@ impl ClientDb { .map_err(to_js_err) } } + +mod integrations; From 8e44570cf835a3cb32c7babfaca470f1bbf23be8 Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Wed, 9 Sep 2026 14:51:31 +0200 Subject: [PATCH 2/3] Run focused Pets E2E on the browser integration branch --- .github/workflows/pets-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pets-e2e.yml b/.github/workflows/pets-e2e.yml index 37f32a802c..ab799cfe08 100644 --- a/.github/workflows/pets-e2e.yml +++ b/.github/workflows/pets-e2e.yml @@ -2,7 +2,7 @@ name: LocalThought API plugins E2E on: push: - branches: [feat/api-plugins, codex/localthought-api-plugins] + branches: [feat/api-plugins, codex/localthought-api-plugins, codex/browser-integrations] permissions: contents: read From a49967074d14f35b333f177bf475b019490a4e0b Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Wed, 9 Sep 2026 15:47:28 +0200 Subject: [PATCH 3/3] Add browser Devonian issue tracker sync demo (#1399) * Add browser Devonian issue tracker sync demo * Use browser tenant connections and test two-way issue sync * Record live proxy OAuth verification and repository access blocker * Run one-way Google Calendar sync entirely in the browser (#1398) * Show one-way Google Calendar imports in a calendar view * Verify browser-only Calendar sync through shared mock proxy --------- Co-authored-by: Michiel de Jong --------- Co-authored-by: Michiel de Jong --- TESTING_COVERAGE.md | 30 ++ .../src/chunks/DevonianDemo/DEVONIAN-LICENSE | 201 +++++++++++ .../src/chunks/DevonianDemo/demo.d.mts | 47 +++ .../src/chunks/DevonianDemo/demo.mjs | 297 ++++++++++++++++ .../src/chunks/DevonianDemo/devonian.js | 328 ++++++++++++++++++ .../chunks/PluginRuns/ConnectLocalThought.tsx | 47 ++- browser/data-browser/src/locales/de.po | 116 +++++++ browser/data-browser/src/locales/en.po | 116 +++++++ browser/data-browser/src/locales/es.po | 116 +++++++ browser/data-browser/src/locales/fr.po | 116 +++++++ .../src/routes/DevonianDemoRoute.tsx | 275 +++++++++++++++ browser/data-browser/src/routes/Router.tsx | 8 + .../e2e/tests/devonian-issue-sync.spec.mts | 182 ++++++++++ .../e2e/tests/google-calendar-import.spec.mts | 169 +++++++++ integrations/github-issues/README.md | 7 + integrations/github-issues/adapter.test.ts | 5 +- integrations/github-issues/devonian/README.md | 129 +++++++ .../github-issues/devonian/bridge.mjs | 243 +++++++++++++ .../github-issues/devonian/bridge.test.mjs | 133 +++++++ integrations/github-issues/devonian/build.mjs | 36 ++ integrations/github-issues/devonian/ports.mjs | 322 +++++++++++++++++ .../github-issues/devonian/ports.test.mjs | 237 +++++++++++++ integrations/github-issues/devonian/proxy.mjs | 199 +++++++++++ .../github-issues/devonian/proxy.test.mjs | 100 ++++++ .../github-issues/devonian/vitest.config.mjs | 14 + integrations/github-issues/tracker-actions.ts | 190 ++++++++++ integrations/localthought/README.md | 50 +++ integrations/localthought/browser.test.ts | 34 ++ integrations/localthought/browser.ts | 102 ++++-- integrations/localthought/calendar.test.ts | 151 ++++++++ integrations/localthought/calendar.ts | 146 ++++++++ integrations/localthought/mock-calendar.mjs | 159 +++++++++ integrations/localthought/mock-github.mjs | 108 ++++++ integrations/localthought/mock-proxy.d.mts | 49 +++ integrations/localthought/mock-proxy.mjs | 163 +++++++-- integrations/localthought/tsconfig.json | 2 +- planning/devonian-issue-sync.md | 21 ++ 37 files changed, 4592 insertions(+), 56 deletions(-) create mode 100644 browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE create mode 100644 browser/data-browser/src/chunks/DevonianDemo/demo.d.mts create mode 100644 browser/data-browser/src/chunks/DevonianDemo/demo.mjs create mode 100644 browser/data-browser/src/chunks/DevonianDemo/devonian.js create mode 100644 browser/data-browser/src/routes/DevonianDemoRoute.tsx create mode 100644 browser/e2e/tests/devonian-issue-sync.spec.mts create mode 100644 browser/e2e/tests/google-calendar-import.spec.mts create mode 100644 integrations/github-issues/devonian/README.md create mode 100644 integrations/github-issues/devonian/bridge.mjs create mode 100644 integrations/github-issues/devonian/bridge.test.mjs create mode 100644 integrations/github-issues/devonian/build.mjs create mode 100644 integrations/github-issues/devonian/ports.mjs create mode 100644 integrations/github-issues/devonian/ports.test.mjs create mode 100644 integrations/github-issues/devonian/proxy.mjs create mode 100644 integrations/github-issues/devonian/proxy.test.mjs create mode 100644 integrations/github-issues/devonian/vitest.config.mjs create mode 100644 integrations/github-issues/tracker-actions.ts create mode 100644 integrations/localthought/calendar.test.ts create mode 100644 integrations/localthought/calendar.ts create mode 100644 integrations/localthought/mock-calendar.mjs create mode 100644 integrations/localthought/mock-github.mjs create mode 100644 integrations/localthought/mock-proxy.d.mts create mode 100644 planning/devonian-issue-sync.md diff --git a/TESTING_COVERAGE.md b/TESTING_COVERAGE.md index dbae936d47..6d2d376406 100644 --- a/TESTING_COVERAGE.md +++ b/TESTING_COVERAGE.md @@ -13,6 +13,24 @@ the companion proxy branch has 39 passing tests including CORS preflight and exposed headers. Live OAuth on the browser path still requires deployment of the companion proxy CORS change and is not yet verified. +`browser/e2e/tests/devonian-issue-sync.spec.mts` exercises tenant-secret entry, +proxy consent, direct HTTP writes and local OPFS storage for two-way issue +creation, comments, close/reopen and reload without duplicate resources. Its +stateful HTTP mock isolates repositories and consumes/rotates connection codes; +it does not substitute the in-page sample transport. + +The browser-only Devonian issue tracker demo has focused tests under +`integrations/github-issues/devonian`: real Devonian lenses with deterministic +connectors exercise bidirectional issue/comment creation and edits, close/reopen, +distinct identical resources, conflicts, missing records and restart/replay. +Transport fixtures cover pagination, label preservation, scoped comment links, +rotating connection codes and refusal to resend uncertain writes. The native +OPFS browser flow was manually verified for creation and comments on both sides, +closing from Atomic, reopening from the sample GitHub side and reloading without +duplicate issues/comments. Live proxy OAuth, +GitHub writes and a guided uncertain-write recovery UI remain unverified/unbuilt; +proxy v40 CORS and browser OAuth are verified, but its GitHub credential returns 404 for the private sandbox. + What is tested, at which layer, and — the part that matters — **what is not**. This exists because the protocol is far better tested than the glue around it, @@ -1066,3 +1084,15 @@ cancelled on teardown. Old plugin-name grants are deliberately not migrated. commit. It failed with the server signer before `Resource::destroy_as` was used; installation deletion must use the same selected identity as create/update. LocalThought: Rust handler tests cover connection binding, request signing, duplicate-page rejection, typed paginated previews, and Calendar UTC date-range validation. Live Calendar OAuth, bounded fetch, review/apply and event table display were verified against proxy v39 (54 records). + +Google Calendar one-way projection: `integrations/localthought/calendar.test.ts` +covers all-day/timed start dates, offset boundaries, exclusive end preservation, +feature notes (including WASM-normalized field names), cancellations without +start data, invalid active events, namespace isolation and repeat import/local +field preservation. `browser/e2e/tests/google-calendar-import.spec.mts` uses the +shared HTTP mock integration-proxy with a paginated Google Calendar, tenant +secret entry and OAuth consent. It covers browser WASM fetching, local +schema/proposal/apply, Calendar display, provider updates, OPFS reload and +stable identities while AtomicServer HTTP/WebSockets are unavailable. Missing +rows in a bounded snapshot are retained, not interpreted as deletions. +Live-provider browser OAuth verification remains separate from this fixture test. diff --git a/browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE b/browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/browser/data-browser/src/chunks/DevonianDemo/demo.d.mts b/browser/data-browser/src/chunks/DevonianDemo/demo.d.mts new file mode 100644 index 0000000000..8055f7e558 --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/demo.d.mts @@ -0,0 +1,47 @@ +import type { Store } from '@tomic/lib'; +export interface Demo { + key: string; + state: { + options: { sample: boolean }; + config: { connection: { table: string } }; + fixture: { + issues?: Array<{ number: number; title: string; state: string }>; + comments?: Array<{ id: number; body: string; issue_url: string }>; + }; + }; +} +export interface Row { + id: string; + value: { title: string; body: string; status: string }; + comments: Array<{ id: string; value: { body: string } }>; +} +export function openDemo( + store: Store, + options: { + sample: boolean; + repository: string; + proxy: string; + }, +): Promise; +export function syncDemo(store: Store, demo: Demo): Promise; +export function demoRows(store: Store, demo: Demo): Promise; +export function editAtomic( + store: Store, + demo: Demo, + command: string, + id?: string, + text?: string, +): Promise; +export function editFixture( + demo: Demo, + command: string, + number?: number, + text?: string, +): Promise; + +export function connectDemo( + store: Store, + options: { repository: string; proxy: string }, + secret: string, +): Promise; +export function resumeDemo(store: Store): Promise; diff --git a/browser/data-browser/src/chunks/DevonianDemo/demo.mjs b/browser/data-browser/src/chunks/DevonianDemo/demo.mjs new file mode 100644 index 0000000000..65615d2b21 --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/demo.mjs @@ -0,0 +1,297 @@ +// @wc-ignore-file +import { BrowserIntegrations } from '../../../../../integrations/localthought/browser'; +import { endpoint } from '../../../../../integrations/github-issues/adapter'; +import { get, set } from 'idb-keyval'; +import { core, server, dataBrowser, Datatype, enableLoro } from '@tomic/lib'; +import * as devonian from './devonian.js'; +import { ensureAgentForDemo } from '../Demo/guestAgent'; +import { buildTableFromSpec } from '../TablePage/createTableFromSpec'; +import { Bridge } from '../../../../../integrations/github-issues/devonian/bridge.mjs'; +import { + AtomicPort, + GitHubPort, +} from '../../../../../integrations/github-issues/devonian/ports.mjs'; +import { + fixtureTransport, + proxyTransport, +} from '../../../../../integrations/github-issues/devonian/proxy.mjs'; + +export async function openDemo(store, options) { + await enableLoro(); + await ensureAgentForDemo(store); + await store.waitForClientDb(10000); + const db = store.getClientDb(); + if (!db || !(await db.waitForReady())) + throw new Error('Enable the browser database on the Sync page first.'); + const repository = options.sample ? 'demo/issues' : options.repository; + endpoint(repository); + const key = `devonian-demo:${JSON.stringify([store.getAgent().subject, repository, options.sample ? 'sample' : new URL(options.proxy).origin])}`; + return navigator.locks.request(key, async () => { + let state = await get(key); + if (!state) { + const actor = store.getAgent().subject; + const drive = await store.newResource({ + noParent: true, + isA: [server.classes.drive], + propVals: { + [core.properties.name]: options.sample + ? 'Devonian sample tracker' + : `GitHub: ${repository}`, + [core.properties.read]: [actor], + [core.properties.write]: [actor], + }, + }); + store.registerLocalOnlyDrive(drive.subject); + await drive.save(); + await store.createDefaultOntology(drive); + const table = await buildTableFromSpec( + store, + { + name: 'Issue Tracker', + rowName: 'Issue', + columns: [ + { name: 'Description', type: 'markdown' }, + { + name: 'Status', + type: 'select', + options: ['Todo', 'Doing', 'Done'], + }, + { name: 'GitHub issue number', type: 'number' }, + ], + views: [ + { + name: 'Board', + kind: 'kanban', + groupByColumn: 'Status', + default: true, + }, + { name: 'All issues', kind: 'table' }, + ], + }, + { + parent: drive.subject, + driveSubject: drive.subject, + addToOntology: async () => {}, + }, + ); + const provenance = await store.newResource({ + parent: drive.subject, + isA: [core.classes.property], + propVals: { + [core.properties.name]: 'GitHub source', + [core.properties.shortname]: 'github-source', + [core.properties.description]: + 'Original author, identity and timestamps from GitHub.', + [core.properties.datatype]: Datatype.JSON, + }, + }); + await provenance.save(); + const folder = await store.newResource({ + parent: drive.subject, + isA: [dataBrowser.classes.folder], + propVals: { [core.properties.name]: 'Comments' }, + }); + await folder.save(); + await drive.set(dataBrowser.properties.commentsFolder, folder.subject); + await drive.save(); + const config = { + connection: { + repository, + drive: drive.subject, + table: table.tableSubject, + rowClass: table.classSubject, + body: table.columns.Description, + status: table.columns.Status, + number: table.columns['GitHub issue number'], + tags: table.tags.Status, + }, + commentsFolder: folder.subject, + provenance: provenance.subject, + }; + state = { + config, + options: { sample: options.sample, proxy: options.proxy, repository }, + base: `https://atomicdata.dev/devonian-bridges/${crypto.randomUUID()}`, + fixture: {}, + journal: {}, + }; + await db.flush(); + await set(key, state); + } + store.registerLocalOnlyDrive(state.config.connection.drive); + store.setDrive(state.config.connection.drive); + return { key, state }; + }); +} + +export async function syncDemo(store, demo) { + return navigator.locks.request(demo.key, async () => { + const state = await get(demo.key); + const save = () => set(demo.key, state); + const call = state.options.sample + ? fixtureTransport(state.fixture, save) + : proxyTransport({ + url: state.options.proxy, + repository: state.options.repository, + journal: state.journal, + save, + dispatch: (path, init) => + client(state.options.proxy).request( + state.config.connection.drive, + store.getAgent().subject, + state.connection, + 'github-issues', + path, + init, + ), + }); + const bridge = new Bridge({ + devonian, + local: new AtomicPort(store, state.config), + remote: new GitHubPort(null, state.config.connection, call), + base: state.base, + snapshot: state.bridge, + save: async snapshot => { + await store.getClientDb().flush(); + state.bridge = snapshot; + await save(); + }, + }); + await bridge.sync(); + demo.state = state; + return Object.keys(bridge.records).length; + }); +} + +export async function demoRows(store, demo) { + const local = new AtomicPort(store, demo.state.config); + const issues = await local.list('issue'); + for (const row of issues) + row.comments = await local.list('comment:ui', { issueId: row.id }); + return issues; +} + +export async function editAtomic(store, demo, command, id, text) { + const port = new AtomicPort(store, demo.state.config); + if (command === 'create') + await port.create( + 'issue', + { title: text, body: '', status: 'Todo' }, + crypto.randomUUID(), + ); + else if (command === 'comment') + await port.create( + 'comment:ui', + { body: text }, + crypto.randomUUID(), + undefined, + { issueId: id }, + ); + else { + const row = await port.get('issue', id); + await port.update('issue', id, { + ...row.value, + status: row.value.status === 'Done' ? 'Todo' : 'Done', + }); + } + await store.getClientDb().flush(); +} + +export async function editFixture(demo, command, number, text) { + return navigator.locks.request(demo.key, async () => { + const state = await get(demo.key); + const call = fixtureTransport(state.fixture, () => set(demo.key, state)); + if (command === 'create') + await call( + 'create_issue', + { title: text, body: '' }, + crypto.randomUUID(), + ); + else if (command === 'comment') + await call('create_comment', { number, body: text }, crypto.randomUUID()); + else { + const issue = state.fixture.issues.find(r => r.number === number); + await call( + 'update_issue', + { + number, + title: issue.title, + body: issue.body, + state: issue.state === 'open' ? 'closed' : 'open', + }, + crypto.randomUUID(), + ); + } + demo.state = state; + }); +} + +const handoffKey = 'devonian-browser-handoff'; +const resumeKey = 'devonian-browser-resume'; +const client = origin => + new BrowserIntegrations( + localStorage, + async () => { + throw new Error('Devonian uses its own resource lenses'); + }, + origin, + ); +export async function connectDemo(store, options, secret) { + const demo = await openDemo(store, { ...options, sample: false }); + const result = await client(options.proxy).start( + demo.state.config.connection.drive, + store.getAgent().subject, + 'github-issues', + `${location.origin}/app/devonian-demo`, + secret, + ); + sessionStorage.setItem( + handoffKey, + JSON.stringify({ key: demo.key, state: result.state }), + ); + location.assign(result.url); +} +export async function resumeDemo(store) { + const url = new URL(location.href); + const callbackCode = url.searchParams.get('connection_code'); + const callbackState = url.searchParams.get('integration_state'); + const handoff = JSON.parse(sessionStorage.getItem(handoffKey) ?? 'null'); + // Save the validated handoff before removing credentials from the URL, so a + // reload while OPFS opens cannot abandon the completed proxy consent. + if (callbackCode || callbackState) { + history.replaceState(null, '', url.pathname); + if (!handoff || callbackState !== handoff.state || !callbackCode) + throw new Error('Invalid connection callback state'); + handoff.code = callbackCode; + sessionStorage.setItem(handoffKey, JSON.stringify(handoff)); + } + const code = handoff?.code; + const stateId = handoff?.state; + const key = code ? handoff.key : sessionStorage.getItem(resumeKey); + if (!key) { + if (callbackCode || callbackState) + throw new Error('Missing browser connection handoff'); + return; + } + const saved = await get(key); + if (!saved) throw new Error('Missing local tracker'); + const demo = await openDemo(store, saved.options); + if (demo.key !== key) throw new Error('Connection belongs to another agent'); + if (code) { + if (!handoff.finished) { + client(saved.options.proxy).finish( + saved.config.connection.drive, + store.getAgent().subject, + stateId, + code, + ); + handoff.finished = true; + sessionStorage.setItem(handoffKey, JSON.stringify(handoff)); + } + demo.state.connection = stateId; + await set(key, demo.state); + sessionStorage.removeItem(handoffKey); + sessionStorage.setItem(resumeKey, key); + } + return demo; +} diff --git a/browser/data-browser/src/chunks/DevonianDemo/devonian.js b/browser/data-browser/src/chunks/DevonianDemo/devonian.js new file mode 100644 index 0000000000..ea269a9670 --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/devonian.js @@ -0,0 +1,328 @@ +// Devonian native resource API, e11104f78ebd151a361171ca0b21489b25e1e2c8. Apache-2.0; see DEVONIAN-LICENSE. + +// ../../localthought/devonian/src/atomic/Resource.ts +import { Datatype, validateDatatype } from "@tomic/lib"; +var IS_A = "https://atomicdata.dev/properties/isA"; +function assertSubject(value) { + if (typeof value !== "string" || /\s/.test(value)) + throw new Error("Expected an absolute URL without whitespace"); + const url = new URL(value); + if (!["http:", "https:"].includes(url.protocol)) { + throw new Error(`Expected an HTTP(S) URL: ${value}`); + } +} +var AtomicSchema = class { + properties = /* @__PURE__ */ new Map([ + [IS_A, Datatype.RESOURCEARRAY] + ]); + property(subject, datatype) { + assertSubject(subject); + if (!Object.values(Datatype).includes(datatype) || datatype === Datatype.UNKNOWN) { + throw new Error(`Unsupported datatype: ${datatype}`); + } + const existing = this.properties.get(subject); + if (existing && existing !== datatype) + throw new Error(`Conflicting property: ${subject}`); + this.properties.set(subject, datatype); + return this; + } + validate(resource) { + assertSubject(resource["@id"]); + this.validateProperties(resource, true); + } + validateProperties(resource, named) { + if (Object.getPrototypeOf(resource) !== Object.prototype) + throw new Error("Expected a plain JSON object"); + for (const [property, value] of Object.entries(resource)) { + if (property === "@id" && named) continue; + assertSubject(property); + const datatype = this.properties.get(property); + if (!datatype) throw new Error(`Unknown property: ${property}`); + if (typeof value === "number" && !Number.isFinite(value)) + throw new Error("Expected a finite number"); + if (value === void 0 || value === null) + throw new Error(`Missing value for ${property}`); + if (datatype === Datatype.ATOMIC_URL) { + this.validateLink(value); + } else if (datatype === Datatype.RESOURCEARRAY) { + if (!Array.isArray(value)) + throw new Error(`Expected a resource array: ${property}`); + for (const link of value) this.validateLink(link); + } else { + if ([Datatype.FLOAT, Datatype.INTEGER, Datatype.TIMESTAMP].includes( + datatype + ) && typeof value !== "number") + throw new Error("Expected a number"); + if (datatype === Datatype.BOOLEAN && typeof value !== "boolean") + throw new Error("Expected a boolean"); + validateDatatype(value, datatype); + } + } + } + validateLink(value) { + if (typeof value === "string") assertSubject(value); + else if (value && typeof value === "object" && !Array.isArray(value)) + this.validateProperties(value, false); + else throw new Error("Expected a resource URL or nested resource"); + } +}; + +// ../../localthought/devonian/src/atomic/Store.ts +var AtomicStore = class _AtomicStore { + constructor(schema) { + this.schema = schema; + } + schema; + resources = /* @__PURE__ */ new Map(); + get(subject) { + const resource = this.resources.get(subject); + return resource && structuredClone(resource); + } + put(resource) { + this.schema.validate(resource); + this.resources.set(resource["@id"], structuredClone(resource)); + } + /** Preserve omitted properties; removals must be listed in unset. */ + patch(subject, patch) { + assertSubject(subject); + if (Object.hasOwn(patch.set ?? {}, "@id") || patch.unset?.includes("@id")) { + throw new Error("A patch cannot change resource identity"); + } + const resource = { + ...this.get(subject) ?? { "@id": subject }, + ...patch.set + }; + for (const property of patch.unset ?? []) { + assertSubject(property); + delete resource[property]; + } + this.put(resource); + return structuredClone(resource); + } + /** Synchronous local transaction. Async work must finish before entering this callback. */ + transaction(operation) { + const previous = this.resources; + this.resources = new Map(previous); + try { + operation(); + } catch (error) { + this.resources = previous; + throw error; + } + } + /** Apply a resource projection as one local transaction. */ + apply(changes) { + const staged = new _AtomicStore(this.schema); + staged.resources = new Map(this.resources); + for (const change of changes) staged.patch(change.subject, change.patch); + this.resources = staged.resources; + } + delete(subject) { + this.resources.delete(subject); + } + /** Optional class filtering; callers can apply additional predicates to the returned copies. */ + all(classSubject) { + return [...this.resources.values()].filter((resource) => { + const classes = resource["https://atomicdata.dev/properties/isA"]; + return !classSubject || Array.isArray(classes) && classes.includes(classSubject); + }).map((resource) => structuredClone(resource)); + } + toJSONAD() { + return JSON.stringify(this.all()); + } + /** Validate the complete snapshot before replacing state. Import does not emit writes. */ + loadJSONAD(json) { + const parsed = JSON.parse(json); + const resources = Array.isArray(parsed) ? parsed : [parsed]; + const next = /* @__PURE__ */ new Map(); + for (const item of resources) { + if (!item || typeof item !== "object" || Array.isArray(item) || typeof item["@id"] !== "string") { + throw new Error("Expected a named JSON-AD resource"); + } + const resource = item; + this.schema.validate(resource); + if (next.has(resource["@id"])) + throw new Error(`Duplicate subject: ${resource["@id"]}`); + next.set(resource["@id"], structuredClone(resource)); + } + this.resources = next; + } +}; + +// ../../localthought/devonian/src/atomic/IdentityMap.ts +var VOCAB = "https://raw.githubusercontent.com/localthought/devonian/main/vocab/"; +var identity = { + class: `${VOCAB}Identity.json`, + scope: `${VOCAB}scope.json`, + entity: `${VOCAB}entity.json`, + localId: `${VOCAB}localId.json`, + idType: `${VOCAB}idType.json`, + resource: `${VOCAB}resource.json` +}; +var AtomicIdentityMap = class { + constructor(store, baseURL) { + this.store = store; + assertSubject(baseURL); + const url = new URL(baseURL); + if (url.search || url.hash) + throw new Error("Identity base URL cannot contain a query or fragment"); + this.base = baseURL.replace(/\/$/, ""); + store.schema.property(identity.scope, Datatype.ATOMIC_URL).property(identity.entity, Datatype.STRING).property(identity.localId, Datatype.STRING).property(identity.idType, Datatype.STRING).property(identity.resource, Datatype.ATOMIC_URL); + } + store; + base; + key(scope, id) { + assertSubject(scope.scope); + if (!scope.entity || typeof id !== "string" && typeof id !== "number" || typeof id === "number" && !Number.isSafeInteger(id) || id === "") { + throw new Error( + "Expected an entity and a nonempty string or safe integer external ID" + ); + } + return encodeURIComponent( + JSON.stringify([scope.scope, scope.entity, typeof id, id]) + ); + } + subjectFor(scope, id) { + return this.lookup(scope, id) ?? `${this.base}/resources/${this.key(scope, id)}`; + } + lookup(scope, id) { + this.key(scope, id); + return this.find(scope).find( + (resource) => resource[identity.localId] === String(id) && resource[identity.idType] === typeof id + )?.[identity.resource]; + } + externalId(scope, subject) { + const resource = this.find(scope).find( + (resource2) => resource2[identity.resource] === subject + ); + if (!resource) return void 0; + return resource[identity.idType] === "number" ? Number(resource[identity.localId]) : resource[identity.localId]; + } + bind(scope, id, subject) { + const key = this.key(scope, id); + assertSubject(subject); + const existing = this.lookup(scope, id); + const reverse = this.externalId(scope, subject); + if (existing !== void 0 && existing !== subject || reverse !== void 0 && reverse !== id) { + throw new Error("Conflicting identity mapping"); + } + if (existing === subject && reverse === id) return; + this.store.put({ + "@id": `${this.base}/identities/${key}`, + [IS_A]: [identity.class], + [identity.scope]: scope.scope, + [identity.entity]: scope.entity, + [identity.localId]: String(id), + [identity.idType]: typeof id, + [identity.resource]: subject + }); + } + find(scope) { + const resources = this.store.all(identity.class).filter( + (resource) => resource[identity.scope] === scope.scope && resource[identity.entity] === scope.entity + ); + const ids = /* @__PURE__ */ new Set(); + const subjects = /* @__PURE__ */ new Set(); + for (const resource of resources) { + const id = resource[identity.localId]; + const type = resource[identity.idType]; + const subject = resource[identity.resource]; + if (typeof id !== "string" || !id || !["string", "number"].includes(String(type)) || typeof subject !== "string" || type === "number" && (!Number.isSafeInteger(Number(id)) || String(Number(id)) !== id)) { + throw new Error("Invalid identity mapping"); + } + const key = JSON.stringify([type, id]); + if (ids.has(key) || subjects.has(subject)) + throw new Error("Conflicting identity mappings in snapshot"); + ids.add(key); + subjects.add(subject); + } + return resources; + } +}; + +// ../../localthought/devonian/src/atomic/Lens.ts +var AtomicLens = class { + constructor(options) { + this.options = options; + assertSubject(options.scope); + if (!options.entity) throw new Error("A lens requires an entity type"); + if (options.identities.store !== options.store) { + throw new Error("Lens and identity map must use the same store"); + } + } + options; + pending = Promise.resolve(); + /** Handle a webhook or fetched record. Never writes back to the connector. */ + ingest(record) { + return this.enqueue(async () => { + const { connector, identities, store, read } = this.options; + const id = connector.id(record); + const subject = identities.subjectFor(this.options, id); + const patch = await read(record, subject); + store.transaction(() => { + store.apply([...patch.related ?? [], { subject, patch }]); + for (const mapping of patch.identities ?? []) + identities.bind(mapping.scope, mapping.id, mapping.subject); + identities.bind(this.options, id, subject); + }); + return subject; + }); + } + /** Publish the latest native state; failures reject and may be retried. */ + publish(subject) { + return this.enqueue(async () => { + const { connector, identities, store, write } = this.options; + const resource = store.get(subject); + if (!resource) throw new Error(`Unknown resource: ${subject}`); + const id = identities.externalId(this.options, subject); + if (id !== void 0) { + const previous = await connector.get(id); + await connector.update(id, await write(resource, previous)); + return id; + } + const key = JSON.stringify([ + this.options.scope, + this.options.entity, + subject + ]); + const created = await connector.create( + await write(resource, void 0), + key + ); + const createdId = connector.id(created); + identities.bind(this.options, createdId, subject); + return createdId; + }); + } + /** Delete native state after an external deletion. Retain identity for replay/recreation. */ + ingestDelete(id) { + return this.enqueue(async () => { + const subject = this.options.identities.lookup(this.options, id); + if (subject) this.options.store.delete(subject); + }); + } + /** Delete externally first, so a failed request leaves native state available for retry. */ + delete(subject) { + return this.enqueue(async () => { + const { connector, identities, store } = this.options; + const id = identities.externalId(this.options, subject); + if (id !== void 0) await connector.delete(id); + store.delete(subject); + }); + } + enqueue(operation) { + const result = this.pending.then(operation); + this.pending = result.catch(() => void 0); + return result; + } +}; +export { + AtomicIdentityMap, + AtomicLens, + AtomicSchema, + AtomicStore, + Datatype, + IS_A, + assertSubject, + identity +}; diff --git a/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx b/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx index 4be4f39df2..1448434a53 100644 --- a/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx +++ b/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx @@ -32,6 +32,10 @@ import { import type { Config } from '../../../../../integrations/localthought/plugin'; import { localImportVerdict } from './localImportVerdict'; import source from '../../../../../integrations/localthought/plugin.js?raw'; +import { + calendarProjection, + calendarFields, +} from '../../../../../integrations/localthought/calendar'; export function ConnectLocalThought({ drive, @@ -130,12 +134,13 @@ export function ConnectLocalThought({ setError(''); try { - const fetched = await proxyRequest(store, 'fetch', { + const response = await proxyRequest(store, 'fetch', { drive, connection: connection.connection, constants, ...(platform === 'google-calendar' ? { calendarRange } : {}), }); + const fetched = calendarProjection(response); if (fetched.platform !== platform) throw new Error('Imported platform did not match this connection'); const schemaStore = localSchemaStore(store); @@ -195,13 +200,47 @@ export function ConnectLocalThought({ [dataBrowser.properties.viewColumns]: columns, }, }); + const calendar = + platform === 'google-calendar' && term.shortname === 'event' + ? await ensureInstallationResource(store, drive, { + parent: destination.subject, + localId: `${identity}:calendar:${term.shortname}`, + isA: [dataBrowser.classes.view], + propVals: { + [core.properties.name]: tableName, + [dataBrowser.properties.viewKind]: 'calendar', + [dataBrowser.properties.viewGroupBy]: + properties[calendarFields.day], + [dataBrowser.properties.viewColumns]: columns, + }, + }) + : undefined; + const existingViews = destination.get( + dataBrowser.properties.tableViews, + ) as string[] | undefined; await destination.set(dataBrowser.properties.tableViews, [ - view.subject, + ...new Set([ + ...(existingViews ?? []), + view.subject, + ...(calendar ? [calendar.subject] : []), + ]), ]); - await destination.set( + const currentDefault = destination.get( dataBrowser.properties.tableDefaultView, - view.subject, ); + + if ( + !currentDefault || + (calendar && + !existingViews?.includes(calendar.subject) && + currentDefault === view.subject) + ) { + await destination.set( + dataBrowser.properties.tableDefaultView, + calendar?.subject ?? view.subject, + ); + } + await destination.save(); destinations[term.shortname] = { table: destination.subject, rowClass }; } diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index e3ca21bfa9..ff10ed5033 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -4704,6 +4704,7 @@ msgstr "" msgid "Syncing…" msgstr "" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "" @@ -6287,6 +6288,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "" #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "" @@ -9661,9 +9663,123 @@ msgid "Events before (UTC)" msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "LocalThought tenant secret" msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." msgstr "" + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "" + +#~ msgid "Connection code" +#~ msgstr "" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "" + +#~ msgid "Open live tracker" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "" + +#~ msgid "Done" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index 3d23fe2d7d..06bff9a6f1 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -4715,6 +4715,7 @@ msgstr "{0}% used" msgid "Syncing…" msgstr "Syncing…" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "Sync now" @@ -6298,6 +6299,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "Keep an encrypted copy of this workspace in {0}. It is sealed on this device, so we store it without being able to read it." #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "Working…" @@ -9700,9 +9702,123 @@ msgid "Events before (UTC)" msgstr "Events before (UTC)" #: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "LocalThought tenant secret" msgstr "LocalThought tenant secret" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." msgstr "The tenant secret is used in this tab. Connection credentials stay in this browser." + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "Synchronized {0} issues and comments." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "Integration proxy URL" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "GitHub repository (owner/repo)" + +#~ msgid "Connection code" +#~ msgstr "Connection code" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." + +#~ msgid "Open live tracker" +#~ msgstr "Open live tracker" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "Connect a real GitHub repository" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "Devonian issue tracker demo" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "Try sample data" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "Open Atomic kanban" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "New issue title or comment" + +#~ msgid "Done" +#~ msgstr "Done" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "Reopen Atomic issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "Close Atomic issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "Add Atomic comment" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "Atomic tracker" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "Atomic issues" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "Reopen sample GitHub issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "Close sample GitHub issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "Add sample GitHub comment" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "Create sample GitHub issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "Create Atomic issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "Sample GitHub tracker" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "Sample GitHub issues" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "Connect GitHub tracker" diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index 0e1c244ed0..c21040df5e 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -4704,6 +4704,7 @@ msgstr "" msgid "Syncing…" msgstr "" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "" @@ -6287,6 +6288,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "" #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "" @@ -9661,9 +9663,123 @@ msgid "Events before (UTC)" msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "LocalThought tenant secret" msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." msgstr "" + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "" + +#~ msgid "Connection code" +#~ msgstr "" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "" + +#~ msgid "Open live tracker" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "" + +#~ msgid "Done" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index b74aceab3e..46b33e9914 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -4704,6 +4704,7 @@ msgstr "" msgid "Syncing…" msgstr "" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "" @@ -6287,6 +6288,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "" #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "" @@ -9661,9 +9663,123 @@ msgid "Events before (UTC)" msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "LocalThought tenant secret" msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." msgstr "" + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "" + +#~ msgid "Connection code" +#~ msgstr "" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "" + +#~ msgid "Open live tracker" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "" + +#~ msgid "Done" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "" diff --git a/browser/data-browser/src/routes/DevonianDemoRoute.tsx b/browser/data-browser/src/routes/DevonianDemoRoute.tsx new file mode 100644 index 0000000000..f961500c59 --- /dev/null +++ b/browser/data-browser/src/routes/DevonianDemoRoute.tsx @@ -0,0 +1,275 @@ +import { createLazyRoute } from '@tanstack/react-router'; +import { useEffect, useRef, useState } from 'react'; +import { useStore } from '@tomic/react'; +import { Main } from '@components/Main'; +import { ContainerWide } from '@components/Containers'; +import { Column, Row as Horizontal } from '@components/Row'; +import { Card } from '@components/Card'; +import { Button } from '@components/Button'; +import { AtomicLink } from '@components/AtomicLink'; +import Field from '@components/forms/Field'; +import { Input, ErrMessage } from '@components/forms/InputStyles'; +import { + openDemo, + connectDemo, + resumeDemo, + syncDemo, + demoRows, + editAtomic, + editFixture, + type Demo, + type Row, +} from '../chunks/DevonianDemo/demo.mjs'; + +function DevonianDemo() { + const store = useStore(); + const [demo, setDemo] = useState(); + const [rows, setRows] = useState([]); + const [proxy, setProxy] = useState('https://localthought.io'); + const [repository, setRepository] = useState(''); + const [secret, setSecret] = useState(''); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(''); + const [status, setStatus] = useState(''); + const [text, setText] = useState(''); + const run = async (work: () => Promise) => { + setBusy(true); + setError(''); + try { + const current = await work(); + if (current) { + setDemo({ ...current }); + setRows(await demoRows(store, current)); + } + } catch (reason) { + setError(reason instanceof Error ? reason.message : String(reason)); + } finally { + setBusy(false); + } + }; + const resumed = useRef(false); + useEffect(() => { + if (resumed.current) return; + resumed.current = true; + void run(() => resumeDemo(store)); + }, [store]); + const start = (sample: boolean) => + run(async () => { + if (!sample) { + const credential = secret; + setSecret(''); + await connectDemo(store, { repository, proxy }, credential); + return; + } + const opened = await openDemo(store, { sample, repository, proxy }); + if (sample) await syncDemo(store, opened); + return opened; + }); + const sync = () => + run(async () => { + if (!demo) return; + const count = await syncDemo(store, demo); + setStatus(`Synchronized ${count} issues and comments.`); + return demo; + }); + const edit = ( + side: 'atomic' | 'fixture', + command: string, + id?: string | number, + ) => + run(async () => { + if (!demo) return; + if (side === 'atomic') + await editAtomic(store, demo, command, id as string, text); + else await editFixture(demo, command, id as number, text); + setText(''); + return demo; + }); + return ( +
+ + +

Devonian issue tracker demo

+

+ Sync issues and comments in both directions. Devonian runs in this + browser; the Atomic tracker is stored on this device. +

+ {!demo && ( + <> + +
+ Connect a real GitHub repository + +

+ Connect through LocalThought. The tenant secret is used in + this tab to start the connection and is never saved. +

+ + setProxy(e.target.value)} + /> + + + setRepository(e.target.value)} + /> + + + setSecret(e.target.value)} + /> + + + +
+ + )} + {demo && ( + <> +

+ {demo.state.options.sample + ? 'Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub.' + : 'Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync.'} +

+ + + + Open Atomic kanban + + + + setText(e.target.value)} + /> + + + + {demo.state.options.sample && ( + + )} + +
+

Atomic tracker

+ + {rows.map(row => ( + + + + {row.value.title} + + {row.value.status} + + + + + {row.comments.map(comment => ( +

{comment.value.body}

+ ))} + + + ))} + +
+ {demo.state.options.sample && ( +
+

Sample GitHub tracker

+ + {demo.state.fixture.issues?.map(issue => ( + + + + #{issue.number} {issue.title} + + {issue.state} + + + + + {demo.state.fixture.comments + ?.filter(c => + c.issue_url.endsWith(`/${issue.number}`), + ) + .map(c => ( +

{c.body}

+ ))} + + + ))} + +
+ )} + + )} +

{status}

+ {error && {error}} + + +
+ ); +} +export const devonianDemoRouteLazy = createLazyRoute('/app/devonian-demo')({ + component: DevonianDemo, +}); diff --git a/browser/data-browser/src/routes/Router.tsx b/browser/data-browser/src/routes/Router.tsx index 5c79967682..176a64ffcb 100644 --- a/browser/data-browser/src/routes/Router.tsx +++ b/browser/data-browser/src/routes/Router.tsx @@ -42,6 +42,13 @@ const DemoRoute = createRoute({ path: pathNames.demo, }).lazy(() => import('./DemoRoute').then(mod => mod.demoRouteLazy)); +const DevonianDemoRoute = createRoute({ + getParentRoute: () => appRoute, + path: '/devonian-demo', +}).lazy(() => + import('./DevonianDemoRoute').then(mod => mod.devonianDemoRouteLazy), +); + const PruneTestsRoute = createRoute({ getParentRoute: () => appRoute, path: pathNames.pruneTests, @@ -91,6 +98,7 @@ const routeTree = rootRoute.addChildren({ SandboxRoute, DevDriveRoute, DemoRoute, + DevonianDemoRoute, InviteRoute, LinkOpenRouter, }), diff --git a/browser/e2e/tests/devonian-issue-sync.spec.mts b/browser/e2e/tests/devonian-issue-sync.spec.mts new file mode 100644 index 0000000000..b79b8e97fa --- /dev/null +++ b/browser/e2e/tests/devonian-issue-sync.spec.mts @@ -0,0 +1,182 @@ +import { test, expect } from '@playwright/test'; +import { + mockProxy, + tenantSecret, +} from '../../../integrations/localthought/mock-proxy.mjs'; +const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:6747'; +const SERVER_URL = process.env.SERVER_URL ?? 'http://localhost:9883'; + +test.use({ serviceWorkers: 'block' }); + +// This is a public fixture secret, never an environment/live credential. +// Exercise the real form, tenant challenge, consent, rotating HTTP transport and OPFS. +test('Devonian syncs issue creation, state and comments both ways through the browser proxy', async ({ + page, +}) => { + test.setTimeout(180_000); + const proxy = mockProxy({ frontendOrigin: new URL(FRONTEND_URL).origin }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const address = proxy.address() as { port: number }; + const proxyOrigin = `http://127.0.0.1:${address.port}`; + const repository = 'demo/two-way'; + const remote = proxy.github; + const initial = remote.createIssue(repository, { + title: 'Created on GitHub', + }); + remote.createComment(repository, initial.number, { + body: 'Initial GitHub comment', + }); + // No AtomicServer WebSocket writes (or Vite HMR reloads) during this journey. + await page.routeWebSocket('**/*', socket => socket.close()); + const forbidden: string[] = []; + const requests: string[] = []; + page.on('request', req => { + if (req.url().startsWith(`${proxyOrigin}/proxy/`)) + requests.push(req.method()); + }); + // Keep static app assets available even when frontend and AtomicServer share an origin. + await page.route('**/*', route => { + const url = new URL(route.request().url()); + if (/^\/(integration-proxy|plugin-run|commit)(\/|$)/.test(url.pathname)) { + forbidden.push(url.pathname); + return route.abort(); + } + if ( + url.origin === new URL(SERVER_URL).origin && + url.origin !== new URL(FRONTEND_URL).origin + ) + return route.abort(); + return route.continue(); + }); + try { + await page.goto(`${FRONTEND_URL}/app/dev-drive`); + await page.waitForURL(/app\/show\?subject=/, { timeout: 60000 }); + await page.goto(`${FRONTEND_URL}/app/devonian-demo`); + await page + .getByText('Connect a real GitHub repository', { exact: true }) + .click(); + await page.getByLabel('Integration proxy URL').fill(proxyOrigin); + await page.getByLabel('GitHub repository (owner/repo)').fill(repository); + await page.getByLabel('LocalThought tenant secret').fill(tenantSecret); + await page + .getByRole('button', { name: 'Connect GitHub tracker', exact: true }) + .click(); + await page + .getByRole('button', { name: 'Connect test account', exact: true }) + .click(); + const sync = async () => { + const button = page.getByRole('button', { + name: 'Sync now', + exact: true, + }); + await button.click(); + await expect(button).toBeEnabled(); + await expect(page.getByRole('alert')).toHaveCount(0); + }; + await expect( + page.getByRole('button', { name: 'Sync now', exact: true }), + ).toBeVisible(); + expect(page.url()).not.toContain('connection_code'); + expect( + await page.evaluate(() => + JSON.stringify({ ...localStorage, ...sessionStorage }), + ), + ).not.toContain(tenantSecret); + await sync(); + const issue = (title: string) => + page + .getByTestId('atomic-issue') + .filter({ has: page.getByRole('link', { name: title, exact: true }) }); + await expect(issue('Created on GitHub')).toContainText( + 'Initial GitHub comment', + ); + + const text = page.getByLabel('New issue title or comment'); + await text.fill('Created in Atomic'); + await page + .getByRole('button', { name: 'Create Atomic issue', exact: true }) + .click(); + await expect(issue('Created in Atomic')).toBeVisible(); + await sync(); + const created = remote + .snapshot(repository) + .issues.find(i => i.title === 'Created in Atomic'); + expect(created).toBeDefined(); + expect(remote.snapshot(repository).issues).toHaveLength(2); + + await text.fill('Comment from Atomic'); + await issue('Created in Atomic') + .getByRole('button', { name: 'Add Atomic comment', exact: true }) + .click(); + await issue('Created in Atomic') + .getByRole('button', { name: 'Close Atomic issue', exact: true }) + .click(); + await sync(); + expect( + remote.snapshot(repository).issues.find(i => i.number === created!.number) + ?.state, + ).toBe('closed'); + expect( + remote + .snapshot(repository) + .comments.filter(c => c.issue_url.endsWith(`/${created!.number}`)) + .map(c => c.body), + ).toEqual(['Comment from Atomic']); + + remote.updateIssue(repository, created!.number, { state: 'open' }); + remote.createComment(repository, created!.number, { + body: 'Reply from GitHub', + }); + remote.updateIssue(repository, initial.number, { state: 'closed' }); + await sync(); + await expect( + issue('Created in Atomic').getByRole('button', { + name: 'Close Atomic issue', + exact: true, + }), + ).toBeVisible(); + await expect(issue('Created in Atomic')).toContainText('Reply from GitHub'); + await expect( + issue('Created on GitHub').getByRole('button', { + name: 'Reopen Atomic issue', + exact: true, + }), + ).toBeVisible(); + await issue('Created on GitHub') + .getByRole('button', { name: 'Reopen Atomic issue', exact: true }) + .click(); + await sync(); + expect( + remote.snapshot(repository).issues.find(i => i.number === initial.number) + ?.state, + ).toBe('open'); + + const before = remote.snapshot(repository); + const subjects = await page + .getByTestId('atomic-issue') + .getByRole('link') + .evaluateAll(links => + links.map(link => link.getAttribute('href')).sort(), + ); + await page.reload(); + await sync(); + await expect(page.getByTestId('atomic-issue')).toHaveCount(2); + await expect(issue('Created in Atomic')).toContainText('Reply from GitHub'); + expect( + await page + .getByTestId('atomic-issue') + .getByRole('link') + .evaluateAll(links => + links.map(link => link.getAttribute('href')).sort(), + ), + ).toEqual(subjects); + expect(remote.snapshot(repository)).toEqual(before); + expect(requests).toEqual(expect.arrayContaining(['GET', 'POST', 'PATCH'])); + expect(forbidden).toEqual([]); + } finally { + proxy.closeAllConnections(); + await new Promise((resolve, reject) => + proxy.close(error => (error ? reject(error) : resolve())), + ); + } +}); diff --git a/browser/e2e/tests/google-calendar-import.spec.mts b/browser/e2e/tests/google-calendar-import.spec.mts new file mode 100644 index 0000000000..22d00c5e81 --- /dev/null +++ b/browser/e2e/tests/google-calendar-import.spec.mts @@ -0,0 +1,169 @@ +import { test, expect } from '@playwright/test'; +import { + mockProxy, + tenantSecret, +} from '../../../integrations/localthought/mock-proxy.mjs'; +const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:6747'; +const SERVER_URL = process.env.SERVER_URL ?? 'http://localhost:9883'; +test.use({ serviceWorkers: 'block' }); + +// Reuse the browser transport and HTTP mock from the Devonian integration. +// The tenant secret is a public fixture. No real provider credentials are used. +test('Calendar imports, refreshes and persists through the browser with AtomicServer unavailable', async ({ + page, +}) => { + test.setTimeout(180_000); + const proxy = mockProxy({ frontendOrigin: new URL(FRONTEND_URL).origin }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const port = (proxy.address() as { port: number }).port; + const proxyOrigin = `http://127.0.0.1:${port}`; + const configuredProxy = + process.env.VITE_INTEGRATION_PROXY_URL || 'https://localthought.io'; + await page.routeWebSocket('**/*', socket => socket.close()); + const forbidden: string[] = []; + const providerMethods: string[] = []; + // Forward the configured proxy to this test's isolated HTTP fixture. All + // tenant proof, consent, code rotation, pagination and WASM code remain real. + await page.route('**/*', async route => { + const request = route.request(); + const url = new URL(request.url()); + + if (/^\/(integration-proxy|plugin-run|commit)(\/|$)/.test(url.pathname)) { + forbidden.push(url.pathname); + + return route.abort(); + } + + if (url.origin === configuredProxy) { + if (url.pathname.startsWith('/proxy/')) + providerMethods.push(request.method()); + const response = await route.fetch({ + url: `${proxyOrigin}${url.pathname}${url.search}`, + maxRedirects: 0, + }); + + return route.fulfill({ response }); + } + + if ( + url.origin === new URL(SERVER_URL).origin && + url.origin !== new URL(FRONTEND_URL).origin + ) + return route.abort(); + + return route.continue(); + }); + + try { + await page.goto(`${FRONTEND_URL}/app/dev-drive`); + await page.waitForURL(/app\/show\?subject=/, { timeout: 60000 }); + + const setup = async () => { + await page + .getByRole('link', { name: 'Integrations', exact: true }) + .click(); + await page + .locator('[data-integration="google-calendar"]') + .getByRole('button', { name: 'Set up connection' }) + .click(); + }; + + await setup(); + await page.getByLabel('LocalThought tenant secret').fill(tenantSecret); + await page + .getByRole('button', { name: 'Install and connect', exact: true }) + .click(); + await page + .getByRole('button', { name: 'Connect test account', exact: true }) + .click(); + await expect( + page.getByRole('button', { name: 'Fetch and preview', exact: true }), + ).toBeVisible(); + expect(page.url()).not.toContain('connection_code'); + expect( + await page.evaluate(() => + JSON.stringify({ ...localStorage, ...sessionStorage }), + ), + ).not.toContain(tenantSecret); + + const apply = async (count: number) => { + await page + .getByRole('button', { name: 'Fetch and preview', exact: true }) + .click(); + await page + .getByRole('button', { name: new RegExp(`^Apply ${count} changes?$`) }) + .click(); + await page + .getByRole('link', { name: 'Open imported records', exact: true }) + .click(); + await expect(page.getByTestId('calendar-view')).toBeVisible(); + }; + + await apply(2); + await expect(page.getByTestId('calendar-event')).toHaveCount(2); + await page.getByText('Calendar timed fixture', { exact: true }).click(); + await expect(page.getByRole('dialog').last()).toContainText( + 'Recurring event', + ); + await expect(page.getByRole('dialog').last()).toContainText( + 'Attendees and RSVP', + ); + await page.keyboard.press('Escape'); + const original = await page.evaluate(async () => { + const store = window.store!; + const table = new URL(location.href).searchParams.get('subject')!; + const result = await store.queryLocalDb({ + drive: store.getDrive()!, + property: 'https://atomicdata.dev/properties/parent', + value: table, + limit: 100, + }); + const subjects = result!.subjects.sort(); + const row = await store.getResource(subjects[0]); + await row.set( + 'https://atomicdata.dev/properties/description', + 'Atomic-only notes', + ); + await row.save(); + await store.getClientDb()!.flush(); + + return { subjects, table, annotated: row.subject }; + }); + await page.reload(); + await expect(page.getByTestId('calendar-event')).toHaveCount(2); + proxy.calendar.events[1].summary = 'Calendar refreshed fixture'; + await setup(); + await apply(1); + await expect( + page.getByText('Calendar refreshed fixture', { exact: true }), + ).toBeVisible(); + await expect(page.getByTestId('calendar-event')).toHaveCount(2); + const restored = await page.evaluate(async saved => { + const store = window.store!; + const result = await store.queryLocalDb({ + drive: store.getDrive()!, + property: 'https://atomicdata.dev/properties/parent', + value: saved.table, + limit: 100, + }); + + return { + subjects: result!.subjects.sort(), + note: (await store.getResource(saved.annotated)).get( + 'https://atomicdata.dev/properties/description', + ), + }; + }, original); + expect(restored.subjects).toEqual(original.subjects); + expect(restored.note).toBe('Atomic-only notes'); + expect(providerMethods).toEqual(['GET', 'GET', 'GET', 'GET']); + expect( + proxy.calendar.requests.filter(r => r.query.pageToken === 'second'), + ).toHaveLength(2); + expect(forbidden).toEqual([]); + } finally { + await new Promise((resolve, reject) => + proxy.close(error => (error ? reject(error) : resolve())), + ); + } +}); diff --git a/integrations/github-issues/README.md b/integrations/github-issues/README.md index f265fa5f0c..e0f80d1254 100644 --- a/integrations/github-issues/README.md +++ b/integrations/github-issues/README.md @@ -1,3 +1,10 @@ +## Browser-only Devonian demo + +Open `/app/devonian-demo` for a browser-side issue/comment sync demo using +Devonian and this integration's mappings. It uses a local-only Atomic drive and +direct integration-proxy requests, with an explicit sample mode. See +[setup, source and the current live proxy CORS limitation](devonian/README.md). + ## Connect in the app Open **Integrations**, enter `owner/repository` and a repository-scoped GitHub diff --git a/integrations/github-issues/adapter.test.ts b/integrations/github-issues/adapter.test.ts index e60052a966..854422b60d 100644 --- a/integrations/github-issues/adapter.test.ts +++ b/integrations/github-issues/adapter.test.ts @@ -3,6 +3,7 @@ import { preview, project, manifest, type Issue } from './adapter.js'; import { validateManifest } from '../../browser/lib/src/plugin-manifest.js'; import { readFile } from 'node:fs/promises'; import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; const issue = (number: number): Issue => ({ number, title: `Issue ${number}`, @@ -18,7 +19,9 @@ describe('GitHub package', () => { }); it('ships exactly the artifact tested by the sandbox', async () => { const built = execFileSync( - './browser/node_modules/.bin/esbuild', + existsSync('./browser/node_modules/.bin/esbuild') + ? './browser/node_modules/.bin/esbuild' + : './browser/node_modules/.pnpm/node_modules/.bin/esbuild', [ 'integrations/github-issues/plugin.ts', '--bundle', diff --git a/integrations/github-issues/devonian/README.md b/integrations/github-issues/devonian/README.md new file mode 100644 index 0000000000..56c895d925 --- /dev/null +++ b/integrations/github-issues/devonian/README.md @@ -0,0 +1,129 @@ +# Browser issue tracker demo + +Open `/app/devonian-demo` in the Atomic web app. Choose **Try sample data**, +create issues or comments on either side, close/reopen an issue, and press +**Sync now**. **Open Atomic kanban** opens the real native tracker, whose issue +pages also have the normal Comments panel. Reload and choose the same mode to +resume saved resources and mappings without importing duplicates. + +The JavaScript runs in the browser. Devonian's native resource lenses run there; +Atomic resources live in a local-only drive in the browser's WASM/OPFS database. +The intermediate graph, scoped identity mappings, baselines and request journal +live in IndexedDB. No Node runtime, custom AtomicServer endpoint, server-side +plugin executor, tenant secret on AtomicServer or server worker is used. Enable +the browser database if it has been disabled. + +## Live GitHub + +Expand **Connect a real GitHub repository**, enter the integration-proxy origin, +`owner/repo`, and your **LocalThought tenant secret**, then choose **Connect GitHub +tracker**. The demo uses #1401's shared `BrowserIntegrations` client to sign the +tenant challenge in the browser, navigate to proxy consent, and bind the return +to this agent and local drive. The secret is cleared from the form and never +persisted. The callback code is removed from the address bar immediately. + +After consent, **Sync now** authorizes two-way writes. The shared transport +serializes requests with Web Locks, consumes each code before dispatch, and +persists rotated credentials in browser localStorage, outside the Atomic graph. +Reloading the tab resumes the local tracker and connection. Use a dedicated +demo repository; writes use the connected GitHub account. + +The browser calls `/proxy/github-issues/repos/{owner}/{repo}/issues...` directly. +The proxy instance must answer unauthenticated OPTIONS preflights, allow the +app's origin and GET/POST/PATCH/DELETE with Authorization and Content-Type, +expose `X-Connection-Code`, preserve query parameters, and allow issue/comment +and label operations in its catalog. CORS headers must cover error responses too. + +**Live verification (2026-09-09):** Heroku release v40 (`bc02f13f`) now +answers browser preflights and exposes `X-Connection-Code`. Tenant challenge, +GitHub OAuth and return to the browser demo succeeded with AtomicServer +unavailable. The first authenticated issue-list request for the private +`ontola/atomic-github-sync-sandbox` returned 404, although the independent +GitHub CLI credential can access it. Repository access for the proxy credential +must be resolved before live two-way writes can be verified. The disposable +CLI-created issue #17 was closed; no issue/comment writes occurred through the proxy. + +## Mapping + +`GitHubPort` reuses `github-issues/adapter.ts`'s projection and request builder. +`tracker-actions.ts` adds scoped issue/comment request construction for the +browser transport. The installed server plugin's actions remain unchanged. + +- Title/body map to the native row's name and Markdown description. +- Open maps to Todo; open with `atomic:doing` maps to Doing; closed maps to Done. + Other labels are preserved. Provision `atomic:doing` before using Doing. +- A comment is a native Message whose `about` points to its issue and whose + `parent` is the drive's Comments folder. Body edits synchronize both ways. +- GitHub identity, author and original timestamps are retained in a separate + provenance property. The importing Atomic agent is distinct from the original + author. GitHub writes use the connected account's authorship. +- The demo creates drive-local task properties so setup works offline. The + intermediate Devonian graph uses the shared task vocabulary and maps native + properties explicitly. Atomic DIDs are scoped external IDs because Devonian's + current native subjects must be HTTP(S) URLs. +- Explicit issue numbers can bind existing rows; matching text never does. + Comments have their own scoped IDs; identical comments remain distinct. + +## Persistence and limits + +A Web Lock serializes syncs. Three-way reconciliation preserves independent +field edits and stops on same-field conflicts. Reloads resume saved operations +before discovering new records. Ingest never automatically publishes an echo. +Missing records are conflicts; neither side is deleted. + +The proxy does not provide idempotent GitHub creation. Writes are journaled +before sending. Successful receipts can be replayed; an uncertain/lost response +stops without resending. An operator must inspect and reconcile that operation; +there is no recovery wizard yet. Do not clear IndexedDB to retry a create. +Concurrent edits during a multi-request status transition may also require +reconciliation. Pre-write reads and verification do not make local/provider +writes one transaction. Sync is explicit while the tab is open, with a +10,000-record scan cap. Clearing browser site data loses the demo and its state. + +## Source and verification + +- `browser/data-browser/src/chunks/DevonianDemo/demo.mjs`: browser entry script. +- `bridge.mjs`: Devonian lenses and checkpointed reconciliation. +- `ports.mjs`: native Atomic and GitHub projections/transports. +- `proxy.mjs`: rotating-code transport and labelled sample fixture. +- `browser/data-browser/src/routes/DevonianDemoRoute.tsx`: demo controls. + +The committed `DevonianDemo/devonian.js` bundle contains only the native resource +API from Devonian main `e11104f78ebd151a361171ca0b21489b25e1e2c8`, with its +Apache-2.0 license alongside. Regenerate at development time: + +```sh +DEVONIAN_PATH=/path/to/devonian node integrations/github-issues/devonian/build.mjs +browser/node_modules/.bin/vitest run --config integrations/github-issues/devonian/vitest.config.mjs +browser/node_modules/.bin/vitest run --config integrations/github-issues/vitest.config.ts +cd browser/data-browser && pnpm typecheck +``` + +Tests use real Devonian lenses and deterministic connectors: creation, identical +content with distinct identities, independent edits, close/reopen, comments, +conflicts, missing resources, replay after lost receipts, label preservation, +pagination, scoped URLs and serialized code rotation. The browser sample flow +was manually verified with native OPFS resources, issue creation on both sides, +comments both ways, closing from Atomic and reopening from the GitHub fixture, +then reloading and resuming the same three issues and two comments without duplication. + +## Playwright two-way regression + +`browser/e2e/tests/devonian-issue-sync.spec.mts` starts an isolated HTTP integration +proxy with stateful, repository-scoped GitHub issue/comment endpoints. It fills +the tenant-secret form with the **public mock secret** (no real credentials), +completes consent, and exercises the live transport mode rather than sample mode. +It verifies creation and comments in both directions, close/reopen in both +directions, matching parent issues, and reload without duplicate resources or +provider writes. Legacy server integration endpoints, commit POSTs and all WebSockets are blocked. +The local run uses an unavailable AtomicServer port to verify browser-only storage. + +With Vite and the built browser/WASM packages available: + +```sh +cd browser/e2e +FRONTEND_URL=http://localhost:6747 playwright test tests/devonian-issue-sync.spec.mts --project chromium +``` + +The test is in the full E2E suite, without a smoke tag. The mock is local test +infrastructure only; the runtime demo uses the real integration-proxy protocol. diff --git a/integrations/github-issues/devonian/bridge.mjs b/integrations/github-issues/devonian/bridge.mjs new file mode 100644 index 0000000000..3f90390a4d --- /dev/null +++ b/integrations/github-issues/devonian/bridge.mjs @@ -0,0 +1,243 @@ +import { reconcileRecord } from '../../../browser/lib/src/plugin-reconcile.js'; + +const p = { + title: 'https://atomicdata.dev/properties/name', + body: 'https://atomicdata.dev/task/v1/body', + status: 'https://atomicdata.dev/task/v1/status', +}; +const tag = 'https://atomicdata.dev/task/v1/'; +const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b); +const copy = value => structuredClone(value); + +/** Single-writer, checkpointed reconciliation. Ports own transport and durable writes. */ +export class Bridge { + constructor({ devonian, local, remote, base, snapshot, save }) { + this.api = devonian; + this.local = local; + this.remote = remote; + this.save = save; + const { AtomicSchema, AtomicStore, AtomicIdentityMap, Datatype } = devonian; + this.store = new AtomicStore( + new AtomicSchema() + .property(p.title, Datatype.STRING) + .property(p.body, Datatype.MARKDOWN) + .property(p.status, Datatype.RESOURCEARRAY), + ); + this.identities = new AtomicIdentityMap(this.store, base); + this.binding = { base, local: local.scope, remote: remote.scope }; + if (snapshot && !equal(snapshot.binding, this.binding)) + throw new Error('State belongs to another connection'); + if (snapshot?.graph) this.store.loadJSONAD(snapshot.graph); + this.records = copy(snapshot?.records ?? {}); + } + + scope(side, entity) { + return { scope: this[side].scope, entity }; + } + id(side, entity, subject) { + return this.identities.externalId(this.scope(side, entity), subject); + } + context(side, entity) { + if (entity === 'issue') return {}; + const parent = entity.slice('comment:'.length); + const issueId = this.id(side, 'issue', parent); + if (issueId === undefined) + throw new Error('Issue must be mapped before comments'); + return { issueId }; + } + async checkpoint() { + await this.save({ + version: 1, + binding: this.binding, + graph: this.store.toJSONAD(), + records: copy(this.records), + }); + } + properties(value) { + return { + [p.body]: value.body, + ...(value.title === undefined + ? {} + : { + [p.title]: value.title, + [p.status]: [`${tag}${value.status.toLowerCase()}`], + }), + }; + } + value(resource, entity) { + if (entity !== 'issue') return { body: resource[p.body] }; + const status = { + [`${tag}todo`]: 'Todo', + [`${tag}doing`]: 'Doing', + [`${tag}done`]: 'Done', + }[resource[p.status]?.[0]]; + if (!status || resource[p.status].length !== 1) + throw new Error('Unsupported task status'); + return { title: resource[p.title], body: resource[p.body], status }; + } + lens(side, entity, operation, metadata) { + const port = this[side], + context = this.context(side, entity); + return new this.api.AtomicLens({ + store: this.store, + identities: this.identities, + ...this.scope(side, entity), + connector: { + id: row => row.id, + get: id => port.get(entity, id, context), + create: (row, key) => + port.create(entity, row.value, key, metadata, context), + update: (id, row) => + port.update( + entity, + id, + row.value, + `${operation}:${side}`, + metadata, + context, + ), + delete: async () => { + throw new Error('Deletion is outside issue sync'); + }, + }, + read: row => ({ set: this.properties(row.value) }), + write: (resource, previous) => ({ + ...previous, + value: this.value(resource, entity), + }), + }); + } + + async sync() { + // Resume saved operations BEFORE discovering their newly-created counterparts. + for (const [subject, record] of Object.entries(this.records)) { + if (record.pending) await this.finish(subject, record); + } + await this.syncEntity('issue'); + for (const [subject, record] of Object.entries(this.records)) { + if (record.entity === 'issue') + await this.syncEntity(`comment:${subject}`); + } + } + + async syncEntity(entity) { + const lists = {}; + for (const side of ['remote', 'local']) { + const rows = await this[side].list(entity, this.context(side, entity)); + lists[side] = new Map(); + for (const row of rows) { + if (lists[side].has(row.id)) + throw new Error('Duplicate external identity'); + lists[side].set(row.id, row); + } + } + // An existing pilot's explicit issue-number column is an identity, never a title match. + for (const row of lists.local.values()) { + if (row.remoteId === undefined) continue; + const scope = this.scope('remote', entity); + const subject = this.identities.subjectFor(scope, row.remoteId); + this.identities.bind(scope, row.remoteId, subject); + this.identities.bind(this.scope('local', entity), row.id, subject); + this.records[subject] ??= { entity }; + } + for (const side of ['remote', 'local']) { + for (const row of lists[side].values()) { + let subject = this.identities.lookup(this.scope(side, entity), row.id); + if (!subject) subject = await this.lens(side, entity).ingest(row); + this.records[subject] ??= { entity }; + } + } + await this.checkpoint(); + for (const [subject, record] of Object.entries(this.records)) { + if (record.entity !== entity) continue; + const rows = {}; + for (const side of ['local', 'remote']) { + const id = this.id(side, entity, subject); + rows[side] = id === undefined ? undefined : lists[side].get(id); + if (id !== undefined && !rows[side]) + throw new Error(`Missing ${side} record: ${subject}`); + } + const decision = reconcileRecord( + record.baseline, + rows.local?.value, + rows.remote?.value, + ); + if (decision.conflicts.length) + throw new Error( + `Conflict on ${subject}: ${decision.conflicts.map(c => c.property).join(', ')}`, + ); + const desired = { + ...(rows.remote?.value ?? rows.local?.value), + ...decision.remote, + }; + const metadata = rows.remote?.metadata; + if ( + rows.local && + rows.remote && + equal(rows.local.value, rows.remote.value) && + equal(rows.local.metadata, metadata) + ) { + record.baseline = copy(desired); + await this.checkpoint(); + continue; + } + record.pending = { + operation: crypto.randomUUID(), + local: rows.local?.value, + remote: rows.remote?.value, + desired, + metadata, + }; + this.store.patch(subject, { set: this.properties(desired) }); + await this.checkpoint(); + await this.finish(subject, record); + } + } + + async finish(subject, record) { + const { pending, entity } = record; + // A retry accepts only the original observation or this operation's exact result. + for (const side of ['local', 'remote']) { + const id = this.id(side, entity, subject); + if (id === undefined) continue; + const row = await this[side].get(entity, id, this.context(side, entity)); + if ( + !equal(row.value, pending[side]) && + !equal(row.value, pending.desired) + ) + throw new Error(`Conflict during saved operation on ${subject}`); + } + for (const side of ['remote', 'local']) { + const id = this.id(side, entity, subject); + const row = + id === undefined + ? undefined + : await this[side].get(entity, id, this.context(side, entity)); + if ( + !row || + !equal(row.value, pending.desired) || + (side === 'local' && !equal(row.metadata, pending.metadata)) + ) { + await this.lens( + side, + entity, + pending.operation, + pending.metadata, + ).publish(subject); + await this.checkpoint(); + } + } + for (const side of ['local', 'remote']) { + const row = await this[side].get( + entity, + this.id(side, entity, subject), + this.context(side, entity), + ); + if (!equal(row.value, pending.desired)) + throw new Error(`Concurrent edit after write on ${subject}`); + } + record.baseline = copy(pending.desired); + delete record.pending; + await this.checkpoint(); + } +} diff --git a/integrations/github-issues/devonian/bridge.test.mjs b/integrations/github-issues/devonian/bridge.test.mjs new file mode 100644 index 0000000000..672a7b1030 --- /dev/null +++ b/integrations/github-issues/devonian/bridge.test.mjs @@ -0,0 +1,133 @@ +import { expect, it } from 'vitest'; +import * as devonian from 'devonian'; +import { Bridge } from './bridge.mjs'; + +function fixture(snapshot) { + let saved = snapshot; + const makePort = scope => ({ + scope, + rows: new Map(), + receipts: new Map(), + writes: 0, + lose: false, + async list(entity) { + return [...this.rows.values()].filter(r => r.entity === entity); + }, + async get(entity, id) { + const row = this.rows.get(id); + if (!row || row.entity !== entity) throw new Error('Missing record'); + return structuredClone(row); + }, + async create(entity, value, key, metadata) { + if (this.receipts.has(key)) return this.receipts.get(key); + const id = this.rows.size + 1; + const row = { id, entity, value: structuredClone(value), metadata }; + this.rows.set(id, row); + this.receipts.set(key, row); + this.writes++; + if (this.lose) { + this.lose = false; + throw new Error('Lost response'); + } + return row; + }, + async update(entity, id, value) { + const row = await this.get(entity, id); + this.rows.set(id, { ...row, value: structuredClone(value) }); + this.writes++; + }, + }); + const local = makePort('https://atomic.example/bridge'); + const remote = makePort('https://github.com/acme/repo'); + const open = () => + new Bridge({ + devonian, + local, + remote, + snapshot: saved, + base: 'https://bridge.example/sync', + save: async s => { + saved = structuredClone(s); + }, + }); + return { local, remote, open, saved: () => saved }; +} +const issue = (id, title = 'Same title') => ({ + id, + entity: 'issue', + value: { title, body: '', status: 'Todo' }, +}); + +it('syncs creation both ways without deduplicating equal content, then no-ops after restart', async () => { + const f = fixture(); + f.local.rows.set(1, issue(1)); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + expect(f.local.rows.size).toBe(2); + expect(f.remote.rows.size).toBe(2); + const writes = f.local.writes + f.remote.writes; + await f.open().sync(); + expect(f.local.writes + f.remote.writes).toBe(writes); +}); + +it('merges independent title/body edits and propagates close/reopen', async () => { + const f = fixture(); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + f.local.rows.get(1).value.title = 'Local title'; + f.remote.rows.get(1).value.body = 'Remote body'; + f.local.rows.get(1).value.status = 'Done'; + await f.open().sync(); + expect(f.remote.rows.get(1).value).toEqual({ + title: 'Local title', + body: 'Remote body', + status: 'Done', + }); + f.remote.rows.get(1).value.status = 'Todo'; + await f.open().sync(); + expect(f.local.rows.get(1).value.status).toBe('Todo'); +}); + +it('syncs comments and edits both ways, preserving source metadata', async () => { + const f = fixture(); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + const parent = Object.keys(f.saved().records)[0]; + const entity = `comment:${parent}`; + f.remote.rows.set(2, { + id: 2, + entity, + value: { body: 'GitHub comment' }, + metadata: { author: 'octocat' }, + }); + f.local.rows.set(2, { id: 2, entity, value: { body: 'Atomic comment' } }); + await f.open().sync(); + expect(f.local.rows.get(3).metadata).toEqual({ author: 'octocat' }); + expect(f.remote.rows.get(3).value.body).toBe('Atomic comment'); + f.local.rows.get(3).value.body = 'Edited'; + await f.open().sync(); + expect(f.remote.rows.get(2).value.body).toBe('Edited'); +}); + +it('stops on same-field conflicts and missing records without deleting either side', async () => { + const f = fixture(); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + f.local.rows.get(1).value.title = 'A'; + f.remote.rows.get(1).value.title = 'B'; + await expect(f.open().sync()).rejects.toThrow('Conflict'); + expect(f.local.rows.get(1).value.title).toBe('A'); + f.remote.rows.delete(1); + await expect(f.open().sync()).rejects.toThrow('Missing'); + expect(f.local.rows.size).toBe(1); +}); + +it('reuses the same create identity after a lost receipt and restart', async () => { + const f = fixture(); + f.local.rows.set(1, issue(1)); + f.remote.lose = true; + await expect(f.open().sync()).rejects.toThrow('Lost response'); + await f.open().sync(); + expect(f.remote.rows.size).toBe(1); + expect(f.remote.writes).toBe(1); +}); diff --git a/integrations/github-issues/devonian/build.mjs b/integrations/github-issues/devonian/build.mjs new file mode 100644 index 0000000000..9a6c4c2051 --- /dev/null +++ b/integrations/github-issues/devonian/build.mjs @@ -0,0 +1,36 @@ +/** Development-time bundle; the demo itself runs entirely in the browser. */ +import { createRequire } from 'node:module'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const require = createRequire(import.meta.url); +const { build } = require( + require.resolve('esbuild', { + paths: [ + resolve(root, 'browser'), + resolve(root, 'browser/node_modules/.pnpm'), + ], + }), +); +if (!process.env.DEVONIAN_PATH) + throw new Error('Set DEVONIAN_PATH to Devonian main'); +await build({ + stdin: { + contents: ['Resource', 'Store', 'IdentityMap', 'Lens'] + .map(name => `export * from './src/atomic/${name}.js';`) + .join('\n'), + resolveDir: resolve(process.env.DEVONIAN_PATH), + loader: 'ts', + }, + bundle: true, + platform: 'browser', + format: 'esm', + external: ['@tomic/lib'], + banner: { + js: '// Devonian native resource API, e11104f78ebd151a361171ca0b21489b25e1e2c8. Apache-2.0; see DEVONIAN-LICENSE.', + }, + outfile: resolve( + root, + 'browser/data-browser/src/chunks/DevonianDemo/devonian.js', + ), +}); diff --git a/integrations/github-issues/devonian/ports.mjs b/integrations/github-issues/devonian/ports.mjs new file mode 100644 index 0000000000..ca5733d489 --- /dev/null +++ b/integrations/github-issues/devonian/ports.mjs @@ -0,0 +1,322 @@ +import { project } from '../adapter.js'; +import { core, dataBrowser } from '../../../browser/lib/src/index.js'; + +export const digest = async value => + Array.from( + new Uint8Array( + await crypto.subtle.digest('SHA-256', new TextEncoder().encode(value)), + ), + byte => byte.toString(16).padStart(2, '0'), + ).join(''); +const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b); + +/** Reuses the GitHub integration mapping with an injected browser proxy transport. */ +export class GitHubPort { + constructor(store, connection, call) { + this.store = store; + this.connection = connection; + this.scope = `https://github.com/${connection.repository}`; + if (!call) + throw new Error('A browser integration-proxy transport is required'); + this.call = call; + } + async request(action, args, key) { + const receipt = await this.call( + action, + args, + await digest(key ?? `${action}:${JSON.stringify(args)}`), + ); + if (receipt.status < 200 || receipt.status >= 300) + throw new Error(`GitHub ${action} returned ${receipt.status}`); + return receipt.body ? JSON.parse(receipt.body) : undefined; + } + row(entity, raw, context = {}) { + if (entity === 'issue') { + if (raw.pull_request) throw new Error('Pull requests are excluded'); + return { + id: raw.number, + value: project(raw), + metadata: this.metadata(raw, { number: raw.number }), + }; + } + if ( + !Number.isSafeInteger(raw.id) || + raw.id <= 0 || + typeof raw.body !== 'string' + ) + throw new Error('Invalid GitHub comment'); + const expected = `https://api.github.com/repos/${this.connection.repository}/issues/${context.issueId}`; + if (raw.issue_url !== expected) + throw new Error('Comment belongs to another issue'); + return { + id: raw.id, + value: { body: raw.body }, + metadata: this.metadata(raw, { commentId: raw.id }), + }; + } + metadata(raw, identity) { + return { + ...identity, + ...(raw.user?.login ? { author: raw.user.login } : {}), + ...(raw.html_url ? { url: raw.html_url } : {}), + ...(raw.created_at ? { createdAt: raw.created_at } : {}), + ...(raw.updated_at ? { updatedAt: raw.updated_at } : {}), + }; + } + async list(entity, context = {}) { + const result = []; + for (let page = 1; page <= 100; page++) { + const raw = await this.request( + entity === 'issue' ? 'list_issues' : 'list_comments', + { + page, + ...(entity === 'issue' ? {} : { number: context.issueId }), + }, + ); + if (!Array.isArray(raw)) throw new Error('Invalid GitHub page'); + for (const r of raw) { + if (entity === 'issue' && 'pull_request' in r) continue; + result.push(this.row(entity, r, context)); + } + if (raw.length < 100) return result; + } + throw new Error('More than 10,000 records in one scan'); + } + async get(entity, id, context) { + const raw = await this.request( + entity === 'issue' ? 'get_issue' : 'get_comment', + entity === 'issue' ? { number: id } : { id }, + ); + const row = this.row(entity, raw, context); + if (row.id !== id) throw new Error('Provider returned another identity'); + return row; + } + async create(entity, value, key, metadata, context) { + const raw = await this.request( + entity === 'issue' ? 'create_issue' : 'create_comment', + entity === 'issue' + ? { title: value.title, body: value.body } + : { number: context.issueId, body: value.body }, + `${key}:create`, + ); + const row = this.row(entity, raw, context); + if (entity === 'issue' && value.status !== 'Todo') + await this.update( + entity, + row.id, + value, + `${key}:initial-state`, + metadata, + context, + ); + return { ...row, value }; + } + async update(entity, id, value, key, _metadata, context) { + if (entity !== 'issue') { + await this.get(entity, id, context); // validate issue membership before writing + await this.request( + 'update_comment', + { id, body: value.body }, + `${key}:body`, + ); + return; + } + if (!['Todo', 'Doing', 'Done'].includes(value.status)) + throw new Error('Unsupported task status'); + const current = await this.get(entity, id, context); + await this.request( + 'update_issue', + { + number: id, + title: value.title, + body: value.body, + state: value.status === 'Done' ? 'closed' : 'open', + }, + `${key}:fields`, + ); + // Only this workflow label is managed. Closing preserves the label like the pilot. + if (value.status === 'Doing' && current.value.status !== 'Doing') { + await this.request('add_doing_label', { number: id }, `${key}:doing`); + } else if (value.status === 'Todo') { + // Closed issues may still carry the doing label, so inspect raw labels. + const raw = await this.request('get_issue', { number: id }); + if ( + raw.labels.some( + l => + (typeof l === 'string' ? l : l.name).toLowerCase() === + 'atomic:doing', + ) + ) { + await this.request('remove_doing_label', { number: id }, `${key}:todo`); + } + } + } +} + +/** Signed SDK writes build on the fetched Loro state, preserving unmanaged fields. */ +export class AtomicPort { + constructor(store, config) { + this.store = store; + this.config = config; + this.connection = config.connection; + this.scope = `https://atomicdata.dev/devonian-local/${encodeURIComponent(this.connection.table)}`; + } + async subjects(property, value) { + const result = await this.store.queryLocalDb({ + drive: this.connection.drive, + property, + value, + limit: 10001, + }); + if ( + !result || + result.count > 10000 || + result.subjects.length !== result.count + ) + throw new Error('Incomplete local database query'); + return result.subjects; + } + async findByLocalId(parent, localId) { + const rows = await Promise.all( + (await this.subjects(core.properties.localId, localId)).map(id => + this.resource(id), + ), + ); + const matches = rows.filter(r => r.get(core.properties.parent) === parent); + if (matches.length > 1) + throw new Error('Duplicate Atomic creation identity'); + return matches[0]; + } + async resource(subject) { + const r = await this.store.getResource(subject); + if (r.error) throw r.error; + r.getLoroDoc(); + return r; + } + row(entity, r, context = {}) { + const c = this.connection; + const classes = r.get(core.properties.isA) ?? []; + if (entity === 'issue') { + if ( + !classes.includes(c.rowClass) || + r.get(core.properties.parent) !== c.table + ) + throw new Error('Resource is not a row of this issue tracker'); + } else if ( + !classes.includes(dataBrowser.classes.message) || + r.get(dataBrowser.properties.about) !== context.issueId + ) + throw new Error('Message belongs to another issue'); + let metadata = r.get(this.config.provenance); + if (typeof metadata === 'string') metadata = JSON.parse(metadata); + if (entity !== 'issue') + return { + id: r.subject, + value: { body: r.get(core.properties.description) ?? '' }, + ...(metadata ? { metadata } : {}), + }; + const statuses = r.get(c.status) ?? [c.tags.Todo]; + const status = Object.keys(c.tags).find(s => c.tags[s] === statuses[0]); + if (statuses.length !== 1 || !status) + throw new Error( + 'Choose exactly one Todo/Doing/Done status; Blocked is unmapped', + ); + const title = r.get(core.properties.name), + body = r.get(c.body) ?? ''; + if (typeof title !== 'string' || !title.trim() || typeof body !== 'string') + throw new Error('Invalid Atomic issue'); + const remoteId = r.get(c.number); + if ( + remoteId !== undefined && + (!Number.isSafeInteger(remoteId) || remoteId <= 0) + ) + throw new Error('Invalid GitHub issue number'); + return { + id: r.subject, + remoteId, + value: { title, body, status }, + ...(metadata ? { metadata } : {}), + }; + } + async list(entity, context = {}) { + const ids = await this.subjects( + entity === 'issue' + ? core.properties.parent + : dataBrowser.properties.about, + entity === 'issue' ? this.connection.table : context.issueId, + ); + const rows = []; + for (const id of ids) { + const resource = await this.resource(id); + if ( + !(resource.get(core.properties.isA) ?? []).includes( + entity === 'issue' + ? this.connection.rowClass + : dataBrowser.classes.message, + ) + ) + continue; + rows.push(this.row(entity, resource, context)); + } + return rows; + } + async get(entity, id, context) { + return this.row(entity, await this.resource(id), context); + } + values(entity, value, metadata, context) { + const c = this.connection; + return { + ...(entity === 'issue' + ? { + [core.properties.name]: value.title, + [c.body]: value.body, + [c.status]: [c.tags[value.status]], + ...(metadata?.number ? { [c.number]: metadata.number } : {}), + } + : { + [core.properties.description]: value.body, + [dataBrowser.properties.about]: context.issueId, + }), + ...(metadata ? { [this.config.provenance]: metadata } : {}), + }; + } + async create(entity, value, key, metadata, context) { + const parent = + entity === 'issue' ? this.connection.table : this.config.commentsFolder; + const localId = `devonian:${await digest(key)}`; + const existing = await this.findByLocalId(parent, localId); + if (existing) { + const row = await this.get(entity, existing.subject, context); + if (!equal(row.value, value)) + throw new Error( + 'Recovered Atomic create was edited; reconcile before retry', + ); + return row; + } + const r = await this.store.newResource({ + parent, + isA: [ + entity === 'issue' + ? this.connection.rowClass + : dataBrowser.classes.message, + ], + propVals: { + ...this.values(entity, value, metadata, context), + [core.properties.localId]: localId, + }, + }); + if ((await r.save()) === 'offline') + throw new Error('AtomicServer disconnected'); + return this.get(entity, r.subject, context); + } + async update(entity, id, value, _key, metadata, context) { + const r = await this.resource(id); + this.row(entity, r, context); + for (const [p, v] of Object.entries( + this.values(entity, value, metadata, context), + )) + await r.set(p, v); + if ((await r.save()) === 'offline') + throw new Error('AtomicServer disconnected'); + } +} diff --git a/integrations/github-issues/devonian/ports.test.mjs b/integrations/github-issues/devonian/ports.test.mjs new file mode 100644 index 0000000000..25590e5ec6 --- /dev/null +++ b/integrations/github-issues/devonian/ports.test.mjs @@ -0,0 +1,237 @@ +import { expect, it } from 'vitest'; +import { GitHubPort, AtomicPort } from './ports.mjs'; +import { + trackerAction, + trackerActions, + trackerOperations, +} from '../tracker-actions.js'; +import { manifest } from '../adapter.js'; +import { validateManifest } from '../../../browser/lib/src/plugin-manifest.js'; + +it('declares and prepares scoped issue/comment actions with validated arguments', () => { + const original = manifest('owner/repo'); + const m = validateManifest({ + ...original, + actions: [...original.actions, ...trackerActions], + operations: [ + ...original.operations, + ...trackerOperations('https://api.github.com/repos/owner/repo/issues'), + ], + }); + for (const [action, args] of [ + ['list_issues', { page: 1 }], + ['list_comments', { number: 5, page: 2 }], + ['create_comment', { number: 5, body: 'Hello' }], + ['get_comment', { id: 9 }], + ['update_comment', { id: 9, body: 'Edit' }], + ['update_issue', { number: 5, title: 'Title', body: '', state: 'closed' }], + ['add_doing_label', { number: 5 }], + ['remove_doing_label', { number: 5 }], + ]) { + const intent = trackerAction('owner/repo', action, args); + expect( + intent.url.startsWith('https://api.github.com/repos/owner/repo/issues'), + ).toBe(true); + expect(m.actions.find(a => a.name === action).operation).toBe( + intent.operation, + ); + expect( + m.operations.some( + o => o.id === intent.operation && o.method === intent.method, + ), + ).toBe(true); + } + expect(() => + trackerAction('owner/repo', 'get_comment', { id: '../escape' }), + ).toThrow(); + expect(() => + trackerAction('owner/repo', 'list_issues', { page: 101 }), + ).toThrow(); + expect(() => + trackerAction('owner/repo', 'create_comment', { number: 1, body: '' }), + ).toThrow(); +}); + +function github() { + const issues = new Map([ + [ + 1, + { number: 1, title: 'First', body: '', state: 'open', labels: ['bug'] }, + ], + ]); + const comments = new Map(); + const receipts = new Map(); + const writes = []; + const port = new GitHubPort( + null, + { repository: 'owner/repo' }, + async (action, args, id) => { + const isWrite = /^(create|update|add|remove)_/.test(action); + if (isWrite && receipts.has(id)) return receipts.get(id); + let value; + if (isWrite) writes.push(action); + switch (action) { + case 'get_issue': + value = issues.get(args.number); + break; + case 'list_issues': + value = [...issues.values()]; + break; + case 'create_issue': + value = { + number: issues.size + 1, + ...args, + state: 'open', + labels: [], + }; + issues.set(value.number, value); + break; + case 'update_issue': + value = issues.get(args.number); + Object.assign(value, args); + break; + case 'add_doing_label': + value = issues.get(args.number); + value.labels.push('atomic:doing'); + break; + case 'remove_doing_label': + value = issues.get(args.number); + value.labels = value.labels.filter(l => l !== 'atomic:doing'); + break; + case 'create_comment': + value = { + id: comments.size + 1, + body: args.body, + issue_url: `https://api.github.com/repos/owner/repo/issues/${args.number}`, + }; + comments.set(value.id, value); + break; + case 'get_comment': + value = comments.get(args.id); + break; + case 'list_comments': + value = [...comments.values()]; + break; + case 'update_comment': + value = comments.get(args.id); + value.body = args.body; + break; + default: + throw new Error(action); + } + const receipt = { status: 200, body: JSON.stringify(value) }; + if (isWrite) receipts.set(id, receipt); + return receipt; + }, + ); + return { port, issues, comments, writes }; +} + +it('creates closed/doing issues and reopens while preserving unrelated labels', async () => { + const { port, issues } = github(); + const value = { title: 'Created closed', body: 'Markdown', status: 'Done' }; + const row = await port.create('issue', value, 'stable-create'); + expect(issues.get(row.id).state).toBe('closed'); + await port.create('issue', value, 'stable-create'); + expect(issues.size).toBe(2); + await port.update('issue', 1, { ...value, status: 'Doing' }, 'doing'); + expect(issues.get(1).labels).toEqual(['bug', 'atomic:doing']); + await port.update('issue', 1, value, 'close'); + await port.update('issue', 1, { ...value, status: 'Todo' }, 'reopen'); + expect(issues.get(1).state).toBe('open'); + expect(issues.get(1).labels).toEqual(['bug']); +}); + +it('creates/edits comments with replay-safe IDs and rejects another issue’s comment', async () => { + const { port, comments } = github(); + const context = { issueId: 1 }; + await port.create( + 'comment:x', + { body: 'Hi' }, + 'comment-create', + undefined, + context, + ); + await port.create( + 'comment:x', + { body: 'Hi' }, + 'comment-create', + undefined, + context, + ); + expect(comments.size).toBe(1); + await port.update( + 'comment:x', + 1, + { body: 'Edited' }, + 'comment-edit', + undefined, + context, + ); + expect(comments.get(1).body).toBe('Edited'); + await expect(port.get('comment:x', 1, { issueId: 2 })).rejects.toThrow( + 'another issue', + ); +}); + +it('fails on provider errors and scans all pages without importing pull requests', async () => { + const f = github(); + const row = f.issues.get(1); + let calls = 0; + const port = new GitHubPort(null, { repository: 'owner/repo' }, async () => ({ + status: 200, + body: JSON.stringify( + ++calls === 1 + ? Array.from({ length: 100 }, (_, i) => ({ + ...row, + number: i + 1, + ...(i === 0 ? { pull_request: {} } : {}), + })) + : [], + ), + })); + expect(await port.list('issue')).toHaveLength(99); + expect(calls).toBe(2); + port.call = async () => ({ status: 429, body: '{}' }); + await expect(port.list('issue')).rejects.toThrow('429'); +}); + +it('Atomic creates reuse native localId after a lost acknowledgement', async () => { + const stored = new Map(); + let lose = true; + const store = { + getServerUrl: () => 'https://atomic.example', + findByLocalId: async (_drive, parent, key) => + [...stored.values()].find(r => r.parent === parent && r.key === key), + newResource: async ({ parent, propVals }) => { + const r = { + subject: `did:ad:${stored.size + 1}`, + parent, + key: propVals['https://atomicdata.dev/properties/localId'], + async save() { + stored.set(r.subject, r); + if (lose) { + lose = false; + throw new Error('Lost acknowledgement'); + } + }, + }; + return r; + }, + }; + const port = new AtomicPort(store, { + connection: { + table: 'did:ad:table', + drive: 'did:ad:drive', + tags: { Todo: 'https://atomicdata.dev/task/v1/todo' }, + }, + }); + const value = { title: 'Same', body: '', status: 'Todo' }; + port.get = async (_entity, id) => ({ id, value }); + port.findByLocalId = (parent, key) => store.findByLocalId('', parent, key); + await expect(port.create('issue', value, 'key')).rejects.toThrow( + 'Lost acknowledgement', + ); + expect((await port.create('issue', value, 'key')).id).toBe('did:ad:1'); + expect(stored.size).toBe(1); +}); diff --git a/integrations/github-issues/devonian/proxy.mjs b/integrations/github-issues/devonian/proxy.mjs new file mode 100644 index 0000000000..e997b41311 --- /dev/null +++ b/integrations/github-issues/devonian/proxy.mjs @@ -0,0 +1,199 @@ +import { endpoint, request } from '../adapter.js'; +import { trackerAction } from '../tracker-actions.js'; + +/** Direct browser transport. Codes stay in tab storage, never in Atomic resources. */ +export function proxyTransport({ + url, + repository, + getCode, + setCode, + journal, + save, + fetcher = fetch, + dispatch, +}) { + const origin = new URL(url); + if ( + origin.protocol !== 'https:' && + !( + origin.protocol === 'http:' && + ['localhost', '127.0.0.1'].includes(origin.hostname) + ) + ) + throw new Error('Use an HTTPS proxy or loopback HTTP'); + const root = endpoint(repository); + let pending = Promise.resolve(); + return (action, args, id) => { + const operation = async () => { + if ( + action === 'get_issue' && + (!Number.isSafeInteger(args.number) || args.number <= 0) + ) + throw new Error('Invalid issue number'); + if ( + action === 'create_issue' && + (typeof args.title !== 'string' || + !args.title.trim() || + args.title.length > 256 || + (args.body !== undefined && typeof args.body !== 'string')) + ) + throw new Error('Invalid issue title or body'); + const intent = + trackerAction(repository, action, args) ?? + (action === 'get_issue' + ? request('get', 'GET', `${root}/${args.number}`, id) + : action === 'create_issue' + ? request('create', 'POST', root, id, args) + : undefined); + if (!intent) throw new Error('Unsupported GitHub action'); + const writes = intent.method !== 'GET'; + const signature = JSON.stringify({ action, args }); + const old = journal[id]; + if (writes && old) { + if (old.signature !== signature) + throw new Error('Operation identity reused with different arguments'); + if (old.receipt) return old.receipt; + throw new Error( + `Uncertain GitHub write (${action}). Inspect its outcome before retrying; it will not be resent.`, + ); + } + if (writes) { + journal[id] = { signature }; + await save(); + } + const target = new URL(intent.url); + let receipt; + let next = true; + if (dispatch) { + receipt = await dispatch(`${target.pathname}${target.search}`, { + method: intent.method, + ...(intent.body ? { body: intent.body } : {}), + }).catch(() => { + throw new Error( + 'Proxy request failed. Check CORS and reconnect; an uncertain write will not be resent.', + ); + }); + } else { + const code = getCode(); + if (!code) + throw new Error( + 'Connect to the proxy or supply a fresh connection code', + ); + setCode(''); + const destination = new URL( + `/proxy/github-issues${target.pathname}${target.search}`, + origin, + ); + const response = await fetcher(destination.href, { + method: intent.method, + redirect: 'error', + credentials: 'omit', + headers: { + Authorization: `Bearer ${code}`, + Accept: 'application/vnd.github+json', + 'Content-Type': 'application/json', + }, + ...(intent.body ? { body: intent.body } : {}), + }).catch(() => { + throw new Error( + 'Proxy request failed. Check CORS and reconnect; an uncertain write will not be resent.', + ); + }); + next = response.headers.get('X-Connection-Code'); + if (next) setCode(next); + receipt = { status: response.status, body: await response.text() }; + } + if (writes && receipt.status >= 200 && receipt.status < 300) { + journal[id].receipt = receipt; + await save(); + } + if (!next) + throw new Error( + 'Proxy did not expose X-Connection-Code. Enable CORS exposure and reconnect.', + ); + return receipt; + }; + const next = pending.then(operation); + pending = next.catch(() => {}); + return next; + }; +} + +/** An explicitly labelled GitHub fixture; no network or real GitHub mutations. */ +export function fixtureTransport(state, save) { + state.issues ??= [ + { + number: 1, + title: 'Welcome from GitHub', + body: 'Edit either tracker, then sync again.', + state: 'open', + labels: ['demo'], + }, + ]; + state.comments ??= []; + state.receipts ??= {}; + return async (action, args, id) => { + if (state.receipts[id]) return state.receipts[id]; + let value; + const issue = state.issues.find(r => r.number === args.number); + const comment = state.comments.find(r => r.id === args.id); + switch (action) { + case 'list_issues': + value = state.issues.slice((args.page - 1) * 100, args.page * 100); + break; + case 'get_issue': + value = issue; + break; + case 'create_issue': + value = { + number: Math.max(0, ...state.issues.map(r => r.number)) + 1, + ...args, + state: 'open', + labels: [], + }; + state.issues.push(value); + break; + case 'update_issue': + value = Object.assign(issue, args); + break; + case 'add_doing_label': + issue.labels = [...new Set([...issue.labels, 'atomic:doing'])]; + value = issue; + break; + case 'remove_doing_label': + issue.labels = issue.labels.filter(l => l !== 'atomic:doing'); + value = issue; + break; + case 'list_comments': + value = state.comments + .filter(c => c.issue_url.endsWith(`/${args.number}`)) + .slice((args.page - 1) * 100, args.page * 100); + break; + case 'get_comment': + value = comment; + break; + case 'create_comment': + value = { + id: Math.max(0, ...state.comments.map(r => r.id)) + 1, + body: args.body, + issue_url: `https://api.github.com/repos/demo/issues/issues/${args.number}`, + user: { login: 'demo-user' }, + }; + state.comments.push(value); + break; + case 'update_comment': + value = Object.assign(comment, { body: args.body }); + break; + default: + throw new Error(`Unsupported fixture action: ${action}`); + } + const receipt = { + status: value ? 200 : 404, + body: JSON.stringify(value ?? {}), + }; + if (!action.startsWith('get_') && !action.startsWith('list_')) + state.receipts[id] = receipt; + await save(); + return receipt; + }; +} diff --git a/integrations/github-issues/devonian/proxy.test.mjs b/integrations/github-issues/devonian/proxy.test.mjs new file mode 100644 index 0000000000..48bf983161 --- /dev/null +++ b/integrations/github-issues/devonian/proxy.test.mjs @@ -0,0 +1,100 @@ +import { expect, it } from 'vitest'; +import { proxyTransport } from './proxy.mjs'; + +it('serializes rotating codes, preserves query strings and checkpoints successful writes', async () => { + let code = 'first'; + const seen = []; + const journal = {}; + let saves = 0; + const call = proxyTransport({ + url: 'https://proxy.example', + repository: 'owner/repo', + journal, + getCode: () => code, + setCode: c => { + code = c; + }, + save: async () => { + saves++; + }, + fetcher: async (url, opts) => { + seen.push({ url, ...opts }); + return new Response('{"id":1}', { + headers: { 'X-Connection-Code': `next-${seen.length}` }, + }); + }, + }); + await Promise.all([ + call('list_issues', { page: 2 }, 'read'), + call('create_comment', { number: 1, body: 'Hello' }, 'write'), + ]); + expect(seen[0].url).toContain( + '/proxy/github-issues/repos/owner/repo/issues?state=all&per_page=100&page=2', + ); + expect(seen.map(r => r.headers.Authorization)).toEqual([ + 'Bearer first', + 'Bearer next-1', + ]); + expect(code).toBe('next-2'); + expect(saves).toBe(2); + await call('create_comment', { number: 1, body: 'Hello' }, 'write'); + expect(seen).toHaveLength(2); + await expect( + call('create_comment', { number: 1, body: 'Changed' }, 'write'), + ).rejects.toThrow('different arguments'); +}); + +it('never retries an uncertain create after a lost response or restart', async () => { + const journal = {}; + let writes = 0; + let code = 'first'; + const options = { + url: 'https://proxy.example', + repository: 'owner/repo', + journal, + getCode: () => code, + setCode: c => { + code = c; + }, + save: async () => {}, + fetcher: async () => { + writes++; + throw new Error('network'); + }, + }; + await expect( + proxyTransport(options)('create_issue', { title: 'Title' }, 'create'), + ).rejects.toThrow('Proxy request failed'); + code = 'reconnected'; + await expect( + proxyTransport(options)('create_issue', { title: 'Title' }, 'create'), + ).rejects.toThrow('Uncertain GitHub write'); + expect(writes).toBe(1); +}); + +it('retains rotated codes on provider errors and identifies missing exposed headers', async () => { + let code = 'first'; + const options = { + url: 'https://proxy.example', + repository: 'owner/repo', + journal: {}, + getCode: () => code, + setCode: c => { + code = c; + }, + save: async () => {}, + fetcher: async () => + new Response('{}', { + status: 403, + headers: { 'X-Connection-Code': 'rotated' }, + }), + }; + expect( + (await proxyTransport(options)('get_issue', { number: 1 }, 'read')).status, + ).toBe(403); + expect(code).toBe('rotated'); + options.fetcher = async () => new Response('{}'); + await expect( + proxyTransport(options)('get_issue', { number: 1 }, 'read'), + ).rejects.toThrow('expose X-Connection-Code'); +}); diff --git a/integrations/github-issues/devonian/vitest.config.mjs b/integrations/github-issues/devonian/vitest.config.mjs new file mode 100644 index 0000000000..1b8f18ff79 --- /dev/null +++ b/integrations/github-issues/devonian/vitest.config.mjs @@ -0,0 +1,14 @@ +import { resolve } from 'node:path'; + +export default { + resolve: { + alias: { + devonian: process.env.DEVONIAN_PATH + ? resolve(process.env.DEVONIAN_PATH, 'build/src/main.js') + : resolve('browser/data-browser/src/chunks/DevonianDemo/devonian.js'), + '@tomic/lib': resolve('browser/lib/src/index.ts'), + vitest: resolve('browser/node_modules/vitest/dist/index.js'), + }, + }, + test: { include: ['integrations/github-issues/devonian/*.test.*'] }, +}; diff --git a/integrations/github-issues/tracker-actions.ts b/integrations/github-issues/tracker-actions.ts new file mode 100644 index 0000000000..47e9096c14 --- /dev/null +++ b/integrations/github-issues/tracker-actions.ts @@ -0,0 +1,190 @@ +/** Additional repository-scoped actions used by the Devonian tracker bridge. */ +import { endpoint, request } from './adapter.js'; + +const integer = { + type: 'integer' as const, + description: 'Positive identifier or page number', +}; +const string = { + type: 'string' as const, + description: 'Issue or comment field value', +}; +const definitions = [ + ['list_issues', 'List issues', 'list', { page: integer }, ['page']], + [ + 'update_issue', + 'Update an issue', + 'update', + { number: integer, title: string, body: string, state: string }, + ['number', 'title', 'body', 'state'], + ], + [ + 'add_doing_label', + 'Mark an issue as doing', + 'doing-add', + { number: integer }, + ['number'], + ], + [ + 'remove_doing_label', + 'Remove the doing label', + 'doing-remove', + { number: integer }, + ['number'], + ], + [ + 'list_comments', + 'List issue comments', + 'comments-list', + { number: integer, page: integer }, + ['number', 'page'], + ], + [ + 'get_comment', + 'Get an issue comment', + 'comments-get', + { id: integer }, + ['id'], + ], + [ + 'create_comment', + 'Create an issue comment', + 'comments-create', + { number: integer, body: string }, + ['number', 'body'], + ], + [ + 'update_comment', + 'Update an issue comment', + 'comments-update', + { id: integer, body: string }, + ['id', 'body'], + ], +] as const; + +export const trackerActions = definitions.map( + ([name, title, operation, properties, required]) => ({ + name, + title, + description: `${title} in this connected repository. Writes require approval.`, + operation, + inputSchema: { + type: 'object', + properties, + required: [...required], + additionalProperties: false, + }, + }), +); + +export function trackerOperations(root: string) { + return [ + { + id: 'comments-list', + method: 'GET', + url: `${root}/{number}/comments`, + effect: 'read', + }, + { + id: 'comments-get', + method: 'GET', + url: `${root}/comments/{id}`, + effect: 'read', + }, + { + id: 'comments-create', + method: 'POST', + url: `${root}/{number}/comments`, + effect: 'write', + }, + { + id: 'comments-update', + method: 'PATCH', + url: `${root}/comments/{id}`, + effect: 'write', + }, + ]; +} + +export function trackerAction( + repository: string, + action: string, + args: Record, +) { + const definition = definitions.find(d => d[0] === action); + if (!definition) return undefined; + const [, , operation, properties, required] = definition; + for (const key of required) { + if (!(key in args)) throw new Error(`Missing ${key}`); + } + for (const [key, value] of Object.entries(args)) { + if (!(key in properties)) throw new Error(`Unexpected ${key}`); + if (['number', 'id', 'page'].includes(key)) { + if ( + !Number.isSafeInteger(value) || + Number(value) <= 0 || + (key === 'page' && Number(value) > 100) + ) + throw new Error(`Invalid ${key}`); + } else if (typeof value !== 'string') throw new Error(`Invalid ${key}`); + } + const root = endpoint(repository); + switch (action) { + case 'list_issues': + return request( + operation, + 'GET', + `${root}?state=all&per_page=100&page=${args.page}&sort=created&direction=asc`, + 'action', + ); + case 'list_comments': + return request( + operation, + 'GET', + `${root}/${args.number}/comments?per_page=100&page=${args.page}`, + 'action', + ); + case 'get_comment': + return request(operation, 'GET', `${root}/comments/${args.id}`, 'action'); + case 'create_comment': + case 'update_comment': + if (!(args.body as string).trim()) + throw new Error('Comment body cannot be empty'); + return request( + operation, + action === 'create_comment' ? 'POST' : 'PATCH', + action === 'create_comment' + ? `${root}/${args.number}/comments` + : `${root}/comments/${args.id}`, + 'action', + { body: args.body }, + ); + case 'add_doing_label': + return request( + operation, + 'POST', + `${root}/${args.number}/labels`, + 'action', + { + labels: ['atomic:doing'], + }, + ); + case 'remove_doing_label': + return request( + operation, + 'DELETE', + `${root}/${args.number}/labels/atomic%3Adoing`, + 'action', + ); + case 'update_issue': + if (!['open', 'closed'].includes(String(args.state))) + throw new Error('Invalid issue state'); + if (!(args.title as string).trim() || (args.title as string).length > 256) + throw new Error('Invalid issue title'); + return request(operation, 'PATCH', `${root}/${args.number}`, 'action', { + title: args.title, + body: args.body, + state: args.state, + }); + } +} diff --git a/integrations/localthought/README.md b/integrations/localthought/README.md index 9ea97f7efc..3674472724 100644 --- a/integrations/localthought/README.md +++ b/integrations/localthought/README.md @@ -82,3 +82,53 @@ An unbounded fetch successfully traversed multiple pages but exceeded the 5,000-record preview limit; the UI now defaults to the next 30 days. Date bounds and recurrence expansion are passed to Syncables as collection query settings. The importer remains a manual snapshot, not a background sync. + +## Calendar view + +Google event imports now install a Calendar view alongside the source table. +The projected date uses the day in Google's supplied start offset (or the +unchanged all-day date), so mixed all-day/timed events share one DATE column. +The original Start and End objects retain timezones and exclusive end values. +The month view shows each event on its start day; it does not draw duration or +multi-day spans. Additional notes identify recurring events, attendees, +reminders and conferencing when those fields are returned by the catalog. +Recurring instances are expanded by the existing bounded provider fetch. + +Use **Fetch and preview** again to refresh, then review and apply changes. +Existing source identities and import baselines prevent duplicates and preserve +Atomic-only fields and local edits. Cancellation records are retained with a +note; absence from a bounded fetch never deletes an Atomic resource. Google +may omit cancelled events from list results, so this is not a deletion feed. +Removed optional provider fields are not cleared by the shared snapshot importer. +This first version deliberately remains manual and one-way. No OAuth write +scope is requested and edits in Atomic do not update Google. + +`calendar.test.ts` exercises mixed dates, offset boundaries, exclusive ends, +recurrence/attendee notes, cancellations, malformed starts, cross-calendar +identity and repeated imports with private local fields. + +## Browser-only Calendar regression + +`browser/e2e/tests/google-calendar-import.spec.mts` starts the shared mock +integration-proxy from #1399 with a synthetic Google Calendar. The test enters +the public fixture tenant secret, completes consent, and exercises real browser +credential rotation, WASM pagination, local schema installation, proposal review, +OPFS application, and Calendar rendering. It refreshes changed provider data +and checks that native identities and Atomic-only notes survive reload. +AtomicServer HTTP and all WebSockets are blocked throughout; only GET requests +are permitted for provider data. The configured LocalThought origin is forwarded +to the isolated HTTP fixture, so no live provider credentials or data are used. + +Run with a dev frontend built from this branch and its matching WASM bundle: + +```sh +FRONTEND_URL=http://127.0.0.1:6747 SERVER_URL=http://127.0.0.1:19999 \ + browser/e2e/node_modules/.bin/playwright test \ + --config browser/e2e/playwright.config.ts \ + browser/e2e/tests/google-calendar-import.spec.mts --project chromium +``` + +If the frontend uses `VITE_INTEGRATION_PROXY_URL`, pass the same value to the +test process. The test forwards that origin to its own fixture. Live Google +OAuth on the browser path still depends on the proxy CORS deployment described +above; this fixture test does not claim live-provider verification. diff --git a/integrations/localthought/browser.test.ts b/integrations/localthought/browser.test.ts index d7194196bf..87a8081208 100644 --- a/integrations/localthought/browser.test.ts +++ b/integrations/localthought/browser.test.ts @@ -124,3 +124,37 @@ it('calls the browser fetch function without binding it to the client', async () const client = new BrowserIntegrations(storage, async () => engine, origin); expect(await client.catalog()).toEqual(['pets']); }); + +it('supports the demo callback and write credentials without using the import engine', async () => { + const { client, http, values } = setup(); + const { state } = await client.start( + 'drive', + 'actor', + 'pets', + 'https://atomic.example/app/devonian-demo', + secret, + ); + client.finish('drive', 'actor', state, 'first'); + http.mockImplementation(async (_url, init) => { + expect(JSON.parse([...values.values()][0]).code).toBeUndefined(); + expect(init?.method).toBe('POST'); + expect(init?.body).toBe('{"title":"new"}'); + return new Response('{"id":1}', { + status: 201, + headers: { 'X-Connection-Code': 'next' }, + }); + }); + await expect( + client.request('drive', 'actor', state, 'github-issues', '/issues'), + ).rejects.toThrow('another platform'); + await expect( + client.request('drive', 'actor', state, 'pets', '//evil.example'), + ).rejects.toThrow('Invalid proxy path'); + expect( + await client.request('drive', 'actor', state, 'pets', '/issues', { + method: 'POST', + body: '{"title":"new"}', + }), + ).toEqual({ status: 201, body: '{"id":1}' }); + expect(JSON.parse([...values.values()][0]).code).toBe('next'); +}); diff --git a/integrations/localthought/browser.ts b/integrations/localthought/browser.ts index 9ed1248fbc..94350c048f 100644 --- a/integrations/localthought/browser.ts +++ b/integrations/localthought/browser.ts @@ -142,7 +142,9 @@ export class BrowserIntegrations { const callback = new URL(returnUrl); if ( callback.origin !== location.origin || - callback.pathname !== '/app/integrations' || + !['/app/integrations', '/app/devonian-demo'].includes( + callback.pathname, + ) || callback.search || callback.hash ) @@ -196,6 +198,72 @@ export class BrowserIntegrations { ); return { connection: state, platform: c.platform }; } + /** Shared rotating-code transport for browser-owned writes as well as reads. */ + async request( + drive: string, + actor: string, + id: string, + platform: string, + path: string, + init: { method?: string; body?: string } = {}, + ): Promise<{ status: number; body: string }> { + if (!navigator.locks) + throw new Error('This browser needs Web Locks for integrations'); + if (!path.startsWith('/') || path.startsWith('//') || /[\\\\#]/.test(path)) + throw new Error('Invalid proxy path'); + const destination = new URL(`/proxy/${platform}${path}`, this.origin); + if (!destination.pathname.startsWith(`/proxy/${platform}/`)) + throw new Error('Invalid proxy path'); + return navigator.locks.request(key + id, async () => { + const c = this.connection(id, drive, actor); + if (c.platform !== platform) + throw new Error('Connection belongs to another platform'); + const response = await this.send( + id, + drive, + actor, + path, + init, + AbortSignal.timeout(30000), + ); + return { status: response.status, body: await limitedText(response) }; + }); + } + private async send( + id: string, + drive: string, + actor: string, + path: string, + init: { method?: string; body?: string }, + signal: AbortSignal, + ) { + const current = this.connection(id, drive, actor); + if (!current.ready || !current.code) + throw new Error('Reconnect before retrying an uncertain request'); + const code = current.code; + delete current.code; + this.storage.setItem(key + id, JSON.stringify(current)); + const response = await this.http( + `${this.origin}/proxy/${current.platform}${path}`, + { + ...init, + headers: { + Authorization: `Bearer ${code}`, + 'Content-Type': 'application/json', + }, + credentials: 'omit', + redirect: 'error', + signal, + }, + ); + const next = response.headers.get('x-connection-code'); + if (!next) + throw new Error( + 'Proxy did not expose a rotated code; reconnect and check CORS', + ); + this.storage.setItem(key + id, JSON.stringify({ ...current, code: next })); + return response; + } async fetchRecords( drive: string, actor: string, @@ -227,32 +295,14 @@ export class BrowserIntegrations { throw new Error('Pagination left the catalog API origin'); if (++requests > 200) throw new Error('Import exceeds 200 requests; narrow its scope'); - const current = this.connection(id, drive, actor); - if (!current.code) - throw new Error('Reconnect before retrying an uncertain request'); - const code = current.code; - delete current.code; - // Consume durably BEFORE dispatch; a lost response must never replay a code. - this.storage.setItem(key + id, JSON.stringify(current)); - const response = await this.http( - `${this.origin}/proxy/${c.platform}${target.pathname}${target.search}`, - { - headers: { Authorization: `Bearer ${code}` }, - credentials: 'omit', - redirect: 'error', - signal, - }, + const response = await this.send( + id, + drive, + actor, + `${target.pathname}${target.search}`, + {}, + signal, ); - const next = response.headers.get('x-connection-code'); - if (next) - this.storage.setItem( - key + id, - JSON.stringify({ ...current, code: next }), - ); - else - throw new Error( - 'Proxy did not expose a rotated code; reconnect and check CORS', - ); const headers = Object.fromEntries( [...response.headers].filter( ([name]) => name !== 'x-connection-code', diff --git a/integrations/localthought/calendar.test.ts b/integrations/localthought/calendar.test.ts new file mode 100644 index 0000000000..1e14884ff4 --- /dev/null +++ b/integrations/localthought/calendar.test.ts @@ -0,0 +1,151 @@ +import { expect, it } from 'vitest'; +import { calendarProjection, calendarFields as fields } from './calendar'; +import { Datatype } from '../../browser/lib/src/index'; +import { run } from './plugin'; +import type { FetchedPlatform } from './schema'; +const fixture = (): FetchedPlatform => ({ + platform: 'google-calendar', + ontology: { + description: '', + terms: [ + { + path: 'event', + kind: 'class', + shortname: 'event', + description: '', + datatype: Datatype.STRING, + requires: [], + recommends: [], + }, + ], + }, + records: [ + { + resource: 'event', + namespace: 'calendar-a', + id: '1', + name: 'Meeting', + values: { + start: { + dateTime: '2026-09-10T00:30:00+02:00', + timeZone: 'Europe/Amsterdam', + }, + end: { dateTime: '2026-09-10T01:30:00+02:00' }, + attendees: [{ email: 'test@example.com' }], + 'recurring-event-id': 'series', + reminders: { useDefault: true }, + }, + }, + ], +}); +it('projects timed events without shifting their day and retains provider structures', () => { + const input = fixture(); + const output = calendarProjection(input); + expect(output.records[0].values).toMatchObject({ + ...input.records[0].values, + [fields.day]: '2026-09-10', + [fields.allDay]: false, + }); + expect(output.records[0].values[fields.notes]).toContain('Recurring'); + expect(output.records[0].values[fields.notes]).toContain('Attendees'); + expect(input.records[0].values[fields.day]).toBeUndefined(); + expect( + output.ontology.terms.find(t => t.shortname === fields.day)?.datatype, + ).toBe(Datatype.DATE); +}); +it('supports all-day dates and preserves exclusive multi-day ends', () => { + const input = fixture(); + input.records[0].values = { + start: { date: '2026-09-10' }, + end: { date: '2026-09-13' }, + }; + expect(calendarProjection(input).records[0].values).toMatchObject({ + [fields.day]: '2026-09-10', + [fields.allDay]: true, + end: { date: '2026-09-13' }, + }); +}); +it('retains cancelled events even when Google omits start', () => { + const input = fixture(); + input.records[0].values = { status: 'cancelled' }; + expect(calendarProjection(input).records[0].values[fields.notes]).toContain( + 'Cancelled', + ); +}); +it('fails the projection on malformed active events instead of silently losing rows', () => { + for (const start of [ + { date: '2026-02-30' }, + { date: '2026-09-10garbage' }, + {}, + { dateTime: '2026-09-10T12:00:00' }, + ]) { + const input = fixture(); + input.records[0].values = { start }; + expect(() => calendarProjection(input)).toThrow(); + } +}); +it('leaves other platforms untouched', () => { + const input = fixture(); + input.platform = 'pets'; + expect(calendarProjection(input)).toBe(input); +}); +it("repeated imports reconcile IDs, preserve local fields and don't delete out-of-window events", () => { + const data = calendarProjection(fixture()); + const properties = Object.fromEntries( + Object.keys(data.records[0].values).map(k => [ + k, + `https://example.com/${k}`, + ]), + ); + const config = { + platform: data.platform, + destinations: { + event: { table: 'did:ad:table', rowClass: 'did:ad:event' }, + }, + properties, + records: data.records, + }; + const first = run({ config, query: () => [], read: () => ({}) }).intents[0]; + if (first.op !== 'create') throw new Error('Expected create'); + const saved = { + ...first.set, + 'https://atomicdata.dev/properties/parent': 'did:ad:table', + 'https://atomicdata.dev/properties/isA': ['did:ad:event'], + 'https://example.com/private': 'my notes', + }; + const host = { + config, + query: (p: string, v: string) => (saved[p] === v ? ['did:ad:row'] : []), + read: () => saved, + }; + expect(run(host).intents).toHaveLength(0); + expect( + run({ ...host, config: { ...config, records: [] } }).intents, + ).toHaveLength(0); + const changed = calendarProjection(fixture()); + changed.records[0].name = 'Updated meeting'; + const result = run({ + ...host, + config: { ...config, records: changed.records }, + }); + expect(result.intents[0]).toMatchObject({ + op: 'set', + subject: 'did:ad:row', + set: { 'https://atomicdata.dev/properties/name': 'Updated meeting' }, + }); + expect(JSON.stringify(result.intents)).not.toContain('my notes'); + const secondCalendar = { ...data.records[0], namespace: 'calendar-b' }; + expect( + run({ ...host, config: { ...config, records: [secondCalendar] } }) + .intents[0].op, + ).toBe('create'); +}); +it('recognizes the lowercase field names produced by the WASM ontology', () => { + const input = fixture(); + delete input.records[0].values['recurring-event-id']; + input.records[0].values.recurringeventid = 'series'; + input.records[0].values.conferencedata = { conferenceId: 'meeting' }; + const notes = calendarProjection(input).records[0].values[fields.notes]; + expect(notes).toContain('Recurring event'); + expect(notes).toContain('Conferencing'); +}); diff --git a/integrations/localthought/calendar.ts b/integrations/localthought/calendar.ts new file mode 100644 index 0000000000..f9706e8792 --- /dev/null +++ b/integrations/localthought/calendar.ts @@ -0,0 +1,146 @@ +// @wc-ignore-file +import { Datatype } from '../../browser/lib/src/index.js'; +import type { JSONValue } from '../../browser/lib/src/value.js'; +import type { FetchedPlatform, Term } from './schema.js'; + +export const calendarFields = { + day: 'atomic-calendar-day', + allDay: 'atomic-calendar-all-day', + notes: 'atomic-calendar-notes', +}; + +/** An additional projection, never a replacement for the provider's fields. + * One DATE column supports both all-day dates and timed events in a single view. + * Timed events use the date in Google's supplied offset; raw start/end retain + * the instant, zone and exclusive-end semantics for future richer rendering. + */ +export function calendarProjection(fetched: FetchedPlatform): FetchedPlatform { + if (fetched.platform !== 'google-calendar') return fetched; + const event = fetched.ontology.terms.find( + t => t.kind === 'class' && t.shortname === 'event', + ); + if (!event) return fetched; + const definitions: [string, Datatype, string][] = [ + [ + calendarFields.day, + Datatype.DATE, + "Start date in the event's supplied offset; all-day dates stay unchanged.", + ], + [ + calendarFields.allDay, + Datatype.BOOLEAN, + 'Whether Google represents this as an all-day event.', + ], + [ + calendarFields.notes, + Datatype.STRING, + 'Display limitations and Google-only features. Edits here remain local.', + ], + ]; + const terms: Term[] = definitions.map( + ([shortname, datatype, description]) => ({ + path: `urn:atomic:google-calendar:${shortname}`, + kind: 'property', + shortname, + datatype, + description, + requires: [], + recommends: [], + }), + ); + if ( + fetched.ontology.terms.some(t => + terms.some(extra => extra.shortname === t.shortname), + ) + ) + throw new Error( + 'Calendar projection property collides with provider ontology', + ); + return { + ...fetched, + ontology: { + ...fetched.ontology, + terms: [ + ...fetched.ontology.terms.map(t => + t === event + ? { + ...t, + recommends: [ + ...t.recommends, + ...terms.map(extra => extra.path), + ], + } + : t, + ), + ...terms, + ], + }, + records: fetched.records.map(row => { + if (row.resource !== 'event') return row; + const start = object(row.values.start); + const end = object(row.values.end); + const date = start.date ?? start.dateTime; + const cancelled = row.values.status === 'cancelled'; + if ( + !cancelled && + (typeof date !== 'string' || !validDay(date.slice(0, 10))) + ) + throw new Error(`Calendar event ${row.id} has no valid start date`); + if ( + start.date !== undefined && + (typeof start.date !== 'string' || !validDay(start.date)) + ) + throw new Error(`Calendar event ${row.id} has an invalid all-day date`); + if ( + start.dateTime !== undefined && + (typeof start.dateTime !== 'string' || + !/^\d{4}-\d{2}-\d{2}T.*(?:Z|[+-]\d{2}:\d{2})$/.test(start.dateTime) || + !Number.isFinite(Date.parse(start.dateTime))) + ) + throw new Error( + `Calendar event ${row.id} has no offset-qualified start time`, + ); + const notes = ['One-way import: edits stay in Atomic']; + if (cancelled) notes.push('Cancelled in Google; retained in Atomic'); + if ( + row.values['recurring-event-id'] || + row.values.recurringEventId || + row.values.recurringeventid || + row.values.recurrence + ) + notes.push('Recurring event: manage the series in Google'); + if (Array.isArray(row.values.attendees) && row.values.attendees.length) + notes.push('Attendees and RSVP: manage in Google'); + if (row.values.reminders) notes.push('Reminders: manage in Google'); + if ( + row.values['conference-data'] || + row.values.conferenceData || + row.values.conferencedata + ) + notes.push('Conferencing: manage in Google'); + if (end.date || end.dateTime) + notes.push('Duration retained in End; shown on start day only'); + const values: Record = { + ...row.values, + [calendarFields.notes]: notes.join('. '), + }; + if (typeof date === 'string' && validDay(date.slice(0, 10))) { + values[calendarFields.day] = date.slice(0, 10); + values[calendarFields.allDay] = typeof start.date === 'string'; + } + return { ...row, values }; + }), + }; +} +function object(value: JSONValue | undefined): Record { + return value && typeof value === 'object' && !Array.isArray(value) + ? value + : {}; +} +function validDay(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const date = new Date(`${value}T00:00:00Z`); + return ( + Number.isFinite(date.getTime()) && date.toISOString().slice(0, 10) === value + ); +} diff --git a/integrations/localthought/mock-calendar.mjs b/integrations/localthought/mock-calendar.mjs new file mode 100644 index 0000000000..13ba1e576a --- /dev/null +++ b/integrations/localthought/mock-calendar.mjs @@ -0,0 +1,159 @@ +/** Synthetic Google Calendar fixtures. No real calendar data or provider writes. */ +const string = { type: 'string' }; +const dateTime = { + type: 'object', + properties: { + date: { ...string, format: 'date' }, + dateTime: { ...string, format: 'date-time' }, + timeZone: string, + }, +}; +export const calendarDocument = { + openapi: '3.0.3', + info: { title: 'Synthetic Calendar', version: 'v3' }, + servers: [{ url: 'https://www.googleapis.com/calendar/v3' }], + paths: { + '/calendars/{calendarId}/events': { + get: { + parameters: [ + { name: 'calendarId', in: 'path', required: true, schema: string }, + ...['pageToken', 'timeMin', 'timeMax'].map(name => ({ + name, + in: 'query', + schema: string, + })), + { name: 'singleEvents', in: 'query', schema: { type: 'boolean' } }, + ], + 'x-pagination': [{ scheme: 'pageToken' }], + responses: { + 200: { + description: 'Events', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + items: { + type: 'array', + items: { $ref: '#/components/schemas/event' }, + }, + nextPageToken: string, + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + paginationSchemes: { + pageToken: { + type: 'pageToken', + request: { queryParameters: { pageToken: { role: 'pageToken' } } }, + response: { bodyFields: { nextPageToken: { role: 'nextPageToken' } } }, + }, + }, + schemas: { + event: { + type: 'object', + properties: { + id: string, + summary: string, + status: string, + start: dateTime, + end: dateTime, + recurringEventId: string, + htmlLink: string, + attendees: { + type: 'array', + items: { + type: 'object', + properties: { email: string, responseStatus: string }, + }, + }, + }, + }, + }, + crudResources: { + event: { + schema: { $ref: '#/components/schemas/event' }, + identity: { + urlTemplate: '/calendars/{calendarId}/events/{eventId}', + bindings: { eventId: { field: 'id' } }, + }, + collections: { + events: { urlTemplate: '/calendars/{calendarId}/events' }, + }, + }, + }, + }, +}; +export function calendarFixture(day = new Date().toISOString().slice(0, 10)) { + const tomorrow = new Date(Date.parse(`${day}T00:00:00Z`) + 86400000) + .toISOString() + .slice(0, 10); + const events = [ + { + id: 'all-day', + summary: 'Calendar all-day fixture', + status: 'confirmed', + start: { date: day }, + end: { date: tomorrow }, + }, + { + id: 'timed', + summary: 'Calendar timed fixture', + status: 'confirmed', + start: { + dateTime: `${day}T00:30:00+02:00`, + timeZone: 'Europe/Amsterdam', + }, + end: { dateTime: `${day}T01:30:00+02:00`, timeZone: 'Europe/Amsterdam' }, + recurringEventId: 'series', + attendees: [ + { email: 'synthetic@example.com', responseStatus: 'accepted' }, + ], + }, + ]; + const requests = []; + return { + events, + requests, + request(method, url) { + requests.push({ + method, + path: url.pathname, + query: Object.fromEntries(url.searchParams), + }); + if (method !== 'GET') return { status: 405, body: {} }; + if ( + !/^\/proxy\/google-calendar\/calendar\/v3\/calendars\/[^/]+\/events$/.test( + url.pathname, + ) + ) + return { status: 404, body: {} }; + // Require the production adapter to send its safety bounds and recurrence expansion. + if ( + !url.searchParams.has('timeMin') || + !url.searchParams.has('timeMax') || + url.searchParams.get('singleEvents') !== 'true' + ) + return { + status: 400, + body: { error: 'Missing bounded recurrence query' }, + }; + const token = url.searchParams.get('pageToken'); + if (token && token !== 'second') return { status: 400, body: {} }; + return { + status: 200, + body: structuredClone( + token + ? { items: events.slice(1) } + : { items: events.slice(0, 1), nextPageToken: 'second' }, + ), + }; + }, + }; +} diff --git a/integrations/localthought/mock-github.mjs b/integrations/localthought/mock-github.mjs new file mode 100644 index 0000000000..629a1e28e1 --- /dev/null +++ b/integrations/localthought/mock-github.mjs @@ -0,0 +1,108 @@ +/** Stateful provider fixture, shared by the HTTP proxy and its test-side driver. */ +export function githubTracker() { + const repositories = new Map(); + const repo = name => { + if (!repositories.has(name)) + repositories.set(name, { issues: [], comments: [] }); + return repositories.get(name); + }; + let id = 0; + const now = () => new Date().toISOString(); + const api = { + snapshot: name => structuredClone(repo(name)), + createIssue(name, input) { + const state = repo(name); + const number = state.issues.length + 1; + const issue = { + id: ++id, + number, + title: input.title, + body: input.body ?? '', + state: 'open', + labels: [], + url: `https://api.github.com/repos/${name}/issues/${number}`, + html_url: `https://github.com/${name}/issues/${number}`, + user: { login: 'mock-user' }, + created_at: now(), + updated_at: now(), + }; + state.issues.push(issue); + return structuredClone(issue); + }, + updateIssue(name, number, input) { + const issue = repo(name).issues.find(i => i.number === number); + if (!issue) return; + for (const field of ['title', 'body', 'state', 'labels']) + if (input[field] !== undefined) issue[field] = input[field]; + issue.updated_at = now(); + return structuredClone(issue); + }, + createComment(name, number, input) { + if (!repo(name).issues.some(i => i.number === number)) return; + const comment = { + id: ++id, + body: input.body, + issue_url: `https://api.github.com/repos/${name}/issues/${number}`, + user: { login: 'mock-commenter' }, + created_at: now(), + updated_at: now(), + }; + repo(name).comments.push(comment); + return structuredClone(comment); + }, + request(method, url, input = {}) { + const match = url.pathname.match( + /^\/proxy\/github-issues\/repos\/([^/]+\/[^/]+)\/issues(?:\/(.*))?$/, + ); + if (!match) return { status: 404, body: {} }; + const [, name, tail = ''] = match; + const state = repo(name); + const page = Number(url.searchParams.get('page') ?? 1); + const size = Number(url.searchParams.get('per_page') ?? 100); + const paginate = rows => rows.slice((page - 1) * size, page * size); + let value; + if (!tail) { + if (method === 'GET') value = paginate(state.issues); + if (method === 'POST') value = api.createIssue(name, input); + } else if (/^comments\/\d+$/.test(tail)) { + const comment = state.comments.find( + c => c.id === Number(tail.split('/')[1]), + ); + if (method === 'GET') value = comment; + if (method === 'PATCH' && comment) + value = Object.assign(comment, { + body: input.body, + updated_at: now(), + }); + } else { + const [numberText, resource, label] = tail.split('/'); + const number = Number(numberText); + const issue = state.issues.find(i => i.number === number); + if (issue && !resource) { + if (method === 'GET') value = issue; + if (method === 'PATCH') value = api.updateIssue(name, number, input); + } else if (issue && resource === 'comments') { + if (method === 'GET') + value = paginate( + state.comments.filter(c => c.issue_url === issue.url), + ); + if (method === 'POST') value = api.createComment(name, number, input); + } else if (issue && resource === 'labels') { + if (method === 'POST') + value = issue.labels = [ + ...new Set([...issue.labels, ...input.labels]), + ]; + if (method === 'DELETE') + value = issue.labels = issue.labels.filter( + l => l !== decodeURIComponent(label), + ); + } + } + return { + status: value === undefined ? 404 : method === 'POST' ? 201 : 200, + body: structuredClone(value ?? {}), + }; + }, + }; + return api; +} diff --git a/integrations/localthought/mock-proxy.d.mts b/integrations/localthought/mock-proxy.d.mts new file mode 100644 index 0000000000..72bdfcb1bd --- /dev/null +++ b/integrations/localthought/mock-proxy.d.mts @@ -0,0 +1,49 @@ +import type { Server } from 'node:http'; +interface Issue { + id: number; + number: number; + title: string; + body: string; + state: string; + labels: string[]; +} +interface Comment { + id: number; + body: string; + issue_url: string; +} +interface GitHubTracker { + snapshot(repository: string): { issues: Issue[]; comments: Comment[] }; + createIssue( + repository: string, + input: { title: string; body?: string }, + ): Issue; + updateIssue( + repository: string, + number: number, + input: Partial, + ): Issue | undefined; + createComment( + repository: string, + number: number, + input: { body: string }, + ): Comment | undefined; +} +interface CalendarFixture { + events: Array<{ + id: string; + summary: string; + status: string; + start: { date?: string; dateTime?: string; timeZone?: string }; + end: { date?: string; dateTime?: string; timeZone?: string }; + }>; + requests: Array<{ + method: string; + path: string; + query: Record; + }>; +} +export const tenantSecret: string; +export function mockProxy(options?: { + frontendOrigin?: string; +}): Server & { github: GitHubTracker; calendar: CalendarFixture }; diff --git a/integrations/localthought/mock-proxy.mjs b/integrations/localthought/mock-proxy.mjs index 650c9aee31..445f8d1749 100644 --- a/integrations/localthought/mock-proxy.mjs +++ b/integrations/localthought/mock-proxy.mjs @@ -1,54 +1,169 @@ /** Local-only integration-proxy fixture. Never deploy this service. */ +import { calendarDocument, calendarFixture } from './mock-calendar.mjs'; +import { githubTracker } from './mock-github.mjs'; import { readFileSync } from 'node:fs'; import { createServer } from 'node:http'; import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; import { pathToFileURL } from 'node:url'; export const tenantSecret = 'bW9jay10ZW5hbnQ.mock-signature'; -const sign = value => createHmac('sha256', tenantSecret).update(value).digest('base64url'); -const equal = (a, b) => typeof a === 'string' && a.length === b.length && timingSafeEqual(Buffer.from(a), Buffer.from(b)); -const pets = ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles'].map((name, i) => ({ id: i + 1, name, species: ['Dog', 'Cat', 'Bird', 'Rabbit', 'Fish'][i], age: i + 1, vaccinated: i % 2 === 0, weight: i + 0.5, updated_at: '2026-09-09T00:00:00Z' })); -export function mockProxy() { +const sign = value => + createHmac('sha256', tenantSecret).update(value).digest('base64url'); +const equal = (a, b) => + typeof a === 'string' && + a.length === b.length && + timingSafeEqual(Buffer.from(a), Buffer.from(b)); +const pets = ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles'].map( + (name, i) => ({ + id: i + 1, + name, + species: ['Dog', 'Cat', 'Bird', 'Rabbit', 'Fish'][i], + age: i + 1, + vaccinated: i % 2 === 0, + weight: i + 0.5, + updated_at: '2026-09-09T00:00:00Z', + }), +); +export function mockProxy({ + frontendOrigin = process.env.MOCK_FRONTEND_ORIGIN ?? 'http://localhost:6747', +} = {}) { + const github = githubTracker(); + const calendar = calendarFixture(); const codes = new Map(); const challenges = new Set(); - const issueCode = platform => { const code = randomBytes(32).toString('base64url'); codes.set(code, platform); return code; }; - return createServer((req, res) => { + const issueCode = platform => { + const code = randomBytes(32).toString('base64url'); + codes.set(code, platform); + return code; + }; + const server = createServer(async (req, res) => { res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); - res.setHeader('Access-Control-Allow-Headers', 'Authorization, Content-Type'); + res.setHeader( + 'Access-Control-Allow-Methods', + 'GET, POST, PATCH, DELETE, OPTIONS', + ); + res.setHeader( + 'Access-Control-Allow-Headers', + 'Authorization, Content-Type', + ); res.setHeader('Access-Control-Expose-Headers', 'X-Connection-Code, Link'); - if (req.method === 'OPTIONS') { res.writeHead(204); return res.end(); } + if (req.method === 'OPTIONS') { + res.writeHead(204); + return res.end(); + } const url = new URL(req.url, 'http://localhost'); - const json = (status, value, headers = {}) => { res.writeHead(status, { 'Content-Type': 'application/json', ...headers }); res.end(JSON.stringify(value)); }; - if (url.pathname === '/catalog') return json(200, ['github-issues', 'google-calendar', 'pets']); - if (url.pathname === '/catalog/pets.yaml') { res.writeHead(200, { 'Content-Type': 'application/yaml' }); return res.end(readFileSync(new URL('./mock-document.json', import.meta.url))); } - if (url.pathname === '/session') { const challenge = randomBytes(32).toString('base64url'); challenges.add(challenge); return json(200, { ts: Math.floor(Date.now() / 1000), nonce: randomBytes(16).toString('hex'), challenge }); } + const json = (status, value, headers = {}) => { + res.writeHead(status, { 'Content-Type': 'application/json', ...headers }); + res.end(JSON.stringify(value)); + }; + if (url.pathname === '/catalog') + return json(200, ['github-issues', 'google-calendar', 'pets']); + if (url.pathname === '/catalog/pets.yaml') { + res.writeHead(200, { 'Content-Type': 'application/yaml' }); + return res.end( + readFileSync(new URL('./mock-document.json', import.meta.url)), + ); + } + if (url.pathname === '/catalog/google-calendar.yaml') + return json(200, calendarDocument); + if (url.pathname === '/session') { + const challenge = randomBytes(32).toString('base64url'); + challenges.add(challenge); + return json(200, { + ts: Math.floor(Date.now() / 1000), + nonce: randomBytes(16).toString('hex'), + challenge, + }); + } if (url.pathname === '/connect') { const p = url.searchParams; const challenge = p.get('challenge'); - if (!challenges.has(challenge) || !equal(p.get('response'), sign(challenge)) || !equal(p.get('user_id_sig'), sign(p.get('user_id') ?? '')) || p.get('tenant_id') !== 'mock-tenant') return json(401, { error: 'Invalid tenant proof' }); + if ( + !challenges.has(challenge) || + !equal(p.get('response'), sign(challenge)) || + !equal(p.get('user_id_sig'), sign(p.get('user_id') ?? '')) || + p.get('tenant_id') !== 'mock-tenant' + ) + return json(401, { error: 'Invalid tenant proof' }); const redirect = new URL(p.get('redirect_uri')); - if (redirect.origin !== (process.env.MOCK_FRONTEND_ORIGIN ?? 'http://localhost:6747') || redirect.pathname !== '/app/integrations') return json(400, { error: 'Invalid callback' }); + if ( + redirect.origin !== frontendOrigin || + !['/app/integrations', '/app/devonian-demo'].includes(redirect.pathname) + ) + return json(400, { error: 'Invalid callback' }); const platform = p.get('platform'); - if (!['github-issues', 'google-calendar', 'pets'].includes(platform)) return json(400, { error: 'Invalid platform' }); - if (req.method === 'GET') { res.writeHead(200, { 'Content-Type': 'text/html' }); return res.end('

Mock integration proxy

Connect your test account.

'); } + if (!['github-issues', 'google-calendar', 'pets'].includes(platform)) + return json(400, { error: 'Invalid platform' }); + if (req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'text/html' }); + return res.end( + '

Mock integration proxy

Connect your test account.

', + ); + } if (req.method !== 'POST') return json(405, {}); challenges.delete(challenge); redirect.searchParams.set('connection_code', issueCode(platform)); - res.writeHead(303, { Location: redirect.href }); return res.end(); + res.writeHead(303, { Location: redirect.href }); + return res.end(); } if (url.pathname.startsWith('/proxy/')) { const code = req.headers.authorization?.replace(/^Bearer /, ''); const platform = codes.get(code); - if (!platform) return json(401, { error: 'Invalid or consumed connection code' }); + if (!platform) + return json(401, { error: 'Invalid or consumed connection code' }); codes.delete(code); - if (req.method !== 'GET' || !url.pathname.startsWith(`/proxy/${platform}/`)) return json(403, {}); - const data = platform === 'pets' ? (url.searchParams.get('page') === '2' ? pets.slice(2) : pets.slice(0, 2)) : platform === 'google-calendar' ? { items: [{ id: 'event-1', summary: 'Team meeting' }] } : [{ id: 42, title: 'Fetched GitHub issue', number: 42, state: 'open' }]; - return json(200, data, { 'X-Connection-Code': issueCode(platform), ...(platform === 'pets' && !url.searchParams.has('page') ? { Link: '; rel="next"' } : {}) }); + const headers = { 'X-Connection-Code': issueCode(platform) }; + if (!url.pathname.startsWith(`/proxy/${platform}/`)) + return json(403, {}, headers); + if (platform === 'github-issues') { + try { + let body = ''; + for await (const chunk of req) { + body += chunk; + if (body.length > 1024 * 1024) return json(413, {}, headers); + } + const result = github.request( + req.method, + url, + body ? JSON.parse(body) : {}, + ); + return json(result.status, result.body, headers); + } catch { + return json(400, { error: 'Invalid request body' }, headers); + } + } + if (platform === 'google-calendar') { + const result = calendar.request(req.method, url); + return json(result.status, result.body, headers); + } + if (req.method !== 'GET') return json(403, {}, headers); + const data = + platform === 'pets' + ? url.searchParams.get('page') === '2' + ? pets.slice(2) + : pets.slice(0, 2) + : { items: [{ id: 'event-1', summary: 'Team meeting' }] }; + return json(200, data, { + ...headers, + ...(platform === 'pets' && !url.searchParams.has('page') + ? { Link: '; rel="next"' } + : {}), + }); } json(404, {}); }); + server.github = github; + server.calendar = calendar; + return server; } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - mockProxy().listen(Number(process.env.MOCK_PROXY_PORT ?? 19090), process.env.MOCK_PROXY_HOST ?? '127.0.0.1', () => console.log('Mock integration proxy listening on http://127.0.0.1:19090')); +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + mockProxy().listen( + Number(process.env.MOCK_PROXY_PORT ?? 19090), + process.env.MOCK_PROXY_HOST ?? '127.0.0.1', + () => + console.log('Mock integration proxy listening on http://127.0.0.1:19090'), + ); } diff --git a/integrations/localthought/tsconfig.json b/integrations/localthought/tsconfig.json index e412e3c473..8a41739785 100644 --- a/integrations/localthought/tsconfig.json +++ b/integrations/localthought/tsconfig.json @@ -8,5 +8,5 @@ "types": ["node"], "noEmit": true }, - "include": ["data.ts", "schema.ts", "plugin.ts", "browser.ts"] + "include": ["calendar.ts", "schema.ts", "plugin.ts", "browser.ts"] } diff --git a/planning/devonian-issue-sync.md b/planning/devonian-issue-sync.md new file mode 100644 index 0000000000..657a8ac58d --- /dev/null +++ b/planning/devonian-issue-sync.md @@ -0,0 +1,21 @@ +# Devonian issue tracker sync + +- [x] Verify Devonian main (`e11104f`) and AtomicServer PR #1394 (`a087a7ca7`). +- [x] Inspect native lenses, GitHub integration actions, comments and test coverage. +- [x] Add regression tests for bidirectional issues/comments, conflict handling and restart recovery. +- [x] Build a browser demo using Devonian, local-only Atomic storage and direct integration-proxy requests. +- [x] Verify 13 focused tests, 7 existing integration tests and frontend typecheck; document setup/limits and update coverage. +- [x] Verify native browser creation/comments both ways, close/reopen and reload without duplicates. +- [x] Verify deployed v40 CORS preflight/exposed headers and browser GitHub OAuth. +- [ ] Verify live two-way writes: proxy credential returns 404 for the private sandbox; resolve repository access first. + +Use Devonian HTTP subjects for the intermediate graph and scoped external mappings +for Atomic DIDs. Persist graph, mappings and request receipts in IndexedDB. Serialize +rotating connection codes; refuse to retry uncertain writes. Tenant authentication and rotating credentials use #1401’s shared BrowserIntegrations client. +No Node runtime, AtomicServer plugin endpoint, tenant secret on AtomicServer, or server scheduler. +Live proxy CORS and OAuth now work; private-sandbox access is the remaining live blocker. +Missing records are conflicts, not deletion requests. + +- [x] Rebase onto #1401 and reuse browser tenant challenge, callback and rotating transport. +- [x] Expand HTTP mock for stateful GitHub issues/comments and add Playwright two-way sync coverage. +- [x] Run 13 sync tests, 11 LocalThought browser tests, 7 existing GitHub integration tests, browser E2E and typechecks.