diff --git a/.dagger/src/index.ts b/.dagger/src/index.ts index c177cf9aef..dc4e0a4b36 100644 --- a/.dagger/src/index.ts +++ b/.dagger/src/index.ts @@ -636,6 +636,10 @@ export class AtomicServer { this.source.directory('plugin-runtime'), ) .withDirectory('/code/wasm', this.source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + this.source.directory('integrations/localthought/syncables'), + ) .withDirectory('/code/server', this.source.directory('server')) .withDirectory('/code/cli', this.source.directory('cli')) .withDirectory('/code/desktop', this.source.directory('desktop')) @@ -713,6 +717,10 @@ export class AtomicServer { .withDirectory('/code/cli', this.source.directory('cli')) .withDirectory('/code/desktop', this.source.directory('desktop')) .withDirectory('/code/wasm', this.source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + this.source.directory('integrations/localthought/syncables'), + ) .withDirectory( '/code/plugin-examples', this.source.directory('plugin-examples'), @@ -1127,7 +1135,9 @@ export class AtomicServer { // Surfaces /app/dev-drive and /app/prunetests in the production // build the e2e tests run against. See `devRoutesEnabled()` in // data-browser/src/config.ts. - buildContainer = buildContainer.withEnvVariable('VITE_E2E', 'true'); + buildContainer = buildContainer + .withEnvVariable('VITE_E2E', 'true') + .withEnvVariable('VITE_INTEGRATION_PROXY_URL', 'http://127.0.0.1:19090'); } return buildContainer.withExec(['pnpm', 'run', 'build']); @@ -1182,6 +1192,10 @@ export class AtomicServer { .withDirectory('/code/cli', source.directory('cli')) .withDirectory('/code/desktop', source.directory('desktop')) .withDirectory('/code/wasm', source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + source.directory('integrations/localthought/syncables'), + ) .withDirectory( '/code/plugin-examples', source.directory('plugin-examples'), @@ -1380,6 +1394,10 @@ export class AtomicServer { .withDirectory('/code/cli', source.directory('cli')) .withDirectory('/code/desktop', source.directory('desktop')) .withDirectory('/code/wasm', source.directory('wasm')) + .withDirectory( + '/code/integrations/localthought/syncables', + source.directory('integrations/localthought/syncables'), + ) .withDirectory( '/code/plugin-examples', source.directory('plugin-examples'), diff --git a/.github/workflows/pets-e2e.yml b/.github/workflows/pets-e2e.yml index 37f32a802c..ab799cfe08 100644 --- a/.github/workflows/pets-e2e.yml +++ b/.github/workflows/pets-e2e.yml @@ -2,7 +2,7 @@ name: LocalThought API plugins E2E on: push: - branches: [feat/api-plugins, codex/localthought-api-plugins] + branches: [feat/api-plugins, codex/localthought-api-plugins, codex/browser-integrations] permissions: contents: read diff --git a/Cargo.lock b/Cargo.lock index 7f5b7ea106..e675c1e34f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,7 +1282,6 @@ dependencies = [ "sha1 0.11.0", "simple-server-timing-header", "static-files 0.3.1", - "syncables", "tempfile", "text-splitter", "tokio", @@ -1345,14 +1344,17 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" name = "atomic-wasm" version = "0.1.0" dependencies = [ + "async-trait", "atomic_lib", "blake3", + "chrono", "console_error_panic_hook", "getrandom 0.3.4", "js-sys", "serde", "serde-wasm-bindgen", "serde_json", + "syncables", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", @@ -13442,7 +13444,6 @@ dependencies = [ [[package]] name = "syncables" version = "0.1.0" -source = "git+https://github.com/localthought/syncables-rs?rev=d48e4d9bad3ed9ec826d1c2040e989171701965d#d48e4d9bad3ed9ec826d1c2040e989171701965d" dependencies = [ "async-trait", "httpdate", @@ -13453,7 +13454,6 @@ dependencies = [ "serde_yaml_ng", "thiserror 2.0.18", "tokio", - "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a575b5bc79..740f704af9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ "plugin-runtime", "tools/cargo-bin", ] -exclude = ["flutter/rust"] +exclude = ["flutter/rust", "integrations/localthought/syncables"] # Debuginfo dominates target/ size: with ~1460 deps (tauri, actix, iroh) the # default `debug = true` produces a multi-GB tree per build flavor, and this diff --git a/TESTING_COVERAGE.md b/TESTING_COVERAGE.md index 6e3c8f1b5a..6d2d376406 100644 --- a/TESTING_COVERAGE.md +++ b/TESTING_COVERAGE.md @@ -1,5 +1,36 @@ # Testing coverage map +LocalThought browser migration: `integrations/localthought/browser.test.ts` +covers tenant HMAC, actor/drive ownership, rotation before dispatch, pagination, +uncertain-response refusal and cross-origin pagination refusal. The real generated +WASM bundle is exercised by `wasm-smoke.mjs` for pagination, typed ontology, +timestamps and provider failures. `browser-smoke.mjs` exercises the complete +mock consent/import/review/OPFS/reload journey with AtomicServer unavailable +(verified locally). Local installation/schema lookup tests reject missing or +incomplete local databases rather than inferring permission to create duplicates. +The companion Syncables branch has 142 passing native tests and a wasm32 build; +the companion proxy branch has 39 passing tests including CORS preflight and +exposed headers. Live OAuth on the browser path still requires deployment of +the companion proxy CORS change and is not yet verified. + +`browser/e2e/tests/devonian-issue-sync.spec.mts` exercises tenant-secret entry, +proxy consent, direct HTTP writes and local OPFS storage for two-way issue +creation, comments, close/reopen and reload without duplicate resources. Its +stateful HTTP mock isolates repositories and consumes/rotates connection codes; +it does not substitute the in-page sample transport. + +The browser-only Devonian issue tracker demo has focused tests under +`integrations/github-issues/devonian`: real Devonian lenses with deterministic +connectors exercise bidirectional issue/comment creation and edits, close/reopen, +distinct identical resources, conflicts, missing records and restart/replay. +Transport fixtures cover pagination, label preservation, scoped comment links, +rotating connection codes and refusal to resend uncertain writes. The native +OPFS browser flow was manually verified for creation and comments on both sides, +closing from Atomic, reopening from the sample GitHub side and reloading without +duplicate issues/comments. Live proxy OAuth, +GitHub writes and a guided uncertain-write recovery UI remain unverified/unbuilt; +proxy v40 CORS and browser OAuth are verified, but its GitHub credential returns 404 for the private sandbox. + What is tested, at which layer, and — the part that matters — **what is not**. This exists because the protocol is far better tested than the glue around it, @@ -65,12 +96,12 @@ return to the same drive, rotating connection codes, two-page Syncables fetch, review/apply, and five displayed records with integer/boolean/float/timestamp properties. Dagger starts the mock for E2E; local runs opt in with `ATOMIC_MOCK_INTEGRATION_PROXY=1` and the README configuration. -`integration_proxy` Rust tests cover actor/drive binding, tenant HMAC, +`browser.test.ts` and the real WASM smoke cover actor/drive binding, tenant HMAC, Syncables pagination/ontology and duplicate-page refusal. The mock's Node test covers invalid tenant proofs and replayed/rotated codes. The mapping tests cover typed proposals, missing identities, repeat imports, local edits and duplicates. -Live catalog OAD endpoints currently return 404 (integration-proxy #25), so live -OAuth and GitHub data fetching are not yet certified. +The historical server path was live-verified for GitHub and Google Calendar. +The new browser path awaits deployment of the companion proxy CORS change. Run it against a production build to catch missing translation catalog entries: Vite dev extracts them automatically and can hide blank production labels. The GitHub setup flow also covers opting into assistant-led automation creation: @@ -1053,3 +1084,15 @@ cancelled on teardown. Old plugin-name grants are deliberately not migrated. commit. It failed with the server signer before `Resource::destroy_as` was used; installation deletion must use the same selected identity as create/update. LocalThought: Rust handler tests cover connection binding, request signing, duplicate-page rejection, typed paginated previews, and Calendar UTC date-range validation. Live Calendar OAuth, bounded fetch, review/apply and event table display were verified against proxy v39 (54 records). + +Google Calendar one-way projection: `integrations/localthought/calendar.test.ts` +covers all-day/timed start dates, offset boundaries, exclusive end preservation, +feature notes (including WASM-normalized field names), cancellations without +start data, invalid active events, namespace isolation and repeat import/local +field preservation. `browser/e2e/tests/google-calendar-import.spec.mts` uses the +shared HTTP mock integration-proxy with a paginated Google Calendar, tenant +secret entry and OAuth consent. It covers browser WASM fetching, local +schema/proposal/apply, Calendar display, provider updates, OPFS reload and +stable identities while AtomicServer HTTP/WebSockets are unavailable. Missing +rows in a bounded snapshot are retained, not interpreted as deletions. +Live-provider browser OAuth verification remains separate from this fixture test. diff --git a/browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE b/browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/DEVONIAN-LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/browser/data-browser/src/chunks/DevonianDemo/demo.d.mts b/browser/data-browser/src/chunks/DevonianDemo/demo.d.mts new file mode 100644 index 0000000000..8055f7e558 --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/demo.d.mts @@ -0,0 +1,47 @@ +import type { Store } from '@tomic/lib'; +export interface Demo { + key: string; + state: { + options: { sample: boolean }; + config: { connection: { table: string } }; + fixture: { + issues?: Array<{ number: number; title: string; state: string }>; + comments?: Array<{ id: number; body: string; issue_url: string }>; + }; + }; +} +export interface Row { + id: string; + value: { title: string; body: string; status: string }; + comments: Array<{ id: string; value: { body: string } }>; +} +export function openDemo( + store: Store, + options: { + sample: boolean; + repository: string; + proxy: string; + }, +): Promise; +export function syncDemo(store: Store, demo: Demo): Promise; +export function demoRows(store: Store, demo: Demo): Promise; +export function editAtomic( + store: Store, + demo: Demo, + command: string, + id?: string, + text?: string, +): Promise; +export function editFixture( + demo: Demo, + command: string, + number?: number, + text?: string, +): Promise; + +export function connectDemo( + store: Store, + options: { repository: string; proxy: string }, + secret: string, +): Promise; +export function resumeDemo(store: Store): Promise; diff --git a/browser/data-browser/src/chunks/DevonianDemo/demo.mjs b/browser/data-browser/src/chunks/DevonianDemo/demo.mjs new file mode 100644 index 0000000000..65615d2b21 --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/demo.mjs @@ -0,0 +1,297 @@ +// @wc-ignore-file +import { BrowserIntegrations } from '../../../../../integrations/localthought/browser'; +import { endpoint } from '../../../../../integrations/github-issues/adapter'; +import { get, set } from 'idb-keyval'; +import { core, server, dataBrowser, Datatype, enableLoro } from '@tomic/lib'; +import * as devonian from './devonian.js'; +import { ensureAgentForDemo } from '../Demo/guestAgent'; +import { buildTableFromSpec } from '../TablePage/createTableFromSpec'; +import { Bridge } from '../../../../../integrations/github-issues/devonian/bridge.mjs'; +import { + AtomicPort, + GitHubPort, +} from '../../../../../integrations/github-issues/devonian/ports.mjs'; +import { + fixtureTransport, + proxyTransport, +} from '../../../../../integrations/github-issues/devonian/proxy.mjs'; + +export async function openDemo(store, options) { + await enableLoro(); + await ensureAgentForDemo(store); + await store.waitForClientDb(10000); + const db = store.getClientDb(); + if (!db || !(await db.waitForReady())) + throw new Error('Enable the browser database on the Sync page first.'); + const repository = options.sample ? 'demo/issues' : options.repository; + endpoint(repository); + const key = `devonian-demo:${JSON.stringify([store.getAgent().subject, repository, options.sample ? 'sample' : new URL(options.proxy).origin])}`; + return navigator.locks.request(key, async () => { + let state = await get(key); + if (!state) { + const actor = store.getAgent().subject; + const drive = await store.newResource({ + noParent: true, + isA: [server.classes.drive], + propVals: { + [core.properties.name]: options.sample + ? 'Devonian sample tracker' + : `GitHub: ${repository}`, + [core.properties.read]: [actor], + [core.properties.write]: [actor], + }, + }); + store.registerLocalOnlyDrive(drive.subject); + await drive.save(); + await store.createDefaultOntology(drive); + const table = await buildTableFromSpec( + store, + { + name: 'Issue Tracker', + rowName: 'Issue', + columns: [ + { name: 'Description', type: 'markdown' }, + { + name: 'Status', + type: 'select', + options: ['Todo', 'Doing', 'Done'], + }, + { name: 'GitHub issue number', type: 'number' }, + ], + views: [ + { + name: 'Board', + kind: 'kanban', + groupByColumn: 'Status', + default: true, + }, + { name: 'All issues', kind: 'table' }, + ], + }, + { + parent: drive.subject, + driveSubject: drive.subject, + addToOntology: async () => {}, + }, + ); + const provenance = await store.newResource({ + parent: drive.subject, + isA: [core.classes.property], + propVals: { + [core.properties.name]: 'GitHub source', + [core.properties.shortname]: 'github-source', + [core.properties.description]: + 'Original author, identity and timestamps from GitHub.', + [core.properties.datatype]: Datatype.JSON, + }, + }); + await provenance.save(); + const folder = await store.newResource({ + parent: drive.subject, + isA: [dataBrowser.classes.folder], + propVals: { [core.properties.name]: 'Comments' }, + }); + await folder.save(); + await drive.set(dataBrowser.properties.commentsFolder, folder.subject); + await drive.save(); + const config = { + connection: { + repository, + drive: drive.subject, + table: table.tableSubject, + rowClass: table.classSubject, + body: table.columns.Description, + status: table.columns.Status, + number: table.columns['GitHub issue number'], + tags: table.tags.Status, + }, + commentsFolder: folder.subject, + provenance: provenance.subject, + }; + state = { + config, + options: { sample: options.sample, proxy: options.proxy, repository }, + base: `https://atomicdata.dev/devonian-bridges/${crypto.randomUUID()}`, + fixture: {}, + journal: {}, + }; + await db.flush(); + await set(key, state); + } + store.registerLocalOnlyDrive(state.config.connection.drive); + store.setDrive(state.config.connection.drive); + return { key, state }; + }); +} + +export async function syncDemo(store, demo) { + return navigator.locks.request(demo.key, async () => { + const state = await get(demo.key); + const save = () => set(demo.key, state); + const call = state.options.sample + ? fixtureTransport(state.fixture, save) + : proxyTransport({ + url: state.options.proxy, + repository: state.options.repository, + journal: state.journal, + save, + dispatch: (path, init) => + client(state.options.proxy).request( + state.config.connection.drive, + store.getAgent().subject, + state.connection, + 'github-issues', + path, + init, + ), + }); + const bridge = new Bridge({ + devonian, + local: new AtomicPort(store, state.config), + remote: new GitHubPort(null, state.config.connection, call), + base: state.base, + snapshot: state.bridge, + save: async snapshot => { + await store.getClientDb().flush(); + state.bridge = snapshot; + await save(); + }, + }); + await bridge.sync(); + demo.state = state; + return Object.keys(bridge.records).length; + }); +} + +export async function demoRows(store, demo) { + const local = new AtomicPort(store, demo.state.config); + const issues = await local.list('issue'); + for (const row of issues) + row.comments = await local.list('comment:ui', { issueId: row.id }); + return issues; +} + +export async function editAtomic(store, demo, command, id, text) { + const port = new AtomicPort(store, demo.state.config); + if (command === 'create') + await port.create( + 'issue', + { title: text, body: '', status: 'Todo' }, + crypto.randomUUID(), + ); + else if (command === 'comment') + await port.create( + 'comment:ui', + { body: text }, + crypto.randomUUID(), + undefined, + { issueId: id }, + ); + else { + const row = await port.get('issue', id); + await port.update('issue', id, { + ...row.value, + status: row.value.status === 'Done' ? 'Todo' : 'Done', + }); + } + await store.getClientDb().flush(); +} + +export async function editFixture(demo, command, number, text) { + return navigator.locks.request(demo.key, async () => { + const state = await get(demo.key); + const call = fixtureTransport(state.fixture, () => set(demo.key, state)); + if (command === 'create') + await call( + 'create_issue', + { title: text, body: '' }, + crypto.randomUUID(), + ); + else if (command === 'comment') + await call('create_comment', { number, body: text }, crypto.randomUUID()); + else { + const issue = state.fixture.issues.find(r => r.number === number); + await call( + 'update_issue', + { + number, + title: issue.title, + body: issue.body, + state: issue.state === 'open' ? 'closed' : 'open', + }, + crypto.randomUUID(), + ); + } + demo.state = state; + }); +} + +const handoffKey = 'devonian-browser-handoff'; +const resumeKey = 'devonian-browser-resume'; +const client = origin => + new BrowserIntegrations( + localStorage, + async () => { + throw new Error('Devonian uses its own resource lenses'); + }, + origin, + ); +export async function connectDemo(store, options, secret) { + const demo = await openDemo(store, { ...options, sample: false }); + const result = await client(options.proxy).start( + demo.state.config.connection.drive, + store.getAgent().subject, + 'github-issues', + `${location.origin}/app/devonian-demo`, + secret, + ); + sessionStorage.setItem( + handoffKey, + JSON.stringify({ key: demo.key, state: result.state }), + ); + location.assign(result.url); +} +export async function resumeDemo(store) { + const url = new URL(location.href); + const callbackCode = url.searchParams.get('connection_code'); + const callbackState = url.searchParams.get('integration_state'); + const handoff = JSON.parse(sessionStorage.getItem(handoffKey) ?? 'null'); + // Save the validated handoff before removing credentials from the URL, so a + // reload while OPFS opens cannot abandon the completed proxy consent. + if (callbackCode || callbackState) { + history.replaceState(null, '', url.pathname); + if (!handoff || callbackState !== handoff.state || !callbackCode) + throw new Error('Invalid connection callback state'); + handoff.code = callbackCode; + sessionStorage.setItem(handoffKey, JSON.stringify(handoff)); + } + const code = handoff?.code; + const stateId = handoff?.state; + const key = code ? handoff.key : sessionStorage.getItem(resumeKey); + if (!key) { + if (callbackCode || callbackState) + throw new Error('Missing browser connection handoff'); + return; + } + const saved = await get(key); + if (!saved) throw new Error('Missing local tracker'); + const demo = await openDemo(store, saved.options); + if (demo.key !== key) throw new Error('Connection belongs to another agent'); + if (code) { + if (!handoff.finished) { + client(saved.options.proxy).finish( + saved.config.connection.drive, + store.getAgent().subject, + stateId, + code, + ); + handoff.finished = true; + sessionStorage.setItem(handoffKey, JSON.stringify(handoff)); + } + demo.state.connection = stateId; + await set(key, demo.state); + sessionStorage.removeItem(handoffKey); + sessionStorage.setItem(resumeKey, key); + } + return demo; +} diff --git a/browser/data-browser/src/chunks/DevonianDemo/devonian.js b/browser/data-browser/src/chunks/DevonianDemo/devonian.js new file mode 100644 index 0000000000..ea269a9670 --- /dev/null +++ b/browser/data-browser/src/chunks/DevonianDemo/devonian.js @@ -0,0 +1,328 @@ +// Devonian native resource API, e11104f78ebd151a361171ca0b21489b25e1e2c8. Apache-2.0; see DEVONIAN-LICENSE. + +// ../../localthought/devonian/src/atomic/Resource.ts +import { Datatype, validateDatatype } from "@tomic/lib"; +var IS_A = "https://atomicdata.dev/properties/isA"; +function assertSubject(value) { + if (typeof value !== "string" || /\s/.test(value)) + throw new Error("Expected an absolute URL without whitespace"); + const url = new URL(value); + if (!["http:", "https:"].includes(url.protocol)) { + throw new Error(`Expected an HTTP(S) URL: ${value}`); + } +} +var AtomicSchema = class { + properties = /* @__PURE__ */ new Map([ + [IS_A, Datatype.RESOURCEARRAY] + ]); + property(subject, datatype) { + assertSubject(subject); + if (!Object.values(Datatype).includes(datatype) || datatype === Datatype.UNKNOWN) { + throw new Error(`Unsupported datatype: ${datatype}`); + } + const existing = this.properties.get(subject); + if (existing && existing !== datatype) + throw new Error(`Conflicting property: ${subject}`); + this.properties.set(subject, datatype); + return this; + } + validate(resource) { + assertSubject(resource["@id"]); + this.validateProperties(resource, true); + } + validateProperties(resource, named) { + if (Object.getPrototypeOf(resource) !== Object.prototype) + throw new Error("Expected a plain JSON object"); + for (const [property, value] of Object.entries(resource)) { + if (property === "@id" && named) continue; + assertSubject(property); + const datatype = this.properties.get(property); + if (!datatype) throw new Error(`Unknown property: ${property}`); + if (typeof value === "number" && !Number.isFinite(value)) + throw new Error("Expected a finite number"); + if (value === void 0 || value === null) + throw new Error(`Missing value for ${property}`); + if (datatype === Datatype.ATOMIC_URL) { + this.validateLink(value); + } else if (datatype === Datatype.RESOURCEARRAY) { + if (!Array.isArray(value)) + throw new Error(`Expected a resource array: ${property}`); + for (const link of value) this.validateLink(link); + } else { + if ([Datatype.FLOAT, Datatype.INTEGER, Datatype.TIMESTAMP].includes( + datatype + ) && typeof value !== "number") + throw new Error("Expected a number"); + if (datatype === Datatype.BOOLEAN && typeof value !== "boolean") + throw new Error("Expected a boolean"); + validateDatatype(value, datatype); + } + } + } + validateLink(value) { + if (typeof value === "string") assertSubject(value); + else if (value && typeof value === "object" && !Array.isArray(value)) + this.validateProperties(value, false); + else throw new Error("Expected a resource URL or nested resource"); + } +}; + +// ../../localthought/devonian/src/atomic/Store.ts +var AtomicStore = class _AtomicStore { + constructor(schema) { + this.schema = schema; + } + schema; + resources = /* @__PURE__ */ new Map(); + get(subject) { + const resource = this.resources.get(subject); + return resource && structuredClone(resource); + } + put(resource) { + this.schema.validate(resource); + this.resources.set(resource["@id"], structuredClone(resource)); + } + /** Preserve omitted properties; removals must be listed in unset. */ + patch(subject, patch) { + assertSubject(subject); + if (Object.hasOwn(patch.set ?? {}, "@id") || patch.unset?.includes("@id")) { + throw new Error("A patch cannot change resource identity"); + } + const resource = { + ...this.get(subject) ?? { "@id": subject }, + ...patch.set + }; + for (const property of patch.unset ?? []) { + assertSubject(property); + delete resource[property]; + } + this.put(resource); + return structuredClone(resource); + } + /** Synchronous local transaction. Async work must finish before entering this callback. */ + transaction(operation) { + const previous = this.resources; + this.resources = new Map(previous); + try { + operation(); + } catch (error) { + this.resources = previous; + throw error; + } + } + /** Apply a resource projection as one local transaction. */ + apply(changes) { + const staged = new _AtomicStore(this.schema); + staged.resources = new Map(this.resources); + for (const change of changes) staged.patch(change.subject, change.patch); + this.resources = staged.resources; + } + delete(subject) { + this.resources.delete(subject); + } + /** Optional class filtering; callers can apply additional predicates to the returned copies. */ + all(classSubject) { + return [...this.resources.values()].filter((resource) => { + const classes = resource["https://atomicdata.dev/properties/isA"]; + return !classSubject || Array.isArray(classes) && classes.includes(classSubject); + }).map((resource) => structuredClone(resource)); + } + toJSONAD() { + return JSON.stringify(this.all()); + } + /** Validate the complete snapshot before replacing state. Import does not emit writes. */ + loadJSONAD(json) { + const parsed = JSON.parse(json); + const resources = Array.isArray(parsed) ? parsed : [parsed]; + const next = /* @__PURE__ */ new Map(); + for (const item of resources) { + if (!item || typeof item !== "object" || Array.isArray(item) || typeof item["@id"] !== "string") { + throw new Error("Expected a named JSON-AD resource"); + } + const resource = item; + this.schema.validate(resource); + if (next.has(resource["@id"])) + throw new Error(`Duplicate subject: ${resource["@id"]}`); + next.set(resource["@id"], structuredClone(resource)); + } + this.resources = next; + } +}; + +// ../../localthought/devonian/src/atomic/IdentityMap.ts +var VOCAB = "https://raw.githubusercontent.com/localthought/devonian/main/vocab/"; +var identity = { + class: `${VOCAB}Identity.json`, + scope: `${VOCAB}scope.json`, + entity: `${VOCAB}entity.json`, + localId: `${VOCAB}localId.json`, + idType: `${VOCAB}idType.json`, + resource: `${VOCAB}resource.json` +}; +var AtomicIdentityMap = class { + constructor(store, baseURL) { + this.store = store; + assertSubject(baseURL); + const url = new URL(baseURL); + if (url.search || url.hash) + throw new Error("Identity base URL cannot contain a query or fragment"); + this.base = baseURL.replace(/\/$/, ""); + store.schema.property(identity.scope, Datatype.ATOMIC_URL).property(identity.entity, Datatype.STRING).property(identity.localId, Datatype.STRING).property(identity.idType, Datatype.STRING).property(identity.resource, Datatype.ATOMIC_URL); + } + store; + base; + key(scope, id) { + assertSubject(scope.scope); + if (!scope.entity || typeof id !== "string" && typeof id !== "number" || typeof id === "number" && !Number.isSafeInteger(id) || id === "") { + throw new Error( + "Expected an entity and a nonempty string or safe integer external ID" + ); + } + return encodeURIComponent( + JSON.stringify([scope.scope, scope.entity, typeof id, id]) + ); + } + subjectFor(scope, id) { + return this.lookup(scope, id) ?? `${this.base}/resources/${this.key(scope, id)}`; + } + lookup(scope, id) { + this.key(scope, id); + return this.find(scope).find( + (resource) => resource[identity.localId] === String(id) && resource[identity.idType] === typeof id + )?.[identity.resource]; + } + externalId(scope, subject) { + const resource = this.find(scope).find( + (resource2) => resource2[identity.resource] === subject + ); + if (!resource) return void 0; + return resource[identity.idType] === "number" ? Number(resource[identity.localId]) : resource[identity.localId]; + } + bind(scope, id, subject) { + const key = this.key(scope, id); + assertSubject(subject); + const existing = this.lookup(scope, id); + const reverse = this.externalId(scope, subject); + if (existing !== void 0 && existing !== subject || reverse !== void 0 && reverse !== id) { + throw new Error("Conflicting identity mapping"); + } + if (existing === subject && reverse === id) return; + this.store.put({ + "@id": `${this.base}/identities/${key}`, + [IS_A]: [identity.class], + [identity.scope]: scope.scope, + [identity.entity]: scope.entity, + [identity.localId]: String(id), + [identity.idType]: typeof id, + [identity.resource]: subject + }); + } + find(scope) { + const resources = this.store.all(identity.class).filter( + (resource) => resource[identity.scope] === scope.scope && resource[identity.entity] === scope.entity + ); + const ids = /* @__PURE__ */ new Set(); + const subjects = /* @__PURE__ */ new Set(); + for (const resource of resources) { + const id = resource[identity.localId]; + const type = resource[identity.idType]; + const subject = resource[identity.resource]; + if (typeof id !== "string" || !id || !["string", "number"].includes(String(type)) || typeof subject !== "string" || type === "number" && (!Number.isSafeInteger(Number(id)) || String(Number(id)) !== id)) { + throw new Error("Invalid identity mapping"); + } + const key = JSON.stringify([type, id]); + if (ids.has(key) || subjects.has(subject)) + throw new Error("Conflicting identity mappings in snapshot"); + ids.add(key); + subjects.add(subject); + } + return resources; + } +}; + +// ../../localthought/devonian/src/atomic/Lens.ts +var AtomicLens = class { + constructor(options) { + this.options = options; + assertSubject(options.scope); + if (!options.entity) throw new Error("A lens requires an entity type"); + if (options.identities.store !== options.store) { + throw new Error("Lens and identity map must use the same store"); + } + } + options; + pending = Promise.resolve(); + /** Handle a webhook or fetched record. Never writes back to the connector. */ + ingest(record) { + return this.enqueue(async () => { + const { connector, identities, store, read } = this.options; + const id = connector.id(record); + const subject = identities.subjectFor(this.options, id); + const patch = await read(record, subject); + store.transaction(() => { + store.apply([...patch.related ?? [], { subject, patch }]); + for (const mapping of patch.identities ?? []) + identities.bind(mapping.scope, mapping.id, mapping.subject); + identities.bind(this.options, id, subject); + }); + return subject; + }); + } + /** Publish the latest native state; failures reject and may be retried. */ + publish(subject) { + return this.enqueue(async () => { + const { connector, identities, store, write } = this.options; + const resource = store.get(subject); + if (!resource) throw new Error(`Unknown resource: ${subject}`); + const id = identities.externalId(this.options, subject); + if (id !== void 0) { + const previous = await connector.get(id); + await connector.update(id, await write(resource, previous)); + return id; + } + const key = JSON.stringify([ + this.options.scope, + this.options.entity, + subject + ]); + const created = await connector.create( + await write(resource, void 0), + key + ); + const createdId = connector.id(created); + identities.bind(this.options, createdId, subject); + return createdId; + }); + } + /** Delete native state after an external deletion. Retain identity for replay/recreation. */ + ingestDelete(id) { + return this.enqueue(async () => { + const subject = this.options.identities.lookup(this.options, id); + if (subject) this.options.store.delete(subject); + }); + } + /** Delete externally first, so a failed request leaves native state available for retry. */ + delete(subject) { + return this.enqueue(async () => { + const { connector, identities, store } = this.options; + const id = identities.externalId(this.options, subject); + if (id !== void 0) await connector.delete(id); + store.delete(subject); + }); + } + enqueue(operation) { + const result = this.pending.then(operation); + this.pending = result.catch(() => void 0); + return result; + } +}; +export { + AtomicIdentityMap, + AtomicLens, + AtomicSchema, + AtomicStore, + Datatype, + IS_A, + assertSubject, + identity +}; diff --git a/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx b/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx index bb33f0ea6e..1448434a53 100644 --- a/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx +++ b/browser/data-browser/src/chunks/PluginRuns/ConnectLocalThought.tsx @@ -3,7 +3,7 @@ import { core, dataBrowser, ensureSchema, - executeServerPlugin, + pluginSchema, useStore, type Resource, } from '@tomic/react'; @@ -12,10 +12,13 @@ import { Column } from '@components/Row'; import Field from '@components/forms/Field'; import { Input, ErrMessage } from '@components/forms/InputStyles'; import { AtomicLink } from '@components/AtomicLink'; -import { pluginClassesFor } from './runScript'; -import { ensureInstallationResource } from './installationResources'; +import { + localSchemaStore, + ensureLocalInstallationResource as ensureInstallationResource, +} from './installationResources'; import { RunPluginDialog } from './RunPluginDialog'; import { + browserIntegrations, connectionKey, platformName, proxyRequest, @@ -27,7 +30,12 @@ import { type FetchedPlatform, } from '../../../../../integrations/localthought/schema'; import type { Config } from '../../../../../integrations/localthought/plugin'; +import { localImportVerdict } from './localImportVerdict'; import source from '../../../../../integrations/localthought/plugin.js?raw'; +import { + calendarProjection, + calendarFields, +} from '../../../../../integrations/localthought/calendar'; export function ConnectLocalThought({ drive, @@ -55,6 +63,7 @@ export function ConnectLocalThought({ start: new Date().toISOString().slice(0, 10), end: new Date(Date.now() + 30 * 86400000).toISOString().slice(0, 10), })); + const [tenantSecret, setTenantSecret] = useState(''); const [busy, setBusy] = useState(false); const [error, setError] = useState(''); const [preview, setPreview] = useState<{ @@ -65,13 +74,10 @@ export function ConnectLocalThought({ const [tables, setTables] = useState([]); useEffect(() => { const controller = new AbortController(); - fetch( - `${store.getServerUrl()}/integration-proxy/platform?platform=${encodeURIComponent(platform)}`, - { signal: controller.signal }, - ) - .then(async response => { - if (!response.ok) throw new Error(await response.text()); - const data = await response.json(); + browserIntegrations() + .describe(platform) + .then(data => { + if (controller.signal.aborted) return; setParameters(data.parameters); setCollections(data.collections); setConstants( @@ -104,7 +110,12 @@ export function ConnectLocalThought({ const result = await proxyRequest<{ url: string; state: string }>( store, 'start', - { drive, platform, returnUrl: `${location.origin}/app/integrations` }, + { + drive, + platform, + tenantSecret, + returnUrl: `${location.origin}/app/integrations`, + }, ); sessionStorage.setItem( 'localthought-pending', @@ -123,15 +134,17 @@ export function ConnectLocalThought({ setError(''); try { - const fetched = await proxyRequest(store, 'fetch', { + const response = await proxyRequest(store, 'fetch', { drive, connection: connection.connection, constants, ...(platform === 'google-calendar' ? { calendarRange } : {}), }); + const fetched = calendarProjection(response); if (fetched.platform !== platform) throw new Error('Imported platform did not match this connection'); - const terms = await pluginClassesFor(store, drive); + const schemaStore = localSchemaStore(store); + const terms = await ensureSchema(schemaStore, drive, pluginSchema()); const name = platformName(platform); const identity = `localthought:${connection.connection}:${JSON.stringify(Object.entries(constants).sort())}`; const resource = await ensureInstallationResource(store, drive, { @@ -145,7 +158,7 @@ export function ConnectLocalThought({ }, }); const schema = await ensureSchema( - store, + schemaStore, drive, platformSchema(platform, fetched.ontology.terms), ); @@ -187,13 +200,47 @@ export function ConnectLocalThought({ [dataBrowser.properties.viewColumns]: columns, }, }); + const calendar = + platform === 'google-calendar' && term.shortname === 'event' + ? await ensureInstallationResource(store, drive, { + parent: destination.subject, + localId: `${identity}:calendar:${term.shortname}`, + isA: [dataBrowser.classes.view], + propVals: { + [core.properties.name]: tableName, + [dataBrowser.properties.viewKind]: 'calendar', + [dataBrowser.properties.viewGroupBy]: + properties[calendarFields.day], + [dataBrowser.properties.viewColumns]: columns, + }, + }) + : undefined; + const existingViews = destination.get( + dataBrowser.properties.tableViews, + ) as string[] | undefined; await destination.set(dataBrowser.properties.tableViews, [ - view.subject, + ...new Set([ + ...(existingViews ?? []), + view.subject, + ...(calendar ? [calendar.subject] : []), + ]), ]); - await destination.set( + const currentDefault = destination.get( dataBrowser.properties.tableDefaultView, - view.subject, ); + + if ( + !currentDefault || + (calendar && + !existingViews?.includes(calendar.subject) && + currentDefault === view.subject) + ) { + await destination.set( + dataBrowser.properties.tableDefaultView, + calendar?.subject ?? view.subject, + ); + } + await destination.save(); destinations[term.shortname] = { table: destination.subject, rowClass }; } @@ -203,24 +250,13 @@ export function ConnectLocalThought({ localthought: { ...config, connection: connection.connection }, }); await resource.save(); - const result = await executeServerPlugin(store, { - drive, - plugin: resource.subject, - source, - input: { - config: { ...config, records: fetched.records }, - trigger: { - kind: 'manual', - at: Date.now(), - subject: resource.subject, - }, - }, + const verdict = await localImportVerdict(store, drive, { + ...config, + records: fetched.records, }); - if (result.error || !result.verdict) - throw new Error(result.error ?? 'Import returned no preview'); setPreview({ resource, - verdict: result.verdict, + verdict, tables: Object.values(destinations).map(d => d.table), }); } catch (reason) { @@ -236,7 +272,21 @@ export function ConnectLocalThought({ Connect your personal account through LocalThought, then return here to preview an import.

- {connection && ( diff --git a/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx b/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx index 7e10caf095..733f9a6170 100644 --- a/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx +++ b/browser/data-browser/src/chunks/PluginRuns/LocalThoughtCatalog.tsx @@ -6,7 +6,12 @@ import { Button } from '@components/Button'; import { Dialog, useDialog } from '@components/Dialog'; import { ErrMessage } from '@components/forms/InputStyles'; import { ConnectLocalThought } from './ConnectLocalThought'; -import { connectionKey, platformName, proxyRequest } from './localThought'; +import { + browserIntegrations, + connectionKey, + platformName, + proxyRequest, +} from './localThought'; const DirectGitHub = lazy(() => import('./ConnectGitHub').then(m => ({ default: m.ConnectGitHub })), @@ -26,20 +31,9 @@ export function LocalThoughtCatalog({ const completing = useRef(false); useEffect(() => { const controller = new AbortController(); - fetch(`${store.getServerUrl()}/integration-proxy/catalog`, { - signal: controller.signal, - }) - .then(async response => { - if (!response.ok) - throw new Error('Could not load the LocalThought catalog'); - const data = await response.json(); - if ( - !Array.isArray(data.platforms) || - data.platforms.some((id: unknown) => typeof id !== 'string') - ) - throw new Error('Invalid integration catalog'); - setPlatforms(data.platforms); - }) + browserIntegrations() + .catalog(controller.signal) + .then(setPlatforms) .catch(reason => { if (!controller.signal.aborted) setError(String(reason)); }); diff --git a/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts b/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts index 81dc6fc793..97ead3f8d7 100644 --- a/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts +++ b/browser/data-browser/src/chunks/PluginRuns/installationResources.test.ts @@ -47,3 +47,37 @@ it('never interprets a failed identity query as permission to create', async () ).rejects.toThrow('offline'); expect(store.newResource).not.toHaveBeenCalled(); }); + +it('uses complete local identities without a server and refuses a missing local DB', async () => { + const { ensureLocalInstallationResource } = + await import('./installationResources'); + const resource = { + get: (p: string) => ({ parent: 'parent', isA: ['class'] })[p], + }; + const queryLocalDb = vi + .fn() + .mockResolvedValue({ subjects: ['saved'], count: 1 }); + const store = { + queryLocalDb, + getResource: async () => resource, + newResource: vi.fn(), + }; + const options = { + parent: 'parent', + localId: 'step', + isA: ['class'], + propVals: {}, + }; + expect( + await ensureLocalInstallationResource(store as never, 'drive', options), + ).toBe(resource); + queryLocalDb.mockResolvedValue(null); + await expect( + ensureLocalInstallationResource(store as never, 'drive', options), + ).rejects.toThrow('Local installation query'); + queryLocalDb.mockResolvedValue({ subjects: [], count: 1 }); + await expect( + ensureLocalInstallationResource(store as never, 'drive', options), + ).rejects.toThrow('Local installation query'); + expect(store.newResource).not.toHaveBeenCalled(); +}); diff --git a/browser/data-browser/src/chunks/PluginRuns/installationResources.ts b/browser/data-browser/src/chunks/PluginRuns/installationResources.ts index e4a7be3b9e..62c808ce40 100644 --- a/browser/data-browser/src/chunks/PluginRuns/installationResources.ts +++ b/browser/data-browser/src/chunks/PluginRuns/installationResources.ts @@ -16,14 +16,12 @@ export async function ensureInstallationResource( isA: string[]; propVals: Record; }, + local = false, ) { const find = async () => { - const ids = await readConnectionSubjects( - store, - drive, - core.properties.localId, - options.localId, - ); + const ids = await ( + local ? readLocalInstallationSubjects : readConnectionSubjects + )(store, drive, core.properties.localId, options.localId); const resources = await Promise.all(ids.map(id => store.getResource(id))); const matches = resources.filter( r => @@ -71,3 +69,60 @@ export async function ensureInstallationResource( throw error; } } + +async function readLocalInstallationSubjects( + store: Store, + drive: string, + property: string, + value: string, +) { + const result = await store.queryLocalDb({ + drive, + property, + value, + limit: 10001, + }); + if (!result || result.count !== result.subjects.length) + throw new Error( + 'Local installation query failed or is incomplete; refusing to create duplicates', + ); + return result.subjects; +} + +/** LocalThought installations use the same identity checks against local OPFS. */ +export function ensureLocalInstallationResource( + store: Store, + drive: string, + options: Parameters[2], +) { + return ensureInstallationResource(store, drive, options, true); +} + +/** Schema recovery must use the same local identity authority as installation. */ +export function localSchemaStore(store: Store) { + return { + getResource: store.getResource.bind(store), + newResource: store.newResource.bind(store), + findByLocalId: async (drive: string, parent: string, localId: string) => { + const subjects = await readLocalInstallationSubjects( + store, + drive, + core.properties.localId, + localId, + ); + const resources = await Promise.all( + subjects.map(s => store.getResource(s)), + ); + const matches = resources.filter( + r => + String(r.get(core.properties.parent)).split('?')[0] === + parent.split('?')[0], + ); + if (matches.length > 1) + throw new Error( + 'Duplicate local schema identity; resolve before importing', + ); + return matches[0]; + }, + }; +} diff --git a/browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts b/browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts new file mode 100644 index 0000000000..7f373d547c --- /dev/null +++ b/browser/data-browser/src/chunks/PluginRuns/localImportVerdict.ts @@ -0,0 +1,49 @@ +// @wc-ignore-file +import { core, type Store } from '@tomic/react'; +import { + run, + type Config, +} from '../../../../../integrations/localthought/plugin'; + +/** The shipped pure mapper gets a local read snapshot, never network or credentials. */ +export async function localImportVerdict( + store: Store, + drive: string, + config: Config, +) { + const rows = new Map>(); + for (const { table } of Object.values(config.destinations)) { + let offset = 0; + for (;;) { + const result = await store.queryLocalDb({ + drive, + property: core.properties.parent, + value: table, + offset, + limit: 1000, + }); + if (!result) + throw new Error('Local database must be available before importing'); + for (const subject of result.subjects) + rows.set(subject, (await store.getResource(subject)).getPropVals()); + offset += result.subjects.length; + if (offset >= result.count) break; + if (!result.subjects.length) + throw new Error('Local import snapshot is incomplete'); + } + } + return JSON.stringify( + run({ + config, + query: (property, value) => + [...rows] + .filter(([, row]) => row[property] === value) + .map(([subject]) => subject), + read: subject => { + const row = rows.get(subject); + if (!row) throw new Error('Missing local import record'); + return row; + }, + }), + ); +} diff --git a/browser/data-browser/src/chunks/PluginRuns/localThought.ts b/browser/data-browser/src/chunks/PluginRuns/localThought.ts index 6dcf43024e..8751989c66 100644 --- a/browser/data-browser/src/chunks/PluginRuns/localThought.ts +++ b/browser/data-browser/src/chunks/PluginRuns/localThought.ts @@ -1,5 +1,5 @@ // @wc-ignore-file -import { signRequest, type Store } from '@tomic/react'; +import { type Store } from '@tomic/react'; export const platformName = (id: string) => ({ @@ -7,25 +7,73 @@ export const platformName = (id: string) => 'google-calendar': 'Google Calendar', pets: 'Pets', })[id] ?? id; +import { + BrowserIntegrations, + type Engine, +} from '../../../../../integrations/localthought/browser'; +import { wasmJsUrl, wasmBinaryUrl } from '../../helpers/wasmUrls'; +let loaded: Promise | undefined; +async function engine(): Promise { + return (loaded ??= (async () => { + const url = wasmJsUrl(); + const module = await import(/* @vite-ignore */ url); + await module.default({ module_or_path: wasmBinaryUrl() }); + if (typeof module.fetchIntegration !== 'function') + throw new Error('Rebuild the WASM bundle and reload Atomic'); + return module; + })().catch(error => { + loaded = undefined; + throw error; + })); +} +export const browserIntegrations = () => + new BrowserIntegrations( + localStorage, + engine, + import.meta.env.VITE_INTEGRATION_PROXY_URL || undefined, + ); export async function proxyRequest( store: Store, action: string, - body: object, + body: { + drive: string; + platform?: string; + returnUrl?: string; + tenantSecret?: string; + state?: string; + connectionCode?: string; + connection?: string; + constants?: Record; + calendarRange?: { start: string; end: string }; + }, ): Promise { - const agent = store.getAgent(); - if (!agent) throw new Error('Sign in before connecting an account'); - const url = `${store.getServerUrl()}/integration-proxy/${action}`; - const response = await fetch(url, { - method: 'POST', - headers: { - ...(await signRequest(url, agent, {})), - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }); - if (!response.ok) throw new Error(await response.text()); - - return response.json(); + const actor = store.getAgent()?.subject; + if (!actor) throw new Error('Sign in before connecting an account'); + const client = browserIntegrations(); + if (action === 'start') + return (await client.start( + body.drive, + actor, + body.platform!, + body.returnUrl!, + body.tenantSecret!, + )) as T; + if (action === 'finish') + return client.finish( + body.drive, + actor, + body.state!, + body.connectionCode!, + ) as T; + if (action === 'fetch') + return client.fetchRecords( + body.drive, + actor, + body.connection!, + body.constants ?? {}, + body.calendarRange, + ); + throw new Error('Unknown browser integration action'); } export interface SavedConnection { connection: string; @@ -34,4 +82,4 @@ export interface SavedConnection { actor: string; } export const connectionKey = (drive: string, actor: string, platform: string) => - `localthought:${JSON.stringify([drive, actor, platform])}`; + `localthought-browser:${JSON.stringify([drive, actor, platform])}`; diff --git a/browser/data-browser/src/locales/de.po b/browser/data-browser/src/locales/de.po index ee9bc15a69..ff10ed5033 100644 --- a/browser/data-browser/src/locales/de.po +++ b/browser/data-browser/src/locales/de.po @@ -4704,6 +4704,7 @@ msgstr "" msgid "Syncing…" msgstr "" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "" @@ -6287,6 +6288,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "" #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "" @@ -9659,3 +9661,125 @@ msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx +msgid "LocalThought tenant secret" +msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "" + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "" + +#~ msgid "Connection code" +#~ msgstr "" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "" + +#~ msgid "Open live tracker" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "" + +#~ msgid "Done" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "" diff --git a/browser/data-browser/src/locales/en.po b/browser/data-browser/src/locales/en.po index 2cbaf4b6de..06bff9a6f1 100644 --- a/browser/data-browser/src/locales/en.po +++ b/browser/data-browser/src/locales/en.po @@ -4715,6 +4715,7 @@ msgstr "{0}% used" msgid "Syncing…" msgstr "Syncing…" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "Sync now" @@ -6298,6 +6299,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "Keep an encrypted copy of this workspace in {0}. It is sealed on this device, so we store it without being able to read it." #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "Working…" @@ -9698,3 +9700,125 @@ msgstr "Events from (UTC)" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "Events before (UTC)" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx +msgid "LocalThought tenant secret" +msgstr "LocalThought tenant secret" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "The tenant secret is used in this tab. Connection credentials stay in this browser." + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "Synchronized {0} issues and comments." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "Integration proxy URL" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "GitHub repository (owner/repo)" + +#~ msgid "Connection code" +#~ msgstr "Connection code" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." + +#~ msgid "Open live tracker" +#~ msgstr "Open live tracker" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "Connect a real GitHub repository" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "Devonian issue tracker demo" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "Try sample data" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "Open Atomic kanban" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "New issue title or comment" + +#~ msgid "Done" +#~ msgstr "Done" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "Reopen Atomic issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "Close Atomic issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "Add Atomic comment" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "Atomic tracker" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "Atomic issues" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "Reopen sample GitHub issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "Close sample GitHub issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "Add sample GitHub comment" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "Create sample GitHub issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "Create Atomic issue" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "Sample GitHub tracker" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "Sample GitHub issues" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "Connect GitHub tracker" diff --git a/browser/data-browser/src/locales/es.po b/browser/data-browser/src/locales/es.po index 73252fe2f3..c21040df5e 100644 --- a/browser/data-browser/src/locales/es.po +++ b/browser/data-browser/src/locales/es.po @@ -4704,6 +4704,7 @@ msgstr "" msgid "Syncing…" msgstr "" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "" @@ -6287,6 +6288,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "" #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "" @@ -9659,3 +9661,125 @@ msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx +msgid "LocalThought tenant secret" +msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "" + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "" + +#~ msgid "Connection code" +#~ msgstr "" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "" + +#~ msgid "Open live tracker" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "" + +#~ msgid "Done" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "" diff --git a/browser/data-browser/src/locales/fr.po b/browser/data-browser/src/locales/fr.po index 7dcbb3f208..46b33e9914 100644 --- a/browser/data-browser/src/locales/fr.po +++ b/browser/data-browser/src/locales/fr.po @@ -4704,6 +4704,7 @@ msgstr "" msgid "Syncing…" msgstr "" +#: src/routes/DevonianDemoRoute.tsx #: src/routes/SyncRoute.tsx msgid "Sync now" msgstr "" @@ -6287,6 +6288,7 @@ msgid "Keep an encrypted copy of this workspace in {0}. It is sealed on this dev msgstr "" #: src/components/Vault/VaultPanel.tsx +#: src/routes/DevonianDemoRoute.tsx msgid "Working…" msgstr "" @@ -9659,3 +9661,125 @@ msgstr "" #: src/chunks/PluginRuns/ConnectLocalThought.tsx msgid "Events before (UTC)" msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +#: src/routes/DevonianDemoRoute.tsx +msgid "LocalThought tenant secret" +msgstr "" + +#: src/chunks/PluginRuns/ConnectLocalThought.tsx +msgid "The tenant secret is used in this tab. Connection credentials stay in this browser." +msgstr "" + +#. 0: count +#: src/routes/DevonianDemoRoute.tsx +msgid "Synchronized {0} issues and comments." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Integration proxy URL" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "GitHub repository (owner/repo)" +msgstr "" + +#~ msgid "Connection code" +#~ msgstr "" + +#~ msgid "The proxy must allow this app’s origin through CORS and expose X-Connection-Code. Use a fresh GitHub connection code from the proxy’s OAuth flow." +#~ msgstr "" + +#~ msgid "Open live tracker" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect a real GitHub repository" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Devonian issue tracker demo" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sync issues and comments in both directions. Devonian runs in this browser; the Atomic tracker is stored on this device." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Try sample data" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Open Atomic kanban" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "New issue title or comment" +msgstr "" + +#~ msgid "Done" +#~ msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add Atomic comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Atomic issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Reopen sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Close sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Add sample GitHub comment" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create sample GitHub issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Create Atomic issue" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub tracker" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Sample GitHub issues" +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect through LocalThought. The tenant secret is used in this tab to start the connection and is never saved." +msgstr "" + +#: src/routes/DevonianDemoRoute.tsx +msgid "Connect GitHub tracker" +msgstr "" diff --git a/browser/data-browser/src/routes/DevonianDemoRoute.tsx b/browser/data-browser/src/routes/DevonianDemoRoute.tsx new file mode 100644 index 0000000000..f961500c59 --- /dev/null +++ b/browser/data-browser/src/routes/DevonianDemoRoute.tsx @@ -0,0 +1,275 @@ +import { createLazyRoute } from '@tanstack/react-router'; +import { useEffect, useRef, useState } from 'react'; +import { useStore } from '@tomic/react'; +import { Main } from '@components/Main'; +import { ContainerWide } from '@components/Containers'; +import { Column, Row as Horizontal } from '@components/Row'; +import { Card } from '@components/Card'; +import { Button } from '@components/Button'; +import { AtomicLink } from '@components/AtomicLink'; +import Field from '@components/forms/Field'; +import { Input, ErrMessage } from '@components/forms/InputStyles'; +import { + openDemo, + connectDemo, + resumeDemo, + syncDemo, + demoRows, + editAtomic, + editFixture, + type Demo, + type Row, +} from '../chunks/DevonianDemo/demo.mjs'; + +function DevonianDemo() { + const store = useStore(); + const [demo, setDemo] = useState(); + const [rows, setRows] = useState([]); + const [proxy, setProxy] = useState('https://localthought.io'); + const [repository, setRepository] = useState(''); + const [secret, setSecret] = useState(''); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(''); + const [status, setStatus] = useState(''); + const [text, setText] = useState(''); + const run = async (work: () => Promise) => { + setBusy(true); + setError(''); + try { + const current = await work(); + if (current) { + setDemo({ ...current }); + setRows(await demoRows(store, current)); + } + } catch (reason) { + setError(reason instanceof Error ? reason.message : String(reason)); + } finally { + setBusy(false); + } + }; + const resumed = useRef(false); + useEffect(() => { + if (resumed.current) return; + resumed.current = true; + void run(() => resumeDemo(store)); + }, [store]); + const start = (sample: boolean) => + run(async () => { + if (!sample) { + const credential = secret; + setSecret(''); + await connectDemo(store, { repository, proxy }, credential); + return; + } + const opened = await openDemo(store, { sample, repository, proxy }); + if (sample) await syncDemo(store, opened); + return opened; + }); + const sync = () => + run(async () => { + if (!demo) return; + const count = await syncDemo(store, demo); + setStatus(`Synchronized ${count} issues and comments.`); + return demo; + }); + const edit = ( + side: 'atomic' | 'fixture', + command: string, + id?: string | number, + ) => + run(async () => { + if (!demo) return; + if (side === 'atomic') + await editAtomic(store, demo, command, id as string, text); + else await editFixture(demo, command, id as number, text); + setText(''); + return demo; + }); + return ( +
+ + +

Devonian issue tracker demo

+

+ Sync issues and comments in both directions. Devonian runs in this + browser; the Atomic tracker is stored on this device. +

+ {!demo && ( + <> + +
+ Connect a real GitHub repository + +

+ Connect through LocalThought. The tenant secret is used in + this tab to start the connection and is never saved. +

+ + setProxy(e.target.value)} + /> + + + setRepository(e.target.value)} + /> + + + setSecret(e.target.value)} + /> + + + +
+ + )} + {demo && ( + <> +

+ {demo.state.options.sample + ? 'Sample mode: the GitHub side below is a browser fixture. No requests are sent to GitHub.' + : 'Live mode: Sync now creates and updates real GitHub issues and comments. Keep this tab open to sync.'} +

+ + + + Open Atomic kanban + + + + setText(e.target.value)} + /> + + + + {demo.state.options.sample && ( + + )} + +
+

Atomic tracker

+ + {rows.map(row => ( + + + + {row.value.title} + + {row.value.status} + + + + + {row.comments.map(comment => ( +

{comment.value.body}

+ ))} + + + ))} + +
+ {demo.state.options.sample && ( +
+

Sample GitHub tracker

+ + {demo.state.fixture.issues?.map(issue => ( + + + + #{issue.number} {issue.title} + + {issue.state} + + + + + {demo.state.fixture.comments + ?.filter(c => + c.issue_url.endsWith(`/${issue.number}`), + ) + .map(c => ( +

{c.body}

+ ))} + + + ))} + +
+ )} + + )} +

{status}

+ {error && {error}} + + +
+ ); +} +export const devonianDemoRouteLazy = createLazyRoute('/app/devonian-demo')({ + component: DevonianDemo, +}); diff --git a/browser/data-browser/src/routes/Router.tsx b/browser/data-browser/src/routes/Router.tsx index 5c79967682..176a64ffcb 100644 --- a/browser/data-browser/src/routes/Router.tsx +++ b/browser/data-browser/src/routes/Router.tsx @@ -42,6 +42,13 @@ const DemoRoute = createRoute({ path: pathNames.demo, }).lazy(() => import('./DemoRoute').then(mod => mod.demoRouteLazy)); +const DevonianDemoRoute = createRoute({ + getParentRoute: () => appRoute, + path: '/devonian-demo', +}).lazy(() => + import('./DevonianDemoRoute').then(mod => mod.devonianDemoRouteLazy), +); + const PruneTestsRoute = createRoute({ getParentRoute: () => appRoute, path: pathNames.pruneTests, @@ -91,6 +98,7 @@ const routeTree = rootRoute.addChildren({ SandboxRoute, DevDriveRoute, DemoRoute, + DevonianDemoRoute, InviteRoute, LinkOpenRouter, }), diff --git a/browser/e2e/tests/devonian-issue-sync.spec.mts b/browser/e2e/tests/devonian-issue-sync.spec.mts new file mode 100644 index 0000000000..b79b8e97fa --- /dev/null +++ b/browser/e2e/tests/devonian-issue-sync.spec.mts @@ -0,0 +1,182 @@ +import { test, expect } from '@playwright/test'; +import { + mockProxy, + tenantSecret, +} from '../../../integrations/localthought/mock-proxy.mjs'; +const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:6747'; +const SERVER_URL = process.env.SERVER_URL ?? 'http://localhost:9883'; + +test.use({ serviceWorkers: 'block' }); + +// This is a public fixture secret, never an environment/live credential. +// Exercise the real form, tenant challenge, consent, rotating HTTP transport and OPFS. +test('Devonian syncs issue creation, state and comments both ways through the browser proxy', async ({ + page, +}) => { + test.setTimeout(180_000); + const proxy = mockProxy({ frontendOrigin: new URL(FRONTEND_URL).origin }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const address = proxy.address() as { port: number }; + const proxyOrigin = `http://127.0.0.1:${address.port}`; + const repository = 'demo/two-way'; + const remote = proxy.github; + const initial = remote.createIssue(repository, { + title: 'Created on GitHub', + }); + remote.createComment(repository, initial.number, { + body: 'Initial GitHub comment', + }); + // No AtomicServer WebSocket writes (or Vite HMR reloads) during this journey. + await page.routeWebSocket('**/*', socket => socket.close()); + const forbidden: string[] = []; + const requests: string[] = []; + page.on('request', req => { + if (req.url().startsWith(`${proxyOrigin}/proxy/`)) + requests.push(req.method()); + }); + // Keep static app assets available even when frontend and AtomicServer share an origin. + await page.route('**/*', route => { + const url = new URL(route.request().url()); + if (/^\/(integration-proxy|plugin-run|commit)(\/|$)/.test(url.pathname)) { + forbidden.push(url.pathname); + return route.abort(); + } + if ( + url.origin === new URL(SERVER_URL).origin && + url.origin !== new URL(FRONTEND_URL).origin + ) + return route.abort(); + return route.continue(); + }); + try { + await page.goto(`${FRONTEND_URL}/app/dev-drive`); + await page.waitForURL(/app\/show\?subject=/, { timeout: 60000 }); + await page.goto(`${FRONTEND_URL}/app/devonian-demo`); + await page + .getByText('Connect a real GitHub repository', { exact: true }) + .click(); + await page.getByLabel('Integration proxy URL').fill(proxyOrigin); + await page.getByLabel('GitHub repository (owner/repo)').fill(repository); + await page.getByLabel('LocalThought tenant secret').fill(tenantSecret); + await page + .getByRole('button', { name: 'Connect GitHub tracker', exact: true }) + .click(); + await page + .getByRole('button', { name: 'Connect test account', exact: true }) + .click(); + const sync = async () => { + const button = page.getByRole('button', { + name: 'Sync now', + exact: true, + }); + await button.click(); + await expect(button).toBeEnabled(); + await expect(page.getByRole('alert')).toHaveCount(0); + }; + await expect( + page.getByRole('button', { name: 'Sync now', exact: true }), + ).toBeVisible(); + expect(page.url()).not.toContain('connection_code'); + expect( + await page.evaluate(() => + JSON.stringify({ ...localStorage, ...sessionStorage }), + ), + ).not.toContain(tenantSecret); + await sync(); + const issue = (title: string) => + page + .getByTestId('atomic-issue') + .filter({ has: page.getByRole('link', { name: title, exact: true }) }); + await expect(issue('Created on GitHub')).toContainText( + 'Initial GitHub comment', + ); + + const text = page.getByLabel('New issue title or comment'); + await text.fill('Created in Atomic'); + await page + .getByRole('button', { name: 'Create Atomic issue', exact: true }) + .click(); + await expect(issue('Created in Atomic')).toBeVisible(); + await sync(); + const created = remote + .snapshot(repository) + .issues.find(i => i.title === 'Created in Atomic'); + expect(created).toBeDefined(); + expect(remote.snapshot(repository).issues).toHaveLength(2); + + await text.fill('Comment from Atomic'); + await issue('Created in Atomic') + .getByRole('button', { name: 'Add Atomic comment', exact: true }) + .click(); + await issue('Created in Atomic') + .getByRole('button', { name: 'Close Atomic issue', exact: true }) + .click(); + await sync(); + expect( + remote.snapshot(repository).issues.find(i => i.number === created!.number) + ?.state, + ).toBe('closed'); + expect( + remote + .snapshot(repository) + .comments.filter(c => c.issue_url.endsWith(`/${created!.number}`)) + .map(c => c.body), + ).toEqual(['Comment from Atomic']); + + remote.updateIssue(repository, created!.number, { state: 'open' }); + remote.createComment(repository, created!.number, { + body: 'Reply from GitHub', + }); + remote.updateIssue(repository, initial.number, { state: 'closed' }); + await sync(); + await expect( + issue('Created in Atomic').getByRole('button', { + name: 'Close Atomic issue', + exact: true, + }), + ).toBeVisible(); + await expect(issue('Created in Atomic')).toContainText('Reply from GitHub'); + await expect( + issue('Created on GitHub').getByRole('button', { + name: 'Reopen Atomic issue', + exact: true, + }), + ).toBeVisible(); + await issue('Created on GitHub') + .getByRole('button', { name: 'Reopen Atomic issue', exact: true }) + .click(); + await sync(); + expect( + remote.snapshot(repository).issues.find(i => i.number === initial.number) + ?.state, + ).toBe('open'); + + const before = remote.snapshot(repository); + const subjects = await page + .getByTestId('atomic-issue') + .getByRole('link') + .evaluateAll(links => + links.map(link => link.getAttribute('href')).sort(), + ); + await page.reload(); + await sync(); + await expect(page.getByTestId('atomic-issue')).toHaveCount(2); + await expect(issue('Created in Atomic')).toContainText('Reply from GitHub'); + expect( + await page + .getByTestId('atomic-issue') + .getByRole('link') + .evaluateAll(links => + links.map(link => link.getAttribute('href')).sort(), + ), + ).toEqual(subjects); + expect(remote.snapshot(repository)).toEqual(before); + expect(requests).toEqual(expect.arrayContaining(['GET', 'POST', 'PATCH'])); + expect(forbidden).toEqual([]); + } finally { + proxy.closeAllConnections(); + await new Promise((resolve, reject) => + proxy.close(error => (error ? reject(error) : resolve())), + ); + } +}); diff --git a/browser/e2e/tests/google-calendar-import.spec.mts b/browser/e2e/tests/google-calendar-import.spec.mts new file mode 100644 index 0000000000..22d00c5e81 --- /dev/null +++ b/browser/e2e/tests/google-calendar-import.spec.mts @@ -0,0 +1,169 @@ +import { test, expect } from '@playwright/test'; +import { + mockProxy, + tenantSecret, +} from '../../../integrations/localthought/mock-proxy.mjs'; +const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:6747'; +const SERVER_URL = process.env.SERVER_URL ?? 'http://localhost:9883'; +test.use({ serviceWorkers: 'block' }); + +// Reuse the browser transport and HTTP mock from the Devonian integration. +// The tenant secret is a public fixture. No real provider credentials are used. +test('Calendar imports, refreshes and persists through the browser with AtomicServer unavailable', async ({ + page, +}) => { + test.setTimeout(180_000); + const proxy = mockProxy({ frontendOrigin: new URL(FRONTEND_URL).origin }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const port = (proxy.address() as { port: number }).port; + const proxyOrigin = `http://127.0.0.1:${port}`; + const configuredProxy = + process.env.VITE_INTEGRATION_PROXY_URL || 'https://localthought.io'; + await page.routeWebSocket('**/*', socket => socket.close()); + const forbidden: string[] = []; + const providerMethods: string[] = []; + // Forward the configured proxy to this test's isolated HTTP fixture. All + // tenant proof, consent, code rotation, pagination and WASM code remain real. + await page.route('**/*', async route => { + const request = route.request(); + const url = new URL(request.url()); + + if (/^\/(integration-proxy|plugin-run|commit)(\/|$)/.test(url.pathname)) { + forbidden.push(url.pathname); + + return route.abort(); + } + + if (url.origin === configuredProxy) { + if (url.pathname.startsWith('/proxy/')) + providerMethods.push(request.method()); + const response = await route.fetch({ + url: `${proxyOrigin}${url.pathname}${url.search}`, + maxRedirects: 0, + }); + + return route.fulfill({ response }); + } + + if ( + url.origin === new URL(SERVER_URL).origin && + url.origin !== new URL(FRONTEND_URL).origin + ) + return route.abort(); + + return route.continue(); + }); + + try { + await page.goto(`${FRONTEND_URL}/app/dev-drive`); + await page.waitForURL(/app\/show\?subject=/, { timeout: 60000 }); + + const setup = async () => { + await page + .getByRole('link', { name: 'Integrations', exact: true }) + .click(); + await page + .locator('[data-integration="google-calendar"]') + .getByRole('button', { name: 'Set up connection' }) + .click(); + }; + + await setup(); + await page.getByLabel('LocalThought tenant secret').fill(tenantSecret); + await page + .getByRole('button', { name: 'Install and connect', exact: true }) + .click(); + await page + .getByRole('button', { name: 'Connect test account', exact: true }) + .click(); + await expect( + page.getByRole('button', { name: 'Fetch and preview', exact: true }), + ).toBeVisible(); + expect(page.url()).not.toContain('connection_code'); + expect( + await page.evaluate(() => + JSON.stringify({ ...localStorage, ...sessionStorage }), + ), + ).not.toContain(tenantSecret); + + const apply = async (count: number) => { + await page + .getByRole('button', { name: 'Fetch and preview', exact: true }) + .click(); + await page + .getByRole('button', { name: new RegExp(`^Apply ${count} changes?$`) }) + .click(); + await page + .getByRole('link', { name: 'Open imported records', exact: true }) + .click(); + await expect(page.getByTestId('calendar-view')).toBeVisible(); + }; + + await apply(2); + await expect(page.getByTestId('calendar-event')).toHaveCount(2); + await page.getByText('Calendar timed fixture', { exact: true }).click(); + await expect(page.getByRole('dialog').last()).toContainText( + 'Recurring event', + ); + await expect(page.getByRole('dialog').last()).toContainText( + 'Attendees and RSVP', + ); + await page.keyboard.press('Escape'); + const original = await page.evaluate(async () => { + const store = window.store!; + const table = new URL(location.href).searchParams.get('subject')!; + const result = await store.queryLocalDb({ + drive: store.getDrive()!, + property: 'https://atomicdata.dev/properties/parent', + value: table, + limit: 100, + }); + const subjects = result!.subjects.sort(); + const row = await store.getResource(subjects[0]); + await row.set( + 'https://atomicdata.dev/properties/description', + 'Atomic-only notes', + ); + await row.save(); + await store.getClientDb()!.flush(); + + return { subjects, table, annotated: row.subject }; + }); + await page.reload(); + await expect(page.getByTestId('calendar-event')).toHaveCount(2); + proxy.calendar.events[1].summary = 'Calendar refreshed fixture'; + await setup(); + await apply(1); + await expect( + page.getByText('Calendar refreshed fixture', { exact: true }), + ).toBeVisible(); + await expect(page.getByTestId('calendar-event')).toHaveCount(2); + const restored = await page.evaluate(async saved => { + const store = window.store!; + const result = await store.queryLocalDb({ + drive: store.getDrive()!, + property: 'https://atomicdata.dev/properties/parent', + value: saved.table, + limit: 100, + }); + + return { + subjects: result!.subjects.sort(), + note: (await store.getResource(saved.annotated)).get( + 'https://atomicdata.dev/properties/description', + ), + }; + }, original); + expect(restored.subjects).toEqual(original.subjects); + expect(restored.note).toBe('Atomic-only notes'); + expect(providerMethods).toEqual(['GET', 'GET', 'GET', 'GET']); + expect( + proxy.calendar.requests.filter(r => r.query.pageToken === 'second'), + ).toHaveLength(2); + expect(forbidden).toEqual([]); + } finally { + await new Promise((resolve, reject) => + proxy.close(error => (error ? reject(error) : resolve())), + ); + } +}); diff --git a/browser/e2e/tests/plugins.spec.ts b/browser/e2e/tests/plugins.spec.ts index 1d4e1f0a6f..1c0f9b0392 100644 --- a/browser/e2e/tests/plugins.spec.ts +++ b/browser/e2e/tests/plugins.spec.ts @@ -28,10 +28,9 @@ test.describe('plugins', () => { !process.env.ATOMIC_MOCK_INTEGRATION_PROXY, 'Run with the documented mock integration-proxy server configuration', ); - await page.getByRole('link', { name: 'Integrations', exact: true }).click(); // CI's browser and server are in different containers. Forward the mock's - // loopback address to the server container; the real connect/rotation logic runs there. + // loopback address to the server container before catalog loading starts. if (process.env.ATOMIC_SERVICE_URL) await page.route('http://127.0.0.1:19090/**', async route => { const target = new URL(route.request().url()); @@ -42,6 +41,7 @@ test.describe('plugins', () => { }); await route.fulfill({ response }); }); + await page.getByRole('link', { name: 'Integrations', exact: true }).click(); const pets = page.locator('[data-integration=pets]'); await expect( pets.getByRole('heading', { name: 'Pets', exact: true }), @@ -52,6 +52,9 @@ test.describe('plugins', () => { await expect( setup.getByRole('button', { name: 'Install and connect', exact: true }), ).toBeVisible(); + await setup + .getByLabel('LocalThought tenant secret') + .fill('bW9jay10ZW5hbnQ.mock-signature'); await setup .getByRole('button', { name: 'Install and connect', exact: true }) .click(); @@ -61,23 +64,11 @@ test.describe('plugins', () => { ).toBeVisible(); await page.getByRole('button', { name: 'Connect test account' }).click(); await expect(page).not.toHaveURL(/connection_code=/); - const [preview] = await Promise.all([ - page.waitForResponse( - response => - response.url().endsWith('/plugin-run') && - response.request().method() === 'POST', - { timeout: 45_000 }, - ), - page.getByRole('button', { name: 'Fetch and preview' }).click(), - ]); - expect(preview.ok(), await preview.text()).toBe(true); + await page.getByRole('button', { name: 'Fetch and preview' }).click(); const review = page.locator('dialog[open]'); - // Installing walks pluginClassesFor, two ensureSchema calls, three - // ensureInstallationResource calls and the actual sandboxed plugin run — - // measured ~24s locally even on a fresh, otherwise-idle server. The - // default 10s expect timeout is tuned for interaction latency, not this - // one-time setup cost. + // The browser creates the local ontology, tables and reviewed proposal. + // Allow the one-time installation more than the interaction timeout. await expect( review.getByRole('button', { name: 'Apply 5 changes', exact: true }), ).toBeEnabled({ timeout: 45_000 }); diff --git a/integrations/github-issues/README.md b/integrations/github-issues/README.md index f265fa5f0c..e0f80d1254 100644 --- a/integrations/github-issues/README.md +++ b/integrations/github-issues/README.md @@ -1,3 +1,10 @@ +## Browser-only Devonian demo + +Open `/app/devonian-demo` for a browser-side issue/comment sync demo using +Devonian and this integration's mappings. It uses a local-only Atomic drive and +direct integration-proxy requests, with an explicit sample mode. See +[setup, source and the current live proxy CORS limitation](devonian/README.md). + ## Connect in the app Open **Integrations**, enter `owner/repository` and a repository-scoped GitHub diff --git a/integrations/github-issues/adapter.test.ts b/integrations/github-issues/adapter.test.ts index e60052a966..854422b60d 100644 --- a/integrations/github-issues/adapter.test.ts +++ b/integrations/github-issues/adapter.test.ts @@ -3,6 +3,7 @@ import { preview, project, manifest, type Issue } from './adapter.js'; import { validateManifest } from '../../browser/lib/src/plugin-manifest.js'; import { readFile } from 'node:fs/promises'; import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; const issue = (number: number): Issue => ({ number, title: `Issue ${number}`, @@ -18,7 +19,9 @@ describe('GitHub package', () => { }); it('ships exactly the artifact tested by the sandbox', async () => { const built = execFileSync( - './browser/node_modules/.bin/esbuild', + existsSync('./browser/node_modules/.bin/esbuild') + ? './browser/node_modules/.bin/esbuild' + : './browser/node_modules/.pnpm/node_modules/.bin/esbuild', [ 'integrations/github-issues/plugin.ts', '--bundle', diff --git a/integrations/github-issues/devonian/README.md b/integrations/github-issues/devonian/README.md new file mode 100644 index 0000000000..56c895d925 --- /dev/null +++ b/integrations/github-issues/devonian/README.md @@ -0,0 +1,129 @@ +# Browser issue tracker demo + +Open `/app/devonian-demo` in the Atomic web app. Choose **Try sample data**, +create issues or comments on either side, close/reopen an issue, and press +**Sync now**. **Open Atomic kanban** opens the real native tracker, whose issue +pages also have the normal Comments panel. Reload and choose the same mode to +resume saved resources and mappings without importing duplicates. + +The JavaScript runs in the browser. Devonian's native resource lenses run there; +Atomic resources live in a local-only drive in the browser's WASM/OPFS database. +The intermediate graph, scoped identity mappings, baselines and request journal +live in IndexedDB. No Node runtime, custom AtomicServer endpoint, server-side +plugin executor, tenant secret on AtomicServer or server worker is used. Enable +the browser database if it has been disabled. + +## Live GitHub + +Expand **Connect a real GitHub repository**, enter the integration-proxy origin, +`owner/repo`, and your **LocalThought tenant secret**, then choose **Connect GitHub +tracker**. The demo uses #1401's shared `BrowserIntegrations` client to sign the +tenant challenge in the browser, navigate to proxy consent, and bind the return +to this agent and local drive. The secret is cleared from the form and never +persisted. The callback code is removed from the address bar immediately. + +After consent, **Sync now** authorizes two-way writes. The shared transport +serializes requests with Web Locks, consumes each code before dispatch, and +persists rotated credentials in browser localStorage, outside the Atomic graph. +Reloading the tab resumes the local tracker and connection. Use a dedicated +demo repository; writes use the connected GitHub account. + +The browser calls `/proxy/github-issues/repos/{owner}/{repo}/issues...` directly. +The proxy instance must answer unauthenticated OPTIONS preflights, allow the +app's origin and GET/POST/PATCH/DELETE with Authorization and Content-Type, +expose `X-Connection-Code`, preserve query parameters, and allow issue/comment +and label operations in its catalog. CORS headers must cover error responses too. + +**Live verification (2026-09-09):** Heroku release v40 (`bc02f13f`) now +answers browser preflights and exposes `X-Connection-Code`. Tenant challenge, +GitHub OAuth and return to the browser demo succeeded with AtomicServer +unavailable. The first authenticated issue-list request for the private +`ontola/atomic-github-sync-sandbox` returned 404, although the independent +GitHub CLI credential can access it. Repository access for the proxy credential +must be resolved before live two-way writes can be verified. The disposable +CLI-created issue #17 was closed; no issue/comment writes occurred through the proxy. + +## Mapping + +`GitHubPort` reuses `github-issues/adapter.ts`'s projection and request builder. +`tracker-actions.ts` adds scoped issue/comment request construction for the +browser transport. The installed server plugin's actions remain unchanged. + +- Title/body map to the native row's name and Markdown description. +- Open maps to Todo; open with `atomic:doing` maps to Doing; closed maps to Done. + Other labels are preserved. Provision `atomic:doing` before using Doing. +- A comment is a native Message whose `about` points to its issue and whose + `parent` is the drive's Comments folder. Body edits synchronize both ways. +- GitHub identity, author and original timestamps are retained in a separate + provenance property. The importing Atomic agent is distinct from the original + author. GitHub writes use the connected account's authorship. +- The demo creates drive-local task properties so setup works offline. The + intermediate Devonian graph uses the shared task vocabulary and maps native + properties explicitly. Atomic DIDs are scoped external IDs because Devonian's + current native subjects must be HTTP(S) URLs. +- Explicit issue numbers can bind existing rows; matching text never does. + Comments have their own scoped IDs; identical comments remain distinct. + +## Persistence and limits + +A Web Lock serializes syncs. Three-way reconciliation preserves independent +field edits and stops on same-field conflicts. Reloads resume saved operations +before discovering new records. Ingest never automatically publishes an echo. +Missing records are conflicts; neither side is deleted. + +The proxy does not provide idempotent GitHub creation. Writes are journaled +before sending. Successful receipts can be replayed; an uncertain/lost response +stops without resending. An operator must inspect and reconcile that operation; +there is no recovery wizard yet. Do not clear IndexedDB to retry a create. +Concurrent edits during a multi-request status transition may also require +reconciliation. Pre-write reads and verification do not make local/provider +writes one transaction. Sync is explicit while the tab is open, with a +10,000-record scan cap. Clearing browser site data loses the demo and its state. + +## Source and verification + +- `browser/data-browser/src/chunks/DevonianDemo/demo.mjs`: browser entry script. +- `bridge.mjs`: Devonian lenses and checkpointed reconciliation. +- `ports.mjs`: native Atomic and GitHub projections/transports. +- `proxy.mjs`: rotating-code transport and labelled sample fixture. +- `browser/data-browser/src/routes/DevonianDemoRoute.tsx`: demo controls. + +The committed `DevonianDemo/devonian.js` bundle contains only the native resource +API from Devonian main `e11104f78ebd151a361171ca0b21489b25e1e2c8`, with its +Apache-2.0 license alongside. Regenerate at development time: + +```sh +DEVONIAN_PATH=/path/to/devonian node integrations/github-issues/devonian/build.mjs +browser/node_modules/.bin/vitest run --config integrations/github-issues/devonian/vitest.config.mjs +browser/node_modules/.bin/vitest run --config integrations/github-issues/vitest.config.ts +cd browser/data-browser && pnpm typecheck +``` + +Tests use real Devonian lenses and deterministic connectors: creation, identical +content with distinct identities, independent edits, close/reopen, comments, +conflicts, missing resources, replay after lost receipts, label preservation, +pagination, scoped URLs and serialized code rotation. The browser sample flow +was manually verified with native OPFS resources, issue creation on both sides, +comments both ways, closing from Atomic and reopening from the GitHub fixture, +then reloading and resuming the same three issues and two comments without duplication. + +## Playwright two-way regression + +`browser/e2e/tests/devonian-issue-sync.spec.mts` starts an isolated HTTP integration +proxy with stateful, repository-scoped GitHub issue/comment endpoints. It fills +the tenant-secret form with the **public mock secret** (no real credentials), +completes consent, and exercises the live transport mode rather than sample mode. +It verifies creation and comments in both directions, close/reopen in both +directions, matching parent issues, and reload without duplicate resources or +provider writes. Legacy server integration endpoints, commit POSTs and all WebSockets are blocked. +The local run uses an unavailable AtomicServer port to verify browser-only storage. + +With Vite and the built browser/WASM packages available: + +```sh +cd browser/e2e +FRONTEND_URL=http://localhost:6747 playwright test tests/devonian-issue-sync.spec.mts --project chromium +``` + +The test is in the full E2E suite, without a smoke tag. The mock is local test +infrastructure only; the runtime demo uses the real integration-proxy protocol. diff --git a/integrations/github-issues/devonian/bridge.mjs b/integrations/github-issues/devonian/bridge.mjs new file mode 100644 index 0000000000..3f90390a4d --- /dev/null +++ b/integrations/github-issues/devonian/bridge.mjs @@ -0,0 +1,243 @@ +import { reconcileRecord } from '../../../browser/lib/src/plugin-reconcile.js'; + +const p = { + title: 'https://atomicdata.dev/properties/name', + body: 'https://atomicdata.dev/task/v1/body', + status: 'https://atomicdata.dev/task/v1/status', +}; +const tag = 'https://atomicdata.dev/task/v1/'; +const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b); +const copy = value => structuredClone(value); + +/** Single-writer, checkpointed reconciliation. Ports own transport and durable writes. */ +export class Bridge { + constructor({ devonian, local, remote, base, snapshot, save }) { + this.api = devonian; + this.local = local; + this.remote = remote; + this.save = save; + const { AtomicSchema, AtomicStore, AtomicIdentityMap, Datatype } = devonian; + this.store = new AtomicStore( + new AtomicSchema() + .property(p.title, Datatype.STRING) + .property(p.body, Datatype.MARKDOWN) + .property(p.status, Datatype.RESOURCEARRAY), + ); + this.identities = new AtomicIdentityMap(this.store, base); + this.binding = { base, local: local.scope, remote: remote.scope }; + if (snapshot && !equal(snapshot.binding, this.binding)) + throw new Error('State belongs to another connection'); + if (snapshot?.graph) this.store.loadJSONAD(snapshot.graph); + this.records = copy(snapshot?.records ?? {}); + } + + scope(side, entity) { + return { scope: this[side].scope, entity }; + } + id(side, entity, subject) { + return this.identities.externalId(this.scope(side, entity), subject); + } + context(side, entity) { + if (entity === 'issue') return {}; + const parent = entity.slice('comment:'.length); + const issueId = this.id(side, 'issue', parent); + if (issueId === undefined) + throw new Error('Issue must be mapped before comments'); + return { issueId }; + } + async checkpoint() { + await this.save({ + version: 1, + binding: this.binding, + graph: this.store.toJSONAD(), + records: copy(this.records), + }); + } + properties(value) { + return { + [p.body]: value.body, + ...(value.title === undefined + ? {} + : { + [p.title]: value.title, + [p.status]: [`${tag}${value.status.toLowerCase()}`], + }), + }; + } + value(resource, entity) { + if (entity !== 'issue') return { body: resource[p.body] }; + const status = { + [`${tag}todo`]: 'Todo', + [`${tag}doing`]: 'Doing', + [`${tag}done`]: 'Done', + }[resource[p.status]?.[0]]; + if (!status || resource[p.status].length !== 1) + throw new Error('Unsupported task status'); + return { title: resource[p.title], body: resource[p.body], status }; + } + lens(side, entity, operation, metadata) { + const port = this[side], + context = this.context(side, entity); + return new this.api.AtomicLens({ + store: this.store, + identities: this.identities, + ...this.scope(side, entity), + connector: { + id: row => row.id, + get: id => port.get(entity, id, context), + create: (row, key) => + port.create(entity, row.value, key, metadata, context), + update: (id, row) => + port.update( + entity, + id, + row.value, + `${operation}:${side}`, + metadata, + context, + ), + delete: async () => { + throw new Error('Deletion is outside issue sync'); + }, + }, + read: row => ({ set: this.properties(row.value) }), + write: (resource, previous) => ({ + ...previous, + value: this.value(resource, entity), + }), + }); + } + + async sync() { + // Resume saved operations BEFORE discovering their newly-created counterparts. + for (const [subject, record] of Object.entries(this.records)) { + if (record.pending) await this.finish(subject, record); + } + await this.syncEntity('issue'); + for (const [subject, record] of Object.entries(this.records)) { + if (record.entity === 'issue') + await this.syncEntity(`comment:${subject}`); + } + } + + async syncEntity(entity) { + const lists = {}; + for (const side of ['remote', 'local']) { + const rows = await this[side].list(entity, this.context(side, entity)); + lists[side] = new Map(); + for (const row of rows) { + if (lists[side].has(row.id)) + throw new Error('Duplicate external identity'); + lists[side].set(row.id, row); + } + } + // An existing pilot's explicit issue-number column is an identity, never a title match. + for (const row of lists.local.values()) { + if (row.remoteId === undefined) continue; + const scope = this.scope('remote', entity); + const subject = this.identities.subjectFor(scope, row.remoteId); + this.identities.bind(scope, row.remoteId, subject); + this.identities.bind(this.scope('local', entity), row.id, subject); + this.records[subject] ??= { entity }; + } + for (const side of ['remote', 'local']) { + for (const row of lists[side].values()) { + let subject = this.identities.lookup(this.scope(side, entity), row.id); + if (!subject) subject = await this.lens(side, entity).ingest(row); + this.records[subject] ??= { entity }; + } + } + await this.checkpoint(); + for (const [subject, record] of Object.entries(this.records)) { + if (record.entity !== entity) continue; + const rows = {}; + for (const side of ['local', 'remote']) { + const id = this.id(side, entity, subject); + rows[side] = id === undefined ? undefined : lists[side].get(id); + if (id !== undefined && !rows[side]) + throw new Error(`Missing ${side} record: ${subject}`); + } + const decision = reconcileRecord( + record.baseline, + rows.local?.value, + rows.remote?.value, + ); + if (decision.conflicts.length) + throw new Error( + `Conflict on ${subject}: ${decision.conflicts.map(c => c.property).join(', ')}`, + ); + const desired = { + ...(rows.remote?.value ?? rows.local?.value), + ...decision.remote, + }; + const metadata = rows.remote?.metadata; + if ( + rows.local && + rows.remote && + equal(rows.local.value, rows.remote.value) && + equal(rows.local.metadata, metadata) + ) { + record.baseline = copy(desired); + await this.checkpoint(); + continue; + } + record.pending = { + operation: crypto.randomUUID(), + local: rows.local?.value, + remote: rows.remote?.value, + desired, + metadata, + }; + this.store.patch(subject, { set: this.properties(desired) }); + await this.checkpoint(); + await this.finish(subject, record); + } + } + + async finish(subject, record) { + const { pending, entity } = record; + // A retry accepts only the original observation or this operation's exact result. + for (const side of ['local', 'remote']) { + const id = this.id(side, entity, subject); + if (id === undefined) continue; + const row = await this[side].get(entity, id, this.context(side, entity)); + if ( + !equal(row.value, pending[side]) && + !equal(row.value, pending.desired) + ) + throw new Error(`Conflict during saved operation on ${subject}`); + } + for (const side of ['remote', 'local']) { + const id = this.id(side, entity, subject); + const row = + id === undefined + ? undefined + : await this[side].get(entity, id, this.context(side, entity)); + if ( + !row || + !equal(row.value, pending.desired) || + (side === 'local' && !equal(row.metadata, pending.metadata)) + ) { + await this.lens( + side, + entity, + pending.operation, + pending.metadata, + ).publish(subject); + await this.checkpoint(); + } + } + for (const side of ['local', 'remote']) { + const row = await this[side].get( + entity, + this.id(side, entity, subject), + this.context(side, entity), + ); + if (!equal(row.value, pending.desired)) + throw new Error(`Concurrent edit after write on ${subject}`); + } + record.baseline = copy(pending.desired); + delete record.pending; + await this.checkpoint(); + } +} diff --git a/integrations/github-issues/devonian/bridge.test.mjs b/integrations/github-issues/devonian/bridge.test.mjs new file mode 100644 index 0000000000..672a7b1030 --- /dev/null +++ b/integrations/github-issues/devonian/bridge.test.mjs @@ -0,0 +1,133 @@ +import { expect, it } from 'vitest'; +import * as devonian from 'devonian'; +import { Bridge } from './bridge.mjs'; + +function fixture(snapshot) { + let saved = snapshot; + const makePort = scope => ({ + scope, + rows: new Map(), + receipts: new Map(), + writes: 0, + lose: false, + async list(entity) { + return [...this.rows.values()].filter(r => r.entity === entity); + }, + async get(entity, id) { + const row = this.rows.get(id); + if (!row || row.entity !== entity) throw new Error('Missing record'); + return structuredClone(row); + }, + async create(entity, value, key, metadata) { + if (this.receipts.has(key)) return this.receipts.get(key); + const id = this.rows.size + 1; + const row = { id, entity, value: structuredClone(value), metadata }; + this.rows.set(id, row); + this.receipts.set(key, row); + this.writes++; + if (this.lose) { + this.lose = false; + throw new Error('Lost response'); + } + return row; + }, + async update(entity, id, value) { + const row = await this.get(entity, id); + this.rows.set(id, { ...row, value: structuredClone(value) }); + this.writes++; + }, + }); + const local = makePort('https://atomic.example/bridge'); + const remote = makePort('https://github.com/acme/repo'); + const open = () => + new Bridge({ + devonian, + local, + remote, + snapshot: saved, + base: 'https://bridge.example/sync', + save: async s => { + saved = structuredClone(s); + }, + }); + return { local, remote, open, saved: () => saved }; +} +const issue = (id, title = 'Same title') => ({ + id, + entity: 'issue', + value: { title, body: '', status: 'Todo' }, +}); + +it('syncs creation both ways without deduplicating equal content, then no-ops after restart', async () => { + const f = fixture(); + f.local.rows.set(1, issue(1)); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + expect(f.local.rows.size).toBe(2); + expect(f.remote.rows.size).toBe(2); + const writes = f.local.writes + f.remote.writes; + await f.open().sync(); + expect(f.local.writes + f.remote.writes).toBe(writes); +}); + +it('merges independent title/body edits and propagates close/reopen', async () => { + const f = fixture(); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + f.local.rows.get(1).value.title = 'Local title'; + f.remote.rows.get(1).value.body = 'Remote body'; + f.local.rows.get(1).value.status = 'Done'; + await f.open().sync(); + expect(f.remote.rows.get(1).value).toEqual({ + title: 'Local title', + body: 'Remote body', + status: 'Done', + }); + f.remote.rows.get(1).value.status = 'Todo'; + await f.open().sync(); + expect(f.local.rows.get(1).value.status).toBe('Todo'); +}); + +it('syncs comments and edits both ways, preserving source metadata', async () => { + const f = fixture(); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + const parent = Object.keys(f.saved().records)[0]; + const entity = `comment:${parent}`; + f.remote.rows.set(2, { + id: 2, + entity, + value: { body: 'GitHub comment' }, + metadata: { author: 'octocat' }, + }); + f.local.rows.set(2, { id: 2, entity, value: { body: 'Atomic comment' } }); + await f.open().sync(); + expect(f.local.rows.get(3).metadata).toEqual({ author: 'octocat' }); + expect(f.remote.rows.get(3).value.body).toBe('Atomic comment'); + f.local.rows.get(3).value.body = 'Edited'; + await f.open().sync(); + expect(f.remote.rows.get(2).value.body).toBe('Edited'); +}); + +it('stops on same-field conflicts and missing records without deleting either side', async () => { + const f = fixture(); + f.remote.rows.set(1, issue(1)); + await f.open().sync(); + f.local.rows.get(1).value.title = 'A'; + f.remote.rows.get(1).value.title = 'B'; + await expect(f.open().sync()).rejects.toThrow('Conflict'); + expect(f.local.rows.get(1).value.title).toBe('A'); + f.remote.rows.delete(1); + await expect(f.open().sync()).rejects.toThrow('Missing'); + expect(f.local.rows.size).toBe(1); +}); + +it('reuses the same create identity after a lost receipt and restart', async () => { + const f = fixture(); + f.local.rows.set(1, issue(1)); + f.remote.lose = true; + await expect(f.open().sync()).rejects.toThrow('Lost response'); + await f.open().sync(); + expect(f.remote.rows.size).toBe(1); + expect(f.remote.writes).toBe(1); +}); diff --git a/integrations/github-issues/devonian/build.mjs b/integrations/github-issues/devonian/build.mjs new file mode 100644 index 0000000000..9a6c4c2051 --- /dev/null +++ b/integrations/github-issues/devonian/build.mjs @@ -0,0 +1,36 @@ +/** Development-time bundle; the demo itself runs entirely in the browser. */ +import { createRequire } from 'node:module'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const require = createRequire(import.meta.url); +const { build } = require( + require.resolve('esbuild', { + paths: [ + resolve(root, 'browser'), + resolve(root, 'browser/node_modules/.pnpm'), + ], + }), +); +if (!process.env.DEVONIAN_PATH) + throw new Error('Set DEVONIAN_PATH to Devonian main'); +await build({ + stdin: { + contents: ['Resource', 'Store', 'IdentityMap', 'Lens'] + .map(name => `export * from './src/atomic/${name}.js';`) + .join('\n'), + resolveDir: resolve(process.env.DEVONIAN_PATH), + loader: 'ts', + }, + bundle: true, + platform: 'browser', + format: 'esm', + external: ['@tomic/lib'], + banner: { + js: '// Devonian native resource API, e11104f78ebd151a361171ca0b21489b25e1e2c8. Apache-2.0; see DEVONIAN-LICENSE.', + }, + outfile: resolve( + root, + 'browser/data-browser/src/chunks/DevonianDemo/devonian.js', + ), +}); diff --git a/integrations/github-issues/devonian/ports.mjs b/integrations/github-issues/devonian/ports.mjs new file mode 100644 index 0000000000..ca5733d489 --- /dev/null +++ b/integrations/github-issues/devonian/ports.mjs @@ -0,0 +1,322 @@ +import { project } from '../adapter.js'; +import { core, dataBrowser } from '../../../browser/lib/src/index.js'; + +export const digest = async value => + Array.from( + new Uint8Array( + await crypto.subtle.digest('SHA-256', new TextEncoder().encode(value)), + ), + byte => byte.toString(16).padStart(2, '0'), + ).join(''); +const equal = (a, b) => JSON.stringify(a) === JSON.stringify(b); + +/** Reuses the GitHub integration mapping with an injected browser proxy transport. */ +export class GitHubPort { + constructor(store, connection, call) { + this.store = store; + this.connection = connection; + this.scope = `https://github.com/${connection.repository}`; + if (!call) + throw new Error('A browser integration-proxy transport is required'); + this.call = call; + } + async request(action, args, key) { + const receipt = await this.call( + action, + args, + await digest(key ?? `${action}:${JSON.stringify(args)}`), + ); + if (receipt.status < 200 || receipt.status >= 300) + throw new Error(`GitHub ${action} returned ${receipt.status}`); + return receipt.body ? JSON.parse(receipt.body) : undefined; + } + row(entity, raw, context = {}) { + if (entity === 'issue') { + if (raw.pull_request) throw new Error('Pull requests are excluded'); + return { + id: raw.number, + value: project(raw), + metadata: this.metadata(raw, { number: raw.number }), + }; + } + if ( + !Number.isSafeInteger(raw.id) || + raw.id <= 0 || + typeof raw.body !== 'string' + ) + throw new Error('Invalid GitHub comment'); + const expected = `https://api.github.com/repos/${this.connection.repository}/issues/${context.issueId}`; + if (raw.issue_url !== expected) + throw new Error('Comment belongs to another issue'); + return { + id: raw.id, + value: { body: raw.body }, + metadata: this.metadata(raw, { commentId: raw.id }), + }; + } + metadata(raw, identity) { + return { + ...identity, + ...(raw.user?.login ? { author: raw.user.login } : {}), + ...(raw.html_url ? { url: raw.html_url } : {}), + ...(raw.created_at ? { createdAt: raw.created_at } : {}), + ...(raw.updated_at ? { updatedAt: raw.updated_at } : {}), + }; + } + async list(entity, context = {}) { + const result = []; + for (let page = 1; page <= 100; page++) { + const raw = await this.request( + entity === 'issue' ? 'list_issues' : 'list_comments', + { + page, + ...(entity === 'issue' ? {} : { number: context.issueId }), + }, + ); + if (!Array.isArray(raw)) throw new Error('Invalid GitHub page'); + for (const r of raw) { + if (entity === 'issue' && 'pull_request' in r) continue; + result.push(this.row(entity, r, context)); + } + if (raw.length < 100) return result; + } + throw new Error('More than 10,000 records in one scan'); + } + async get(entity, id, context) { + const raw = await this.request( + entity === 'issue' ? 'get_issue' : 'get_comment', + entity === 'issue' ? { number: id } : { id }, + ); + const row = this.row(entity, raw, context); + if (row.id !== id) throw new Error('Provider returned another identity'); + return row; + } + async create(entity, value, key, metadata, context) { + const raw = await this.request( + entity === 'issue' ? 'create_issue' : 'create_comment', + entity === 'issue' + ? { title: value.title, body: value.body } + : { number: context.issueId, body: value.body }, + `${key}:create`, + ); + const row = this.row(entity, raw, context); + if (entity === 'issue' && value.status !== 'Todo') + await this.update( + entity, + row.id, + value, + `${key}:initial-state`, + metadata, + context, + ); + return { ...row, value }; + } + async update(entity, id, value, key, _metadata, context) { + if (entity !== 'issue') { + await this.get(entity, id, context); // validate issue membership before writing + await this.request( + 'update_comment', + { id, body: value.body }, + `${key}:body`, + ); + return; + } + if (!['Todo', 'Doing', 'Done'].includes(value.status)) + throw new Error('Unsupported task status'); + const current = await this.get(entity, id, context); + await this.request( + 'update_issue', + { + number: id, + title: value.title, + body: value.body, + state: value.status === 'Done' ? 'closed' : 'open', + }, + `${key}:fields`, + ); + // Only this workflow label is managed. Closing preserves the label like the pilot. + if (value.status === 'Doing' && current.value.status !== 'Doing') { + await this.request('add_doing_label', { number: id }, `${key}:doing`); + } else if (value.status === 'Todo') { + // Closed issues may still carry the doing label, so inspect raw labels. + const raw = await this.request('get_issue', { number: id }); + if ( + raw.labels.some( + l => + (typeof l === 'string' ? l : l.name).toLowerCase() === + 'atomic:doing', + ) + ) { + await this.request('remove_doing_label', { number: id }, `${key}:todo`); + } + } + } +} + +/** Signed SDK writes build on the fetched Loro state, preserving unmanaged fields. */ +export class AtomicPort { + constructor(store, config) { + this.store = store; + this.config = config; + this.connection = config.connection; + this.scope = `https://atomicdata.dev/devonian-local/${encodeURIComponent(this.connection.table)}`; + } + async subjects(property, value) { + const result = await this.store.queryLocalDb({ + drive: this.connection.drive, + property, + value, + limit: 10001, + }); + if ( + !result || + result.count > 10000 || + result.subjects.length !== result.count + ) + throw new Error('Incomplete local database query'); + return result.subjects; + } + async findByLocalId(parent, localId) { + const rows = await Promise.all( + (await this.subjects(core.properties.localId, localId)).map(id => + this.resource(id), + ), + ); + const matches = rows.filter(r => r.get(core.properties.parent) === parent); + if (matches.length > 1) + throw new Error('Duplicate Atomic creation identity'); + return matches[0]; + } + async resource(subject) { + const r = await this.store.getResource(subject); + if (r.error) throw r.error; + r.getLoroDoc(); + return r; + } + row(entity, r, context = {}) { + const c = this.connection; + const classes = r.get(core.properties.isA) ?? []; + if (entity === 'issue') { + if ( + !classes.includes(c.rowClass) || + r.get(core.properties.parent) !== c.table + ) + throw new Error('Resource is not a row of this issue tracker'); + } else if ( + !classes.includes(dataBrowser.classes.message) || + r.get(dataBrowser.properties.about) !== context.issueId + ) + throw new Error('Message belongs to another issue'); + let metadata = r.get(this.config.provenance); + if (typeof metadata === 'string') metadata = JSON.parse(metadata); + if (entity !== 'issue') + return { + id: r.subject, + value: { body: r.get(core.properties.description) ?? '' }, + ...(metadata ? { metadata } : {}), + }; + const statuses = r.get(c.status) ?? [c.tags.Todo]; + const status = Object.keys(c.tags).find(s => c.tags[s] === statuses[0]); + if (statuses.length !== 1 || !status) + throw new Error( + 'Choose exactly one Todo/Doing/Done status; Blocked is unmapped', + ); + const title = r.get(core.properties.name), + body = r.get(c.body) ?? ''; + if (typeof title !== 'string' || !title.trim() || typeof body !== 'string') + throw new Error('Invalid Atomic issue'); + const remoteId = r.get(c.number); + if ( + remoteId !== undefined && + (!Number.isSafeInteger(remoteId) || remoteId <= 0) + ) + throw new Error('Invalid GitHub issue number'); + return { + id: r.subject, + remoteId, + value: { title, body, status }, + ...(metadata ? { metadata } : {}), + }; + } + async list(entity, context = {}) { + const ids = await this.subjects( + entity === 'issue' + ? core.properties.parent + : dataBrowser.properties.about, + entity === 'issue' ? this.connection.table : context.issueId, + ); + const rows = []; + for (const id of ids) { + const resource = await this.resource(id); + if ( + !(resource.get(core.properties.isA) ?? []).includes( + entity === 'issue' + ? this.connection.rowClass + : dataBrowser.classes.message, + ) + ) + continue; + rows.push(this.row(entity, resource, context)); + } + return rows; + } + async get(entity, id, context) { + return this.row(entity, await this.resource(id), context); + } + values(entity, value, metadata, context) { + const c = this.connection; + return { + ...(entity === 'issue' + ? { + [core.properties.name]: value.title, + [c.body]: value.body, + [c.status]: [c.tags[value.status]], + ...(metadata?.number ? { [c.number]: metadata.number } : {}), + } + : { + [core.properties.description]: value.body, + [dataBrowser.properties.about]: context.issueId, + }), + ...(metadata ? { [this.config.provenance]: metadata } : {}), + }; + } + async create(entity, value, key, metadata, context) { + const parent = + entity === 'issue' ? this.connection.table : this.config.commentsFolder; + const localId = `devonian:${await digest(key)}`; + const existing = await this.findByLocalId(parent, localId); + if (existing) { + const row = await this.get(entity, existing.subject, context); + if (!equal(row.value, value)) + throw new Error( + 'Recovered Atomic create was edited; reconcile before retry', + ); + return row; + } + const r = await this.store.newResource({ + parent, + isA: [ + entity === 'issue' + ? this.connection.rowClass + : dataBrowser.classes.message, + ], + propVals: { + ...this.values(entity, value, metadata, context), + [core.properties.localId]: localId, + }, + }); + if ((await r.save()) === 'offline') + throw new Error('AtomicServer disconnected'); + return this.get(entity, r.subject, context); + } + async update(entity, id, value, _key, metadata, context) { + const r = await this.resource(id); + this.row(entity, r, context); + for (const [p, v] of Object.entries( + this.values(entity, value, metadata, context), + )) + await r.set(p, v); + if ((await r.save()) === 'offline') + throw new Error('AtomicServer disconnected'); + } +} diff --git a/integrations/github-issues/devonian/ports.test.mjs b/integrations/github-issues/devonian/ports.test.mjs new file mode 100644 index 0000000000..25590e5ec6 --- /dev/null +++ b/integrations/github-issues/devonian/ports.test.mjs @@ -0,0 +1,237 @@ +import { expect, it } from 'vitest'; +import { GitHubPort, AtomicPort } from './ports.mjs'; +import { + trackerAction, + trackerActions, + trackerOperations, +} from '../tracker-actions.js'; +import { manifest } from '../adapter.js'; +import { validateManifest } from '../../../browser/lib/src/plugin-manifest.js'; + +it('declares and prepares scoped issue/comment actions with validated arguments', () => { + const original = manifest('owner/repo'); + const m = validateManifest({ + ...original, + actions: [...original.actions, ...trackerActions], + operations: [ + ...original.operations, + ...trackerOperations('https://api.github.com/repos/owner/repo/issues'), + ], + }); + for (const [action, args] of [ + ['list_issues', { page: 1 }], + ['list_comments', { number: 5, page: 2 }], + ['create_comment', { number: 5, body: 'Hello' }], + ['get_comment', { id: 9 }], + ['update_comment', { id: 9, body: 'Edit' }], + ['update_issue', { number: 5, title: 'Title', body: '', state: 'closed' }], + ['add_doing_label', { number: 5 }], + ['remove_doing_label', { number: 5 }], + ]) { + const intent = trackerAction('owner/repo', action, args); + expect( + intent.url.startsWith('https://api.github.com/repos/owner/repo/issues'), + ).toBe(true); + expect(m.actions.find(a => a.name === action).operation).toBe( + intent.operation, + ); + expect( + m.operations.some( + o => o.id === intent.operation && o.method === intent.method, + ), + ).toBe(true); + } + expect(() => + trackerAction('owner/repo', 'get_comment', { id: '../escape' }), + ).toThrow(); + expect(() => + trackerAction('owner/repo', 'list_issues', { page: 101 }), + ).toThrow(); + expect(() => + trackerAction('owner/repo', 'create_comment', { number: 1, body: '' }), + ).toThrow(); +}); + +function github() { + const issues = new Map([ + [ + 1, + { number: 1, title: 'First', body: '', state: 'open', labels: ['bug'] }, + ], + ]); + const comments = new Map(); + const receipts = new Map(); + const writes = []; + const port = new GitHubPort( + null, + { repository: 'owner/repo' }, + async (action, args, id) => { + const isWrite = /^(create|update|add|remove)_/.test(action); + if (isWrite && receipts.has(id)) return receipts.get(id); + let value; + if (isWrite) writes.push(action); + switch (action) { + case 'get_issue': + value = issues.get(args.number); + break; + case 'list_issues': + value = [...issues.values()]; + break; + case 'create_issue': + value = { + number: issues.size + 1, + ...args, + state: 'open', + labels: [], + }; + issues.set(value.number, value); + break; + case 'update_issue': + value = issues.get(args.number); + Object.assign(value, args); + break; + case 'add_doing_label': + value = issues.get(args.number); + value.labels.push('atomic:doing'); + break; + case 'remove_doing_label': + value = issues.get(args.number); + value.labels = value.labels.filter(l => l !== 'atomic:doing'); + break; + case 'create_comment': + value = { + id: comments.size + 1, + body: args.body, + issue_url: `https://api.github.com/repos/owner/repo/issues/${args.number}`, + }; + comments.set(value.id, value); + break; + case 'get_comment': + value = comments.get(args.id); + break; + case 'list_comments': + value = [...comments.values()]; + break; + case 'update_comment': + value = comments.get(args.id); + value.body = args.body; + break; + default: + throw new Error(action); + } + const receipt = { status: 200, body: JSON.stringify(value) }; + if (isWrite) receipts.set(id, receipt); + return receipt; + }, + ); + return { port, issues, comments, writes }; +} + +it('creates closed/doing issues and reopens while preserving unrelated labels', async () => { + const { port, issues } = github(); + const value = { title: 'Created closed', body: 'Markdown', status: 'Done' }; + const row = await port.create('issue', value, 'stable-create'); + expect(issues.get(row.id).state).toBe('closed'); + await port.create('issue', value, 'stable-create'); + expect(issues.size).toBe(2); + await port.update('issue', 1, { ...value, status: 'Doing' }, 'doing'); + expect(issues.get(1).labels).toEqual(['bug', 'atomic:doing']); + await port.update('issue', 1, value, 'close'); + await port.update('issue', 1, { ...value, status: 'Todo' }, 'reopen'); + expect(issues.get(1).state).toBe('open'); + expect(issues.get(1).labels).toEqual(['bug']); +}); + +it('creates/edits comments with replay-safe IDs and rejects another issue’s comment', async () => { + const { port, comments } = github(); + const context = { issueId: 1 }; + await port.create( + 'comment:x', + { body: 'Hi' }, + 'comment-create', + undefined, + context, + ); + await port.create( + 'comment:x', + { body: 'Hi' }, + 'comment-create', + undefined, + context, + ); + expect(comments.size).toBe(1); + await port.update( + 'comment:x', + 1, + { body: 'Edited' }, + 'comment-edit', + undefined, + context, + ); + expect(comments.get(1).body).toBe('Edited'); + await expect(port.get('comment:x', 1, { issueId: 2 })).rejects.toThrow( + 'another issue', + ); +}); + +it('fails on provider errors and scans all pages without importing pull requests', async () => { + const f = github(); + const row = f.issues.get(1); + let calls = 0; + const port = new GitHubPort(null, { repository: 'owner/repo' }, async () => ({ + status: 200, + body: JSON.stringify( + ++calls === 1 + ? Array.from({ length: 100 }, (_, i) => ({ + ...row, + number: i + 1, + ...(i === 0 ? { pull_request: {} } : {}), + })) + : [], + ), + })); + expect(await port.list('issue')).toHaveLength(99); + expect(calls).toBe(2); + port.call = async () => ({ status: 429, body: '{}' }); + await expect(port.list('issue')).rejects.toThrow('429'); +}); + +it('Atomic creates reuse native localId after a lost acknowledgement', async () => { + const stored = new Map(); + let lose = true; + const store = { + getServerUrl: () => 'https://atomic.example', + findByLocalId: async (_drive, parent, key) => + [...stored.values()].find(r => r.parent === parent && r.key === key), + newResource: async ({ parent, propVals }) => { + const r = { + subject: `did:ad:${stored.size + 1}`, + parent, + key: propVals['https://atomicdata.dev/properties/localId'], + async save() { + stored.set(r.subject, r); + if (lose) { + lose = false; + throw new Error('Lost acknowledgement'); + } + }, + }; + return r; + }, + }; + const port = new AtomicPort(store, { + connection: { + table: 'did:ad:table', + drive: 'did:ad:drive', + tags: { Todo: 'https://atomicdata.dev/task/v1/todo' }, + }, + }); + const value = { title: 'Same', body: '', status: 'Todo' }; + port.get = async (_entity, id) => ({ id, value }); + port.findByLocalId = (parent, key) => store.findByLocalId('', parent, key); + await expect(port.create('issue', value, 'key')).rejects.toThrow( + 'Lost acknowledgement', + ); + expect((await port.create('issue', value, 'key')).id).toBe('did:ad:1'); + expect(stored.size).toBe(1); +}); diff --git a/integrations/github-issues/devonian/proxy.mjs b/integrations/github-issues/devonian/proxy.mjs new file mode 100644 index 0000000000..e997b41311 --- /dev/null +++ b/integrations/github-issues/devonian/proxy.mjs @@ -0,0 +1,199 @@ +import { endpoint, request } from '../adapter.js'; +import { trackerAction } from '../tracker-actions.js'; + +/** Direct browser transport. Codes stay in tab storage, never in Atomic resources. */ +export function proxyTransport({ + url, + repository, + getCode, + setCode, + journal, + save, + fetcher = fetch, + dispatch, +}) { + const origin = new URL(url); + if ( + origin.protocol !== 'https:' && + !( + origin.protocol === 'http:' && + ['localhost', '127.0.0.1'].includes(origin.hostname) + ) + ) + throw new Error('Use an HTTPS proxy or loopback HTTP'); + const root = endpoint(repository); + let pending = Promise.resolve(); + return (action, args, id) => { + const operation = async () => { + if ( + action === 'get_issue' && + (!Number.isSafeInteger(args.number) || args.number <= 0) + ) + throw new Error('Invalid issue number'); + if ( + action === 'create_issue' && + (typeof args.title !== 'string' || + !args.title.trim() || + args.title.length > 256 || + (args.body !== undefined && typeof args.body !== 'string')) + ) + throw new Error('Invalid issue title or body'); + const intent = + trackerAction(repository, action, args) ?? + (action === 'get_issue' + ? request('get', 'GET', `${root}/${args.number}`, id) + : action === 'create_issue' + ? request('create', 'POST', root, id, args) + : undefined); + if (!intent) throw new Error('Unsupported GitHub action'); + const writes = intent.method !== 'GET'; + const signature = JSON.stringify({ action, args }); + const old = journal[id]; + if (writes && old) { + if (old.signature !== signature) + throw new Error('Operation identity reused with different arguments'); + if (old.receipt) return old.receipt; + throw new Error( + `Uncertain GitHub write (${action}). Inspect its outcome before retrying; it will not be resent.`, + ); + } + if (writes) { + journal[id] = { signature }; + await save(); + } + const target = new URL(intent.url); + let receipt; + let next = true; + if (dispatch) { + receipt = await dispatch(`${target.pathname}${target.search}`, { + method: intent.method, + ...(intent.body ? { body: intent.body } : {}), + }).catch(() => { + throw new Error( + 'Proxy request failed. Check CORS and reconnect; an uncertain write will not be resent.', + ); + }); + } else { + const code = getCode(); + if (!code) + throw new Error( + 'Connect to the proxy or supply a fresh connection code', + ); + setCode(''); + const destination = new URL( + `/proxy/github-issues${target.pathname}${target.search}`, + origin, + ); + const response = await fetcher(destination.href, { + method: intent.method, + redirect: 'error', + credentials: 'omit', + headers: { + Authorization: `Bearer ${code}`, + Accept: 'application/vnd.github+json', + 'Content-Type': 'application/json', + }, + ...(intent.body ? { body: intent.body } : {}), + }).catch(() => { + throw new Error( + 'Proxy request failed. Check CORS and reconnect; an uncertain write will not be resent.', + ); + }); + next = response.headers.get('X-Connection-Code'); + if (next) setCode(next); + receipt = { status: response.status, body: await response.text() }; + } + if (writes && receipt.status >= 200 && receipt.status < 300) { + journal[id].receipt = receipt; + await save(); + } + if (!next) + throw new Error( + 'Proxy did not expose X-Connection-Code. Enable CORS exposure and reconnect.', + ); + return receipt; + }; + const next = pending.then(operation); + pending = next.catch(() => {}); + return next; + }; +} + +/** An explicitly labelled GitHub fixture; no network or real GitHub mutations. */ +export function fixtureTransport(state, save) { + state.issues ??= [ + { + number: 1, + title: 'Welcome from GitHub', + body: 'Edit either tracker, then sync again.', + state: 'open', + labels: ['demo'], + }, + ]; + state.comments ??= []; + state.receipts ??= {}; + return async (action, args, id) => { + if (state.receipts[id]) return state.receipts[id]; + let value; + const issue = state.issues.find(r => r.number === args.number); + const comment = state.comments.find(r => r.id === args.id); + switch (action) { + case 'list_issues': + value = state.issues.slice((args.page - 1) * 100, args.page * 100); + break; + case 'get_issue': + value = issue; + break; + case 'create_issue': + value = { + number: Math.max(0, ...state.issues.map(r => r.number)) + 1, + ...args, + state: 'open', + labels: [], + }; + state.issues.push(value); + break; + case 'update_issue': + value = Object.assign(issue, args); + break; + case 'add_doing_label': + issue.labels = [...new Set([...issue.labels, 'atomic:doing'])]; + value = issue; + break; + case 'remove_doing_label': + issue.labels = issue.labels.filter(l => l !== 'atomic:doing'); + value = issue; + break; + case 'list_comments': + value = state.comments + .filter(c => c.issue_url.endsWith(`/${args.number}`)) + .slice((args.page - 1) * 100, args.page * 100); + break; + case 'get_comment': + value = comment; + break; + case 'create_comment': + value = { + id: Math.max(0, ...state.comments.map(r => r.id)) + 1, + body: args.body, + issue_url: `https://api.github.com/repos/demo/issues/issues/${args.number}`, + user: { login: 'demo-user' }, + }; + state.comments.push(value); + break; + case 'update_comment': + value = Object.assign(comment, { body: args.body }); + break; + default: + throw new Error(`Unsupported fixture action: ${action}`); + } + const receipt = { + status: value ? 200 : 404, + body: JSON.stringify(value ?? {}), + }; + if (!action.startsWith('get_') && !action.startsWith('list_')) + state.receipts[id] = receipt; + await save(); + return receipt; + }; +} diff --git a/integrations/github-issues/devonian/proxy.test.mjs b/integrations/github-issues/devonian/proxy.test.mjs new file mode 100644 index 0000000000..48bf983161 --- /dev/null +++ b/integrations/github-issues/devonian/proxy.test.mjs @@ -0,0 +1,100 @@ +import { expect, it } from 'vitest'; +import { proxyTransport } from './proxy.mjs'; + +it('serializes rotating codes, preserves query strings and checkpoints successful writes', async () => { + let code = 'first'; + const seen = []; + const journal = {}; + let saves = 0; + const call = proxyTransport({ + url: 'https://proxy.example', + repository: 'owner/repo', + journal, + getCode: () => code, + setCode: c => { + code = c; + }, + save: async () => { + saves++; + }, + fetcher: async (url, opts) => { + seen.push({ url, ...opts }); + return new Response('{"id":1}', { + headers: { 'X-Connection-Code': `next-${seen.length}` }, + }); + }, + }); + await Promise.all([ + call('list_issues', { page: 2 }, 'read'), + call('create_comment', { number: 1, body: 'Hello' }, 'write'), + ]); + expect(seen[0].url).toContain( + '/proxy/github-issues/repos/owner/repo/issues?state=all&per_page=100&page=2', + ); + expect(seen.map(r => r.headers.Authorization)).toEqual([ + 'Bearer first', + 'Bearer next-1', + ]); + expect(code).toBe('next-2'); + expect(saves).toBe(2); + await call('create_comment', { number: 1, body: 'Hello' }, 'write'); + expect(seen).toHaveLength(2); + await expect( + call('create_comment', { number: 1, body: 'Changed' }, 'write'), + ).rejects.toThrow('different arguments'); +}); + +it('never retries an uncertain create after a lost response or restart', async () => { + const journal = {}; + let writes = 0; + let code = 'first'; + const options = { + url: 'https://proxy.example', + repository: 'owner/repo', + journal, + getCode: () => code, + setCode: c => { + code = c; + }, + save: async () => {}, + fetcher: async () => { + writes++; + throw new Error('network'); + }, + }; + await expect( + proxyTransport(options)('create_issue', { title: 'Title' }, 'create'), + ).rejects.toThrow('Proxy request failed'); + code = 'reconnected'; + await expect( + proxyTransport(options)('create_issue', { title: 'Title' }, 'create'), + ).rejects.toThrow('Uncertain GitHub write'); + expect(writes).toBe(1); +}); + +it('retains rotated codes on provider errors and identifies missing exposed headers', async () => { + let code = 'first'; + const options = { + url: 'https://proxy.example', + repository: 'owner/repo', + journal: {}, + getCode: () => code, + setCode: c => { + code = c; + }, + save: async () => {}, + fetcher: async () => + new Response('{}', { + status: 403, + headers: { 'X-Connection-Code': 'rotated' }, + }), + }; + expect( + (await proxyTransport(options)('get_issue', { number: 1 }, 'read')).status, + ).toBe(403); + expect(code).toBe('rotated'); + options.fetcher = async () => new Response('{}'); + await expect( + proxyTransport(options)('get_issue', { number: 1 }, 'read'), + ).rejects.toThrow('expose X-Connection-Code'); +}); diff --git a/integrations/github-issues/devonian/vitest.config.mjs b/integrations/github-issues/devonian/vitest.config.mjs new file mode 100644 index 0000000000..1b8f18ff79 --- /dev/null +++ b/integrations/github-issues/devonian/vitest.config.mjs @@ -0,0 +1,14 @@ +import { resolve } from 'node:path'; + +export default { + resolve: { + alias: { + devonian: process.env.DEVONIAN_PATH + ? resolve(process.env.DEVONIAN_PATH, 'build/src/main.js') + : resolve('browser/data-browser/src/chunks/DevonianDemo/devonian.js'), + '@tomic/lib': resolve('browser/lib/src/index.ts'), + vitest: resolve('browser/node_modules/vitest/dist/index.js'), + }, + }, + test: { include: ['integrations/github-issues/devonian/*.test.*'] }, +}; diff --git a/integrations/github-issues/tracker-actions.ts b/integrations/github-issues/tracker-actions.ts new file mode 100644 index 0000000000..47e9096c14 --- /dev/null +++ b/integrations/github-issues/tracker-actions.ts @@ -0,0 +1,190 @@ +/** Additional repository-scoped actions used by the Devonian tracker bridge. */ +import { endpoint, request } from './adapter.js'; + +const integer = { + type: 'integer' as const, + description: 'Positive identifier or page number', +}; +const string = { + type: 'string' as const, + description: 'Issue or comment field value', +}; +const definitions = [ + ['list_issues', 'List issues', 'list', { page: integer }, ['page']], + [ + 'update_issue', + 'Update an issue', + 'update', + { number: integer, title: string, body: string, state: string }, + ['number', 'title', 'body', 'state'], + ], + [ + 'add_doing_label', + 'Mark an issue as doing', + 'doing-add', + { number: integer }, + ['number'], + ], + [ + 'remove_doing_label', + 'Remove the doing label', + 'doing-remove', + { number: integer }, + ['number'], + ], + [ + 'list_comments', + 'List issue comments', + 'comments-list', + { number: integer, page: integer }, + ['number', 'page'], + ], + [ + 'get_comment', + 'Get an issue comment', + 'comments-get', + { id: integer }, + ['id'], + ], + [ + 'create_comment', + 'Create an issue comment', + 'comments-create', + { number: integer, body: string }, + ['number', 'body'], + ], + [ + 'update_comment', + 'Update an issue comment', + 'comments-update', + { id: integer, body: string }, + ['id', 'body'], + ], +] as const; + +export const trackerActions = definitions.map( + ([name, title, operation, properties, required]) => ({ + name, + title, + description: `${title} in this connected repository. Writes require approval.`, + operation, + inputSchema: { + type: 'object', + properties, + required: [...required], + additionalProperties: false, + }, + }), +); + +export function trackerOperations(root: string) { + return [ + { + id: 'comments-list', + method: 'GET', + url: `${root}/{number}/comments`, + effect: 'read', + }, + { + id: 'comments-get', + method: 'GET', + url: `${root}/comments/{id}`, + effect: 'read', + }, + { + id: 'comments-create', + method: 'POST', + url: `${root}/{number}/comments`, + effect: 'write', + }, + { + id: 'comments-update', + method: 'PATCH', + url: `${root}/comments/{id}`, + effect: 'write', + }, + ]; +} + +export function trackerAction( + repository: string, + action: string, + args: Record, +) { + const definition = definitions.find(d => d[0] === action); + if (!definition) return undefined; + const [, , operation, properties, required] = definition; + for (const key of required) { + if (!(key in args)) throw new Error(`Missing ${key}`); + } + for (const [key, value] of Object.entries(args)) { + if (!(key in properties)) throw new Error(`Unexpected ${key}`); + if (['number', 'id', 'page'].includes(key)) { + if ( + !Number.isSafeInteger(value) || + Number(value) <= 0 || + (key === 'page' && Number(value) > 100) + ) + throw new Error(`Invalid ${key}`); + } else if (typeof value !== 'string') throw new Error(`Invalid ${key}`); + } + const root = endpoint(repository); + switch (action) { + case 'list_issues': + return request( + operation, + 'GET', + `${root}?state=all&per_page=100&page=${args.page}&sort=created&direction=asc`, + 'action', + ); + case 'list_comments': + return request( + operation, + 'GET', + `${root}/${args.number}/comments?per_page=100&page=${args.page}`, + 'action', + ); + case 'get_comment': + return request(operation, 'GET', `${root}/comments/${args.id}`, 'action'); + case 'create_comment': + case 'update_comment': + if (!(args.body as string).trim()) + throw new Error('Comment body cannot be empty'); + return request( + operation, + action === 'create_comment' ? 'POST' : 'PATCH', + action === 'create_comment' + ? `${root}/${args.number}/comments` + : `${root}/comments/${args.id}`, + 'action', + { body: args.body }, + ); + case 'add_doing_label': + return request( + operation, + 'POST', + `${root}/${args.number}/labels`, + 'action', + { + labels: ['atomic:doing'], + }, + ); + case 'remove_doing_label': + return request( + operation, + 'DELETE', + `${root}/${args.number}/labels/atomic%3Adoing`, + 'action', + ); + case 'update_issue': + if (!['open', 'closed'].includes(String(args.state))) + throw new Error('Invalid issue state'); + if (!(args.title as string).trim() || (args.title as string).length > 256) + throw new Error('Invalid issue title'); + return request(operation, 'PATCH', `${root}/${args.number}`, 'action', { + title: args.title, + body: args.body, + state: args.state, + }); + } +} diff --git a/integrations/localthought/README.md b/integrations/localthought/README.md index d6f4504702..3674472724 100644 --- a/integrations/localthought/README.md +++ b/integrations/localthought/README.md @@ -1,40 +1,67 @@ -# LocalThought API plugins - -AtomicServer reads `/catalog` from `https://localthought.io` and offers one -connection card per advertised platform. It loads `/catalog/{platform}.yaml` -for collection discovery, required scope parameters, pagination and ontology. - -`syncables-rs` is pinned in `server/Cargo.toml`. Its `SyncClient` receives the -catalog's overlaid OpenAPI document and a host-owned transport that forwards -GET requests through `/proxy/{platform}/{path}`, preserving query parameters -and pagination headers. Rotating connection codes are held in AtomicServer's -secret store, scoped to a connection, drive and authenticated agent. The -browser carries a one-time OAuth return code to signed completion and removes -it from the address bar. Tenant secrets and provider tokens never enter graph -resources, plugin source or browser storage. - -Each platform gets drive-local classes and properties derived by Syncables. -The platform ID prefixes every generated schema identity. Scalars retain their -datatypes; RFC3339 timestamps become milliseconds; arrays and nested objects -use Atomic JSON properties. API-required fields are recommended in Atomic -because nullable/partial API representations may omit them. Each resource type -gets a table and view. The shared sandbox importer proposes changes for review, -uses provider/resource/namespace/ID for reconciliation, and preserves local edits. -A failed collection aborts the preview instead of applying a partial snapshot. - -This is a manual read/import flow, with limits of 200 requests, 5,000 records, -10 MB per page/document and 120 seconds per import. No provider writes or -background syncing are enabled. The existing direct GitHub token workflow -remains available explicitly in the GitHub connection dialog. - -## Server configuration - -- `TENANT_SECRET`: the LocalThought tenant secret, stored only on AtomicServer. -- `ATOMIC_INTEGRATION_FRONTEND_ORIGIN`: exact frontend origin, e.g. - `https://your-atomic-app.example` (no trailing slash). Returns must use its - `/app/integrations` path. -- `ATOMIC_INTEGRATION_PROXY_URL`: optional; defaults to `https://localthought.io`. - Loopback HTTP is accepted for development. +# LocalThought browser integrations + +The LocalThought flow runs entirely in the browser: catalog discovery, tenant +challenge signing, OAuth return handling, paginated Syncables reads, ontology +creation, proposal review and local Store/OPFS writes. No AtomicServer HTTP +instance is needed. LocalThought remains the remote OAuth and API proxy. + +Open Integrations, select a platform and paste your `TENANT_SECRET`. It is used +in tab memory to sign the handoff and is not persisted. OAuth returns to the +same frontend `/app/integrations` page. The short-lived return is bound to the +agent, drive and proxy; its code is removed from the address bar immediately. +Connection codes are stored in this browser's localStorage, outside the synced +graph, and may be read by code running on this frontend origin. Clearing site +data requires reconnecting. Existing server-held connections require reconnecting. +Web Locks serialize rotating codes across tabs; a request consumes its code +before dispatch and saves the replacement before processing data. Uncertain +requests cannot silently replay credentials. + +Syncables is vendored temporarily under `syncables/` with upstream provenance in +`UPSTREAM.md`; the matching upstream branch is `codex/browser-integrations`. +`wasm/src/integrations.rs` exposes its in-memory engine through wasm-bindgen. +The shipped pure import mapper reads a local snapshot and produces the existing +reviewed intents; user-edited plugin source is not executed on this path. +Local edits and repeated imports retain the existing reconciliation behavior. + +## Build and proxy requirements + +- Build `atomic-wasm` using `cd browser/data-browser && pnpm build:wasm`. +- Set `VITE_INTEGRATION_PROXY_URL` at frontend build/dev time to override the + default `https://localthought.io`. HTTPS or loopback HTTP origins only. +- Deploy the companion integration-proxy CORS change. It handles preflights for + explicit Authorization headers and exposes `X-Connection-Code`, `Link`, + pagination/count headers, `ETag` and `Retry-After`. Cookie credentials are not + enabled; login and consent use top-level navigation. +- Native AtomicServer's `TENANT_SECRET`, `ATOMIC_INTEGRATION_PROXY_URL` and + `ATOMIC_INTEGRATION_FRONTEND_ORIGIN` no longer configure this flow. Its + `/integration-proxy/*` handlers and Syncables dependency have been removed. + +Limits remain 200 requests, 5,000 records, 10 MB per page/document and 120 +seconds of network work. Calendar imports require explicit UTC date bounds. +Imports are manual; closed tabs do not run schedules. Other legacy integrations, +server plugin execution, actions and schedules are outside this migration. + +## Checks + +```sh +cargo check -p atomic-wasm --target wasm32-unknown-unknown +browser/node_modules/.bin/vitest run --config integrations/localthought/vitest.config.ts +node integrations/localthought/wasm-smoke.mjs # after building wasm/pkg +``` + +For the browser-only mock journey (no AtomicServer on port 19999): + +```sh +MOCK_PROXY_PORT=19091 MOCK_FRONTEND_ORIGIN=http://localhost:6748 node integrations/localthought/mock-proxy.mjs +# Separate terminal, browser/data-browser: +VITE_INTEGRATION_PROXY_URL=http://127.0.0.1:19091 VITE_ATOMIC_SERVER_URL=http://127.0.0.1:19999 pnpm exec vite --host 127.0.0.1 --port 6748 +# Repository root: +node integrations/localthought/browser-smoke.mjs +``` + +The mock is test-only. It uses synthetic credentials and data; never deploy it. + +## Historical server-flow verification Live verification on 2026-09-09 succeeded against proxy Heroku release v38 (`5960ae43`): OAuth returned to AtomicServer, Syncables fetched 29 issue/PR @@ -56,43 +83,52 @@ An unbounded fetch successfully traversed multiple pages but exceeded the bounds and recurrence expansion are passed to Syncables as collection query settings. The importer remains a manual snapshot, not a background sync. -## Local mock and tests - -The mock is test-only, with a synthetic tenant, consent screen, expiring-by-use -challenges, rotating single-use codes, and an OpenAPI document serving five -Pets over two pages. Never deploy it as a real credential service. - -```sh -node integrations/localthought/mock-proxy.mjs -``` - -Start AtomicServer with: - -```sh -TENANT_SECRET=bW9jay10ZW5hbnQ.mock-signature \ -ATOMIC_INTEGRATION_PROXY_URL=http://127.0.0.1:19090 \ -ATOMIC_INTEGRATION_FRONTEND_ORIGIN=http://localhost:6747 \ -ATOMICSERVER_SKIP_JS_BUILD=true \ -cargo run -p atomic-server --features light,wasm-plugins -- --port 9883 -``` - -Run one Vite server with `VITE_ATOMIC_SERVER_URL=http://localhost:9883`, then: - -```sh -cd browser/e2e -ATOMIC_MOCK_INTEGRATION_PROXY=1 pnpm exec playwright test tests/plugins.spec.ts \ - --project chromium --grep 'Pets imports' -``` - -Additional checks from the repository root: +## Calendar view + +Google event imports now install a Calendar view alongside the source table. +The projected date uses the day in Google's supplied start offset (or the +unchanged all-day date), so mixed all-day/timed events share one DATE column. +The original Start and End objects retain timezones and exclusive end values. +The month view shows each event on its start day; it does not draw duration or +multi-day spans. Additional notes identify recurring events, attendees, +reminders and conferencing when those fields are returned by the catalog. +Recurring instances are expanded by the existing bounded provider fetch. + +Use **Fetch and preview** again to refresh, then review and apply changes. +Existing source identities and import baselines prevent duplicates and preserve +Atomic-only fields and local edits. Cancellation records are retained with a +note; absence from a bounded fetch never deletes an Atomic resource. Google +may omit cancelled events from list results, so this is not a deletion feed. +Removed optional provider fields are not cleared by the shared snapshot importer. +This first version deliberately remains manual and one-way. No OAuth write +scope is requested and edits in Atomic do not update Google. + +`calendar.test.ts` exercises mixed dates, offset boundaries, exclusive ends, +recurrence/attendee notes, cancellations, malformed starts, cross-calendar +identity and repeated imports with private local fields. + +## Browser-only Calendar regression + +`browser/e2e/tests/google-calendar-import.spec.mts` starts the shared mock +integration-proxy from #1399 with a synthetic Google Calendar. The test enters +the public fixture tenant secret, completes consent, and exercises real browser +credential rotation, WASM pagination, local schema installation, proposal review, +OPFS application, and Calendar rendering. It refreshes changed provider data +and checks that native identities and Atomic-only notes survive reload. +AtomicServer HTTP and all WebSockets are blocked throughout; only GET requests +are permitted for provider data. The configured LocalThought origin is forwarded +to the isolated HTTP fixture, so no live provider credentials or data are used. + +Run with a dev frontend built from this branch and its matching WASM bundle: ```sh -node --test integrations/localthought/mock-proxy.test.mjs -browser/node_modules/.bin/vitest run --config integrations/localthought/vitest.config.ts -browser/node_modules/.bin/tsc -p integrations/localthought/tsconfig.json -ATOMICSERVER_SKIP_JS_BUILD=true cargo test -p atomic-server --lib \ - --features light,wasm-plugins integration_proxy +FRONTEND_URL=http://127.0.0.1:6747 SERVER_URL=http://127.0.0.1:19999 \ + browser/e2e/node_modules/.bin/playwright test \ + --config browser/e2e/playwright.config.ts \ + browser/e2e/tests/google-calendar-import.spec.mts --project chromium ``` -Dagger's E2E server starts the mock alongside AtomicServer, and the focused -GitHub Actions workflow runs the typed Pets journey on the API-plugin branches. +If the frontend uses `VITE_INTEGRATION_PROXY_URL`, pass the same value to the +test process. The test forwards that origin to its own fixture. Live Google +OAuth on the browser path still depends on the proxy CORS deployment described +above; this fixture test does not claim live-provider verification. diff --git a/integrations/localthought/browser-smoke.mjs b/integrations/localthought/browser-smoke.mjs new file mode 100644 index 0000000000..87ce370fcf --- /dev/null +++ b/integrations/localthought/browser-smoke.mjs @@ -0,0 +1,63 @@ +/** Run against Vite + the mock proxy, with VITE_ATOMIC_SERVER_URL on a closed port. */ +import { chromium } from '../../browser/e2e/node_modules/@playwright/test/index.mjs'; +const browser = await chromium.launch({ headless: true }); +const page = await browser.newPage(); +page.setDefaultTimeout(45000); +await page.route('http://127.0.0.1:19999/**', route => route.abort()); +const errors = []; +page.on('pageerror', e => errors.push(e.message)); +try { + console.log('Opening offline dev drive'); + await page.goto('http://localhost:6748/app/dev-drive'); + await page.waitForURL(/app\/show\?subject=/, { timeout: 120000 }); + console.log('Opening integrations'); + await page.getByRole('link', { name: 'Integrations', exact: true }).click(); + console.log('Opening Pets'); + const pets = page.locator('[data-integration=pets]'); + await pets.getByRole('button', { name: 'Set up connection' }).click(); + await page + .getByLabel('LocalThought tenant secret') + .fill('bW9jay10ZW5hbnQ.mock-signature'); + await page + .getByRole('button', { name: 'Install and connect', exact: true }) + .click(); + await page.getByRole('button', { name: 'Connect test account' }).click(); + console.log('Fetching preview'); + await page.getByRole('button', { name: 'Fetch and preview' }).click(); + console.log('Applying preview'); + await page + .getByRole('button', { name: 'Apply 5 changes', exact: true }) + .click(); + await page + .getByRole('link', { name: 'Open imported records', exact: true }) + .click(); + for (const name of ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles']) + await page + .getByRole('main') + .getByText(name, { exact: true }) + .first() + .waitFor(); + await page.reload(); + for (const name of ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles']) + await page + .getByRole('main') + .getByText(name, { exact: true }) + .first() + .waitFor(); + console.log( + 'Browser-only OAuth, WASM import, review, OPFS apply and reload passed', + ); +} catch (error) { + console.error( + error.message, + '\nURL:', + page.url(), + '\nUI:', + (await page.locator('body').innerText()).slice(-7000), + '\nErrors:', + errors, + ); + process.exitCode = 1; +} finally { + await browser.close(); +} diff --git a/integrations/localthought/browser.test.ts b/integrations/localthought/browser.test.ts new file mode 100644 index 0000000000..87a8081208 --- /dev/null +++ b/integrations/localthought/browser.test.ts @@ -0,0 +1,160 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { BrowserIntegrations, proxyOrigin, sign, type Engine } from './browser'; +const secret = 'bW9jay10ZW5hbnQ.mock-signature'; +const origin = 'https://proxy.example'; +function setup() { + const values = new Map(); + const storage = { + getItem: (k: string) => values.get(k) ?? null, + setItem: (k: string, v: string) => { + values.set(k, v); + }, + } as Storage; + vi.stubGlobal('location', { origin: 'https://atomic.example' }); + vi.stubGlobal('navigator', { + locks: { request: (_: string, f: () => unknown) => f() }, + }); + const http = vi.fn(async (url: string, init?: RequestInit) => { + expect(init?.credentials).toBe('omit'); + expect(init?.redirect).toBe('error'); + if (url.endsWith('/catalog')) return new Response('["pets"]'); + if (url.endsWith('/session')) + return Response.json({ ts: 1, nonce: 'nonce', challenge: 'challenge' }); + return new Response('{}'); + }); + const engine: Engine = { + describeIntegration: async () => + JSON.stringify({ upstream: 'https://pets.example' }), + fetchIntegration: async (_text, _platform, _constants, _range, fetch) => { + const first = JSON.parse(await fetch('https://pets.example/pets')); + expect(first.headers['x-connection-code']).toBeUndefined(); + await fetch('https://pets.example/pets?page=2'); + return '{"records":[]}'; + }, + }; + const client = new BrowserIntegrations( + storage, + async () => engine, + origin, + http as typeof fetch, + ); + const start = () => + client.start( + 'drive', + 'actor', + 'pets', + 'https://atomic.example/app/integrations', + secret, + ); + return { values, storage, http, engine, client, start }; +} +afterEach(() => vi.unstubAllGlobals()); +it('matches the tenant HMAC protocol and rejects non-origin proxy URLs', async () => { + expect(await sign('key', 'The quick brown fox jumps over the lazy dog')).toBe( + '97yD9DBThCSxMpjmqm-xQ-9NWaFJRhdZl0edvC0aPNg', + ); + expect(() => proxyOrigin('https://proxy.example/path')).toThrow(); + expect(() => proxyOrigin('http://proxy.example')).toThrow(); +}); +it('binds returns to actor, drive and expiry without saving tenant secrets', async () => { + const { client, start, values } = setup(); + const { state, url } = await start(); + expect(url).not.toContain(secret); + expect([...values.values()].join()).not.toContain(secret); + expect(() => client.finish('other', 'actor', state, 'code')).toThrow(); + expect(() => client.finish('drive', 'other', state, 'code')).toThrow(); + expect(client.finish('drive', 'actor', state, 'code').platform).toBe('pets'); + expect(() => client.finish('drive', 'actor', state, 'code')).toThrow(); +}); +it('consumes before dispatch and preserves rotation and pagination', async () => { + const { client, start, http, values } = setup(); + const { state } = await start(); + client.finish('drive', 'actor', state, 'first'); + const codes: string[] = []; + http.mockImplementation(async (url, init) => { + if (url.includes('/catalog/')) return new Response('{}'); + expect(JSON.parse([...values.values()][0]).code).toBeUndefined(); + codes.push((init?.headers as Record).Authorization); + return new Response('[]', { + headers: { + 'x-connection-code': 'second', + link: '; rel=next', + }, + }); + }); + await client.fetchRecords('drive', 'actor', state, {}); + expect(codes).toEqual(['Bearer first', 'Bearer second']); + expect(http.mock.calls.at(-1)?.[0]).toBe(`${origin}/proxy/pets/pets?page=2`); +}); +it('never retries an uncertain consumed credential', async () => { + const { client, start, http } = setup(); + const { state } = await start(); + client.finish('drive', 'actor', state, 'first'); + http.mockImplementation(async url => { + if (url.includes('/catalog/')) return new Response('{}'); + throw new Error('connection lost'); + }); + await expect( + client.fetchRecords('drive', 'actor', state, {}), + ).rejects.toThrow('lost'); + const calls = http.mock.calls.length; + await expect( + client.fetchRecords('drive', 'actor', state, {}), + ).rejects.toThrow('Reconnect'); + expect(http.mock.calls).toHaveLength(calls); +}); +it('rejects pagination to a different provider before spending a credential', async () => { + const { client, start, engine, values } = setup(); + const { state } = await start(); + client.finish('drive', 'actor', state, 'first'); + engine.fetchIntegration = async (_t, _p, _c, _r, fetch) => + fetch('https://evil.example/pets'); + await expect( + client.fetchRecords('drive', 'actor', state, {}), + ).rejects.toThrow('origin'); + expect(JSON.parse([...values.values()][0]).code).toBe('first'); +}); + +it('calls the browser fetch function without binding it to the client', async () => { + const { storage, engine } = setup(); + vi.stubGlobal('fetch', function (this: unknown) { + expect(this).not.toBeInstanceOf(BrowserIntegrations); + return Promise.resolve(new Response('["pets"]')); + }); + const client = new BrowserIntegrations(storage, async () => engine, origin); + expect(await client.catalog()).toEqual(['pets']); +}); + +it('supports the demo callback and write credentials without using the import engine', async () => { + const { client, http, values } = setup(); + const { state } = await client.start( + 'drive', + 'actor', + 'pets', + 'https://atomic.example/app/devonian-demo', + secret, + ); + client.finish('drive', 'actor', state, 'first'); + http.mockImplementation(async (_url, init) => { + expect(JSON.parse([...values.values()][0]).code).toBeUndefined(); + expect(init?.method).toBe('POST'); + expect(init?.body).toBe('{"title":"new"}'); + return new Response('{"id":1}', { + status: 201, + headers: { 'X-Connection-Code': 'next' }, + }); + }); + await expect( + client.request('drive', 'actor', state, 'github-issues', '/issues'), + ).rejects.toThrow('another platform'); + await expect( + client.request('drive', 'actor', state, 'pets', '//evil.example'), + ).rejects.toThrow('Invalid proxy path'); + expect( + await client.request('drive', 'actor', state, 'pets', '/issues', { + method: 'POST', + body: '{"title":"new"}', + }), + ).toEqual({ status: 201, body: '{"id":1}' }); + expect(JSON.parse([...values.values()][0]).code).toBe('next'); +}); diff --git a/integrations/localthought/browser.ts b/integrations/localthought/browser.ts new file mode 100644 index 0000000000..94350c048f --- /dev/null +++ b/integrations/localthought/browser.ts @@ -0,0 +1,328 @@ +/** Browser-owned credentials. Never store these in Atomic graph resources. */ +export const DEFAULT_PROXY = 'https://localthought.io'; +const key = 'localthought-browser-v1:'; +export interface Connection { + drive: string; + actor: string; + platform: string; + origin: string; + expires: number; + code?: string; + ready: boolean; +} +export interface Engine { + describeIntegration(text: string): Promise; + fetchIntegration( + text: string, + platform: string, + constants: string, + range: string | undefined, + fetch: (url: string) => Promise, + ): Promise; +} +export function proxyOrigin(value = DEFAULT_PROXY): string { + const u = new URL(value); + if ( + u.origin !== value || + (u.protocol !== 'https:' && + !( + u.protocol === 'http:' && + ['localhost', '127.0.0.1'].includes(u.hostname) + )) + ) + throw new Error('Proxy must be an HTTPS origin or localhost'); + return value; +} +const base64url = (bytes: Uint8Array) => + btoa(String.fromCharCode(...bytes)) + .replaceAll('+', '-') + .replaceAll('/', '_') + .replaceAll('=', ''); +export async function sign(secret: string, text: string) { + const encoder = new TextEncoder(); + const k = await crypto.subtle.importKey( + 'raw', + encoder.encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + return base64url( + new Uint8Array(await crypto.subtle.sign('HMAC', k, encoder.encode(text))), + ); +} +async function limitedText(response: Response): Promise { + if (!response.body) throw new Error('Empty proxy response'); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let size = 0, + text = ''; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > 10 * 1024 * 1024) + throw new Error('Proxy response exceeds 10 MB'); + text += decoder.decode(value, { stream: true }); + } + return text + decoder.decode(); + } finally { + await reader.cancel(); + } +} +export class BrowserIntegrations { + constructor( + private storage: Storage, + private engine: () => Promise, + readonly origin = DEFAULT_PROXY, + private http: typeof fetch = (...args) => fetch(...args), + ) { + proxyOrigin(origin); + } + private async get(path: string, signal?: AbortSignal) { + const response = await this.http(`${this.origin}${path}`, { + credentials: 'omit', + redirect: 'error', + signal: signal ?? AbortSignal.timeout(30000), + }); + if (!response.ok) + throw new Error(`LocalThought returned HTTP ${response.status}`); + return limitedText(response); + } + async catalog(signal?: AbortSignal): Promise { + const names = JSON.parse(await this.get('/catalog', signal)); + if ( + !Array.isArray(names) || + names.length > 200 || + names.some(s => typeof s !== 'string' || !/^[a-z0-9-]{1,80}$/.test(s)) + ) + throw new Error('Invalid platform catalog'); + return names; + } + private async document(platform: string) { + if (!/^[a-z0-9-]{1,80}$/.test(platform)) + throw new Error('Invalid platform'); + return this.get(`/catalog/${platform}.yaml`); + } + async describe(platform: string) { + return JSON.parse( + await ( + await this.engine() + ).describeIntegration(await this.document(platform)), + ) as { + parameters: string[]; + collections: string[]; + upstream: string; + }; + } + async start( + drive: string, + actor: string, + platform: string, + returnUrl: string, + secret: string, + ) { + if (!(await this.catalog()).includes(platform)) + throw new Error('Unknown platform'); + const encoded = secret.split('.')[0]; + let tenant: string; + try { + tenant = new TextDecoder('utf-8', { fatal: true }).decode( + Uint8Array.from( + atob(encoded.replaceAll('-', '+').replaceAll('_', '/')), + c => c.charCodeAt(0), + ), + ); + } catch { + throw new Error('Invalid tenant secret'); + } + if (!tenant || !secret.includes('.')) + throw new Error('Invalid tenant secret'); + const callback = new URL(returnUrl); + if ( + callback.origin !== location.origin || + !['/app/integrations', '/app/devonian-demo'].includes( + callback.pathname, + ) || + callback.search || + callback.hash + ) + throw new Error('Invalid integration return URL'); + const state = base64url(crypto.getRandomValues(new Uint8Array(32))); + callback.searchParams.set('integration_state', state); + callback.searchParams.set('platform', platform); + const challenge = JSON.parse(await this.get('/session')); + const url = new URL(`${this.origin}/connect`); + for (const [k, v] of Object.entries({ + redirect_uri: callback.href, + platform, + ts: String(challenge.ts), + nonce: challenge.nonce, + challenge: challenge.challenge, + tenant_id: tenant, + user_id: actor, + user_id_sig: await sign(secret, actor), + response: await sign(secret, challenge.challenge), + })) + url.searchParams.set(k, v as string); + // Only the short-lived handoff survives navigation, never the tenant secret. + this.storage.setItem( + key + state, + JSON.stringify({ + drive, + actor, + platform, + origin: this.origin, + expires: Date.now() + 600000, + ready: false, + } satisfies Connection), + ); + return { state, url: url.href }; + } + private connection(id: string, drive: string, actor: string) { + const raw = this.storage.getItem(key + id); + if (!raw) throw new Error('Reconnect your account'); + const c: Connection = JSON.parse(raw); + if (c.drive !== drive || c.actor !== actor || c.origin !== this.origin) + throw new Error('Connection belongs to another drive, agent or proxy'); + return c; + } + finish(drive: string, actor: string, state: string, code: string) { + const c = this.connection(state, drive, actor); + if (c.ready || c.expires < Date.now() || !code || code.length > 4096) + throw new Error('Invalid, expired or completed connection'); + this.storage.setItem( + key + state, + JSON.stringify({ ...c, ready: true, code }), + ); + return { connection: state, platform: c.platform }; + } + /** Shared rotating-code transport for browser-owned writes as well as reads. */ + async request( + drive: string, + actor: string, + id: string, + platform: string, + path: string, + init: { method?: string; body?: string } = {}, + ): Promise<{ status: number; body: string }> { + if (!navigator.locks) + throw new Error('This browser needs Web Locks for integrations'); + if (!path.startsWith('/') || path.startsWith('//') || /[\\\\#]/.test(path)) + throw new Error('Invalid proxy path'); + const destination = new URL(`/proxy/${platform}${path}`, this.origin); + if (!destination.pathname.startsWith(`/proxy/${platform}/`)) + throw new Error('Invalid proxy path'); + return navigator.locks.request(key + id, async () => { + const c = this.connection(id, drive, actor); + if (c.platform !== platform) + throw new Error('Connection belongs to another platform'); + const response = await this.send( + id, + drive, + actor, + path, + init, + AbortSignal.timeout(30000), + ); + return { status: response.status, body: await limitedText(response) }; + }); + } + private async send( + id: string, + drive: string, + actor: string, + path: string, + init: { method?: string; body?: string }, + signal: AbortSignal, + ) { + const current = this.connection(id, drive, actor); + if (!current.ready || !current.code) + throw new Error('Reconnect before retrying an uncertain request'); + const code = current.code; + delete current.code; + this.storage.setItem(key + id, JSON.stringify(current)); + const response = await this.http( + `${this.origin}/proxy/${current.platform}${path}`, + { + ...init, + headers: { + Authorization: `Bearer ${code}`, + 'Content-Type': 'application/json', + }, + credentials: 'omit', + redirect: 'error', + signal, + }, + ); + const next = response.headers.get('x-connection-code'); + if (!next) + throw new Error( + 'Proxy did not expose a rotated code; reconnect and check CORS', + ); + this.storage.setItem(key + id, JSON.stringify({ ...current, code: next })); + return response; + } + async fetchRecords( + drive: string, + actor: string, + id: string, + constants: Record, + range?: { start: string; end: string }, + ) { + // Web Locks serialize rotating credentials across tabs as well as UI actions. + if (!navigator.locks) + throw new Error('This browser needs Web Locks for integrations'); + return navigator.locks.request(key + id, async () => { + const c = this.connection(id, drive, actor); + if (!c.ready || !c.code) throw new Error('Reconnect your account'); + const text = await this.document(c.platform); + const engine = await this.engine(); + const { upstream } = JSON.parse(await engine.describeIntegration(text)); + const base = new URL(upstream); + let requests = 0; + const signal = AbortSignal.timeout(120000); + const transport = async (raw: string) => { + signal.throwIfAborted(); + const target = new URL(raw); + if ( + target.origin !== base.origin || + target.username || + target.password || + target.hash + ) + throw new Error('Pagination left the catalog API origin'); + if (++requests > 200) + throw new Error('Import exceeds 200 requests; narrow its scope'); + const response = await this.send( + id, + drive, + actor, + `${target.pathname}${target.search}`, + {}, + signal, + ); + const headers = Object.fromEntries( + [...response.headers].filter( + ([name]) => name !== 'x-connection-code', + ), + ); + return JSON.stringify({ + status: response.status, + headers, + body: await limitedText(response), + }); + }; + const result = await engine.fetchIntegration( + text, + c.platform, + JSON.stringify(constants), + range ? JSON.stringify(range) : undefined, + transport, + ); + signal.throwIfAborted(); + return JSON.parse(result); + }); + } +} diff --git a/integrations/localthought/calendar.test.ts b/integrations/localthought/calendar.test.ts new file mode 100644 index 0000000000..1e14884ff4 --- /dev/null +++ b/integrations/localthought/calendar.test.ts @@ -0,0 +1,151 @@ +import { expect, it } from 'vitest'; +import { calendarProjection, calendarFields as fields } from './calendar'; +import { Datatype } from '../../browser/lib/src/index'; +import { run } from './plugin'; +import type { FetchedPlatform } from './schema'; +const fixture = (): FetchedPlatform => ({ + platform: 'google-calendar', + ontology: { + description: '', + terms: [ + { + path: 'event', + kind: 'class', + shortname: 'event', + description: '', + datatype: Datatype.STRING, + requires: [], + recommends: [], + }, + ], + }, + records: [ + { + resource: 'event', + namespace: 'calendar-a', + id: '1', + name: 'Meeting', + values: { + start: { + dateTime: '2026-09-10T00:30:00+02:00', + timeZone: 'Europe/Amsterdam', + }, + end: { dateTime: '2026-09-10T01:30:00+02:00' }, + attendees: [{ email: 'test@example.com' }], + 'recurring-event-id': 'series', + reminders: { useDefault: true }, + }, + }, + ], +}); +it('projects timed events without shifting their day and retains provider structures', () => { + const input = fixture(); + const output = calendarProjection(input); + expect(output.records[0].values).toMatchObject({ + ...input.records[0].values, + [fields.day]: '2026-09-10', + [fields.allDay]: false, + }); + expect(output.records[0].values[fields.notes]).toContain('Recurring'); + expect(output.records[0].values[fields.notes]).toContain('Attendees'); + expect(input.records[0].values[fields.day]).toBeUndefined(); + expect( + output.ontology.terms.find(t => t.shortname === fields.day)?.datatype, + ).toBe(Datatype.DATE); +}); +it('supports all-day dates and preserves exclusive multi-day ends', () => { + const input = fixture(); + input.records[0].values = { + start: { date: '2026-09-10' }, + end: { date: '2026-09-13' }, + }; + expect(calendarProjection(input).records[0].values).toMatchObject({ + [fields.day]: '2026-09-10', + [fields.allDay]: true, + end: { date: '2026-09-13' }, + }); +}); +it('retains cancelled events even when Google omits start', () => { + const input = fixture(); + input.records[0].values = { status: 'cancelled' }; + expect(calendarProjection(input).records[0].values[fields.notes]).toContain( + 'Cancelled', + ); +}); +it('fails the projection on malformed active events instead of silently losing rows', () => { + for (const start of [ + { date: '2026-02-30' }, + { date: '2026-09-10garbage' }, + {}, + { dateTime: '2026-09-10T12:00:00' }, + ]) { + const input = fixture(); + input.records[0].values = { start }; + expect(() => calendarProjection(input)).toThrow(); + } +}); +it('leaves other platforms untouched', () => { + const input = fixture(); + input.platform = 'pets'; + expect(calendarProjection(input)).toBe(input); +}); +it("repeated imports reconcile IDs, preserve local fields and don't delete out-of-window events", () => { + const data = calendarProjection(fixture()); + const properties = Object.fromEntries( + Object.keys(data.records[0].values).map(k => [ + k, + `https://example.com/${k}`, + ]), + ); + const config = { + platform: data.platform, + destinations: { + event: { table: 'did:ad:table', rowClass: 'did:ad:event' }, + }, + properties, + records: data.records, + }; + const first = run({ config, query: () => [], read: () => ({}) }).intents[0]; + if (first.op !== 'create') throw new Error('Expected create'); + const saved = { + ...first.set, + 'https://atomicdata.dev/properties/parent': 'did:ad:table', + 'https://atomicdata.dev/properties/isA': ['did:ad:event'], + 'https://example.com/private': 'my notes', + }; + const host = { + config, + query: (p: string, v: string) => (saved[p] === v ? ['did:ad:row'] : []), + read: () => saved, + }; + expect(run(host).intents).toHaveLength(0); + expect( + run({ ...host, config: { ...config, records: [] } }).intents, + ).toHaveLength(0); + const changed = calendarProjection(fixture()); + changed.records[0].name = 'Updated meeting'; + const result = run({ + ...host, + config: { ...config, records: changed.records }, + }); + expect(result.intents[0]).toMatchObject({ + op: 'set', + subject: 'did:ad:row', + set: { 'https://atomicdata.dev/properties/name': 'Updated meeting' }, + }); + expect(JSON.stringify(result.intents)).not.toContain('my notes'); + const secondCalendar = { ...data.records[0], namespace: 'calendar-b' }; + expect( + run({ ...host, config: { ...config, records: [secondCalendar] } }) + .intents[0].op, + ).toBe('create'); +}); +it('recognizes the lowercase field names produced by the WASM ontology', () => { + const input = fixture(); + delete input.records[0].values['recurring-event-id']; + input.records[0].values.recurringeventid = 'series'; + input.records[0].values.conferencedata = { conferenceId: 'meeting' }; + const notes = calendarProjection(input).records[0].values[fields.notes]; + expect(notes).toContain('Recurring event'); + expect(notes).toContain('Conferencing'); +}); diff --git a/integrations/localthought/calendar.ts b/integrations/localthought/calendar.ts new file mode 100644 index 0000000000..f9706e8792 --- /dev/null +++ b/integrations/localthought/calendar.ts @@ -0,0 +1,146 @@ +// @wc-ignore-file +import { Datatype } from '../../browser/lib/src/index.js'; +import type { JSONValue } from '../../browser/lib/src/value.js'; +import type { FetchedPlatform, Term } from './schema.js'; + +export const calendarFields = { + day: 'atomic-calendar-day', + allDay: 'atomic-calendar-all-day', + notes: 'atomic-calendar-notes', +}; + +/** An additional projection, never a replacement for the provider's fields. + * One DATE column supports both all-day dates and timed events in a single view. + * Timed events use the date in Google's supplied offset; raw start/end retain + * the instant, zone and exclusive-end semantics for future richer rendering. + */ +export function calendarProjection(fetched: FetchedPlatform): FetchedPlatform { + if (fetched.platform !== 'google-calendar') return fetched; + const event = fetched.ontology.terms.find( + t => t.kind === 'class' && t.shortname === 'event', + ); + if (!event) return fetched; + const definitions: [string, Datatype, string][] = [ + [ + calendarFields.day, + Datatype.DATE, + "Start date in the event's supplied offset; all-day dates stay unchanged.", + ], + [ + calendarFields.allDay, + Datatype.BOOLEAN, + 'Whether Google represents this as an all-day event.', + ], + [ + calendarFields.notes, + Datatype.STRING, + 'Display limitations and Google-only features. Edits here remain local.', + ], + ]; + const terms: Term[] = definitions.map( + ([shortname, datatype, description]) => ({ + path: `urn:atomic:google-calendar:${shortname}`, + kind: 'property', + shortname, + datatype, + description, + requires: [], + recommends: [], + }), + ); + if ( + fetched.ontology.terms.some(t => + terms.some(extra => extra.shortname === t.shortname), + ) + ) + throw new Error( + 'Calendar projection property collides with provider ontology', + ); + return { + ...fetched, + ontology: { + ...fetched.ontology, + terms: [ + ...fetched.ontology.terms.map(t => + t === event + ? { + ...t, + recommends: [ + ...t.recommends, + ...terms.map(extra => extra.path), + ], + } + : t, + ), + ...terms, + ], + }, + records: fetched.records.map(row => { + if (row.resource !== 'event') return row; + const start = object(row.values.start); + const end = object(row.values.end); + const date = start.date ?? start.dateTime; + const cancelled = row.values.status === 'cancelled'; + if ( + !cancelled && + (typeof date !== 'string' || !validDay(date.slice(0, 10))) + ) + throw new Error(`Calendar event ${row.id} has no valid start date`); + if ( + start.date !== undefined && + (typeof start.date !== 'string' || !validDay(start.date)) + ) + throw new Error(`Calendar event ${row.id} has an invalid all-day date`); + if ( + start.dateTime !== undefined && + (typeof start.dateTime !== 'string' || + !/^\d{4}-\d{2}-\d{2}T.*(?:Z|[+-]\d{2}:\d{2})$/.test(start.dateTime) || + !Number.isFinite(Date.parse(start.dateTime))) + ) + throw new Error( + `Calendar event ${row.id} has no offset-qualified start time`, + ); + const notes = ['One-way import: edits stay in Atomic']; + if (cancelled) notes.push('Cancelled in Google; retained in Atomic'); + if ( + row.values['recurring-event-id'] || + row.values.recurringEventId || + row.values.recurringeventid || + row.values.recurrence + ) + notes.push('Recurring event: manage the series in Google'); + if (Array.isArray(row.values.attendees) && row.values.attendees.length) + notes.push('Attendees and RSVP: manage in Google'); + if (row.values.reminders) notes.push('Reminders: manage in Google'); + if ( + row.values['conference-data'] || + row.values.conferenceData || + row.values.conferencedata + ) + notes.push('Conferencing: manage in Google'); + if (end.date || end.dateTime) + notes.push('Duration retained in End; shown on start day only'); + const values: Record = { + ...row.values, + [calendarFields.notes]: notes.join('. '), + }; + if (typeof date === 'string' && validDay(date.slice(0, 10))) { + values[calendarFields.day] = date.slice(0, 10); + values[calendarFields.allDay] = typeof start.date === 'string'; + } + return { ...row, values }; + }), + }; +} +function object(value: JSONValue | undefined): Record { + return value && typeof value === 'object' && !Array.isArray(value) + ? value + : {}; +} +function validDay(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const date = new Date(`${value}T00:00:00Z`); + return ( + Number.isFinite(date.getTime()) && date.toISOString().slice(0, 10) === value + ); +} diff --git a/integrations/localthought/mock-calendar.mjs b/integrations/localthought/mock-calendar.mjs new file mode 100644 index 0000000000..13ba1e576a --- /dev/null +++ b/integrations/localthought/mock-calendar.mjs @@ -0,0 +1,159 @@ +/** Synthetic Google Calendar fixtures. No real calendar data or provider writes. */ +const string = { type: 'string' }; +const dateTime = { + type: 'object', + properties: { + date: { ...string, format: 'date' }, + dateTime: { ...string, format: 'date-time' }, + timeZone: string, + }, +}; +export const calendarDocument = { + openapi: '3.0.3', + info: { title: 'Synthetic Calendar', version: 'v3' }, + servers: [{ url: 'https://www.googleapis.com/calendar/v3' }], + paths: { + '/calendars/{calendarId}/events': { + get: { + parameters: [ + { name: 'calendarId', in: 'path', required: true, schema: string }, + ...['pageToken', 'timeMin', 'timeMax'].map(name => ({ + name, + in: 'query', + schema: string, + })), + { name: 'singleEvents', in: 'query', schema: { type: 'boolean' } }, + ], + 'x-pagination': [{ scheme: 'pageToken' }], + responses: { + 200: { + description: 'Events', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + items: { + type: 'array', + items: { $ref: '#/components/schemas/event' }, + }, + nextPageToken: string, + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + paginationSchemes: { + pageToken: { + type: 'pageToken', + request: { queryParameters: { pageToken: { role: 'pageToken' } } }, + response: { bodyFields: { nextPageToken: { role: 'nextPageToken' } } }, + }, + }, + schemas: { + event: { + type: 'object', + properties: { + id: string, + summary: string, + status: string, + start: dateTime, + end: dateTime, + recurringEventId: string, + htmlLink: string, + attendees: { + type: 'array', + items: { + type: 'object', + properties: { email: string, responseStatus: string }, + }, + }, + }, + }, + }, + crudResources: { + event: { + schema: { $ref: '#/components/schemas/event' }, + identity: { + urlTemplate: '/calendars/{calendarId}/events/{eventId}', + bindings: { eventId: { field: 'id' } }, + }, + collections: { + events: { urlTemplate: '/calendars/{calendarId}/events' }, + }, + }, + }, + }, +}; +export function calendarFixture(day = new Date().toISOString().slice(0, 10)) { + const tomorrow = new Date(Date.parse(`${day}T00:00:00Z`) + 86400000) + .toISOString() + .slice(0, 10); + const events = [ + { + id: 'all-day', + summary: 'Calendar all-day fixture', + status: 'confirmed', + start: { date: day }, + end: { date: tomorrow }, + }, + { + id: 'timed', + summary: 'Calendar timed fixture', + status: 'confirmed', + start: { + dateTime: `${day}T00:30:00+02:00`, + timeZone: 'Europe/Amsterdam', + }, + end: { dateTime: `${day}T01:30:00+02:00`, timeZone: 'Europe/Amsterdam' }, + recurringEventId: 'series', + attendees: [ + { email: 'synthetic@example.com', responseStatus: 'accepted' }, + ], + }, + ]; + const requests = []; + return { + events, + requests, + request(method, url) { + requests.push({ + method, + path: url.pathname, + query: Object.fromEntries(url.searchParams), + }); + if (method !== 'GET') return { status: 405, body: {} }; + if ( + !/^\/proxy\/google-calendar\/calendar\/v3\/calendars\/[^/]+\/events$/.test( + url.pathname, + ) + ) + return { status: 404, body: {} }; + // Require the production adapter to send its safety bounds and recurrence expansion. + if ( + !url.searchParams.has('timeMin') || + !url.searchParams.has('timeMax') || + url.searchParams.get('singleEvents') !== 'true' + ) + return { + status: 400, + body: { error: 'Missing bounded recurrence query' }, + }; + const token = url.searchParams.get('pageToken'); + if (token && token !== 'second') return { status: 400, body: {} }; + return { + status: 200, + body: structuredClone( + token + ? { items: events.slice(1) } + : { items: events.slice(0, 1), nextPageToken: 'second' }, + ), + }; + }, + }; +} diff --git a/integrations/localthought/mock-github.mjs b/integrations/localthought/mock-github.mjs new file mode 100644 index 0000000000..629a1e28e1 --- /dev/null +++ b/integrations/localthought/mock-github.mjs @@ -0,0 +1,108 @@ +/** Stateful provider fixture, shared by the HTTP proxy and its test-side driver. */ +export function githubTracker() { + const repositories = new Map(); + const repo = name => { + if (!repositories.has(name)) + repositories.set(name, { issues: [], comments: [] }); + return repositories.get(name); + }; + let id = 0; + const now = () => new Date().toISOString(); + const api = { + snapshot: name => structuredClone(repo(name)), + createIssue(name, input) { + const state = repo(name); + const number = state.issues.length + 1; + const issue = { + id: ++id, + number, + title: input.title, + body: input.body ?? '', + state: 'open', + labels: [], + url: `https://api.github.com/repos/${name}/issues/${number}`, + html_url: `https://github.com/${name}/issues/${number}`, + user: { login: 'mock-user' }, + created_at: now(), + updated_at: now(), + }; + state.issues.push(issue); + return structuredClone(issue); + }, + updateIssue(name, number, input) { + const issue = repo(name).issues.find(i => i.number === number); + if (!issue) return; + for (const field of ['title', 'body', 'state', 'labels']) + if (input[field] !== undefined) issue[field] = input[field]; + issue.updated_at = now(); + return structuredClone(issue); + }, + createComment(name, number, input) { + if (!repo(name).issues.some(i => i.number === number)) return; + const comment = { + id: ++id, + body: input.body, + issue_url: `https://api.github.com/repos/${name}/issues/${number}`, + user: { login: 'mock-commenter' }, + created_at: now(), + updated_at: now(), + }; + repo(name).comments.push(comment); + return structuredClone(comment); + }, + request(method, url, input = {}) { + const match = url.pathname.match( + /^\/proxy\/github-issues\/repos\/([^/]+\/[^/]+)\/issues(?:\/(.*))?$/, + ); + if (!match) return { status: 404, body: {} }; + const [, name, tail = ''] = match; + const state = repo(name); + const page = Number(url.searchParams.get('page') ?? 1); + const size = Number(url.searchParams.get('per_page') ?? 100); + const paginate = rows => rows.slice((page - 1) * size, page * size); + let value; + if (!tail) { + if (method === 'GET') value = paginate(state.issues); + if (method === 'POST') value = api.createIssue(name, input); + } else if (/^comments\/\d+$/.test(tail)) { + const comment = state.comments.find( + c => c.id === Number(tail.split('/')[1]), + ); + if (method === 'GET') value = comment; + if (method === 'PATCH' && comment) + value = Object.assign(comment, { + body: input.body, + updated_at: now(), + }); + } else { + const [numberText, resource, label] = tail.split('/'); + const number = Number(numberText); + const issue = state.issues.find(i => i.number === number); + if (issue && !resource) { + if (method === 'GET') value = issue; + if (method === 'PATCH') value = api.updateIssue(name, number, input); + } else if (issue && resource === 'comments') { + if (method === 'GET') + value = paginate( + state.comments.filter(c => c.issue_url === issue.url), + ); + if (method === 'POST') value = api.createComment(name, number, input); + } else if (issue && resource === 'labels') { + if (method === 'POST') + value = issue.labels = [ + ...new Set([...issue.labels, ...input.labels]), + ]; + if (method === 'DELETE') + value = issue.labels = issue.labels.filter( + l => l !== decodeURIComponent(label), + ); + } + } + return { + status: value === undefined ? 404 : method === 'POST' ? 201 : 200, + body: structuredClone(value ?? {}), + }; + }, + }; + return api; +} diff --git a/integrations/localthought/mock-proxy.d.mts b/integrations/localthought/mock-proxy.d.mts new file mode 100644 index 0000000000..72bdfcb1bd --- /dev/null +++ b/integrations/localthought/mock-proxy.d.mts @@ -0,0 +1,49 @@ +import type { Server } from 'node:http'; +interface Issue { + id: number; + number: number; + title: string; + body: string; + state: string; + labels: string[]; +} +interface Comment { + id: number; + body: string; + issue_url: string; +} +interface GitHubTracker { + snapshot(repository: string): { issues: Issue[]; comments: Comment[] }; + createIssue( + repository: string, + input: { title: string; body?: string }, + ): Issue; + updateIssue( + repository: string, + number: number, + input: Partial, + ): Issue | undefined; + createComment( + repository: string, + number: number, + input: { body: string }, + ): Comment | undefined; +} +interface CalendarFixture { + events: Array<{ + id: string; + summary: string; + status: string; + start: { date?: string; dateTime?: string; timeZone?: string }; + end: { date?: string; dateTime?: string; timeZone?: string }; + }>; + requests: Array<{ + method: string; + path: string; + query: Record; + }>; +} +export const tenantSecret: string; +export function mockProxy(options?: { + frontendOrigin?: string; +}): Server & { github: GitHubTracker; calendar: CalendarFixture }; diff --git a/integrations/localthought/mock-proxy.mjs b/integrations/localthought/mock-proxy.mjs index 8560e618b7..445f8d1749 100644 --- a/integrations/localthought/mock-proxy.mjs +++ b/integrations/localthought/mock-proxy.mjs @@ -1,48 +1,169 @@ /** Local-only integration-proxy fixture. Never deploy this service. */ +import { calendarDocument, calendarFixture } from './mock-calendar.mjs'; +import { githubTracker } from './mock-github.mjs'; import { readFileSync } from 'node:fs'; import { createServer } from 'node:http'; import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; import { pathToFileURL } from 'node:url'; export const tenantSecret = 'bW9jay10ZW5hbnQ.mock-signature'; -const sign = value => createHmac('sha256', tenantSecret).update(value).digest('base64url'); -const equal = (a, b) => typeof a === 'string' && a.length === b.length && timingSafeEqual(Buffer.from(a), Buffer.from(b)); -const pets = ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles'].map((name, i) => ({ id: i + 1, name, species: ['Dog', 'Cat', 'Bird', 'Rabbit', 'Fish'][i], age: i + 1, vaccinated: i % 2 === 0, weight: i + 0.5, updated_at: '2026-09-09T00:00:00Z' })); -export function mockProxy() { +const sign = value => + createHmac('sha256', tenantSecret).update(value).digest('base64url'); +const equal = (a, b) => + typeof a === 'string' && + a.length === b.length && + timingSafeEqual(Buffer.from(a), Buffer.from(b)); +const pets = ['Rex', 'Whiskers', 'Tweety', 'Nibbles', 'Bubbles'].map( + (name, i) => ({ + id: i + 1, + name, + species: ['Dog', 'Cat', 'Bird', 'Rabbit', 'Fish'][i], + age: i + 1, + vaccinated: i % 2 === 0, + weight: i + 0.5, + updated_at: '2026-09-09T00:00:00Z', + }), +); +export function mockProxy({ + frontendOrigin = process.env.MOCK_FRONTEND_ORIGIN ?? 'http://localhost:6747', +} = {}) { + const github = githubTracker(); + const calendar = calendarFixture(); const codes = new Map(); const challenges = new Set(); - const issueCode = platform => { const code = randomBytes(32).toString('base64url'); codes.set(code, platform); return code; }; - return createServer((req, res) => { + const issueCode = platform => { + const code = randomBytes(32).toString('base64url'); + codes.set(code, platform); + return code; + }; + const server = createServer(async (req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader( + 'Access-Control-Allow-Methods', + 'GET, POST, PATCH, DELETE, OPTIONS', + ); + res.setHeader( + 'Access-Control-Allow-Headers', + 'Authorization, Content-Type', + ); + res.setHeader('Access-Control-Expose-Headers', 'X-Connection-Code, Link'); + if (req.method === 'OPTIONS') { + res.writeHead(204); + return res.end(); + } + const url = new URL(req.url, 'http://localhost'); - const json = (status, value, headers = {}) => { res.writeHead(status, { 'Content-Type': 'application/json', ...headers }); res.end(JSON.stringify(value)); }; - if (url.pathname === '/catalog') return json(200, ['github-issues', 'google-calendar', 'pets']); - if (url.pathname === '/catalog/pets.yaml') { res.writeHead(200, { 'Content-Type': 'application/yaml' }); return res.end(readFileSync(new URL('./mock-document.json', import.meta.url))); } - if (url.pathname === '/session') { const challenge = randomBytes(32).toString('base64url'); challenges.add(challenge); return json(200, { ts: Math.floor(Date.now() / 1000), nonce: randomBytes(16).toString('hex'), challenge }); } + const json = (status, value, headers = {}) => { + res.writeHead(status, { 'Content-Type': 'application/json', ...headers }); + res.end(JSON.stringify(value)); + }; + if (url.pathname === '/catalog') + return json(200, ['github-issues', 'google-calendar', 'pets']); + if (url.pathname === '/catalog/pets.yaml') { + res.writeHead(200, { 'Content-Type': 'application/yaml' }); + return res.end( + readFileSync(new URL('./mock-document.json', import.meta.url)), + ); + } + if (url.pathname === '/catalog/google-calendar.yaml') + return json(200, calendarDocument); + if (url.pathname === '/session') { + const challenge = randomBytes(32).toString('base64url'); + challenges.add(challenge); + return json(200, { + ts: Math.floor(Date.now() / 1000), + nonce: randomBytes(16).toString('hex'), + challenge, + }); + } if (url.pathname === '/connect') { const p = url.searchParams; const challenge = p.get('challenge'); - if (!challenges.has(challenge) || !equal(p.get('response'), sign(challenge)) || !equal(p.get('user_id_sig'), sign(p.get('user_id') ?? '')) || p.get('tenant_id') !== 'mock-tenant') return json(401, { error: 'Invalid tenant proof' }); + if ( + !challenges.has(challenge) || + !equal(p.get('response'), sign(challenge)) || + !equal(p.get('user_id_sig'), sign(p.get('user_id') ?? '')) || + p.get('tenant_id') !== 'mock-tenant' + ) + return json(401, { error: 'Invalid tenant proof' }); const redirect = new URL(p.get('redirect_uri')); - if (redirect.origin !== (process.env.MOCK_FRONTEND_ORIGIN ?? 'http://localhost:6747') || redirect.pathname !== '/app/integrations') return json(400, { error: 'Invalid callback' }); + if ( + redirect.origin !== frontendOrigin || + !['/app/integrations', '/app/devonian-demo'].includes(redirect.pathname) + ) + return json(400, { error: 'Invalid callback' }); const platform = p.get('platform'); - if (!['github-issues', 'google-calendar', 'pets'].includes(platform)) return json(400, { error: 'Invalid platform' }); - if (req.method === 'GET') { res.writeHead(200, { 'Content-Type': 'text/html' }); return res.end('

Mock integration proxy

Connect your test account.

'); } + if (!['github-issues', 'google-calendar', 'pets'].includes(platform)) + return json(400, { error: 'Invalid platform' }); + if (req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'text/html' }); + return res.end( + '

Mock integration proxy

Connect your test account.

', + ); + } if (req.method !== 'POST') return json(405, {}); challenges.delete(challenge); redirect.searchParams.set('connection_code', issueCode(platform)); - res.writeHead(303, { Location: redirect.href }); return res.end(); + res.writeHead(303, { Location: redirect.href }); + return res.end(); } if (url.pathname.startsWith('/proxy/')) { const code = req.headers.authorization?.replace(/^Bearer /, ''); const platform = codes.get(code); - if (!platform) return json(401, { error: 'Invalid or consumed connection code' }); + if (!platform) + return json(401, { error: 'Invalid or consumed connection code' }); codes.delete(code); - if (req.method !== 'GET' || !url.pathname.startsWith(`/proxy/${platform}/`)) return json(403, {}); - const data = platform === 'pets' ? (url.searchParams.get('page') === '2' ? pets.slice(2) : pets.slice(0, 2)) : platform === 'google-calendar' ? { items: [{ id: 'event-1', summary: 'Team meeting' }] } : [{ id: 42, title: 'Fetched GitHub issue', number: 42, state: 'open' }]; - return json(200, data, { 'X-Connection-Code': issueCode(platform), ...(platform === 'pets' && !url.searchParams.has('page') ? { Link: '; rel="next"' } : {}) }); + const headers = { 'X-Connection-Code': issueCode(platform) }; + if (!url.pathname.startsWith(`/proxy/${platform}/`)) + return json(403, {}, headers); + if (platform === 'github-issues') { + try { + let body = ''; + for await (const chunk of req) { + body += chunk; + if (body.length > 1024 * 1024) return json(413, {}, headers); + } + const result = github.request( + req.method, + url, + body ? JSON.parse(body) : {}, + ); + return json(result.status, result.body, headers); + } catch { + return json(400, { error: 'Invalid request body' }, headers); + } + } + if (platform === 'google-calendar') { + const result = calendar.request(req.method, url); + return json(result.status, result.body, headers); + } + if (req.method !== 'GET') return json(403, {}, headers); + const data = + platform === 'pets' + ? url.searchParams.get('page') === '2' + ? pets.slice(2) + : pets.slice(0, 2) + : { items: [{ id: 'event-1', summary: 'Team meeting' }] }; + return json(200, data, { + ...headers, + ...(platform === 'pets' && !url.searchParams.has('page') + ? { Link: '; rel="next"' } + : {}), + }); } json(404, {}); }); + server.github = github; + server.calendar = calendar; + return server; } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - mockProxy().listen(Number(process.env.MOCK_PROXY_PORT ?? 19090), process.env.MOCK_PROXY_HOST ?? '127.0.0.1', () => console.log('Mock integration proxy listening on http://127.0.0.1:19090')); +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + mockProxy().listen( + Number(process.env.MOCK_PROXY_PORT ?? 19090), + process.env.MOCK_PROXY_HOST ?? '127.0.0.1', + () => + console.log('Mock integration proxy listening on http://127.0.0.1:19090'), + ); } diff --git a/integrations/localthought/syncables/Cargo.toml b/integrations/localthought/syncables/Cargo.toml new file mode 100644 index 0000000000..d162e2ac8d --- /dev/null +++ b/integrations/localthought/syncables/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "syncables" +version = "0.1.0" +edition = "2021" +rust-version = "1.82" +description = "Reads an OpenAPI document to run a mock API server and a client that syncs a local copy of the API's dataset." +authors = ["Michiel de Jong "] +license = "Apache-2.0" +repository = "https://github.com/localthought/syncables-rs" +readme = "README.md" +keywords = ["openapi", "pagination", "sync", "mock-server", "local-first"] +categories = ["web-programming", "api-bindings"] + +[dependencies] +async-trait = "0.1" +httpdate = "1" +indexmap = { version = "2", features = ["serde"] } +percent-encoding = "2" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +serde_yaml_ng = "0.10" +thiserror = "2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +tokio = { version = "1", features = ["fs"] } + +[dev-dependencies] +uuid = { version = "1", features = ["v4"] } +tokio = { version = "1", features = ["full"] } + +[lints.rust] +missing_docs = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +pedantic = { level = "warn", priority = -1 } +module_name_repetitions = "allow" +# The module tree deliberately mirrors the TypeScript original's `src/` +# layout one-to-one, which puts `client/client.rs` inside `client/`. +module_inception = "allow" +# "OpenAPI", "JSONPath" and friends are proper nouns, not code. +doc_markdown = "allow" +# `schema` vs. `scheme` are the spec's own two words, not a typo. +similar_names = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +must_use_candidate = "allow" diff --git a/integrations/localthought/syncables/LICENSE b/integrations/localthought/syncables/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/integrations/localthought/syncables/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/integrations/localthought/syncables/README.md b/integrations/localthought/syncables/README.md new file mode 100644 index 0000000000..f48da88994 --- /dev/null +++ b/integrations/localthought/syncables/README.md @@ -0,0 +1,209 @@ +# syncables-rs + +A Rust port of [localthought/syncables](https://github.com/localthought/syncables). + +Reads an OpenAPI document and gives you: + +- a **mock API server** that implements it, backed by a real (in-memory) + CRUD store per resource, seeded with fake data generated from the + document's schemas; +- an **API client** that talks to any server implementing that OpenAPI + document and keeps a local copy of each resource collection in sync. + +Alongside that port, the crate also carries a second, unrelated surface: +a **sync engine** (`SyncClient`) that reads a document's +[CRUD Causality Extension](https://github.com/pondersource/openapi-extensions/tree/main/spec/crud-causality) +(`components.crudResources`) and syncs records — including nested +collections, walked once per parent record — into a host-provided +`Storage` implementation, deriving a neutral, Atomic-Data-shaped ontology +along the way without the crate itself depending on `atomic_lib`. This is +new scope, not part of the original TypeScript port; see +[Sync engine](#sync-engine) below. + +## Port status + +This section covers the original TypeScript port only — see +[Sync engine](#sync-engine) below for that surface's status. + +This crate is **scaffolding**. The module tree mirrors the TypeScript +original's `src/` one-to-one, and the following are ported and tested: + +| Area | Module | Status | +| --- | --- | --- | +| Document loading | `openapi::load`, `openapi::resolve_refs` | ported | +| Overlays | `openapi::overlay` | ported | +| OpenAPI type surface | `openapi::types` | ported | +| Resource discovery | `resources::discover` | ported | +| Path routing | `routing::router` | ported | +| Fake data | `fake_data::generate` | ported | +| Pagination | `pagination::{types, validate, autodetect, items, request_builder, response_parser}` | ported | +| Mock server store | `mock_server::store` | ported | +| Client storage | `client::storage` | ported | +| Mock server handler | `mock_server::server` | **scaffolded** — public surface only | +| Client sync/writes | `client::client` | **scaffolded** — public surface only | + +The two scaffolded modules carry their full ported public API, doc +comments and constants; their function bodies are `todo!()`, each naming +the TypeScript source file and function it is to be ported from. Nothing +in this crate silently returns a wrong answer — the parts that exist are +covered by tests, including acceptance tests against unmodified real-world +documents. + +## Usage + +```sh +cargo build +cargo test +cargo clippy --all-targets -- -D warnings +cargo fmt --all --check +``` + +Once the client and mock server are ported, the shape will be: + +```rust,ignore +use syncables::{create_api_client, create_mock_server, load_open_api_document, ApiClientOptions}; + +let document = load_open_api_document("./petstore.yaml").await?; + +let server = create_mock_server(document.clone()); +let address = server.listen(None).await?; + +let client = create_api_client(document, ApiClientOptions::new(address.url)); +client.sync().await?; // pulls every discovered resource collection into local storage + +let pets = client.list("/pets").await?; +``` + +A "resource" is any pair of an OpenAPI collection path and its matching +item path, e.g. `/pets` and `/pets/{petId}`. Paths without that pairing +(health checks, one-off actions, etc.) are served from their documented +examples/schemas but aren't treated as syncable resources. + +## Sync engine + +New scope, not part of the original TypeScript port; tracked by +[issues #1–#9](https://github.com/localthought/syncables-rs/issues/1). +Where the port above discovers resources from plain collection/item path +pairing, the sync engine derives a richer resource model from a +document's [CRUD Causality Extension](https://github.com/pondersource/openapi-extensions/tree/main/spec/crud-causality) +(`components.crudResources`) — including nested collections, like a +repository's issues and each issue's comments — and drives a full, +paginated read of every collection into a host-provided `Storage`, +deriving a neutral ontology (Classes and Properties, Atomic-Data-shaped +but not Atomic-Data-typed) along the way. + +| Area | Module | Status | +| --- | --- | --- | +| Resource model (`crudResources`, `x-crud`) | `sync::resource_model` | ported and tested | +| Binding configured constants into path templates | `sync::constants` | ported and tested | +| Credentials, API base URL | `sync::credentials` | ported and tested | +| Ontology derivation | `sync::ontology` | ported and tested | +| `Storage` trait, `InMemoryStorage` | `sync::storage` | ported and tested | +| `SyncClient::sync()` — full read | `sync::client` | ported and tested | +| `SyncClient` — local-first write-back | `sync::client` | **not implemented** — [#9](https://github.com/localthought/syncables-rs/issues/9) | + +```rust,ignore +use std::sync::Arc; +use syncables::{ClientConfig, Credentials, InMemoryStorage, SyncClient}; + +let config = ClientConfig { + document: "./github-issues.openapi.yaml".into(), + overlays: vec!["./auth-overlay.yaml".into(), "./crud-causality-overlay.yaml".into()], + credentials: Credentials::Bearer(std::env::var("GITHUB_TOKEN")?), + constants: [("owner".to_string(), "localthought".to_string()), + ("repo".to_string(), "test-repo-1".to_string())].into(), + ontology_base_url: "https://my-ontologies.com".to_string(), +}; + +let client = SyncClient::new(config, Arc::new(my_fetch_impl))?; +let storage = InMemoryStorage::new(); +let report = client.sync(&storage).await?; // walks issues, then each issue's comments +``` + +`SyncClient::new` takes an `Arc` (the same injectable-transport +trait `ApiClient` above uses) alongside `ClientConfig` — the crate has no +HTTP client dependency of its own, so a host supplies one. This is the one +deliberate divergence from the `ClientConfig`/`SyncClient` contract +[`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +is already written against in its `src/syncables.rs`, which otherwise this +module matches field-for-field; that module is meant to be deleted once +reflector-rs points its `use`s here instead. + +## Differences from the TypeScript original + +The port keeps the original's names, comments and behaviour wherever it +can. Where Rust forced a decision, it went like this: + +- **`Record` → `serde_json::Map`**, and + `unknown` → `serde_json::Value`. Maps that need document order + (schema properties, paths, collections) use `IndexMap`, and + `serde_json` is built with `preserve_order`: `locate_items_field` picks + the *first* array-typed property, so order is load-bearing. +- **The injectable `fetch` becomes a `Fetch` trait** + (`client::client::Fetch`). As in the original, it is the only extension + point — there is no built-in notion of auth, so authenticating means + supplying an implementation that adds the right header to every request. +- **`StorageAdapter` is an `async_trait`**, with `InMemoryStorageAdapter` + as the default, matching the original's pluggable adapter. +- **Pagination roles stay strings** rather than becoming closed enums: the + spec allows `x-` extension roles, and `validate` is what decides + validity, so an unknown role has to survive parsing to be reported. + `PaginationSchemeObject::type` likewise deserializes leniently, so one + malformed scheme (e.g. Giphy's former `type: offset`) does not fail the + whole document. +- **Errors are a `thiserror` enum** (`syncables::Error`) instead of thrown + strings; every fallible function returns `syncables::Result`. +- **Numbers are `f64`** in `PaginationResponseState`, mirroring JavaScript + number semantics for values read out of arbitrary JSON bodies. + +## Tests + +`tests/unit/` mirrors the original's `__tests__/unit/` layout module for +module. Cargo builds it as one test binary (`tests/unit/main.rs`), so the +module tree is declared there rather than discovered per file. + +`tests/fixtures/real-world/` holds real OpenAPI documents and pagination +overlays vendored unmodified from apis.guru and localthought/overlays (see +the header comment in each file for provenance). The acceptance tests run +the ported pipeline against these and deliberately document real quirks +rather than working around them. When extending them, keep that spirit: +assert what actually happens against the unmodified real document, not an +idealized result. + +## NLnet milestone 1 + +The TypeScript original is the reference implementation for +[milestone 1](https://github.com/tubsproject/syncables/blob/main/nlnet-milestones.md#1-syncables) +of the project's NLnet grant. This port tracks it. + +## Generative AI use + +syncables-rs is developed collaboratively with **Claude Code** +(Anthropic), an agentic coding assistant: a human directs the design and +reviews, edits, and tests the changes it proposes before they're +committed. + +As an NLnet-funded project, this follows +[NLnet's Generative AI policy](https://nlnet.nl/foundation/policies/generativeAI/): + +- Commits produced with AI assistance carry a `Claude-Session: ` + trailer identifying the session that produced them. +- [`docs/ai-logs/`](docs/ai-logs) holds prompt/output disclosure logs, + redacted for secrets and personal information. +- AI-drafted content is reviewed and edited by a human before being + committed; it is not represented as unassisted human work. + +## License + +Apache-2.0, matching the original. + +## Browser / WASM + +`cargo check --target wasm32-unknown-unknown --lib` builds the engine without +Tokio filesystem or native runtime dependencies. Load catalog text with +`openapi::load::parse_yaml`, pass the resulting value to +`load_open_api_document`, then call `SyncClient::sync_document(&doc, &storage)`. +This path never opens `ClientConfig.document` or `overlays`; apply overlays in +memory before calling it. File sources return an explicit unsupported error +in WASM. Implement browser `Fetch` with `#[async_trait(?Send)]`; it may hold +a JS callback. Native `Fetch` keeps its Send + Sync contract. diff --git a/integrations/localthought/syncables/UPSTREAM.md b/integrations/localthought/syncables/UPSTREAM.md new file mode 100644 index 0000000000..1cd8d62970 --- /dev/null +++ b/integrations/localthought/syncables/UPSTREAM.md @@ -0,0 +1 @@ +Vendored from localthought/syncables-rs commit 0ab3521 (codex/browser-integrations), based on d48e4d9bad3ed9ec826d1c2040e989171701965d. This snapshot adds sync_document and target-specific transport/filesystem support. Replace with a pinned upstream dependency after that branch is merged. Apache-2.0; see LICENSE. diff --git a/integrations/localthought/syncables/src/client/client.rs b/integrations/localthought/syncables/src/client/client.rs new file mode 100644 index 0000000000..152700df4c --- /dev/null +++ b/integrations/localthought/syncables/src/client/client.rs @@ -0,0 +1,369 @@ +//! The API client: talks to any server implementing the document, and +//! keeps a local copy of each resource collection in sync. +//! +//! Reads are served from local storage ([`StorageAdapter`]; +//! [`InMemoryStorageAdapter`] is the default). `sync()` and the standalone +//! `paginate()` both walk every page of a paginated GET operation before +//! returning. +//! +//! `create`/`update`/`remove` are local-first: each writes to storage +//! immediately and returns without waiting on the network, then applies +//! itself against the server in the background via a per-record write +//! queue (keyed by `{resource}:{id}`, one write in flight at a time so +//! writes to the same record land in server order), retrying failures with +//! exponential backoff. +//! +//! Nothing in this crate reads an OpenAPI document's +//! `security`/`securitySchemes` — there is no built-in notion of auth. +//! [`ApiClientOptions::fetch`] is the only extension point, so +//! authenticating (a bearer token, an API key from an env var, etc.) means +//! passing a [`Fetch`] implementation that adds the right header to every +//! request. +//! +//! # Port status +//! +//! Scaffolding. The public surface, options and types below are ported; +//! the sync/write/pagination machinery is not yet implemented. + +// The `async` on the stubs below is part of the ported API surface, +// not an accident of the current `todo!()` bodies. +#![allow(clippy::unused_async)] + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use crate::error::Result; +use crate::openapi::types::OpenApiDocument; +use crate::resources::discover::{discover_resources, ResourceRoute}; + +use super::storage::{InMemoryStorageAdapter, StorageAdapter}; + +/// Hard ceiling on pages walked in one traversal, mirroring the original's +/// `MAX_PAGES`. +pub const MAX_PAGES: usize = 50; + +/// One outgoing HTTP request. +#[derive(Debug, Clone)] +pub struct HttpRequest { + /// HTTP method. + pub method: String, + /// Absolute URL. + pub url: String, + /// Request headers. + pub headers: IndexMap, + /// Request body, if any. + pub body: Option>, +} + +/// One HTTP response. +#[derive(Debug, Clone)] +pub struct HttpResponse { + /// HTTP status code. + pub status: u16, + /// Response headers. + pub headers: IndexMap, + /// Response body. + pub body: Vec, +} + +/// The client's only extension point for how requests reach the network. +/// +/// This is the Rust equivalent of the original's injectable `fetch`. +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait Fetch: FetchBounds { + /// Sends one request and returns the response. + async fn fetch(&self, request: HttpRequest) -> Result; +} + +/// How a failed background write is retried. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RetryOptions { + /// Delay before the first retry of a failed write. Doubles on each + /// subsequent attempt. Default 200ms. + pub base_delay: Duration, + /// Ceiling for the exponential backoff between retries. Default 30s. + pub max_delay: Duration, + /// Stop auto-retrying a write after this many attempts. Default is + /// unlimited (keep retrying until it succeeds). + pub max_attempts: Option, +} + +impl Default for RetryOptions { + fn default() -> Self { + Self { + base_delay: Duration::from_millis(200), + max_delay: Duration::from_secs(30), + max_attempts: None, + } + } +} + +/// How to build an [`ApiClient`]. +pub struct ApiClientOptions { + /// Base URL of the server to talk to. + pub base_url: String, + /// Where the local copy lives. Defaults to [`InMemoryStorageAdapter`]. + pub storage: Option>, + /// How requests reach the network. + pub fetch: Option>, + /// How failed background writes are retried. + pub retry: RetryOptions, + /// Record property that holds a resource's identity — the value used + /// as the local storage key, read back from a create response to + /// reconcile the server-assigned id, and substituted into the item + /// URL's path variable. + /// + /// Defaults to `id`. Set this when the API addresses a resource by a + /// different field (e.g. GitHub issues are keyed by `number`, not the + /// global `id` the payload also carries). + pub identity_field: String, +} + +impl ApiClientOptions { + /// Options pointing at `base_url`, with everything else defaulted. + pub fn new(base_url: impl Into) -> Self { + Self { + base_url: base_url.into(), + storage: None, + fetch: None, + retry: RetryOptions::default(), + identity_field: "id".to_string(), + } + } +} + +/// Options for one [`ApiClient::paginate`] traversal. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PaginateOptions { + /// Page size to request. Falls back to the server's own default when + /// omitted. + pub page_size: Option, +} + +/// What one [`ApiClient::sync`] changed. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SyncResult { + /// Collection paths whose local copy was actually added to, updated, + /// or pruned by this sync. + pub changed: Vec, +} + +/// Called after each sync while polling. +pub type SyncCallback = Box; +/// Called when a sync fails while polling. +pub type ErrorCallback = Box; + +/// How [`ApiClient::start_polling`] should behave. +pub struct PollOptions { + /// How often to call `sync()`. An initial sync runs immediately. + pub interval: Duration, + /// Called after every sync while polling, including ones where nothing + /// changed. + pub on_sync: Option, + /// Called when a sync fails while polling; polling continues on the + /// next interval. + pub on_error: Option, +} + +/// Cancels a running poll loop. +#[derive(Debug)] +pub struct PollingHandle { + _private: (), +} + +impl PollingHandle { + /// Stops future polling. Does not cancel a sync already in flight. + pub fn stop(self) { + todo!("port src/client/client.ts: startPolling") + } +} + +/// Which kind of write is still pending. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PendingWriteType { + /// A `create` not yet confirmed by the server. + Create, + /// An `update` not yet confirmed by the server. + Update, + /// A `remove` not yet confirmed by the server. + Delete, +} + +/// A write local storage already reflects, but the server has not yet +/// confirmed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PendingWriteInfo { + /// Collection path the write belongs to. + pub resource: String, + /// The id the write is filed under locally. For an unsettled `create`, + /// this is the client-generated id, not (yet) whatever the server + /// assigns. + pub id: String, + /// Which kind of write this is. + pub write_type: PendingWriteType, + /// How many attempts to reach the server have failed so far. + /// + /// If [`RetryOptions::max_attempts`] is set and reached, this stops + /// growing and the write stops auto-retrying — it stays listed until + /// `create`/`update`/`remove` is called again for the same record. + pub attempts: u32, + /// The most recent failure, if at least one attempt has failed. + pub last_error: Option, +} + +/// A local-first client for a server implementing the document. +pub struct ApiClient { + document: Arc, + routes: Arc>, + storage: Arc, + fetch: Option>, + options: Arc, +} + +impl ApiClient { + /// Collection paths this client knows how to sync. + pub fn resources(&self) -> Vec { + self.routes + .iter() + .map(|route| route.collection_path.clone()) + .collect() + } + + /// The document this client was built from. + pub fn document(&self) -> &OpenApiDocument { + &self.document + } + + /// Where the local copy lives. + pub fn storage(&self) -> &Arc { + &self.storage + } + + /// Pulls every discovered resource collection into local storage. + /// + /// Safe to call repeatedly: for a non-paginated collection it + /// conditionally re-fetches, keyed by exact request URL, sending + /// `If-None-Match`/`If-Modified-Since` from the prior response's + /// `ETag`/`Last-Modified` and treating a `304` as "nothing to do". + /// Even on a fresh `200`, or for a paginated collection, it only + /// touches storage for items that actually changed. + pub async fn sync(&self) -> Result { + let _ = &self.fetch; + let _ = &self.options; + todo!("port src/client/client.ts: sync") + } + + /// Calls [`Self::sync`] on `options.interval`, skipping a tick if the + /// previous sync is still running. + // `options` is consumed by the poll loop once this is implemented. + #[allow(clippy::needless_pass_by_value)] + pub fn start_polling(&self, options: PollOptions) -> PollingHandle { + let _ = options; + todo!("port src/client/client.ts: startPolling") + } + + /// Every locally held record of `resource`. + pub async fn list(&self, resource: &str) -> Result>> { + self.storage.list(resource).await + } + + /// One locally held record, by id. + pub async fn get(&self, resource: &str, id: &str) -> Result>> { + self.storage.get(resource, id).await + } + + /// Writes `data` to local storage immediately, under a client-generated + /// id (or `data[identity_field]`, if already set) and returns without + /// waiting on the network. + /// + /// The write to the server happens in the background and is retried on + /// failure — see [`Self::pending_writes`] for its outcome so far. If + /// the server assigns a different id than the one used locally, the + /// record is moved to it once the write settles. + pub async fn create( + &self, + resource: &str, + data: Map, + ) -> Result> { + let _ = (resource, data); + todo!("port src/client/client.ts: create") + } + + /// Merges `data` into the local copy of `id` immediately and returns + /// without waiting on the network; the corresponding `PUT` (or `PATCH`, + /// per [`ResourceRoute::update_method`]) is sent, and retried on + /// failure, in the background. + pub async fn update( + &self, + resource: &str, + id: &str, + data: Map, + ) -> Result> { + let _ = (resource, id, data); + todo!("port src/client/client.ts: update") + } + + /// Removes `id` from local storage immediately and returns without + /// waiting on the network; the corresponding `DELETE` is sent, and + /// retried on failure, in the background. + pub async fn remove(&self, resource: &str, id: &str) -> Result<()> { + let _ = (resource, id); + todo!("port src/client/client.ts: remove") + } + + /// Writes not yet confirmed by the server, across every resource (or + /// just `resource`, if given). + pub fn pending_writes(&self, resource: Option<&str>) -> Vec { + let _ = resource; + todo!("port src/client/client.ts: pendingWrites") + } + + /// Fetches every item from a GET list operation at `path`, walking + /// every page per its resolved pagination scheme (explicit + /// `x-pagination` or auto-detected from `components.paginationSchemes`). + /// + /// `path` need not be a discovered resource — any GET operation in the + /// document works, e.g. a search/listing endpoint with no paired item + /// route. + pub async fn paginate( + &self, + path: &str, + options: PaginateOptions, + ) -> Result>> { + let _ = (path, options); + todo!("port src/client/client.ts: paginate") + } +} + +/// Builds a client for `document` against the server in `options`. +pub fn create_api_client(document: OpenApiDocument, options: ApiClientOptions) -> ApiClient { + let routes = discover_resources(&document.paths); + let storage = options + .storage + .clone() + .unwrap_or_else(|| Arc::new(InMemoryStorageAdapter::new())); + let fetch = options.fetch.clone(); + ApiClient { + document: Arc::new(document), + routes: Arc::new(routes), + storage, + fetch, + options: Arc::new(options), + } +} + +/// Native transports must be thread safe; browser transports run on one JS thread. +#[cfg(not(target_arch = "wasm32"))] +pub trait FetchBounds: Send + Sync {} +#[cfg(not(target_arch = "wasm32"))] +impl FetchBounds for T {} +/// Browser transports may own JavaScript callbacks. +#[cfg(target_arch = "wasm32")] +pub trait FetchBounds {} +#[cfg(target_arch = "wasm32")] +impl FetchBounds for T {} diff --git a/integrations/localthought/syncables/src/client/mod.rs b/integrations/localthought/syncables/src/client/mod.rs new file mode 100644 index 0000000000..27ea15e9c5 --- /dev/null +++ b/integrations/localthought/syncables/src/client/mod.rs @@ -0,0 +1,5 @@ +//! An API client that talks to any server implementing the OpenAPI +//! document and keeps a local copy of each resource collection in sync. + +pub mod client; +pub mod storage; diff --git a/integrations/localthought/syncables/src/client/storage.rs b/integrations/localthought/syncables/src/client/storage.rs new file mode 100644 index 0000000000..65b2a20bef --- /dev/null +++ b/integrations/localthought/syncables/src/client/storage.rs @@ -0,0 +1,84 @@ +//! Where the client keeps its local copy of each resource collection. + +use std::collections::HashMap; +use std::sync::Mutex; + +use async_trait::async_trait; +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use crate::error::Result; + +/// Pluggable local storage for the client's copy of a collection. +/// +/// Implement this to persist somewhere other than memory; +/// [`InMemoryStorageAdapter`] is the default. +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait StorageAdapter: Send + Sync { + /// Every record held for `resource`. + async fn list(&self, resource: &str) -> Result>>; + /// One record, by id. + async fn get(&self, resource: &str, id: &str) -> Result>>; + /// Inserts or replaces a record. + async fn put(&self, resource: &str, id: &str, value: Map) -> Result<()>; + /// Removes a record. + async fn delete(&self, resource: &str, id: &str) -> Result<()>; +} + +/// One collection per resource path, each keyed by record id. +type Collections = HashMap>>; + +/// The default [`StorageAdapter`]: everything in process memory. +#[derive(Debug, Default)] +pub struct InMemoryStorageAdapter { + collections: Mutex, +} + +impl InMemoryStorageAdapter { + /// An adapter with no collections. + pub fn new() -> Self { + Self::default() + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl StorageAdapter for InMemoryStorageAdapter { + async fn list(&self, resource: &str) -> Result>> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + Ok(collections + .entry(resource.to_string()) + .or_default() + .values() + .cloned() + .collect()) + } + + async fn get(&self, resource: &str, id: &str) -> Result>> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + Ok(collections + .entry(resource.to_string()) + .or_default() + .get(id) + .cloned()) + } + + async fn put(&self, resource: &str, id: &str, value: Map) -> Result<()> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + collections + .entry(resource.to_string()) + .or_default() + .insert(id.to_string(), value); + Ok(()) + } + + async fn delete(&self, resource: &str, id: &str) -> Result<()> { + let mut collections = self.collections.lock().expect("storage mutex poisoned"); + collections + .entry(resource.to_string()) + .or_default() + .shift_remove(id); + Ok(()) + } +} diff --git a/integrations/localthought/syncables/src/error.rs b/integrations/localthought/syncables/src/error.rs new file mode 100644 index 0000000000..ebddbd6c05 --- /dev/null +++ b/integrations/localthought/syncables/src/error.rs @@ -0,0 +1,95 @@ +//! The crate's error type. + +use std::path::PathBuf; + +use thiserror::Error; + +/// Anything that can go wrong loading a document or talking to a server. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum Error { + /// Reading a document from disk failed. + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), + + /// A document could not be parsed as YAML or JSON. + #[error("could not parse document: {0}")] + Yaml(#[from] serde_yaml_ng::Error), + + /// A document did not fit the OpenAPI type surface this crate expects. + #[error("could not read document: {0}")] + Json(#[from] serde_json::Error), + + /// A document or overlay file could not be read or parsed. Wraps the + /// underlying [`Error::Io`] or [`Error::Yaml`] with the path that caused + /// it, since a bare i/o or parse error doesn't otherwise name the file. + #[error("could not load \"{path}\": {source}")] + FileLoad { + /// The file that could not be loaded. + path: PathBuf, + /// The underlying read or parse failure. + #[source] + source: Box, + }, + + /// An overlay used a JSONPath target outside the supported subset. + #[error( + "unsupported overlay target \"{0}\": only \"$\", simple dot-paths like \ + \"$.components...\", and quoted bracket segments like \ + \"$.paths['/pets'].get\" are supported" + )] + UnsupportedOverlayTarget(String), + + /// An overlay target segment resolved to something that isn't an object. + #[error("overlay target segment \"{0}\" does not resolve to an object")] + OverlayTargetNotAnObject(String), + + /// An overlay tried to `remove` the document root. + #[error("overlay cannot remove the document root")] + OverlayRemovesRoot, + + /// The client could not reach the server, or the server rejected the request. + #[error("http error: {0}")] + Http(String), + + /// A resource path was asked for that the document does not declare. + #[error("unknown resource \"{0}\"")] + UnknownResource(String), + + /// The document declares no `components.crudResources`; the + /// CRUD-causality overlay hasn't been applied. + #[error("document declares no crudResources; apply the CRUD-causality overlay")] + NoCrudResources, + + /// A constant named a parameter the document does not declare (path or + /// query, on any operation). A typo'd key here would otherwise sync + /// nothing, or scope the sync far wider than intended. + #[error("constant \"{0}\" does not name a parameter this document declares")] + UnknownConstant(String), + + /// A path template variable is bound by neither a constant, a parent + /// record's context provider, nor the resource's own identity binding. + #[error( + "path variable \"{0}\" is not bound by a constant, a parent record, \ + or an identity binding" + )] + UnboundContextParam(String), + + /// Two differently-named resources or fields normalized to the same + /// ontology shortname — the ontology would then mint one term for two + /// distinct things. + #[error( + "\"{first}\" and \"{second}\" both normalize to the ontology shortname \"{shortname}\"" + )] + ShortnameCollision { + /// The shortname both names normalized to. + shortname: String, + /// The name that claimed the shortname first. + first: String, + /// The name that collided with it. + second: String, + }, +} + +/// `Result` specialized to this crate's [`Error`]. +pub type Result = std::result::Result; diff --git a/integrations/localthought/syncables/src/fake_data/generate.rs b/integrations/localthought/syncables/src/fake_data/generate.rs new file mode 100644 index 0000000000..9602c95912 --- /dev/null +++ b/integrations/localthought/syncables/src/fake_data/generate.rs @@ -0,0 +1,83 @@ +//! Schema-to-value synthesis, shared by the mock server for seeding and +//! for example responses. +//! +//! A schema's own fixed `example` is reused verbatim on every call, so +//! callers generating more than one item from the same schema (resource +//! seeding, paginated list generation) must inject their own unique `id` +//! afterward rather than relying on the generated value to differ per item. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde_json::{Map, Number, Value}; + +use crate::openapi::types::SchemaObject; + +static STRING_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Synthesizes a value matching a JSON Schema, preferring an `example` on +/// the schema itself when present. +pub fn generate_from_schema(schema: Option<&SchemaObject>) -> Value { + let Some(schema) = schema else { + return Value::Null; + }; + if let Some(example) = &schema.example { + return example.clone(); + } + if let Some(enum_values) = &schema.enum_values { + if let Some(first) = enum_values.first() { + return first.clone(); + } + } + if let Some(all_of) = &schema.all_of { + let mut merged = Map::new(); + for sub in all_of { + if let Value::Object(object) = generate_from_schema(Some(sub)) { + merged.extend(object); + } + } + return Value::Object(merged); + } + if let Some(one_of) = &schema.one_of { + if let Some(first) = one_of.first() { + return generate_from_schema(Some(first)); + } + } + if let Some(any_of) = &schema.any_of { + if let Some(first) = any_of.first() { + return generate_from_schema(Some(first)); + } + } + + match schema.schema_type.as_deref() { + Some("string") => Value::String(generate_string(schema)), + Some("integer" | "number") => { + Number::from_f64(schema.minimum.unwrap_or(1.0)).map_or(Value::Null, Value::Number) + } + Some("boolean") => Value::Bool(true), + Some("array") => Value::Array(vec![generate_from_schema(schema.items.as_deref())]), + Some("object") | None => Value::Object(generate_object(schema)), + Some(_) => Value::Null, + } +} + +fn generate_object(schema: &SchemaObject) -> Map { + let mut result = Map::new(); + for (key, property_schema) in schema.properties.iter().flatten() { + result.insert(key.clone(), generate_from_schema(Some(property_schema))); + } + result +} + +fn generate_string(schema: &SchemaObject) -> String { + match schema.format.as_deref() { + Some("date-time") => "1970-01-01T00:00:00.000Z".to_string(), + Some("date") => "1970-01-01".to_string(), + Some("uuid") => "00000000-0000-4000-8000-000000000000".to_string(), + Some("email") => "user@example.com".to_string(), + Some("uri" | "url") => "https://example.com".to_string(), + _ => { + let next = STRING_COUNTER.fetch_add(1, Ordering::Relaxed) + 1; + format!("string-{next}") + } + } +} diff --git a/integrations/localthought/syncables/src/fake_data/mod.rs b/integrations/localthought/syncables/src/fake_data/mod.rs new file mode 100644 index 0000000000..8c6a87ef33 --- /dev/null +++ b/integrations/localthought/syncables/src/fake_data/mod.rs @@ -0,0 +1,3 @@ +//! Synthesizing values from OpenAPI schemas. + +pub mod generate; diff --git a/integrations/localthought/syncables/src/lib.rs b/integrations/localthought/syncables/src/lib.rs new file mode 100644 index 0000000000..0b9e3fc9f3 --- /dev/null +++ b/integrations/localthought/syncables/src/lib.rs @@ -0,0 +1,81 @@ +//! Reads an OpenAPI document and gives you: +//! +//! - a **mock API server** ([`create_mock_server`]) that implements it, +//! backed by a real (in-memory) CRUD store per resource, seeded with +//! fake data generated from the document's schemas; +//! - an **API client** ([`create_api_client`]) that talks to any server +//! implementing that OpenAPI document and keeps a local copy of each +//! resource collection in sync. +//! +//! Both understand the [OpenAPI Pagination Schemes Extension](https://github.com/pondersource/openapi-pagination-schemes-extension) +//! when a document declares `components.paginationSchemes`. +//! +//! A "resource" is any pair of an OpenAPI collection path and its matching +//! item path, e.g. `/pets` and `/pets/{petId}`. Paths without that pairing +//! (health checks, one-off actions, etc.) are served from their documented +//! examples/schemas but aren't treated as syncable resources. +//! +//! # Port status +//! +//! This crate is a port of [localthought/syncables](https://github.com/localthought/syncables) +//! (TypeScript) and is **scaffolding**: the module layout mirrors the +//! original's `src/` one-to-one, and the document, resource, pagination +//! and storage layers are ported. The mock server's request handler +//! ([`mock_server::server`]) and the client's sync/write machinery +//! ([`client::client`]) carry their full public surface but are not +//! implemented yet — those functions `todo!()`, each naming the +//! TypeScript source it is to be ported from. + +pub mod client; +pub mod error; +pub mod fake_data; +pub mod mock_server; +pub mod openapi; +pub mod pagination; +pub mod resources; +pub mod routing; +pub mod sync; + +pub use crate::error::{Error, Result}; + +pub use crate::openapi::load::{load_open_api_document, OpenApiSource}; +pub use crate::openapi::overlay::{ + apply_overlay, load_open_api_document_with_overlays, load_overlay, OverlayAction, + OverlayDocument, +}; +pub use crate::openapi::resolve_refs::resolve_refs; +pub use crate::openapi::types::{ + OpenApiDocument, OperationObject, ParameterObject, SchemaObject, ServerObject, +}; + +pub use crate::resources::discover::{discover_resources, ResourceRoute}; + +pub use crate::fake_data::generate::generate_from_schema; + +pub use crate::mock_server::server::{create_mock_server, MockServer}; + +pub use crate::client::client::{ + create_api_client, ApiClient, ApiClientOptions, PaginateOptions, PollOptions, PollingHandle, + SyncResult, +}; +pub use crate::client::storage::{InMemoryStorageAdapter, StorageAdapter}; + +pub use crate::pagination::autodetect::{resolve_effective_scheme, EffectiveScheme}; +pub use crate::pagination::types::{ + AutoDetectObject, PaginationApplicationObject, PaginationResponseState, PaginationSchemeObject, + PaginationSchemesMap, RequestRole, ResponseRole, SchemeType, +}; +pub use crate::pagination::validate::validate_pagination_scheme; + +pub use crate::sync::client::{ClientConfig, SyncClient, SyncError, SyncReport}; +pub use crate::sync::constants::{bind_url, validate_constants}; +pub use crate::sync::credentials::{base_url, Credentials}; +pub use crate::sync::ontology::{ + derive_ontology, ontology_shortname, Ontology, OntologyTerm, TermKind, +}; +pub use crate::sync::resource_model::{ + crud_operation, discover_resource_model, AddedField, CollectionMembership, ContextProvider, + CrudAction, CrudOperation, CrudResourceObject, IdentityBindingObject, ManagedCollection, + ResourceCollectionObject, ResourceIdentityObject, ResourceModel, +}; +pub use crate::sync::storage::{InMemoryStorage, Record, Storage, StorageError}; diff --git a/integrations/localthought/syncables/src/mock_server/mod.rs b/integrations/localthought/syncables/src/mock_server/mod.rs new file mode 100644 index 0000000000..68e080ed5e --- /dev/null +++ b/integrations/localthought/syncables/src/mock_server/mod.rs @@ -0,0 +1,6 @@ +//! A mock API server that implements an OpenAPI document, backed by a real +//! (in-memory) CRUD store per resource, seeded with fake data generated +//! from the document's schemas. + +pub mod server; +pub mod store; diff --git a/integrations/localthought/syncables/src/mock_server/server.rs b/integrations/localthought/syncables/src/mock_server/server.rs new file mode 100644 index 0000000000..2dd754b1b5 --- /dev/null +++ b/integrations/localthought/syncables/src/mock_server/server.rs @@ -0,0 +1,91 @@ +//! The mock server: an HTTP request handler generated from an OpenAPI +//! document. +//! +//! For a `GET` operation it first checks whether a pagination scheme +//! applies (see [`crate::pagination::autodetect`]) and, if so, serves it +//! as a paginated list; otherwise it treats the match as a resource +//! ([`ResourceStore`], CRUD semantics based on collection vs. item path +//! and HTTP method) or falls back to serving the operation's documented +//! example/generated schema response verbatim. Resource collections are +//! lazily seeded with [`SEED_COUNT`] fake records on first `GET`. +//! +//! # Port status +//! +//! Scaffolding. The types, constants and the public surface below are +//! ported; the request handler itself is not yet implemented. + +// The `async` on the stubs below is part of the ported API surface, +// not an accident of the current `todo!()` bodies. +#![allow(clippy::unused_async)] + +use std::sync::{Arc, Mutex}; + +use crate::error::Result; +use crate::openapi::types::OpenApiDocument; +use crate::resources::discover::{discover_resources, ResourceRoute}; + +use super::store::ResourceStore; + +/// Fake records seeded into a resource collection on its first `GET`. +pub const SEED_COUNT: usize = 3; +/// Total fake items generated for a paginated list endpoint, once per path template. +pub const PAGINATED_TOTAL_COUNT: usize = 7; +/// Page size used when the request doesn't specify one. +pub const DEFAULT_PAGE_SIZE: usize = 3; + +/// Where a started mock server is listening. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerAddress { + /// The bound TCP port. + pub port: u16, + /// The base URL clients should use, e.g. `http://127.0.0.1:8080`. + pub url: String, +} + +/// A mock server built from an OpenAPI document. +#[derive(Clone)] +pub struct MockServer { + document: Arc, + resources: Arc>, + store: Arc>, +} + +impl MockServer { + /// The document this server implements. + pub fn document(&self) -> &OpenApiDocument { + &self.document + } + + /// The resources discovered in the document. + pub fn resources(&self) -> &[ResourceRoute] { + &self.resources + } + + /// The backing CRUD store. + pub fn store(&self) -> &Mutex { + &self.store + } + + /// Binds a port and starts serving. + /// + /// Pass `None` for `port` to bind an ephemeral one. + pub async fn listen(&self, port: Option) -> Result { + let _ = port; + todo!("port src/mock-server/server.ts: handleRequest and the HTTP listener") + } + + /// Stops serving. + pub async fn close(&self) -> Result<()> { + todo!("port src/mock-server/server.ts: server shutdown") + } +} + +/// Builds a mock server for `document`. +pub fn create_mock_server(document: OpenApiDocument) -> MockServer { + let resources = discover_resources(&document.paths); + MockServer { + document: Arc::new(document), + resources: Arc::new(resources), + store: Arc::new(Mutex::new(ResourceStore::new())), + } +} diff --git a/integrations/localthought/syncables/src/mock_server/store.rs b/integrations/localthought/syncables/src/mock_server/store.rs new file mode 100644 index 0000000000..003c1c3919 --- /dev/null +++ b/integrations/localthought/syncables/src/mock_server/store.rs @@ -0,0 +1,89 @@ +//! The mock server's in-memory CRUD store, one collection per resource path. + +use std::collections::HashMap; +use std::time::SystemTime; + +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +#[derive(Debug, Clone)] +struct CollectionMeta { + version: u64, + last_modified: String, +} + +/// One in-memory `Map` per collection path, with CRUD semantics based on +/// collection vs. item path and HTTP method. +#[derive(Debug, Default)] +pub struct ResourceStore { + collections: HashMap>>, + meta: HashMap, +} + +impl ResourceStore { + /// A store with no collections. + pub fn new() -> Self { + Self::default() + } + + /// Whether `resource`'s collection has been created (seeded) yet. + pub fn has(&self, resource: &str) -> bool { + self.collections.contains_key(resource) + } + + /// Every record in `resource`'s collection, in insertion order. + pub fn list(&mut self, resource: &str) -> Vec> { + self.collection(resource).values().cloned().collect() + } + + /// One record, by id. + pub fn get(&mut self, resource: &str, id: &str) -> Option> { + self.collection(resource).get(id).cloned() + } + + /// Inserts or replaces a record. + pub fn put(&mut self, resource: &str, id: &str, value: Map) { + self.collection(resource).insert(id.to_string(), value); + self.touch(resource); + } + + /// Removes a record, reporting whether it existed. + pub fn delete(&mut self, resource: &str, id: &str) -> bool { + let deleted = self.collection(resource).shift_remove(id).is_some(); + if deleted { + self.touch(resource); + } + deleted + } + + /// Weak ETag for the current state of `resource`'s collection, once it + /// has been populated at least once. + pub fn etag(&self, resource: &str) -> Option { + self.meta + .get(resource) + .map(|meta| format!("W/\"{}\"", meta.version)) + } + + /// RFC 7231 HTTP-date of the last mutation to `resource`'s collection, + /// if any. + pub fn last_modified(&self, resource: &str) -> Option { + self.meta + .get(resource) + .map(|meta| meta.last_modified.clone()) + } + + fn touch(&mut self, resource: &str) { + let version = self.meta.get(resource).map_or(0, |meta| meta.version) + 1; + self.meta.insert( + resource.to_string(), + CollectionMeta { + version, + last_modified: httpdate::fmt_http_date(SystemTime::now()), + }, + ); + } + + fn collection(&mut self, resource: &str) -> &mut IndexMap> { + self.collections.entry(resource.to_string()).or_default() + } +} diff --git a/integrations/localthought/syncables/src/openapi/load.rs b/integrations/localthought/syncables/src/openapi/load.rs new file mode 100644 index 0000000000..7df664c9a3 --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/load.rs @@ -0,0 +1,91 @@ +//! Loading an OpenAPI document from a file path or an in-memory value. + +use std::path::Path; + +use serde_json::Value; + +use super::resolve_refs::resolve_refs; +use super::types::OpenApiDocument; +use crate::error::{Error, Result}; + +/// Where a document comes from: a file path, or an already-parsed value. +/// +/// The TypeScript original takes `string | Record`; +/// this is the same union, made explicit. +#[derive(Debug, Clone)] +pub enum OpenApiSource<'a> { + /// A path to a YAML or JSON document on disk. + Path(&'a Path), + /// An in-memory document. + Value(Value), +} + +impl<'a> From<&'a str> for OpenApiSource<'a> { + fn from(path: &'a str) -> Self { + Self::Path(Path::new(path)) + } +} + +impl<'a> From<&'a Path> for OpenApiSource<'a> { + fn from(path: &'a Path) -> Self { + Self::Path(path) + } +} + +impl From for OpenApiSource<'_> { + fn from(value: Value) -> Self { + Self::Value(value) + } +} + +/// Loads an OpenAPI document from a JSON/YAML file path or an in-memory +/// value, and resolves all local `$ref`s. +/// +/// YAML is a superset of JSON, so the file format does not need to be +/// detected separately. +pub async fn load_open_api_document<'a>( + source: impl Into> + Send, +) -> Result { + let raw = match source.into() { + OpenApiSource::Path(path) => load_yaml_file(path).await?, + OpenApiSource::Value(value) => value, + }; + let resolved = resolve_refs(&raw); + serde_json::from_value(resolved).map_err(Error::from) +} + +/// Parses YAML (or JSON) text into a [`Value`]. +pub fn parse_yaml(text: &str) -> Result { + serde_yaml_ng::from_str(text).map_err(Error::from) +} + +/// Reads and parses a YAML/JSON file, wrapping any i/o or parse failure in +/// [`Error::FileLoad`] so it names the offending path — a bare +/// [`std::io::Error`] from a failed read doesn't otherwise mention which +/// file was missing or unreadable. +#[cfg(not(target_arch = "wasm32"))] +pub(crate) async fn load_yaml_file(path: &Path) -> Result { + async { + let text = tokio::fs::read_to_string(path).await?; + parse_yaml(&text) + } + .await + .map_err(|source| Error::FileLoad { + path: path.to_path_buf(), + source: Box::new(source), + }) +} + +#[cfg(target_arch = "wasm32")] +pub(crate) async fn load_yaml_file(path: &Path) -> Result { + Err(Error::FileLoad { + path: path.to_path_buf(), + source: Box::new( + std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Use an in-memory OpenAPI document in the browser", + ) + .into(), + ), + }) +} diff --git a/integrations/localthought/syncables/src/openapi/mod.rs b/integrations/localthought/syncables/src/openapi/mod.rs new file mode 100644 index 0000000000..81904227c8 --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/mod.rs @@ -0,0 +1,12 @@ +//! Reading an OpenAPI document and preparing it for everything downstream. +//! +//! [`load`] reads a document from a file path or an in-memory value and +//! passes it through [`resolve_refs`], which inlines all local `#/...` +//! JSON-pointer `$ref`s. Everything downstream assumes refs are already +//! resolved; [`types`] holds the minimal OpenAPI type surface actually +//! used (not a full spec typing). + +pub mod load; +pub mod overlay; +pub mod resolve_refs; +pub mod types; diff --git a/integrations/localthought/syncables/src/openapi/overlay.rs b/integrations/localthought/syncables/src/openapi/overlay.rs new file mode 100644 index 0000000000..20751b120c --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/overlay.rs @@ -0,0 +1,209 @@ +//! An intentionally minimal [OpenAPI Overlay](https://spec.openapis.org/overlay/v1.0.0.html) +//! implementation: `update`/`remove` actions against `$`, plain dot-paths +//! like `$.components`, and quoted bracket segments like +//! `$.paths['/pets/{petId}'].get` — not the full JSONPath grammar (no +//! wildcards, filters, or numeric/array indexing). + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use super::load::{load_open_api_document, load_yaml_file, OpenApiSource}; +use super::types::OpenApiDocument; +use crate::error::{Error, Result}; + +/// A single overlay action. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OverlayAction { + /// JSONPath target — `$`, a dot-path such as `$.components`, or a + /// dot-path with quoted bracket segments such as + /// `$.paths['/pets/{petId}'].get`. + pub target: String, + /// Object to deep-merge onto the target. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub update: Option>, + /// When true, delete the target instead of merging. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remove: Option, +} + +/// Metadata of an overlay document. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OverlayInfo { + /// Overlay title. + pub title: String, + /// Overlay version. + pub version: String, +} + +/// An OpenAPI Overlay document. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OverlayDocument { + /// Overlay specification version. + pub overlay: String, + /// Overlay metadata. + pub info: OverlayInfo, + /// Actions to apply, in order. + pub actions: Vec, +} + +/// Loads an OpenAPI Overlay document from a YAML/JSON file path or value. +pub async fn load_overlay<'a>( + source: impl Into> + Send, +) -> Result { + let raw = match source.into() { + OpenApiSource::Path(path) => load_yaml_file(path).await?, + OpenApiSource::Value(value) => value, + }; + serde_json::from_value(raw).map_err(Error::from) +} + +/// Loads an OpenAPI document and applies a list of Overlays to it, in the +/// order given — a later overlay may refine what an earlier one added. Each +/// overlay is loaded from a file path. +/// +/// Overlays are applied to the document after its own `$ref`s are resolved +/// (mirroring the TypeScript original's `buildDocumentFrom` in +/// `src/sync/document.ts` of `localthought/reflector`), so an overlay's +/// `update` can safely assume there are no refs left to chase. +pub async fn load_open_api_document_with_overlays<'a>( + document: impl Into> + Send, + overlay_paths: &[PathBuf], +) -> Result { + let document = load_open_api_document(document).await?; + let mut value = serde_json::to_value(document).map_err(Error::from)?; + for path in overlay_paths { + let overlay = load_overlay(path.as_path()).await?; + value = apply_overlay(&value, &overlay)?; + } + serde_json::from_value(value).map_err(Error::from) +} + +fn deep_merge_value(existing: Option<&Value>, incoming: &Value) -> Value { + match (existing, incoming) { + (Some(Value::Object(existing)), Value::Object(incoming)) => { + let mut merged = existing.clone(); + for (key, value) in incoming { + merged.insert(key.clone(), deep_merge_value(merged.get(key), value)); + } + Value::Object(merged) + } + _ => incoming.clone(), + } +} + +/// Tokenizes the intentionally small subset of Overlay JSONPath targets this +/// crate supports into property segments: `$` (the document root), a +/// dot-path like `$.components.schemas.Foo`, and quoted bracket segments +/// like `$.paths['/repos/{owner}/{repo}/issues'].get` — needed because a +/// path template contains slashes and braces that a plain `.split('.')` +/// would mangle. No wildcards, filters, or numeric/array indexing. +fn parse_target(target: &str) -> Result> { + let unsupported = || Error::UnsupportedOverlayTarget(target.to_string()); + + if target == "$" { + return Ok(Vec::new()); + } + if !target.starts_with('$') { + return Err(unsupported()); + } + + let mut segments = Vec::new(); + let bytes = target.as_bytes(); + let mut index = 1; + while index < bytes.len() { + match bytes[index] { + b'.' => { + index += 1; + let start = index; + while index < bytes.len() && bytes[index] != b'.' && bytes[index] != b'[' { + index += 1; + } + if index == start { + return Err(unsupported()); + } + segments.push(target[start..index].to_string()); + } + b'[' => { + let quote = *bytes.get(index + 1).ok_or_else(unsupported)?; + if quote != b'\'' && quote != b'"' { + return Err(unsupported()); + } + let key_start = index + 2; + let end = target[key_start..] + .find(quote as char) + .map(|offset| key_start + offset) + .ok_or_else(unsupported)?; + segments.push(target[key_start..end].to_string()); + index = end + 1; + if bytes.get(index) != Some(&b']') { + return Err(unsupported()); + } + index += 1; + } + _ => return Err(unsupported()), + } + } + Ok(segments) +} + +fn navigate<'v>( + root: &'v mut Value, + segments: &[String], + create_missing: bool, +) -> Result> { + let mut node = root; + for segment in segments { + let object = node + .as_object_mut() + .ok_or_else(|| Error::OverlayTargetNotAnObject(segment.clone()))?; + if !object.contains_key(segment) { + if !create_missing { + return Ok(None); + } + object.insert(segment.clone(), Value::Object(Map::new())); + } + let child = object + .get_mut(segment) + .expect("segment inserted or already present"); + if !child.is_object() { + return Err(Error::OverlayTargetNotAnObject(segment.clone())); + } + node = child; + } + Ok(Some(node)) +} + +/// Applies an overlay to a document, returning a new document. +/// +/// Supports `update` (deep-merged onto the target) and `remove` actions. +pub fn apply_overlay(document: &Value, overlay: &OverlayDocument) -> Result { + let mut result = document.clone(); + + for action in &overlay.actions { + let segments = parse_target(&action.target)?; + if action.remove == Some(true) { + let Some((key, parent_segments)) = segments.split_last() else { + return Err(Error::OverlayRemovesRoot); + }; + if let Some(parent) = navigate(&mut result, parent_segments, false)? { + if let Some(object) = parent.as_object_mut() { + object.shift_remove(key); + } + } + } else if let Some(update) = &action.update { + let target = navigate(&mut result, &segments, true)? + .expect("navigate with create_missing always yields a node"); + let object = target + .as_object_mut() + .ok_or_else(|| Error::OverlayTargetNotAnObject(action.target.clone()))?; + for (key, value) in update { + let merged = deep_merge_value(object.get(key), value); + object.insert(key.clone(), merged); + } + } + } + + Ok(result) +} diff --git a/integrations/localthought/syncables/src/openapi/resolve_refs.rs b/integrations/localthought/syncables/src/openapi/resolve_refs.rs new file mode 100644 index 0000000000..7a9c24a36a --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/resolve_refs.rs @@ -0,0 +1,92 @@ +//! Resolves local `#/...` JSON pointer `$ref`s in place. +//! +//! Each `{ "$ref": ... }` node is replaced with the value it points to. +//! Reused (non-cyclic) targets are resolved once and cached so diamond +//! references share a result; a target still being resolved when it is +//! referenced again is a genuine cycle, so that occurrence is left +//! unresolved to avoid recursing forever. Non-local refs (external files, +//! URLs) are left unresolved too, since there is nothing in the document +//! to resolve them against — real-world documents sometimes use these in +//! vendor extensions unrelated to the schemas this crate cares about. + +use std::collections::{HashMap, HashSet}; + +use serde_json::Value; + +/// Resolves every local `$ref` in `document`, returning a new value. +pub fn resolve_refs(document: &Value) -> Value { + let mut resolver = Resolver { + root: document, + resolving: HashSet::new(), + resolved: HashMap::new(), + }; + resolver.walk(document) +} + +/// A JSON pointer identifying a node, used as the cache/cycle key. +/// +/// The TypeScript original keys its `Set`/`Map` on object *identity*; +/// `serde_json` values are cloned rather than shared, so the pointer that +/// reached a node stands in for that identity here. +type NodeKey = String; + +struct Resolver<'a> { + root: &'a Value, + resolving: HashSet, + resolved: HashMap, +} + +impl Resolver<'_> { + fn resolve_pointer(&self, reference: &str) -> Option<&Value> { + let trimmed = reference.strip_prefix("#/")?; + let mut node = self.root; + for segment in trimmed.split('/') { + let segment = segment.replace("~1", "/").replace("~0", "~"); + node = node.get(&segment)?; + } + Some(node) + } + + fn walk(&mut self, node: &Value) -> Value { + match node { + Value::Array(items) => Value::Array(items.iter().map(|i| self.walk(i)).collect()), + Value::Object(object) => { + if let Some(Value::String(reference)) = object.get("$ref") { + return self.walk_ref(node, &reference.clone()); + } + let mut result = serde_json::Map::new(); + for (key, value) in object { + result.insert(key.clone(), self.walk(value)); + } + Value::Object(result) + } + other => other.clone(), + } + } + + fn walk_ref(&mut self, node: &Value, reference: &str) -> Value { + if !reference.starts_with("#/") { + return node.clone(); + } + let Some(target) = self.resolve_pointer(reference) else { + return Value::Null; + }; + if !target.is_object() && !target.is_array() { + return target.clone(); + } + let key: NodeKey = reference.to_string(); + if let Some(cached) = self.resolved.get(&key) { + return cached.clone(); + } + if self.resolving.contains(&key) { + return target.clone(); + } + + let target = target.clone(); + self.resolving.insert(key.clone()); + let result = self.walk(&target); + self.resolving.remove(&key); + self.resolved.insert(key, result.clone()); + result + } +} diff --git a/integrations/localthought/syncables/src/openapi/types.rs b/integrations/localthought/syncables/src/openapi/types.rs new file mode 100644 index 0000000000..ebdc5c7305 --- /dev/null +++ b/integrations/localthought/syncables/src/openapi/types.rs @@ -0,0 +1,250 @@ +//! The minimal OpenAPI type surface this crate actually uses — not a full +//! spec typing. Every struct keeps an `extensions` catch-all so a document +//! round-trips through these types without losing vendor extensions +//! (`x-pagination` in particular is read back out of `OperationObject`). + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::pagination::types::PaginationSchemesMap; + +/// A JSON object with insertion order preserved. +/// +/// Order is load-bearing: [`crate::pagination::items::locate_items_field`] +/// picks the *first* array-typed property of a response schema. +pub type JsonMap = IndexMap; + +/// A JSON Schema subset, as it appears inside an OpenAPI document. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct SchemaObject { + /// `type` keyword (`string`, `integer`, `object`, `array`, ...). + #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] + pub schema_type: Option, + /// `format` keyword (`date-time`, `uuid`, `email`, ...). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, + /// Properties of an object schema, in document order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub properties: Option>, + /// Names of required properties. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option>, + /// Item schema of an array schema. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, + /// Allowed values; the first is used when generating fake data. + #[serde(rename = "enum", default, skip_serializing_if = "Option::is_none")] + pub enum_values: Option>, + /// A fixed example, preferred over synthesis when generating fake data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, + /// Lower bound for numeric schemas. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub minimum: Option, + /// `oneOf` branches. + #[serde(rename = "oneOf", default, skip_serializing_if = "Option::is_none")] + pub one_of: Option>, + /// `anyOf` branches. + #[serde(rename = "anyOf", default, skip_serializing_if = "Option::is_none")] + pub any_of: Option>, + /// `allOf` branches, merged when generating data or listing properties. + #[serde(rename = "allOf", default, skip_serializing_if = "Option::is_none")] + pub all_of: Option>, + /// Any other keyword present on the schema. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One entry of a `content` map, keyed by media type. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct MediaTypeObject { + /// Schema of the payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// A fixed example payload, preferred over the schema when serving mocks. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, +} + +/// A documented response for one status code. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ResponseObject { + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Response payloads, keyed by media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option>, +} + +/// A documented request body. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RequestBodyObject { + /// Whether the body is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option, + /// Request payloads, keyed by media type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option>, +} + +/// Where a parameter is carried. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ParameterLocation { + /// Query string parameter. + Query, + /// Path template variable. + Path, + /// Request header. + Header, + /// Cookie. + Cookie, +} + +/// A single operation parameter. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ParameterObject { + /// Parameter name. + pub name: String, + /// Where the parameter is carried. + #[serde(rename = "in")] + pub location: ParameterLocation, + /// Whether the parameter is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option, + /// Schema of the parameter's value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// Any other key present on the parameter. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One HTTP operation on a path. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OperationObject { + /// `operationId`, when the document declares one. + #[serde( + rename = "operationId", + default, + skip_serializing_if = "Option::is_none" + )] + pub operation_id: Option, + /// Declared parameters. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option>, + /// Declared request body. + #[serde( + rename = "requestBody", + default, + skip_serializing_if = "Option::is_none" + )] + pub request_body: Option, + /// Documented responses, keyed by status code. + #[serde(default)] + pub responses: IndexMap, + /// Any other key on the operation — notably `x-pagination`. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// The set of operations declared on one path template. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct PathItem { + /// `GET` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub get: Option, + /// `PUT` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub put: Option, + /// `POST` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub post: Option, + /// `PATCH` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub patch: Option, + /// `DELETE` operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delete: Option, + /// Any other key on the path item. + #[serde(flatten)] + pub extensions: JsonMap, +} + +impl PathItem { + /// The operation declared for `method` (case-insensitive), if any. + pub fn operation(&self, method: &str) -> Option<&OperationObject> { + match method.to_ascii_uppercase().as_str() { + "GET" => self.get.as_ref(), + "PUT" => self.put.as_ref(), + "POST" => self.post.as_ref(), + "PATCH" => self.patch.as_ref(), + "DELETE" => self.delete.as_ref(), + _ => None, + } + } +} + +/// Document metadata. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct InfoObject { + /// Document title. + pub title: String, + /// Document version. + pub version: String, +} + +/// One entry of the document's top-level `servers` list. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ServerObject { + /// Root URL of the API, e.g. `https://api.github.com`. + pub url: String, + /// Any other key on the server entry. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// The `components` section, narrowed to what this crate reads. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ComponentsObject { + /// Reusable schemas. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schemas: Option>, + /// Pagination schemes, per the OpenAPI Pagination Schemes Extension. + #[serde( + rename = "paginationSchemes", + default, + skip_serializing_if = "Option::is_none" + )] + pub pagination_schemes: Option, + /// Any other key under `components`. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// An OpenAPI document with all local `$ref`s already resolved. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OpenApiDocument { + /// OpenAPI version string. + #[serde(default)] + pub openapi: String, + /// Document metadata. + #[serde(default)] + pub info: InfoObject, + /// Path templates, in document order. + #[serde(default)] + pub paths: IndexMap, + /// The API's base URL(s). The credential layer targets requests at the + /// first entry rather than any URL configured separately, so a document + /// can't be pointed at the wrong host by mistake. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub servers: Option>, + /// Reusable components. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub components: Option, + /// Any other top-level key. + #[serde(flatten)] + pub extensions: JsonMap, +} diff --git a/integrations/localthought/syncables/src/pagination/autodetect.rs b/integrations/localthought/syncables/src/pagination/autodetect.rs new file mode 100644 index 0000000000..c604c0e98f --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/autodetect.rs @@ -0,0 +1,169 @@ +//! Resolving which pagination scheme applies to an operation. + +use std::collections::HashSet; + +use indexmap::IndexMap; +use serde_json::Value; + +use super::types::{ + AutoDetect, AutoDetectObject, PaginationApplicationObject, PaginationSchemeObject, +}; +use super::validate::validate_pagination_scheme; +use crate::openapi::types::{OpenApiDocument, OperationObject, ParameterLocation}; + +/// The scheme that applies to an operation, and the name it is declared under. +#[derive(Debug, Clone, PartialEq)] +pub struct EffectiveScheme { + /// Key in `components.paginationSchemes`. + pub scheme_name: String, + /// The scheme itself, with any per-operation overrides merged in. + pub scheme: PaginationSchemeObject, +} + +/// Schemes that fail validation (spec §9) are excluded here rather than +/// thrown on eagerly — one malformed scheme in a document (see e.g. +/// Giphy's `type: offset`, which isn't a valid scheme type) shouldn't +/// prevent using the rest of the document or its other schemes. +fn valid_schemes(document: &OpenApiDocument) -> IndexMap { + document + .components + .as_ref() + .and_then(|c| c.pagination_schemes.as_ref()) + .into_iter() + .flatten() + .filter(|(name, scheme)| validate_pagination_scheme(name, scheme).is_empty()) + .map(|(name, scheme)| (name.clone(), scheme.clone())) + .collect() +} + +fn query_param_names(operation: &OperationObject) -> HashSet<&str> { + operation + .parameters + .as_deref() + .unwrap_or_default() + .iter() + .filter(|parameter| parameter.location == ParameterLocation::Query) + .map(|parameter| parameter.name.as_str()) + .collect() +} + +fn body_field_names(operation: &OperationObject) -> HashSet<&str> { + operation + .request_body + .as_ref() + .and_then(|body| body.content.as_ref()) + .and_then(|content| content.get("application/json")) + .and_then(|media| media.schema.as_ref()) + .and_then(|schema| schema.properties.as_ref()) + .into_iter() + .flatten() + .map(|(name, _)| name.as_str()) + .collect() +} + +/// Default auto-detection rules (spec §6.2/§6.3): a dimension only +/// contributes to the match when the scheme actually declares fields for +/// it — an empty declaration isn't treated as vacuously satisfied, or +/// every scheme with no query parameters would match every operation. +fn auto_detect_matches(scheme: &PaginationSchemeObject, operation: &OperationObject) -> bool { + let options = match &scheme.auto_detect { + Some(AutoDetect::Enabled(false)) => return false, + Some(AutoDetect::Options(options)) => (**options).clone(), + _ => AutoDetectObject::default(), + }; + let require_all = options.require_all.unwrap_or(true); + let mut results: Vec = Vec::new(); + + let request = scheme.request.as_ref(); + + if options.match_query_params.unwrap_or(true) { + let required: Vec<&String> = request + .and_then(|r| r.query_parameters.as_ref()) + .into_iter() + .flatten() + .map(|(name, _)| name) + .collect(); + if !required.is_empty() { + let declared = query_param_names(operation); + results.push(required.iter().all(|name| declared.contains(name.as_str()))); + } + } + + if options.match_body_fields.unwrap_or(true) { + let required: Vec<&String> = request + .and_then(|r| r.body_fields.as_ref()) + .into_iter() + .flatten() + .map(|(name, _)| name) + .collect(); + if !required.is_empty() { + let declared = body_field_names(operation); + results.push(required.iter().all(|name| declared.contains(name.as_str()))); + } + } + + if results.is_empty() { + return false; + } + if require_all { + results.iter().all(|matched| *matched) + } else { + results.iter().any(|matched| *matched) + } +} + +fn deep_merge(base: &Value, overrides: &Value) -> Value { + match (base, overrides) { + (Value::Object(base), Value::Object(overrides)) => { + let mut result = base.clone(); + for (key, value) in overrides { + let merged = match result.get(key) { + Some(existing) if existing.is_object() && value.is_object() => { + deep_merge(existing, value) + } + _ => value.clone(), + }; + result.insert(key.clone(), merged); + } + Value::Object(result) + } + _ => overrides.clone(), + } +} + +/// Resolves the pagination scheme that applies to an operation: an +/// explicit `x-pagination` application (with overrides merged in) takes +/// priority, falling back to auto-detection against the document's valid +/// `paginationSchemes`. +pub fn resolve_effective_scheme( + document: &OpenApiDocument, + operation: &OperationObject, +) -> Option { + let schemes = valid_schemes(document); + + if let Some(explicit) = operation.extensions.get("x-pagination") { + let applications: Vec = + serde_json::from_value(explicit.clone()).unwrap_or_default(); + let application = applications.into_iter().next()?; + let base = schemes.get(&application.scheme)?; + let scheme = match &application.overrides { + Some(overrides) => { + let merged = deep_merge(&serde_json::to_value(base).ok()?, overrides); + serde_json::from_value(merged).ok()? + } + None => base.clone(), + }; + return Some(EffectiveScheme { + scheme_name: application.scheme, + scheme, + }); + } + + schemes + .into_iter() + .find(|(_, scheme)| auto_detect_matches(scheme, operation)) + .map(|(scheme_name, scheme)| EffectiveScheme { + scheme_name, + scheme, + }) +} diff --git a/integrations/localthought/syncables/src/pagination/items.rs b/integrations/localthought/syncables/src/pagination/items.rs new file mode 100644 index 0000000000..af0f3d14f1 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/items.rs @@ -0,0 +1,91 @@ +//! Locating which response property actually holds the list of items. +//! +//! The extension itself only describes pagination metadata, not where +//! items live, so this excludes whatever fields the scheme claims as +//! metadata, then picks the remaining array-typed property (falling back +//! to common envelope names). This is also what makes real enveloped +//! responses (e.g. `{ data: [...], meta, pagination }`) work at all, +//! pagination or not. + +use std::collections::HashSet; + +use indexmap::IndexMap; + +use super::types::PaginationSchemeObject; +use crate::openapi::types::SchemaObject; + +/// Field names real-world APIs commonly wrap a list response in, tried +/// when the response schema itself doesn't unambiguously point at one +/// array property. +const COMMON_ITEMS_FIELDS: [&str; 5] = ["items", "data", "results", "records", "content"]; + +/// Flattens a schema's own properties together with any `allOf` branches' +/// properties into one map. +/// +/// Real paginated response schemas often compose a shared "paging" base +/// schema with a branch that adds the concrete `items` property (e.g. +/// Spotify's `PagingSimplifiedAlbumObject`). +pub fn effective_properties(schema: Option<&SchemaObject>) -> IndexMap { + let Some(schema) = schema else { + return IndexMap::new(); + }; + if let Some(all_of) = &schema.all_of { + let mut merged = IndexMap::new(); + for branch in all_of { + merged.extend(effective_properties(Some(branch))); + } + return merged; + } + schema.properties.clone().unwrap_or_default() +} + +/// The top-level field names a pagination scheme claims for its own metadata. +fn metadata_field_roots(scheme: Option<&PaginationSchemeObject>) -> HashSet { + scheme + .and_then(|s| s.response.as_ref()) + .and_then(|r| r.body_fields.as_ref()) + .into_iter() + .flatten() + .map(|(key, _)| key.split('.').next().unwrap_or(key).to_string()) + .collect() +} + +/// Finds the property in a response schema that holds the actual list of +/// items: the first array-typed property that isn't claimed by the +/// pagination scheme as a metadata field, falling back to common +/// enveloping field names. +pub fn locate_items_field( + schema: Option<&SchemaObject>, + scheme: Option<&PaginationSchemeObject>, +) -> Option { + let properties = effective_properties(schema); + let excluded = metadata_field_roots(scheme); + + for (name, property_schema) in &properties { + if !excluded.contains(name) && property_schema.schema_type.as_deref() == Some("array") { + return Some(name.clone()); + } + } + + COMMON_ITEMS_FIELDS + .iter() + .find(|name| properties.contains_key(**name) && !excluded.contains(**name)) + .map(|name| (*name).to_string()) +} + +/// The schema of a single item, given the schema of the whole (enveloped) +/// response. +pub fn item_schema_for( + schema: Option<&SchemaObject>, + scheme: Option<&PaginationSchemeObject>, +) -> Option { + if let Some(schema) = schema { + if schema.schema_type.as_deref() == Some("array") { + return schema.items.as_deref().cloned(); + } + } + let field = locate_items_field(schema, scheme)?; + effective_properties(schema) + .get(&field) + .and_then(|property| property.items.as_deref().cloned()) +} diff --git a/integrations/localthought/syncables/src/pagination/mod.rs b/integrations/localthought/syncables/src/pagination/mod.rs new file mode 100644 index 0000000000..cd61a08926 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/mod.rs @@ -0,0 +1,20 @@ +//! The [OpenAPI Pagination Schemes Extension](https://github.com/pondersource/openapi-pagination-schemes-extension) +//! (`components.paginationSchemes`), applied to third-party documents via +//! [OpenAPI Overlays](https://spec.openapis.org/overlay/v1.0.0.html). +//! +//! **Pagination is orthogonal to the collection/item resource model.** In +//! real APIs, the paths that pair into a "resource" (batch-get-by-IDs +//! style, e.g. Giphy's `/gifs`, Spotify's `/albums`) are often *not* the +//! paginated ones — real pagination usually lives on separate search/list +//! endpoints (`/gifs/trending`, `/artists/{id}/albums`) that have no +//! sibling item path and are therefore invisible to +//! [`crate::resources::discover`]. So pagination support in both the mock +//! server and `paginate()` operates on *any* GET operation matched by a +//! scheme, not just discovered resources. + +pub mod autodetect; +pub mod items; +pub mod request_builder; +pub mod response_parser; +pub mod types; +pub mod validate; diff --git a/integrations/localthought/syncables/src/pagination/request_builder.rs b/integrations/localthought/syncables/src/pagination/request_builder.rs new file mode 100644 index 0000000000..318b247d3e --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/request_builder.rs @@ -0,0 +1,144 @@ +//! Building the query for one page request, and computing the next cursor. + +use super::types::{PaginationQuery, PaginationResponseState, PaginationSchemeObject, SchemeType}; + +/// Hard ceiling on pages walked in one paginated traversal, so a +/// misconfigured `Link` header — or any response that always claims +/// another page exists — cannot spin forever. Mirrors +/// [`crate::client::client::MAX_PAGES`], the equivalent constant for the +/// crate's existing `ApiClient` surface. +pub const MAX_PAGES: usize = 50; + +/// Where the client is in a paginated traversal, independent of scheme type. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PageCursor { + /// Item offset, for offset-based schemes. + pub offset: Option, + /// Page number, for page-based schemes. + pub page: Option, + /// Opaque token, for token/cursor-based schemes. + pub page_token: Option, +} + +fn fields_with_role(scheme: &PaginationSchemeObject, role: &str) -> Vec { + scheme + .request + .as_ref() + .and_then(|request| request.query_parameters.as_ref()) + .into_iter() + .flatten() + .filter(|(_, field)| field.role.as_deref() == Some(role)) + .map(|(name, _)| name.clone()) + .collect() +} + +/// Builds the query parameters for one page request from the current cursor. +pub fn build_query( + scheme: &PaginationSchemeObject, + cursor: &PageCursor, + page_size: Option, +) -> PaginationQuery { + let mut query = PaginationQuery::new(); + + if let Some(page_size) = page_size { + for name in fields_with_role(scheme, "pageSize") { + query.insert(name, page_size.to_string()); + } + } + for name in fields_with_role(scheme, "offset") { + query.insert(name, cursor.offset.unwrap_or(0).to_string()); + } + for name in fields_with_role(scheme, "page") { + query.insert(name, cursor.page.unwrap_or(1).to_string()); + } + if let Some(page_token) = &cursor.page_token { + for name in fields_with_role(scheme, "pageToken") + .into_iter() + .chain(fields_with_role(scheme, "cursor")) + { + query.insert(name, page_token.clone()); + } + } + + query +} + +/// Computes the cursor for the next page from the previous cursor and the +/// parsed response state. +/// +/// Returns `None` when there is no next page, or when the scheme type is +/// `nextLink` — for that type the caller should follow +/// [`PaginationResponseState::next_link`] directly rather than rebuilding +/// query parameters. +pub fn next_cursor( + scheme: &PaginationSchemeObject, + cursor: &PageCursor, + state: &PaginationResponseState, + items_returned: u64, +) -> Option { + if !state.has_next_page { + return None; + } + + match scheme.typed()? { + SchemeType::PageToken => state.next_page_token.clone().map(|token| PageCursor { + page_token: Some(token), + ..PageCursor::default() + }), + SchemeType::NextLink => None, + SchemeType::PageNumber => { + if !fields_with_role(scheme, "offset").is_empty() { + return Some(PageCursor { + offset: Some(cursor.offset.unwrap_or(0) + items_returned), + ..PageCursor::default() + }); + } + if !fields_with_role(scheme, "page").is_empty() { + return Some(PageCursor { + page: Some(cursor.page.unwrap_or(1) + 1), + ..PageCursor::default() + }); + } + None + } + } +} + +/// What a paginated traversal should do next, after parsing one page's +/// response. +#[derive(Debug, Clone, PartialEq)] +pub enum PageStep { + /// Follow this absolute URL directly (a `nextLink` scheme) — built from + /// the response's own `Link` header, not rebuilt from the operation's + /// path template. + FollowLink(String), + /// Request the next page with this cursor. + NextPage(PageCursor), + /// No more pages: the response says so, or [`MAX_PAGES`] was reached. + Done, +} + +/// Decides the next step of a paginated traversal, wrapping [`next_cursor`] +/// with the two cases it deliberately doesn't handle: a `nextLink` scheme +/// (whose next page is a URL, not a cursor) and the [`MAX_PAGES`] cap. +/// +/// `pages_fetched` is the number of pages fetched so far, including the one +/// `state` describes (i.e. after fetching the first page, pass `1`). +pub fn next_step( + scheme: &PaginationSchemeObject, + cursor: &PageCursor, + state: &PaginationResponseState, + items_returned: u64, + pages_fetched: usize, +) -> PageStep { + if !state.has_next_page || pages_fetched >= MAX_PAGES { + return PageStep::Done; + } + if scheme.typed() == Some(SchemeType::NextLink) { + return state + .next_link + .clone() + .map_or(PageStep::Done, PageStep::FollowLink); + } + next_cursor(scheme, cursor, state, items_returned).map_or(PageStep::Done, PageStep::NextPage) +} diff --git a/integrations/localthought/syncables/src/pagination/response_parser.rs b/integrations/localthought/syncables/src/pagination/response_parser.rs new file mode 100644 index 0000000000..90e007aa42 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/response_parser.rs @@ -0,0 +1,237 @@ +//! Parsing pagination state back out of a server response. + +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use super::types::{PaginationResponseState, PaginationSchemeObject, ResponseRole, SchemeType}; + +/// Reads a dot-separated path out of a JSON object, e.g. `pagination.total_count`. +pub fn read_nested_field<'v>(body: &'v Value, path: &str) -> Option<&'v Value> { + let mut node = body; + for segment in path.split('.') { + node = node.as_object()?.get(segment)?; + } + Some(node) +} + +/// Writes a value at a dot-separated path, creating intermediate objects +/// as needed. +pub fn set_nested_field(body: &mut Map, path: &str, value: Value) { + let segments: Vec<&str> = path.split('.').collect(); + let Some((last, parents)) = segments.split_last() else { + return; + }; + + let mut node = body; + for segment in parents { + let entry = node + .entry((*segment).to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if !entry.is_object() { + *entry = Value::Object(Map::new()); + } + node = entry.as_object_mut().expect("just ensured object"); + } + node.insert((*last).to_string(), value); +} + +/// Parses an RFC 8288 `Link` header value and extracts the URL with +/// `rel="next"`. +/// +/// Example: `; rel="next", <...>; rel="prev"` +pub fn parse_link_header(header: &str) -> Option { + if header.is_empty() { + return None; + } + for part in split_links(header) { + let part = part.trim_start(); + let Some(rest) = part.strip_prefix('<') else { + continue; + }; + let Some(end) = rest.find('>') else { + continue; + }; + let (url, attrs) = rest.split_at(end); + if rel_is_next(&attrs[1..]) { + return Some(url.to_string()); + } + } + None +} + +/// Splits on commas that begin a new `` link-value, mirroring the +/// original's `/,\s*(?=<)/` lookahead. +fn split_links(header: &str) -> Vec<&str> { + let bytes = header.as_bytes(); + let mut parts = Vec::new(); + let mut start = 0; + for (index, byte) in bytes.iter().enumerate() { + if *byte != b',' { + continue; + } + let after = header[index + 1..].trim_start(); + if after.starts_with('<') { + parts.push(&header[start..index]); + start = index + 1; + } + } + parts.push(&header[start..]); + parts +} + +/// Finds `rel=next` (quoted or bare) among a link-value's attributes. +fn rel_is_next(attrs: &str) -> bool { + let lowered = attrs.to_ascii_lowercase(); + let mut search = lowered.as_str(); + while let Some(index) = search.find("rel") { + let before_is_boundary = index == 0 + || !search.as_bytes()[index - 1].is_ascii_alphanumeric() + && search.as_bytes()[index - 1] != b'_'; + let after = search[index + 3..].trim_start(); + if before_is_boundary { + if let Some(value) = after.strip_prefix('=') { + let value = value.trim_start().trim_start_matches('"'); + let value = value + .split(|c: char| c == '"' || c == ';' || c == ',' || c.is_whitespace()) + .next() + .unwrap_or(""); + if value == "next" { + return true; + } + } + } + search = &search[index + 3..]; + } + false +} + +fn to_string_or_none(value: Option<&Value>) -> Option { + match value? { + Value::Null => None, + Value::String(s) if s.is_empty() => None, + Value::String(s) => Some(s.clone()), + other => Some(other.to_string()), + } +} + +fn to_number_or_none(value: Option<&Value>) -> Option { + match value? { + Value::Number(n) => n.as_f64(), + Value::String(s) => s.trim().parse::().ok(), + Value::Bool(b) => Some(f64::from(u8::from(*b))), + _ => None, + } +} + +fn extract_by_role( + scheme: &PaginationSchemeObject, + body: &Value, + headers: &IndexMap, +) -> IndexMap { + let mut roles = IndexMap::new(); + + let response = scheme.response.as_ref(); + + for (path, field) in response + .and_then(|r| r.body_fields.as_ref()) + .into_iter() + .flatten() + { + let Some(role) = &field.role else { continue }; + if let Some(value) = read_nested_field(body, path) { + roles.insert(role.clone(), value.clone()); + } + } + + for (name, field) in response + .and_then(|r| r.headers.as_ref()) + .into_iter() + .flatten() + { + let Some(role) = &field.role else { continue }; + let raw = headers + .get(name) + .or_else(|| headers.get(&name.to_lowercase())) + .or_else(|| headers.get(&name.to_uppercase())); + let Some(raw) = raw else { continue }; + if role == "nextLink" { + if let Some(parsed) = parse_link_header(raw) { + roles.insert("nextLink".to_string(), Value::String(parsed)); + } + } else { + roles.insert(role.clone(), Value::String(raw.clone())); + } + } + + roles +} + +/// A `nextLink`/`nextPageToken` value is a strong, type-agnostic signal +/// that another page exists — real APIs sometimes include one even on a +/// scheme whose `type` is `pageNumber` (e.g. Spotify's offset-based +/// endpoints all carry a `next` URL). Checking it first, ahead of the +/// type-specific counting rules, means traversal still terminates +/// correctly for those schemes even without a `currentPage`/`totalPages` +/// role declared. +/// +/// `totalCount` (role: `all` per spec §4.5) is checked next against +/// `items_fetched_so_far`, which the *caller* tracks — some real schemes +/// (e.g. Giphy's) report `totalCount` and `pageSize` but no `currentPage` +/// at all, so there's nothing here to compute "current page * pageSize" +/// from; the client already knows exactly how many items it has pulled +/// across all pages so far, which is the more direct signal anyway. +fn derive_has_next_page( + scheme_type: Option, + state: &PaginationResponseState, + items_fetched_so_far: Option, +) -> bool { + if state.next_link.is_some() || state.next_page_token.is_some() { + return true; + } + if let (Some(total_count), Some(fetched)) = (state.total_count, items_fetched_so_far) { + #[allow(clippy::cast_precision_loss)] + return (fetched as f64) < total_count; + } + if scheme_type == Some(SchemeType::PageNumber) { + if let (Some(current_page), Some(total_pages)) = (state.current_page, state.total_pages) { + return current_page < total_pages; + } + if let (Some(current_page), Some(total_count), Some(page_size)) = + (state.current_page, state.total_count, state.page_size) + { + return current_page * page_size < total_count; + } + } + false +} + +/// Parses a server response into pagination state, per the resolved scheme. +/// +/// `items_fetched_so_far` — the cumulative item count across all pages +/// fetched so far, including this one — lets `has_next_page` be derived +/// from a plain `totalCount` field even when no `currentPage` role is +/// declared. +pub fn parse_pagination_state( + scheme: &PaginationSchemeObject, + body: &Value, + headers: &IndexMap, + items_fetched_so_far: Option, +) -> PaginationResponseState { + let roles = extract_by_role(scheme, body, headers); + + let mut state = PaginationResponseState { + next_page_token: to_string_or_none( + roles + .get("nextPageToken") + .or_else(|| roles.get("nextCursor")), + ), + next_link: to_string_or_none(roles.get("nextLink")), + current_page: to_number_or_none(roles.get("currentPage")), + total_count: to_number_or_none(roles.get("totalCount")), + total_pages: to_number_or_none(roles.get("totalPages")), + page_size: to_number_or_none(roles.get("pageSize")), + has_next_page: false, + }; + state.has_next_page = derive_has_next_page(scheme.typed(), &state, items_fetched_so_far); + state +} diff --git a/integrations/localthought/syncables/src/pagination/types.rs b/integrations/localthought/syncables/src/pagination/types.rs new file mode 100644 index 0000000000..be15cfad62 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/types.rs @@ -0,0 +1,255 @@ +//! Type definitions for the [OpenAPI Pagination Schemes Extension](https://github.com/pondersource/openapi-pagination-schemes-extension). +//! +//! Spec version 0.1.0. Field names, optionality, and enum values are taken +//! verbatim from the spec so a [`PaginationSchemeObject`] can be lifted +//! from an OAS document without transformation. +//! +//! Roles are modelled as string newtypes rather than closed enums: the +//! spec allows `x-`-prefixed extension roles alongside the standard ones, +//! and [`crate::pagination::validate`] is what decides validity, so an +//! unknown role has to survive deserialization to be reported. + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Which family of pagination a scheme belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SchemeType { + /// Page- or offset-numbered traversal. + #[serde(rename = "pageNumber")] + PageNumber, + /// Opaque-token traversal. + #[serde(rename = "pageToken")] + PageToken, + /// Follow-the-link traversal. + #[serde(rename = "nextLink")] + NextLink, +} + +/// The role a request field plays. Spec-defined values are `page`, +/// `pageSize`, `offset`, `pageToken` and `cursor`. +pub type RequestRole = String; + +/// The role a response field plays. Spec-defined values are +/// `nextPageToken`, `nextCursor`, `nextLink`, `totalCount`, `totalPages`, +/// `pageSize` and `currentPage`. +pub type ResponseRole = String; + +/// One declared request field. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RequestFieldObject { + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Schema of the field's value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// The field's role in the scheme. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + /// Whether the field is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub required: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// Request-side fields a scheme declares, grouped by where they travel. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RequestPaginationFieldsObject { + /// Query parameters. + #[serde( + rename = "queryParameters", + default, + skip_serializing_if = "Option::is_none" + )] + pub query_parameters: Option>, + /// Request body fields. + #[serde( + rename = "bodyFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub body_fields: Option>, + /// Request headers. + /// + /// Part of the type surface (mirroring the spec); nothing reads it yet. + #[serde( + rename = "headerFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub header_fields: Option>, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// One declared response field. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ResponseFieldObject { + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Schema of the field's value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// The field's role in the scheme. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// Response-side fields a scheme declares, grouped by where they travel. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ResponsePaginationFieldsObject { + /// Response body fields; keys may be dotted paths into nested objects. + #[serde( + rename = "bodyFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub body_fields: Option>, + /// Response headers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub headers: Option>, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// Fine-grained auto-detection options. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct AutoDetectObject { + /// Match the scheme's declared query parameters against the operation's. + #[serde( + rename = "matchQueryParams", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_query_params: Option, + /// Match the scheme's declared body fields against the operation's. + #[serde( + rename = "matchBodyFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_body_fields: Option, + /// Part of the type surface (mirroring the spec); nothing reads it yet. + #[serde( + rename = "matchResponseFields", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_response_fields: Option, + /// Part of the type surface (mirroring the spec); nothing reads it yet. + #[serde( + rename = "matchHeaders", + default, + skip_serializing_if = "Option::is_none" + )] + pub match_headers: Option, + /// Require every considered dimension to match, rather than any. + #[serde( + rename = "requireAll", + default, + skip_serializing_if = "Option::is_none" + )] + pub require_all: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// `autoDetect` is either a plain toggle or a set of options. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum AutoDetect { + /// `autoDetect: true` / `autoDetect: false`. + Enabled(bool), + /// `autoDetect: { ... }`. + Options(Box), +} + +/// One entry of `components.paginationSchemes`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PaginationSchemeObject { + /// Which family of pagination this scheme belongs to. + /// + /// Deserialized leniently: an invalid value (e.g. Giphy's `offset`) + /// has to survive parsing so [`crate::pagination::validate`] can + /// report it rather than failing the whole document. + #[serde(rename = "type")] + pub scheme_type: Value, + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Whether and how the scheme may be auto-detected. + #[serde( + rename = "autoDetect", + default, + skip_serializing_if = "Option::is_none" + )] + pub auto_detect: Option, + /// Request-side declared fields. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request: Option, + /// Response-side declared fields. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub response: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +impl PaginationSchemeObject { + /// The scheme's type, when it is one the spec defines. + pub fn typed(&self) -> Option { + serde_json::from_value(self.scheme_type.clone()).ok() + } +} + +/// One entry of an operation's `x-pagination` array. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PaginationApplicationObject { + /// Name of the scheme in `components.paginationSchemes`. + pub scheme: String, + /// Per-operation overrides, deep-merged onto the named scheme. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overrides: Option, + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// `x-`-prefixed extension keys. + #[serde(flatten)] + pub extensions: IndexMap, +} + +/// `components.paginationSchemes`, keyed by scheme name. +pub type PaginationSchemesMap = IndexMap; + +/// Everything derivable from a server response about the state of pagination. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PaginationResponseState { + /// Token to request the next page with, if the response carried one. + pub next_page_token: Option, + /// URL of the next page, if the response carried one. + pub next_link: Option, + /// The page number this response represents. + pub current_page: Option, + /// Total number of items across all pages. + pub total_count: Option, + /// Total number of pages. + pub total_pages: Option, + /// Number of items per page. + pub page_size: Option, + /// Whether another page exists after this one. + pub has_next_page: bool, +} + +/// Query parameters to send for a single page request. +pub type PaginationQuery = IndexMap; diff --git a/integrations/localthought/syncables/src/pagination/validate.rs b/integrations/localthought/syncables/src/pagination/validate.rs new file mode 100644 index 0000000000..faa1ae1b18 --- /dev/null +++ b/integrations/localthought/syncables/src/pagination/validate.rs @@ -0,0 +1,112 @@ +//! Validating a pagination scheme against the extension's own rules (§9). + +use indexmap::IndexMap; + +use super::types::{PaginationSchemeObject, RequestFieldObject, ResponseFieldObject}; + +const SCHEME_TYPES: [&str; 3] = ["pageNumber", "pageToken", "nextLink"]; +const REQUEST_ROLES: [&str; 5] = ["page", "pageSize", "offset", "pageToken", "cursor"]; +const RESPONSE_ROLES: [&str; 7] = [ + "nextPageToken", + "nextCursor", + "nextLink", + "totalCount", + "totalPages", + "pageSize", + "currentPage", +]; + +fn is_extension_key(key: &str) -> bool { + key.starts_with("x-") +} + +fn check_request_roles( + errors: &mut Vec, + path: &str, + section: &str, + fields: Option<&IndexMap>, +) { + for (field_name, field) in fields.into_iter().flatten() { + if let Some(role) = &field.role { + if !is_extension_key(role) && !REQUEST_ROLES.contains(&role.as_str()) { + errors.push(format!( + "{path}.request.{section}.{field_name}.role is not a valid request role (got \"{role}\")" + )); + } + } + } +} + +fn check_response_roles( + errors: &mut Vec, + path: &str, + section: &str, + fields: Option<&IndexMap>, +) { + for (field_name, field) in fields.into_iter().flatten() { + if let Some(role) = &field.role { + if !is_extension_key(role) && !RESPONSE_ROLES.contains(&role.as_str()) { + errors.push(format!( + "{path}.response.{section}.{field_name}.role is not a valid response role (got \"{role}\")" + )); + } + } + } +} + +/// Validates a single scheme against spec section 9. +/// +/// Returns a list of human-readable errors (each naming the offending +/// location), empty if the scheme is valid. Schemes with errors are +/// excluded from auto-detection rather than thrown on — one malformed +/// scheme in a document shouldn't prevent using the others. +pub fn validate_pagination_scheme(name: &str, scheme: &PaginationSchemeObject) -> Vec { + let mut errors = Vec::new(); + let path = format!("paginationSchemes.{name}"); + + let declared_type = scheme + .scheme_type + .as_str() + .map_or_else(|| scheme.scheme_type.to_string(), str::to_string); + if !SCHEME_TYPES.contains(&declared_type.as_str()) { + errors.push(format!( + "{path}.type must be one of pageNumber, pageToken, or nextLink (got \"{declared_type}\")" + )); + } + + if scheme.request.is_none() && scheme.response.is_none() { + errors.push(format!( + "{path} must define at least one of \"request\" or \"response\"" + )); + } + + let request = scheme.request.as_ref(); + check_request_roles( + &mut errors, + &path, + "queryParameters", + request.and_then(|r| r.query_parameters.as_ref()), + ); + check_request_roles( + &mut errors, + &path, + "bodyFields", + request.and_then(|r| r.body_fields.as_ref()), + ); + + let response = scheme.response.as_ref(); + check_response_roles( + &mut errors, + &path, + "bodyFields", + response.and_then(|r| r.body_fields.as_ref()), + ); + check_response_roles( + &mut errors, + &path, + "headers", + response.and_then(|r| r.headers.as_ref()), + ); + + errors +} diff --git a/integrations/localthought/syncables/src/resources/discover.rs b/integrations/localthought/syncables/src/resources/discover.rs new file mode 100644 index 0000000000..44fb66e66c --- /dev/null +++ b/integrations/localthought/syncables/src/resources/discover.rs @@ -0,0 +1,106 @@ +//! Pairing collection paths with their item paths. +//! +//! This pairing is the core concept the rest of the crate builds on: a +//! "resource" only exists where a collection path (`/pets`) has a direct +//! item-path child (`/pets/{petId}`). Paths without such a pairing +//! (health checks, one-off actions) are not resources and are handled +//! separately as raw request/response passthroughs. + +use indexmap::IndexMap; + +use crate::openapi::types::PathItem; + +/// HTTP method a background `update` sends. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum UpdateMethod { + /// Full replace. + Put, + /// Partial update, for APIs (e.g. GitHub) that expose no `PUT`. + Patch, +} + +impl UpdateMethod { + /// The method name as it goes on the wire. + pub fn as_str(self) -> &'static str { + match self { + Self::Put => "PUT", + Self::Patch => "PATCH", + } + } +} + +/// A collection path paired with its item path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResourceRoute { + /// The collection path, e.g. `/pets`. + pub collection_path: String, + /// The item path, e.g. `/pets/{petId}`. + pub item_path: String, + /// Name of the item path's trailing variable, e.g. `petId`. + pub item_param: String, + /// HTTP method a background `update` sends. + /// + /// Derived from the item path's declared operations: `PUT` when the + /// item path has a `put` operation (a full replace), otherwise + /// `PATCH` when it only offers a partial update. + pub update_method: UpdateMethod, +} + +/// Chooses the update method for an item path from the operations it +/// declares: prefer `PUT` (replace) when present, fall back to `PATCH` +/// when the path only offers a partial update, and default to `PUT` when +/// neither is declared. +fn update_method_for(paths: &IndexMap, item_path: &str) -> UpdateMethod { + match paths.get(item_path) { + Some(item) if item.put.is_none() && item.patch.is_some() => UpdateMethod::Patch, + _ => UpdateMethod::Put, + } +} + +/// Pairs each collection path with its item path so mock server and +/// client can apply CRUD semantics. +pub fn discover_resources(paths: &IndexMap) -> Vec { + let all_paths: Vec<&String> = paths.keys().collect(); + + let mut resources = Vec::new(); + for collection_path in all_paths.iter().filter(|p| !is_item_path(p)) { + let item_path = all_paths + .iter() + .find(|p| is_item_path(p) && is_direct_child(collection_path, p)); + if let Some(item_path) = item_path { + resources.push(ResourceRoute { + collection_path: (*collection_path).clone(), + item_path: (*item_path).clone(), + item_param: extract_param_name(item_path), + update_method: update_method_for(paths, item_path), + }); + } + } + resources +} + +fn is_item_path(path: &str) -> bool { + path.split('/') + .rfind(|s| !s.is_empty()) + .is_some_and(|last| last.starts_with('{') && last.ends_with('}')) +} + +fn extract_param_name(item_path: &str) -> String { + let last = item_path.split('/').rfind(|s| !s.is_empty()).unwrap_or(""); + last.get(1..last.len().saturating_sub(1)) + .unwrap_or("") + .to_string() +} + +fn is_direct_child(collection_path: &str, candidate: &str) -> bool { + let collection: Vec<&str> = collection_path + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + let candidate: Vec<&str> = candidate.split('/').filter(|s| !s.is_empty()).collect(); + candidate.len() == collection.len() + 1 + && collection + .iter() + .enumerate() + .all(|(index, segment)| candidate.get(index) == Some(segment)) +} diff --git a/integrations/localthought/syncables/src/resources/mod.rs b/integrations/localthought/syncables/src/resources/mod.rs new file mode 100644 index 0000000000..03d24596b1 --- /dev/null +++ b/integrations/localthought/syncables/src/resources/mod.rs @@ -0,0 +1,3 @@ +//! Turning `document.paths` into the list of syncable resources. + +pub mod discover; diff --git a/integrations/localthought/syncables/src/routing/mod.rs b/integrations/localthought/syncables/src/routing/mod.rs new file mode 100644 index 0000000000..65ca30eb38 --- /dev/null +++ b/integrations/localthought/syncables/src/routing/mod.rs @@ -0,0 +1,3 @@ +//! Matching incoming request paths against OpenAPI path templates. + +pub mod router; diff --git a/integrations/localthought/syncables/src/routing/router.rs b/integrations/localthought/syncables/src/routing/router.rs new file mode 100644 index 0000000000..8ded223da7 --- /dev/null +++ b/integrations/localthought/syncables/src/routing/router.rs @@ -0,0 +1,59 @@ +//! Matching a concrete request path against OpenAPI path templates. + +use std::borrow::Cow; + +use indexmap::IndexMap; +use percent_encoding::percent_decode_str; + +/// A template that matched, together with the path variables it bound. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RouteMatch { + /// The OpenAPI path template that matched, e.g. `/pets/{petId}`. + pub template: String, + /// Path variables bound by the match, percent-decoded. + pub params: IndexMap, +} + +fn decode(segment: &str) -> String { + match percent_decode_str(segment).decode_utf8() { + Ok(Cow::Borrowed(value)) => value.to_string(), + Ok(Cow::Owned(value)) => value, + Err(_) => segment.to_string(), + } +} + +fn match_template(template: &str, actual: &str) -> Option> { + let template_segments: Vec<&str> = template.split('/').filter(|s| !s.is_empty()).collect(); + let actual_segments: Vec<&str> = actual.split('/').filter(|s| !s.is_empty()).collect(); + if template_segments.len() != actual_segments.len() { + return None; + } + + let mut params = IndexMap::new(); + for (index, template_segment) in template_segments.iter().enumerate() { + let actual_segment = actual_segments.get(index).copied().unwrap_or(""); + if let Some(name) = template_segment + .strip_prefix('{') + .and_then(|s| s.strip_suffix('}')) + { + params.insert(name.to_string(), decode(actual_segment)); + } else if *template_segment != actual_segment { + return None; + } + } + Some(params) +} + +/// Returns the first template in `templates` that matches `actual`. +pub fn find_route>(templates: &[S], actual: &str) -> Option { + for template in templates { + let template = template.as_ref(); + if let Some(params) = match_template(template, actual) { + return Some(RouteMatch { + template: template.to_string(), + params, + }); + } + } + None +} diff --git a/integrations/localthought/syncables/src/sync/client.rs b/integrations/localthought/syncables/src/sync/client.rs new file mode 100644 index 0000000000..237cd80021 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/client.rs @@ -0,0 +1,520 @@ +//! `SyncClient`: the read half of [issue #9](https://github.com/localthought/syncables-rs/issues/9) +//! — "the call reflector-rs makes." Loads the document and its overlays +//! (#2), derives the resource model (#3) and validates the configured +//! constants (#6), derives and stores the ontology (#8) before any record, +//! then walks every managed collection — following pagination to the end +//! (#4) and binding constants (#6) — putting each record into a +//! host-provided [`Storage`] (#7). +//! +//! **Local-first writes are out of scope here.** The issue explicitly +//! allows that: "Steps beyond \[the full read\] can land later." `create`/ +//! `update`/`remove`, the per-record write queue and retry/backoff, and +//! reading back `x-crud`'s `addedFields` from a create response are a +//! separate, later piece of #9. +//! +//! `ClientConfig`/`SyncReport`/`SyncError` are copied field-for-field from +//! the contract [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +//! is already written against, in its `src/syncables.rs`; that module is +//! meant to be deleted once reflector-rs points its `use`s here instead. +//! One deliberate divergence: that stub's `ClientConfig` carries no way to +//! actually reach the network, and `SyncClient::new` takes only a +//! `ClientConfig`. This crate has no HTTP client dependency (mirroring +//! [`crate::client::client`]'s existing design, where +//! [`ApiClientOptions::fetch`](crate::client::client::ApiClientOptions::fetch) +//! is the host's only extension point for making requests), so +//! [`SyncClient::new`] additionally takes a [`Fetch`] implementation. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; + +use indexmap::IndexMap; +use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; +use serde_json::{Map, Value}; + +use crate::client::client::{Fetch, HttpRequest}; +use crate::error::Error; +use crate::openapi::overlay::load_open_api_document_with_overlays; +use crate::openapi::types::{OpenApiDocument, SchemaObject}; +use crate::pagination::autodetect::resolve_effective_scheme; +use crate::pagination::items::locate_items_field; +use crate::pagination::request_builder::{build_query, next_step, PageCursor, PageStep}; +use crate::pagination::response_parser::parse_pagination_state; +use crate::pagination::types::PaginationSchemeObject; + +use super::constants::{bind_url, validate_constants}; +use super::credentials::{base_url, Credentials}; +use super::ontology::derive_ontology; +use super::resource_model::{ + discover_resource_model, ContextProvider, ManagedCollection, ResourceModel, +}; +use super::storage::{Record, Storage, StorageError}; + +/// Everything the engine needs to derive and run a sync. +#[derive(Clone, Debug)] +pub struct ClientConfig { + /// Path to the OpenAPI document describing the API. + pub document: PathBuf, + /// Overlays applied to that document, in order. + pub overlays: Vec, + /// The credential sent to the API. + pub credentials: Credentials, + /// Values bound into the document's path and query parameters. This is + /// what narrows a sync to one issue tracker (`owner`/`repo`) instead of + /// every tracker the credential can reach. + pub constants: BTreeMap, + /// Canonical base URL the derived ontology's terms are minted under, + /// e.g. `https://my-ontologies.com`. No trailing slash. + pub ontology_base_url: String, +} + +/// What one [`SyncClient::sync`] did. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SyncReport { + /// Records read from the API and written to storage, per resource. + pub read: BTreeMap, + /// Ontology terms stored. + pub ontology_terms: usize, + /// Non-fatal problems: one collection failing does not abandon the + /// rest of the sync. + pub errors: Vec, +} + +/// Errors the engine itself raises. +#[derive(Debug)] +pub enum SyncError { + /// The document or its overlays could not be loaded or reconciled. + Document(String), + /// The API rejected or failed a request. + Transport(String), + /// The [`Storage`] the host supplied failed. + Storage(StorageError), + /// This build of the engine has no behaviour behind the given call yet. + NotImplemented(&'static str), +} + +impl std::fmt::Display for SyncError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SyncError::Document(message) => write!(f, "OpenAPI document error: {message}"), + SyncError::Transport(message) => write!(f, "transport error: {message}"), + SyncError::Storage(error) => write!(f, "storage error: {error}"), + SyncError::NotImplemented(what) => write!(f, "not implemented yet: {what}"), + } + } +} + +impl std::error::Error for SyncError {} + +impl From for SyncError { + /// Every failure this crate's document/overlay/resource-model/ontology + /// pipeline raises is, from the engine's point of view, a problem with + /// the document or its configuration. + fn from(error: Error) -> Self { + SyncError::Document(error.to_string()) + } +} + +/// The sync engine. +pub struct SyncClient { + config: ClientConfig, + fetch: Arc, +} + +impl std::fmt::Debug for SyncClient { + /// `Fetch` implementations aren't required to be `Debug`, so this + /// shows the configuration only. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyncClient") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl SyncClient { + /// Builds a client for `config`, reaching the network through `fetch`. + /// + /// Errors if `config.ontology_base_url` is empty — ontology terms need + /// a canonical, resolvable base URL. + pub fn new(config: ClientConfig, fetch: Arc) -> Result { + if config.ontology_base_url.is_empty() { + return Err(SyncError::Document( + "ontology_base_url is required: ontology terms need a canonical, resolvable base URL" + .to_string(), + )); + } + Ok(SyncClient { config, fetch }) + } + + /// The configuration this client was built from. + #[must_use] + pub fn config(&self) -> &ClientConfig { + &self.config + } + + /// Reads everything the document describes — narrowed by + /// [`ClientConfig::constants`] — into `storage`, and stores the + /// ontology derived from the document alongside it, before any record. + /// + /// One collection failing partway through — a 404, an unparseable + /// response, a storage error — is recorded in the returned + /// [`SyncReport::errors`] and does not abandon the rest of the sync. + pub async fn sync(&self, storage: &dyn Storage) -> Result { + let document = load_open_api_document_with_overlays( + self.config.document.as_path(), + &self.config.overlays, + ) + .await?; + self.sync_document(&document, storage).await + } + + /// Sync an already loaded, resolved document without filesystem access. + pub async fn sync_document( + &self, + document: &OpenApiDocument, + storage: &dyn Storage, + ) -> Result { + let model = discover_resource_model(document)?; + validate_constants(document, &model, &self.config.constants)?; + + // Fail before writing anything if there's nowhere to sync from — + // no point minting an ontology for a sync that can't run at all. + let base = base_url(document) + .ok_or_else(|| SyncError::Document("document declares no servers".to_string()))?; + + let ontology = derive_ontology(document)?; + storage + .put_ontology(&ontology) + .await + .map_err(SyncError::Storage)?; + + let mut report = SyncReport { + ontology_terms: ontology.terms.len(), + ..SyncReport::default() + }; + self.walk_all(document, &model, base, storage, &mut report) + .await; + Ok(report) + } + + /// Walks every managed collection in dependency order: a collection + /// whose context parameters are all constants runs once; one with a + /// parent-record-provided parameter (see + /// [`ResourceModel::provider_for`]) runs once per parent record + /// already read, its own records feeding any collection nested under + /// it in turn. + async fn walk_all( + &self, + document: &OpenApiDocument, + model: &ResourceModel, + base: &str, + storage: &dyn Storage, + report: &mut SyncReport, + ) { + let mut records_by_collection: BTreeMap>> = BTreeMap::new(); + let mut pending: Vec<&ManagedCollection> = model.collections.iter().collect(); + + loop { + let mut still_pending = Vec::new(); + let mut progressed = false; + + for collection in pending { + let provider_params: Vec<(&str, &ContextProvider)> = collection + .context_params + .iter() + .filter(|param| !self.config.constants.contains_key(param.as_str())) + .filter_map(|param| { + model + .provider_for(param) + .map(|provider| (param.as_str(), provider)) + }) + .collect(); + + let ready = provider_params + .iter() + .all(|(_, provider)| records_by_collection.contains_key(&provider.collection)); + if !ready { + still_pending.push(collection); + continue; + } + progressed = true; + + let mut collection_records = Vec::new(); + for values in binding_combinations( + &self.config.constants, + &provider_params, + &records_by_collection, + ) { + match self + .walk_collection(document, base, collection, &values) + .await + { + Ok(records) => { + let namespace = collection + .context_params + .iter() + .map(|param| values.get(param).cloned().unwrap_or_default()) + .collect::>() + .join("/"); + for record in &records { + let id = record + .get(&collection.id_field) + .map(json_to_string) + .unwrap_or_default(); + let stored = Record { + namespace: namespace.clone(), + resource: collection.resource.clone(), + id, + value: record.clone(), + }; + if let Err(error) = storage.put(&stored).await { + report.errors.push(format!("{}: {error}", collection.name)); + } + } + collection_records.extend(records); + } + Err(message) => report + .errors + .push(format!("{}: {message}", collection.name)), + } + } + + *report.read.entry(collection.resource.clone()).or_insert(0) += + collection_records.len(); + records_by_collection.insert(collection.name.clone(), collection_records); + } + + if still_pending.is_empty() { + break; + } + if !progressed { + // Unreachable once `validate_constants` has passed against + // the same model, but never spin: report and stop rather + // than loop forever if it somehow is. + for collection in still_pending { + report.errors.push(format!( + "{}: could not resolve its context parameters", + collection.name + )); + } + break; + } + pending = still_pending; + } + } + + /// Fetches every item of one managed collection under `values`, + /// walking every page per its resolved pagination scheme. + async fn walk_collection( + &self, + document: &OpenApiDocument, + base: &str, + collection: &ManagedCollection, + values: &BTreeMap, + ) -> std::result::Result>, String> { + let operation = document + .paths + .get(&collection.collection_url) + .and_then(|item| item.get.as_ref()) + .ok_or_else(|| format!("{} declares no GET operation", collection.collection_url))?; + + let effective = resolve_effective_scheme(document, operation); + let response_schema = operation + .responses + .get("200") + .and_then(|response| response.content.as_ref()) + .and_then(|content| content.get("application/json")) + .and_then(|media| media.schema.as_ref()); + + let path = + bind_url(&collection.collection_url, values).map_err(|error| error.to_string())?; + + let mut items = Vec::new(); + let mut cursor = PageCursor::default(); + let mut pages_fetched = 0usize; + let mut next_url: Option = None; + + loop { + let request_url = if let Some(url) = next_url.take() { + url + } else { + let mut query = collection.list_query.clone(); + if let Some(effective) = &effective { + for (name, value) in build_query(&effective.scheme, &cursor, None) { + query.insert(name, value); + } + } + request_url(base, &path, &query) + }; + + let mut headers = IndexMap::new(); + if let Some(authorization) = self.config.credentials.authorization_header() { + headers.insert("Authorization".to_string(), authorization); + } + let diagnostic_url = redacted_request_url(&request_url); + let response = self + .fetch + .fetch(HttpRequest { + method: "GET".to_string(), + url: request_url, + headers, + body: None, + }) + .await + .map_err(|error| error.to_string())?; + + if !(200..300).contains(&response.status) { + return Err(format!( + "GET {} responded {}", + diagnostic_url, response.status + )); + } + + let body: Value = + serde_json::from_slice(&response.body).map_err(|error| error.to_string())?; + let page_items = response_items( + response_schema, + effective.as_ref().map(|e| &e.scheme), + &body, + )?; + let page_count = u64::try_from(page_items.len()).unwrap_or(u64::MAX); + items.extend(page_items); + pages_fetched += 1; + + let Some(effective) = &effective else { break }; + let items_so_far = u64::try_from(items.len()).unwrap_or(u64::MAX); + let state = parse_pagination_state( + &effective.scheme, + &body, + &response.headers, + Some(items_so_far), + ); + match next_step( + &effective.scheme, + &cursor, + &state, + page_count, + pages_fetched, + ) { + PageStep::Done => break, + PageStep::NextPage(next) => cursor = next, + PageStep::FollowLink(url) => next_url = Some(url), + } + } + + Ok(items) + } +} + +/// Returns a request URL that is useful in an error message without exposing +/// values sent as query parameters. Authentication normally travels in a +/// header, but OpenAPI also permits query-parameter API keys. +fn redacted_request_url(url: &str) -> String { + match url.split_once('?') { + Some((base, _)) => format!("{base}?"), + None => url.to_owned(), + } +} + +/// Every combination of constants plus one parent record's value per +/// `provider_params` entry — the cartesian product across however many +/// ancestor collections `collection` is nested under. Empty +/// `provider_params` yields exactly the constants unchanged (a root +/// collection, walked once); a provider whose collection has no records +/// yet read yields no combinations at all (nothing to nest under). +fn binding_combinations( + constants: &BTreeMap, + provider_params: &[(&str, &ContextProvider)], + records_by_collection: &BTreeMap>>, +) -> Vec> { + let mut combinations = vec![constants.clone()]; + for (param, provider) in provider_params { + let empty = Vec::new(); + let parent_records = records_by_collection + .get(&provider.collection) + .unwrap_or(&empty); + let mut next = Vec::new(); + for combination in &combinations { + for record in parent_records { + let Some(value) = record.get(&provider.field) else { + continue; + }; + let mut extended = combination.clone(); + extended.insert((*param).to_string(), json_to_string(value)); + next.push(extended); + } + } + combinations = next; + } + combinations +} + +/// The array of item objects in a paginated response: the whole body when +/// its schema is itself an array, otherwise the property +/// [`locate_items_field`] finds. +fn response_items( + schema: Option<&SchemaObject>, + scheme: Option<&PaginationSchemeObject>, + body: &Value, +) -> std::result::Result>, String> { + let is_bare_array = schema.and_then(|s| s.schema_type.as_deref()) == Some("array"); + let array = if is_bare_array { + body.as_array() + } else if let Some(field) = locate_items_field(schema, scheme) { + body.get(&field).and_then(Value::as_array) + } else { + body.as_array() + }; + let array = + array.ok_or_else(|| "could not locate the items array in the response".to_string())?; + Ok(array + .iter() + .filter_map(|item| item.as_object().cloned()) + .collect()) +} + +/// Renders a JSON scalar as a plain string, for use as a record id or a +/// namespace segment. +fn json_to_string(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +/// Characters percent-encoded in a query string component — the reserved +/// characters that would otherwise be mistaken for delimiters (`&`, `=`, +/// `#`, `+`), not every non-alphanumeric character: a query parameter name +/// like `per_page` should round-trip as `per_page`, not `per%5Fpage`. +const QUERY_COMPONENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'%') + .add(b'&') + .add(b'\'') + .add(b'+') + .add(b'<') + .add(b'>') + .add(b'=') + .add(b'`'); + +/// Builds the absolute URL for one page request. +fn request_url(base: &str, path: &str, query: &IndexMap) -> String { + let mut url = format!("{base}{path}"); + if !query.is_empty() { + let pairs: Vec = query + .iter() + .map(|(name, value)| { + format!( + "{}={}", + utf8_percent_encode(name, QUERY_COMPONENT), + utf8_percent_encode(value, QUERY_COMPONENT) + ) + }) + .collect(); + url.push('?'); + url.push_str(&pairs.join("&")); + } + url +} diff --git a/integrations/localthought/syncables/src/sync/constants.rs b/integrations/localthought/syncables/src/sync/constants.rs new file mode 100644 index 0000000000..0c13c38994 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/constants.rs @@ -0,0 +1,129 @@ +//! Binding configured constants into a resource model's path templates — +//! what scopes a sync to one issue tracker (`owner`/`repo`) instead of +//! every tracker the credential can reach, per +//! [issue #6](https://github.com/localthought/syncables-rs/issues/6). +//! +//! A GitHub token can read every repository its owner can reach; narrowing +//! that down has to be part of the sync's configuration, not a filter +//! applied after the fact — fetching every tracker and discarding most of +//! it is both slow and a data-handling problem. + +use std::collections::{BTreeMap, HashSet}; + +use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; + +use crate::error::{Error, Result}; +use crate::openapi::types::OpenApiDocument; + +use super::resource_model::{path_variables, ResourceModel}; + +/// Characters percent-encoded when a value is substituted into a URL path +/// segment — everything outside what's safe unescaped in a path segment, +/// mirroring the `url` crate's `PATH_SEGMENT_ENCODE_SET` (not pulled in as +/// a dependency for one constant). +const PATH_SEGMENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'<') + .add(b'>') + .add(b'?') + .add(b'`') + .add(b'{') + .add(b'}') + .add(b'/') + .add(b'%'); + +/// Every parameter name declared by some operation in the document — path +/// or query, on any method — the set a constant is allowed to name. +fn declared_parameter_names(document: &OpenApiDocument) -> HashSet<&str> { + document + .paths + .values() + .flat_map(|item| { + [ + item.get.as_ref(), + item.put.as_ref(), + item.post.as_ref(), + item.patch.as_ref(), + item.delete.as_ref(), + ] + }) + .flatten() + .flat_map(|operation| operation.parameters.iter().flatten()) + .map(|parameter| parameter.name.as_str()) + .collect() +} + +/// A path variable is resolvable if a constant supplies it, or a parent +/// record's context provider does. +fn is_resolvable(model: &ResourceModel, constants: &BTreeMap, param: &str) -> bool { + constants.contains_key(param) || model.provider_for(param).is_some() +} + +/// Validates `constants` against `document` and `model`, before any request +/// is made: +/// +/// - every constant must name a parameter the document actually declares +/// ([`Error::UnknownConstant`]) — a typo'd key would otherwise silently +/// sync nothing, or scope the sync far wider than intended; +/// - every path variable a managed collection needs — in its list URL, or +/// in its own item URL beyond what its identity binding already supplies +/// — must be resolvable by a constant or a parent record's provider +/// ([`Error::UnboundContextParam`]). +pub fn validate_constants( + document: &OpenApiDocument, + model: &ResourceModel, + constants: &BTreeMap, +) -> Result<()> { + let declared = declared_parameter_names(document); + for key in constants.keys() { + if !declared.contains(key.as_str()) { + return Err(Error::UnknownConstant(key.clone())); + } + } + + for collection in &model.collections { + for param in &collection.context_params { + if !is_resolvable(model, constants, param) { + return Err(Error::UnboundContextParam(param.clone())); + } + } + for param in path_variables(&collection.item_url) { + if collection.identity_params.contains(¶m) { + continue; + } + if !is_resolvable(model, constants, ¶m) { + return Err(Error::UnboundContextParam(param)); + } + } + } + Ok(()) +} + +/// Substitutes every `{param}` in `template` with its percent-encoded value +/// from `values`. +/// +/// Errors with [`Error::UnboundContextParam`] if `template` names a +/// variable `values` has no entry for — [`validate_constants`] is meant to +/// have already ruled this out for every template the resource model +/// declares, so this only fires on a template built outside that check. +pub fn bind_url(template: &str, values: &BTreeMap) -> Result { + let mut bound = String::with_capacity(template.len()); + let mut rest = template; + while let Some(start) = rest.find('{') { + bound.push_str(&rest[..start]); + let Some(end) = rest[start..].find('}') else { + bound.push_str(&rest[start..]); + return Ok(bound); + }; + let name = &rest[start + 1..start + end]; + let value = values + .get(name) + .ok_or_else(|| Error::UnboundContextParam(name.to_string()))?; + bound.push_str(&utf8_percent_encode(value, PATH_SEGMENT).to_string()); + rest = &rest[start + end + 1..]; + } + bound.push_str(rest); + Ok(bound) +} diff --git a/integrations/localthought/syncables/src/sync/credentials.rs b/integrations/localthought/syncables/src/sync/credentials.rs new file mode 100644 index 0000000000..9a89c23368 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/credentials.rs @@ -0,0 +1,70 @@ +//! Credentials presented to the API being reflected, and the base URL they +//! are presented against. +//! +//! Ported from `Credentials` in the syncables-rs API contract mirrored at +//! [`localthought/reflector-rs`'s `src/syncables.rs`](https://github.com/localthought/reflector-rs/blob/main/src/syncables.rs), +//! and from `StaticTokenManager` in +//! [`localthought/reflector`](https://github.com/localthought/reflector)'s +//! `src/oauth/static-token.ts`, which is what actually attaches the header +//! and retargets a request at the real API base in the TypeScript +//! original. + +use crate::openapi::types::OpenApiDocument; + +/// Credentials presented to the API being reflected. +/// +/// Only a static bearer token is modelled so far: that is what the GitHub +/// auth overlay's `http`/`bearer` security scheme asks for. An interactive +/// OAuth profile — derived from the document the way the TypeScript +/// Reflector derives Google Calendar's — is tracked separately (see +/// [issue #5](https://github.com/localthought/syncables-rs/issues/5)). +#[derive(Clone, PartialEq, Eq)] +pub enum Credentials { + /// Sent as `Authorization: Bearer `. + Bearer(String), + /// No credential — only useful against a public, unauthenticated API. + Anonymous, +} + +impl std::fmt::Debug for Credentials { + /// Never renders the secret, so `{:?}` on a config that holds one is + /// safe to log. A host logging its resolved configuration at startup is + /// exactly the scenario this guards against. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Credentials::Bearer(_) => f.write_str("Bearer()"), + Credentials::Anonymous => f.write_str("Anonymous"), + } + } +} + +impl Credentials { + /// The `Authorization` header value to send with every request, if any. + /// + /// [`Credentials::Anonymous`] sends no `Authorization` header at all — + /// distinct in principle from an empty bearer token, which would still + /// be presented. + #[must_use] + pub fn authorization_header(&self) -> Option { + match self { + Credentials::Bearer(token) => Some(format!("Bearer {token}")), + Credentials::Anonymous => None, + } + } +} + +/// The API's base URL, read from the document's `servers` — never from +/// separate configuration, so a document can't be pointed at the wrong host +/// by a configuration mismatch. +/// +/// Returns the first declared server, matching the TypeScript original's +/// `StaticTokenManager`/`TokenManager`, which likewise target whatever +/// `servers[0]` names. `None` if the document declares no servers at all. +#[must_use] +pub fn base_url(document: &OpenApiDocument) -> Option<&str> { + document + .servers + .as_ref()? + .first() + .map(|server| server.url.as_str()) +} diff --git a/integrations/localthought/syncables/src/sync/mod.rs b/integrations/localthought/syncables/src/sync/mod.rs new file mode 100644 index 0000000000..8d3819ded1 --- /dev/null +++ b/integrations/localthought/syncables/src/sync/mod.rs @@ -0,0 +1,24 @@ +//! The sync engine surface: `SyncClient`, `ClientConfig`, [`storage::Storage`] +//! and friends. +//! +//! This is a *different* surface from the crate's existing +//! [`crate::client`]/[`crate::mock_server`] pair (a port of the +//! TypeScript `syncables` package). It is new scope, tracked by +//! [localthought/syncables-rs#1](https://github.com/localthought/syncables-rs/issues/1) +//! and the issues under it: a generic engine that reads an OpenAPI +//! document plus a resource model derived from it, and syncs records into +//! a host-provided [`storage::Storage`] implementation. +//! [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +//! is the first intended host. +//! +//! [`storage`] (issue #7), [`credentials`] (issue #5), [`resource_model`] +//! (issue #3), [`constants`] (issue #6), [`ontology`] (issue #8) and +//! [`client`] (issue #9 — the read half; local-first writes are a +//! follow-up) exist so far. + +pub mod client; +pub mod constants; +pub mod credentials; +pub mod ontology; +pub mod resource_model; +pub mod storage; diff --git a/integrations/localthought/syncables/src/sync/ontology.rs b/integrations/localthought/syncables/src/sync/ontology.rs new file mode 100644 index 0000000000..e8094f6a5d --- /dev/null +++ b/integrations/localthought/syncables/src/sync/ontology.rs @@ -0,0 +1,273 @@ +//! Deriving an Atomic Data ontology from the document's `crudResources` +//! extension, per [issue #8](https://github.com/localthought/syncables-rs/issues/8): +//! a Class per resource and a Property per field of that resource's schema. +//! +//! **This crate must not depend on `atomic_lib`.** Terms are handed over as +//! a neutral description — a path, a kind, a shortname, a description, an +//! optional datatype URL, and cross-references by path — not as Atomic Data +//! `Resource`s. Rendering those into `Resource`s at their public/internal +//! subjects is the host's job: see `AtomicStorage` in +//! [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs)'s +//! `src/store.rs`, which is written against exactly this shape. +//! +//! The engine never invents an origin for a term's identity: [`Ontology`] +//! and [`OntologyTerm`] carry only relative paths (no leading slash), which +//! the host mints under its own `ClientConfig::ontology_base_url`. + +use std::collections::HashSet; + +use indexmap::IndexMap; +use serde_json::Value; + +use crate::error::{Error, Result}; +use crate::openapi::types::{OpenApiDocument, SchemaObject}; + +use super::resource_model::{crud_resources, CrudResourceObject}; + +const DATATYPE_STRING: &str = "https://atomicdata.dev/datatypes/string"; +const DATATYPE_INTEGER: &str = "https://atomicdata.dev/datatypes/integer"; +const DATATYPE_FLOAT: &str = "https://atomicdata.dev/datatypes/float"; +const DATATYPE_BOOLEAN: &str = "https://atomicdata.dev/datatypes/boolean"; +const DATATYPE_TIMESTAMP: &str = "https://atomicdata.dev/datatypes/timestamp"; +const DATATYPE_DATE: &str = "https://atomicdata.dev/datatypes/date"; + +/// Whether a term describes a Class or a Property. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TermKind { + /// A resource, e.g. `issue`. + Class, + /// A field of a resource's schema, e.g. `title`. + Property, +} + +/// One term of the ontology derived from an OpenAPI document. +/// +/// Paths are relative to a host-supplied base URL and are the term's +/// identity: `github-issues/property/title` is published (by the host, not +/// this crate) as e.g. `https://my-ontologies.com/github-issues/property/title`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OntologyTerm { + /// Path under the ontology's own path, without a leading slash. + pub path: String, + /// Whether this is a Class or a Property. + pub kind: TermKind, + /// Atomic Data shortname — lowercase, `-`-separated. + pub shortname: String, + /// Human-readable description. + pub description: String, + /// For a Property: the Atomic Data datatype URL its values carry. + /// `None` for a Class, or for a Property whose schema type this crate's + /// mapping can't place — omitted rather than guessed; the host falls + /// back to inferring from the JSON value. + pub datatype: Option, + /// For a Class: the paths (or absolute URLs) of its required + /// properties, from the schema's `required` list. + pub requires: Vec, + /// For a Class: the paths (or absolute URLs) of its recommended + /// (present but not required) properties. + pub recommends: Vec, +} + +/// The ontology derived from one OpenAPI document. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Ontology { + /// Path of the ontology resource itself, e.g. `github-issues`. + pub path: String, + /// Atomic Data shortname for the ontology itself. + pub shortname: String, + /// Human-readable description. + pub description: String, + /// Every term the ontology declares, Classes and Properties mixed, in + /// the order their resources appear in `crudResources`. + pub terms: Vec, +} + +/// Normalizes an OpenAPI name into the Atomic Data shortname used by the +/// generated ontology: lowercase, `-`-separated, with every run of +/// non-alphanumeric characters collapsed to one `-` (`state_reason` → +/// `state-reason`, `updated_at` → `updated-at`). +/// +/// Hosts use this when matching raw API record keys or resource names to the +/// ontology terms returned by [`derive_ontology`]. +pub fn ontology_shortname(name: &str) -> String { + let mut slug = String::with_capacity(name.len()); + let mut pending_dash = false; + for ch in name.chars() { + if ch.is_ascii_alphanumeric() { + if pending_dash && !slug.is_empty() { + slug.push('-'); + } + pending_dash = false; + slug.push(ch.to_ascii_lowercase()); + } else { + pending_dash = true; + } + } + slug +} + +/// Claims `original`'s slug in `claimed`, reusing the existing slug if +/// `original` already claimed one (properties are shared across every +/// resource that has a same-named field) and erroring if a *different* +/// name has already claimed the same slug. +fn claim_shortname(claimed: &mut IndexMap, original: &str) -> Result { + let candidate = ontology_shortname(original); + match claimed.get(&candidate) { + Some(existing) if existing == original => Ok(candidate), + Some(existing) => Err(Error::ShortnameCollision { + shortname: candidate, + first: existing.clone(), + second: original.to_string(), + }), + None => { + claimed.insert(candidate.clone(), original.to_string()); + Ok(candidate) + } + } +} + +/// Maps a property schema's `type`/`format` to an Atomic Data datatype URL. +/// A type this mapping doesn't recognize is `None` rather than guessed. +fn datatype_url(schema: &SchemaObject) -> Option { + match (schema.schema_type.as_deref(), schema.format.as_deref()) { + (Some("string"), Some("date-time")) => Some(DATATYPE_TIMESTAMP.to_string()), + (Some("string"), Some("date")) => Some(DATATYPE_DATE.to_string()), + (Some("string"), _) => Some(DATATYPE_STRING.to_string()), + (Some("integer"), _) => Some(DATATYPE_INTEGER.to_string()), + (Some("number"), _) => Some(DATATYPE_FLOAT.to_string()), + (Some("boolean"), _) => Some(DATATYPE_BOOLEAN.to_string()), + _ => None, + } +} + +/// A property schema's own `description`, if the document gives one — not +/// a typed field on [`SchemaObject`], so it's read from the catch-all. +fn schema_description(schema: &SchemaObject) -> Option { + schema + .extensions + .get("description")? + .as_str() + .map(str::to_string) +} + +/// Resolves a `crudResources..schema` to the [`SchemaObject`] it +/// names. The overlay declares it as a `$ref` (`{ $ref: '#/components/schemas/issue' }`) +/// — added to the document *after* `resolve_refs` already ran (overlays +/// apply on top of an already-resolved document), so it's never inlined +/// automatically and has to be resolved here by name instead. An inline +/// schema (no `$ref`) is also accepted. +fn resource_schema( + document: &OpenApiDocument, + resource: &CrudResourceObject, +) -> Option { + let raw = resource.extensions.get("schema")?; + if let Some(pointer) = raw.get("$ref").and_then(Value::as_str) { + let name = pointer.strip_prefix("#/components/schemas/")?; + document + .components + .as_ref()? + .schemas + .as_ref()? + .get(name) + .cloned() + } else { + serde_json::from_value(raw.clone()).ok() + } +} + +/// Derives the ontology from `document`'s `components.crudResources`: one +/// Class per resource, and one Property per field of that resource's +/// schema — shared across every resource that has a same-named field, +/// rather than minted again per resource. +/// +/// Returns [`Error::NoCrudResources`] if the document declares none, and +/// [`Error::ShortnameCollision`] if two differently-named resources or +/// fields would normalize to the same slug. +pub fn derive_ontology(document: &OpenApiDocument) -> Result { + let resources = crud_resources(document)?; + + let title = document.info.title.trim(); + let ontology_path = if title.is_empty() { + "ontology".to_string() + } else { + ontology_shortname(title) + }; + let description = if title.is_empty() { + "Derived from an OpenAPI document.".to_string() + } else { + format!("Derived from the \"{title}\" OpenAPI document.") + }; + + let mut terms: Vec = Vec::new(); + let mut class_shortnames = IndexMap::new(); + let mut property_shortnames = IndexMap::new(); + // shortname -> index into `terms`, so a field shared across resources + // reuses its one Property term instead of minting a duplicate. + let mut property_terms: IndexMap = IndexMap::new(); + + for (resource_name, resource) in &resources { + let class_shortname = claim_shortname(&mut class_shortnames, resource_name)?; + let schema = resource_schema(document, resource); + + let required: HashSet<&str> = schema + .as_ref() + .and_then(|s| s.required.as_deref()) + .into_iter() + .flatten() + .map(String::as_str) + .collect(); + + let mut requires = Vec::new(); + let mut recommends = Vec::new(); + for (field_name, field_schema) in schema.iter().flat_map(|s| s.properties.iter()).flatten() + { + // Validate the shortname (and any collision) before ever + // consulting `property_terms`, so two different field names + // that happen to normalize the same way can never be silently + // merged into one reused term. + let shortname = claim_shortname(&mut property_shortnames, field_name)?; + let path = if let Some(&index) = property_terms.get(&shortname) { + terms[index].path.clone() + } else { + let path = format!("{ontology_path}/property/{shortname}"); + property_terms.insert(shortname.clone(), terms.len()); + terms.push(OntologyTerm { + path: path.clone(), + kind: TermKind::Property, + shortname, + description: schema_description(field_schema) + .unwrap_or_else(|| format!("`{field_name}` of `{resource_name}`.")), + datatype: datatype_url(field_schema), + requires: Vec::new(), + recommends: Vec::new(), + }); + path + }; + if required.contains(field_name.as_str()) { + requires.push(path); + } else { + recommends.push(path); + } + } + + terms.push(OntologyTerm { + path: format!("{ontology_path}/class/{class_shortname}"), + kind: TermKind::Class, + shortname: class_shortname, + description: resource + .description + .clone() + .unwrap_or_else(|| format!("The `{resource_name}` resource.")), + datatype: None, + requires, + recommends, + }); + } + + Ok(Ontology { + path: ontology_path.clone(), + shortname: ontology_path, + description, + terms, + }) +} diff --git a/integrations/localthought/syncables/src/sync/resource_model.rs b/integrations/localthought/syncables/src/sync/resource_model.rs new file mode 100644 index 0000000000..829619031e --- /dev/null +++ b/integrations/localthought/syncables/src/sync/resource_model.rs @@ -0,0 +1,426 @@ +//! Deriving the resource model from `components.crudResources` — the +//! [CRUD Causality Extension](https://github.com/pondersource/openapi-extensions/tree/main/spec/crud-causality) +//! overlay's contribution to a document, and the `x-crud` block it adds to +//! each operation. +//! +//! Ported from `discoverResourceModel` in +//! [`localthought/reflector`](https://github.com/localthought/reflector)'s +//! `src/sync/resources.ts`, which is the reference for the traversal below +//! (a resource's identity binding, a collection's context parameters, and +//! how a nested collection's parent is resolved). That file also derives a +//! client-generated-id policy for Google Calendar's resources; GitHub's +//! `addedFields` are all server-assigned, so this port doesn't carry that +//! part over — see [issue #3](https://github.com/localthought/syncables-rs/issues/3). + +use indexmap::IndexMap; +use serde::de::Error as _; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; + +use crate::error::{Error, Result}; +use crate::openapi::types::{JsonMap, OpenApiDocument, OperationObject, SchemaObject}; + +// --- The raw `crudResources` shape, as the overlay declares it ------------ + +/// One resource declared under `components.crudResources`, keyed by its +/// resource name (e.g. `issue`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CrudResourceObject { + /// Human-readable description of the resource. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// How a single item of this resource is addressed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identity: Option, + /// The collections that list, and are written through, this resource. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collections: Option>, + /// Any other key on the resource, notably `schema`. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// A resource's single-item URL template and how its path variables map to +/// record fields. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResourceIdentityObject { + /// Single-item URL template, e.g. `/repos/{owner}/{repo}/issues/{issue_number}`. + #[serde(rename = "urlTemplate")] + pub url_template: String, + /// Path variable to record field, e.g. `issue_number` binds to `number` + /// — the resource's URL identity need not be the payload's own `id`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bindings: Option>, + /// Any other key on the identity. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One entry of a [`ResourceIdentityObject::bindings`] map. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct IdentityBindingObject { + /// The record field the path variable is read from (and reconciled + /// against, on write). + pub field: String, + /// Any other key on the binding. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// One entry of a [`CrudResourceObject::collections`] map. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ResourceCollectionObject { + /// Collection URL template, e.g. `/repos/{owner}/{repo}/issues`. + #[serde(rename = "urlTemplate")] + pub url_template: String, + /// Fixed query parameters added to every list request against this + /// collection — e.g. GitHub's issues list returns only open issues by + /// default, so the overlay declares `{ state: all }` to include closed + /// ones too. + #[serde( + rename = "x-list-query", + default, + skip_serializing_if = "Option::is_none" + )] + pub list_query: Option>, + /// Any other key on the collection. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// Reads `document.components.crudResources`, keyed by resource name. +/// +/// Returns [`Error::NoCrudResources`] if the document declares none — the +/// CRUD-causality overlay hasn't been applied. +pub(super) fn crud_resources( + document: &OpenApiDocument, +) -> Result> { + let raw = document + .components + .as_ref() + .and_then(|components| components.extensions.get("crudResources")) + .ok_or(Error::NoCrudResources)?; + serde_json::from_value(raw.clone()).map_err(Error::from) +} + +// --- The derived resource model -------------------------------------------- + +/// One resource collection the sync engine can walk, derived from one +/// `crudResources..collections.` entry and its resource's +/// `identity`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManagedCollection { + /// The collection's key, e.g. `issues`. + pub name: String, + /// The resource key in `crudResources`, e.g. `issue`. + pub resource: String, + /// Collection URL template, e.g. `/repos/{owner}/{repo}/issues`. + pub collection_url: String, + /// Single-item URL template (the resource's identity), e.g. + /// `/repos/{owner}/{repo}/issues/{issue_number}`. Empty if the resource + /// declares no `identity`. + pub item_url: String, + /// Record field that carries the item's own id (from the identity + /// binding whose path variable the item URL — not the collection + /// URL — uses), usually `id` but not always: GitHub's issues are + /// addressed by `number`. + pub id_field: String, + /// Path variables in `collection_url` that must be supplied by a + /// parent record or a configured constant (see + /// [issue #6](https://github.com/localthought/syncables-rs/issues/6)). + pub context_params: Vec, + /// Path variables in `item_url` that this resource's own identity + /// binding supplies from the record itself (e.g. `issue_number`, from + /// `number`) — not from a constant or a parent record. + pub identity_params: Vec, + /// Fixed query parameters to add to the list request, from the + /// collection's `x-list-query`. + pub list_query: IndexMap, +} + +/// How a collection's context variable is filled: enumerate `collection` +/// and read `field` off each of its records. E.g. `issue_number` is +/// provided by listing `issues` and reading each record's `number`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContextProvider { + /// The collection to enumerate. + pub collection: String, + /// The record field whose value fills the context variable. + pub field: String, +} + +/// The resource model derived from `components.crudResources`: every +/// managed collection, and how a nested collection's unresolved context +/// variable is resolved from a parent's own records. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ResourceModel { + /// Every managed collection, in document order. + pub collections: Vec, + providers: IndexMap, +} + +impl ResourceModel { + /// The managed collection named `name`, if any. + #[must_use] + pub fn by_name(&self, name: &str) -> Option<&ManagedCollection> { + self.collections + .iter() + .find(|collection| collection.name == name) + } + + /// The collection+field that supplies values for context path variable + /// `param`, if any resource's identity binds it. + #[must_use] + pub fn provider_for(&self, param: &str) -> Option<&ContextProvider> { + self.providers.get(param) + } +} + +/// The path variables (`{...}` segments) in a URL template, in order. +pub(super) fn path_variables(template: &str) -> Vec { + let mut variables = Vec::new(); + let mut rest = template; + while let Some(start) = rest.find('{') { + let Some(end) = rest[start..].find('}') else { + break; + }; + variables.push(rest[start + 1..start + end].to_string()); + rest = &rest[start + end + 1..]; + } + variables +} + +/// Whether any of the resource's own collection URLs contains `{param}` — +/// used to tell an identity binding that names the item's own id apart +/// from one that merely repeats a parent-scoping context variable. +fn collection_url_has(resource: &CrudResourceObject, param: &str) -> bool { + let needle = format!("{{{param}}}"); + resource + .collections + .iter() + .flatten() + .any(|(_, collection)| collection.url_template.contains(&needle)) +} + +/// Renders a JSON scalar the way GitHub's `x-list-query` values are meant +/// to be sent — as the literal query string value. +fn stringify(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + Value::Null => String::new(), + other => other.to_string(), + } +} + +/// Builds the resource model from `components.crudResources`. Each resource +/// may declare an `identity` (its single-item URL and the binding from a +/// path variable to a record field) and one or more `collections` (list +/// URLs). This emits one [`ManagedCollection`] per declared collection, +/// across every resource — nothing about issues, comments, calendars or +/// events is compiled in. +pub fn discover_resource_model(document: &OpenApiDocument) -> Result { + let resources = crud_resources(document)?; + + let mut collections = Vec::new(); + // resource name -> its first managed collection name (used as an + // enumeration source for that resource's own identity bindings). + let mut collection_of_resource: IndexMap = IndexMap::new(); + // path variable -> (resource, field), from every resource's identity bindings. + let mut bindings: Vec<(String, String, String)> = Vec::new(); + + for (resource_name, resource) in &resources { + let item_url = resource + .identity + .as_ref() + .map(|identity| identity.url_template.as_str()) + .unwrap_or_default(); + + let mut id_field = "id".to_string(); + let mut identity_params = Vec::new(); + if let Some(identity_bindings) = resource + .identity + .as_ref() + .and_then(|identity| identity.bindings.as_ref()) + { + for (param, binding) in identity_bindings { + bindings.push((param.clone(), resource_name.clone(), binding.field.clone())); + // The variable bound in the item URL — as opposed to one + // that merely repeats a parent-scoping context variable + // this resource's own collections also carry — is this + // resource's own id field. + let is_own_identity = item_url.contains(&format!("{{{param}}}")) + && !collection_url_has(resource, param); + if is_own_identity { + id_field.clone_from(&binding.field); + identity_params.push(param.clone()); + } + } + } + + let Some(resource_collections) = &resource.collections else { + continue; + }; + for (name, collection) in resource_collections { + collection_of_resource + .entry(resource_name.clone()) + .or_insert_with(|| name.clone()); + + let list_query = collection + .list_query + .iter() + .flatten() + .map(|(key, value)| (key.clone(), stringify(value))) + .collect(); + + collections.push(ManagedCollection { + name: name.clone(), + resource: resource_name.clone(), + collection_url: collection.url_template.clone(), + item_url: item_url.to_string(), + id_field: id_field.clone(), + context_params: path_variables(&collection.url_template), + identity_params: identity_params.clone(), + list_query, + }); + } + } + + let mut providers = IndexMap::new(); + for (param, resource_name, field) in bindings { + // A resource can supply a context value only if it is itself + // enumerable (has a managed collection). + if let Some(collection) = collection_of_resource.get(&resource_name) { + providers.entry(param).or_insert_with(|| ContextProvider { + collection: collection.clone(), + field, + }); + } + } + + Ok(ResourceModel { + collections, + providers, + }) +} + +// --- `x-crud`, the operation-level half of the extension ------------------- + +/// Which CRUD action an operation performs, from its `x-crud.action`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum CrudAction { + /// Lists a collection's items. + List, + /// Reads one item. + Read, + /// Creates an item. + Create, + /// Updates an item. + Update, + /// Deletes an item. + Delete, +} + +/// One field a create response adds beyond what the client sent — read back +/// from the response and merged into the record. GitHub's issue `number` is +/// server-assigned this way; the client never supplies it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AddedField { + /// The field's schema. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, + /// Where the field's value comes from, e.g. `server`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// Human-readable description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Any other key on the added field. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// Which collections a write affects, from `memberOf`/`removesFrom`: either +/// a fixed list of collection names, or every collection the resource +/// belongs to (`"*"`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CollectionMembership { + /// Only the named collections. + Named(Vec), + /// Every collection the resource belongs to. + All, +} + +impl Serialize for CollectionMembership { + fn serialize(&self, serializer: S) -> std::result::Result { + match self { + CollectionMembership::Named(names) => names.serialize(serializer), + CollectionMembership::All => serializer.serialize_str("*"), + } + } +} + +impl<'de> Deserialize<'de> for CollectionMembership { + fn deserialize>(deserializer: D) -> std::result::Result { + match Value::deserialize(deserializer)? { + Value::String(marker) if marker == "*" => Ok(CollectionMembership::All), + value @ Value::Array(_) => { + let names = Vec::::deserialize(value).map_err(D::Error::custom)?; + Ok(CollectionMembership::Named(names)) + } + other => Err(D::Error::custom(format!( + "expected \"*\" or an array of collection names, got {other}" + ))), + } + } +} + +/// The `x-crud` annotation on one operation, declaring what it does in +/// terms of the resource model rather than of any particular API. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct CrudOperation { + /// Which action the operation performs. + pub action: CrudAction, + /// The resource this operation concerns, e.g. `issue`. + pub resource: String, + /// The collection a `list` operation reads, or a `create` adds to via + /// `url.source: template`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collection: Option, + /// How an `update` is sent — GitHub exposes PATCH and no PUT for + /// issues, so this is `patch` rather than the default PUT semantics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mode: Option, + /// How a PATCH body is interpreted, e.g. `merge`. + #[serde( + rename = "patchFormat", + default, + skip_serializing_if = "Option::is_none" + )] + pub patch_format: Option, + /// Fields a `create` response adds beyond what the client sent. + #[serde(rename = "addedFields", default)] + pub added_fields: IndexMap, + /// Collections a `create` adds the new record to. + #[serde(rename = "memberOf", default)] + pub member_of: Vec, + /// Collections a `delete` removes the record from. + #[serde( + rename = "removesFrom", + default, + skip_serializing_if = "Option::is_none" + )] + pub removes_from: Option, + /// Any other key on the annotation. + #[serde(flatten)] + pub extensions: JsonMap, +} + +/// Reads the `x-crud` annotation off one operation, if it declares one. +pub fn crud_operation(operation: &OperationObject) -> Result> { + operation + .extensions + .get("x-crud") + .map(|raw| serde_json::from_value(raw.clone()).map_err(Error::from)) + .transpose() +} diff --git a/integrations/localthought/syncables/src/sync/storage.rs b/integrations/localthought/syncables/src/sync/storage.rs new file mode 100644 index 0000000000..193c44407c --- /dev/null +++ b/integrations/localthought/syncables/src/sync/storage.rs @@ -0,0 +1,221 @@ +//! Host-provided persistence for records read by the sync engine. +//! +//! The engine (built in +//! [issue #9](https://github.com/localthought/syncables-rs/issues/9)) never +//! decides where the local-first copy of a synced dataset lives — it talks +//! to whatever implements [`Storage`]. [`InMemoryStorage`] is a reference +//! implementation used by this crate's own tests; +//! [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs)'s +//! `AtomicStorage` (`src/store.rs`) is the intended real-world +//! implementation, storing records in an Atomic Data `Storelike`. + +use std::collections::HashMap; +use std::error::Error as StdError; +use std::sync::Mutex; + +use async_trait::async_trait; +use thiserror::Error; + +use super::ontology::Ontology; + +/// One record read from or written to a host's [`Storage`]. +/// +/// Deliberately plain JSON: `value` is exactly the record's own fields. +/// The trait must not require `atomic_lib`, or any other host-side model, +/// to construct a `Record` — and this crate itself takes no `atomic_lib` +/// dependency. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Record { + /// Separates otherwise-identical `resource`/`id` pairs that belong to + /// different parents — e.g. every issue's comments share the resource + /// name `issueComment`, so without a namespace two issues' comment + /// sets would overwrite each other. The engine derives the namespace + /// from the resource model's context parameters + /// ([issue #3](https://github.com/localthought/syncables-rs/issues/3)) + /// and is consistent about it across a run, since a host indexes by + /// it. + pub namespace: String, + /// The resource name, e.g. `issue` or `issueComment`. + pub resource: String, + /// The record's id within its `namespace`/`resource`. + pub id: String, + /// The record's own fields, as plain JSON. + pub value: serde_json::Map, +} + +/// Something a host's [`Storage`] implementation failed to do. +/// +/// A `StorageError` is fatal for the one call it came from, not for the +/// sync as a whole: the engine +/// ([issue #9](https://github.com/localthought/syncables-rs/issues/9)) +/// collects errors like this into `SyncReport::errors` and continues +/// syncing the rest of the dataset. +#[derive(Debug, Error)] +#[error("{message}")] +pub struct StorageError { + /// What went wrong, in a form suitable for logging or for surfacing + /// in a `SyncReport::errors` list. + pub message: String, + /// The underlying error from the host's storage backend, if any. + #[source] + pub source: Option>, +} + +impl StorageError { + /// Builds a [`StorageError`] with no underlying source error. + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + source: None, + } + } + + /// Builds a [`StorageError`] that wraps an underlying error from the + /// host's storage backend. + pub fn with_source( + message: impl Into, + source: impl StdError + Send + Sync + 'static, + ) -> Self { + Self { + message: message.into(), + source: Some(Box::new(source)), + } + } +} + +/// Host-provided persistence for the records and ontology a sync produces. +/// +/// The engine talks to this trait rather than deciding for itself where +/// the local-first copy of a dataset lives — [`InMemoryStorage`] is a +/// reference implementation used by this crate's own tests, and +/// `AtomicStorage` in +/// [`localthought/reflector-rs`](https://github.com/localthought/reflector-rs) +/// (`src/store.rs`) is the intended real-world one, storing records in an +/// Atomic Data `Storelike` under `internal:///` +/// subjects. +/// +/// # Contract +/// +/// [`Storage::put_ontology`] is called once per sync, before any +/// [`Storage::put`] — this is documented behavior for callers of this +/// trait, not something enforced by the trait or by any implementation of +/// it. That ordering is the engine's responsibility +/// ([issue #9](https://github.com/localthought/syncables-rs/issues/9)); +/// `reflector-rs`'s `AtomicStorage` is written against it. +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait Storage: Send + Sync { + /// Inserts or replaces a record, keyed by its `namespace`, `resource` + /// and `id`. + async fn put(&self, record: &Record) -> Result<(), StorageError>; + + /// One record, by `namespace`, `resource` and id — `Ok(None)` if no + /// such record has been [`put`](Storage::put). + async fn get( + &self, + namespace: &str, + resource: &str, + id: &str, + ) -> Result, StorageError>; + + /// Every record held for `namespace`/`resource`. + async fn list(&self, namespace: &str, resource: &str) -> Result, StorageError>; + + /// Removes a record, by `namespace`, `resource` and id. Removing a + /// record that isn't present is not an error. + async fn delete(&self, namespace: &str, resource: &str, id: &str) -> Result<(), StorageError>; + + /// Stores the ontology derived from the synced document. Called once + /// per sync, before any [`put`](Storage::put) — see the trait-level + /// "Contract" section. + async fn put_ontology(&self, ontology: &Ontology) -> Result<(), StorageError>; +} + +/// Key a [`Record`] is stored under: its `namespace`, `resource` and `id`. +type RecordKey = (String, String, String); + +/// A reference [`Storage`] implementation that keeps everything in process +/// memory, analogous to [`crate::client::storage::InMemoryStorageAdapter`] +/// but namespace-keyed. Used by this crate's own tests; hosts that need +/// persistence implement [`Storage`] themselves. +#[derive(Debug, Default)] +pub struct InMemoryStorage { + records: Mutex>, + ontologies: Mutex>, +} + +impl InMemoryStorage { + /// A store with no records and no ontology. + pub fn new() -> Self { + Self::default() + } + + /// Every ontology passed to [`Storage::put_ontology`] so far, in call + /// order. Exposed for tests to assert `put_ontology` was called. + pub fn ontologies(&self) -> Vec { + self.ontologies + .lock() + .expect("ontologies mutex poisoned") + .clone() + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl Storage for InMemoryStorage { + async fn put(&self, record: &Record) -> Result<(), StorageError> { + let key = ( + record.namespace.clone(), + record.resource.clone(), + record.id.clone(), + ); + self.records + .lock() + .expect("records mutex poisoned") + .insert(key, record.clone()); + Ok(()) + } + + async fn get( + &self, + namespace: &str, + resource: &str, + id: &str, + ) -> Result, StorageError> { + let key = (namespace.to_string(), resource.to_string(), id.to_string()); + Ok(self + .records + .lock() + .expect("records mutex poisoned") + .get(&key) + .cloned()) + } + + async fn list(&self, namespace: &str, resource: &str) -> Result, StorageError> { + Ok(self + .records + .lock() + .expect("records mutex poisoned") + .values() + .filter(|record| record.namespace == namespace && record.resource == resource) + .cloned() + .collect()) + } + + async fn delete(&self, namespace: &str, resource: &str, id: &str) -> Result<(), StorageError> { + let key = (namespace.to_string(), resource.to_string(), id.to_string()); + self.records + .lock() + .expect("records mutex poisoned") + .remove(&key); + Ok(()) + } + + async fn put_ontology(&self, ontology: &Ontology) -> Result<(), StorageError> { + self.ontologies + .lock() + .expect("ontologies mutex poisoned") + .push(ontology.clone()); + Ok(()) + } +} diff --git a/integrations/localthought/tsconfig.json b/integrations/localthought/tsconfig.json index 7de173b3fc..8a41739785 100644 --- a/integrations/localthought/tsconfig.json +++ b/integrations/localthought/tsconfig.json @@ -8,5 +8,5 @@ "types": ["node"], "noEmit": true }, - "include": ["data.ts", "schema.ts", "plugin.ts"] + "include": ["calendar.ts", "schema.ts", "plugin.ts", "browser.ts"] } diff --git a/integrations/localthought/wasm-smoke.mjs b/integrations/localthought/wasm-smoke.mjs new file mode 100644 index 0000000000..4381e77c1a --- /dev/null +++ b/integrations/localthought/wasm-smoke.mjs @@ -0,0 +1,72 @@ +/** Real WASM engine, no AtomicServer or provider network required. */ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import init, { + describeIntegration, + fetchIntegration, +} from '../../wasm/pkg/atomic_wasm.js'; +await init({ + module_or_path: await readFile( + new URL('../../wasm/pkg/atomic_wasm_bg.wasm', import.meta.url), + ), +}); +const document = await readFile( + new URL('./mock-document.json', import.meta.url), + 'utf8', +); +const description = JSON.parse(await describeIntegration(document)); +assert.deepEqual(description.collections, ['pets']); +const urls = []; +const output = JSON.parse( + await fetchIntegration(document, 'pets', '{}', undefined, async url => { + urls.push(url); + const second = url.includes('page=2'); + return JSON.stringify({ + status: 200, + headers: second + ? {} + : { link: '; rel="next"' }, + body: JSON.stringify([ + { + id: second ? 2 : 1, + name: 'Pet', + age: 3, + vaccinated: true, + weight: 2.5, + updated_at: '2026-09-09T00:00:00Z', + }, + ]), + }); + }), +); +assert.deepEqual(urls, [ + 'https://pets.example/pets', + 'https://pets.example/pets?page=2', +]); +assert.equal(output.records.length, 2); +assert.equal(output.records[0].values.age, 3); +assert.equal(output.records[0].values.vaccinated, true); +assert.equal(output.records[0].values['updated-at'], 1788912000000); +assert.equal( + output.ontology.terms.find(t => t.shortname === 'age').datatype, + 'https://atomicdata.dev/datatypes/integer', +); +await assert.rejects( + fetchIntegration(document, 'pets', '{}', undefined, async () => + JSON.stringify({ status: 503, headers: {}, body: '{}' }), + ), + /Import incomplete/, +); +await assert.rejects( + fetchIntegration(document, 'pets', '{}', undefined, async () => + JSON.stringify({ + status: 200, + headers: { link: '; rel="next"' }, + body: JSON.stringify([{ id: 1, name: 'Repeated' }]), + }), + ), + /Import incomplete/, +); +console.log( + 'Real WASM pagination, typed ontology, timestamps and failure refusal passed', +); diff --git a/planning/api-plugins.md b/planning/api-plugins.md index ddb818503c..e657a5ce93 100644 --- a/planning/api-plugins.md +++ b/planning/api-plugins.md @@ -66,3 +66,12 @@ Give only that bulk test a 30-second budget; retain all compilation assertions. Run 34352390180 was canceled before jobs started when another branch replaced it in the default single pending slot. Set `queue: max` on main-pipeline so pending validations can wait sequentially instead of displacing one another. + +## Browser migration + +`codex/browser-integrations` moves the LocalThought flow off AtomicServer. +Catalog parsing and pagination run in the Atomic WASM bundle; the browser owns +the tenant handoff and rotating connection code, then maps fetched records into +locally reviewed proposals. Companion branches in Syncables and integration-proxy +provide WASM compatibility and CORS. See `integrations/localthought/README.md`. +Legacy direct integrations, action infrastructure and scheduling remain separate. diff --git a/planning/devonian-issue-sync.md b/planning/devonian-issue-sync.md new file mode 100644 index 0000000000..657a8ac58d --- /dev/null +++ b/planning/devonian-issue-sync.md @@ -0,0 +1,21 @@ +# Devonian issue tracker sync + +- [x] Verify Devonian main (`e11104f`) and AtomicServer PR #1394 (`a087a7ca7`). +- [x] Inspect native lenses, GitHub integration actions, comments and test coverage. +- [x] Add regression tests for bidirectional issues/comments, conflict handling and restart recovery. +- [x] Build a browser demo using Devonian, local-only Atomic storage and direct integration-proxy requests. +- [x] Verify 13 focused tests, 7 existing integration tests and frontend typecheck; document setup/limits and update coverage. +- [x] Verify native browser creation/comments both ways, close/reopen and reload without duplicates. +- [x] Verify deployed v40 CORS preflight/exposed headers and browser GitHub OAuth. +- [ ] Verify live two-way writes: proxy credential returns 404 for the private sandbox; resolve repository access first. + +Use Devonian HTTP subjects for the intermediate graph and scoped external mappings +for Atomic DIDs. Persist graph, mappings and request receipts in IndexedDB. Serialize +rotating connection codes; refuse to retry uncertain writes. Tenant authentication and rotating credentials use #1401’s shared BrowserIntegrations client. +No Node runtime, AtomicServer plugin endpoint, tenant secret on AtomicServer, or server scheduler. +Live proxy CORS and OAuth now work; private-sandbox access is the remaining live blocker. +Missing records are conflicts, not deletion requests. + +- [x] Rebase onto #1401 and reuse browser tenant challenge, callback and rotating transport. +- [x] Expand HTTP mock for stateful GitHub issues/comments and add Playwright two-way sync coverage. +- [x] Run 13 sync tests, 11 LocalThought browser tests, 7 existing GitHub integration tests, browser E2E and typechecks. diff --git a/server/Cargo.toml b/server/Cargo.toml index 9f7fbdcac8..3230bb8dee 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -33,7 +33,6 @@ static-files = "0.3.1" walkdir = "2" [dependencies] -syncables = { git = "https://github.com/localthought/syncables-rs", rev = "d48e4d9bad3ed9ec826d1c2040e989171701965d" } tempfile = "3" async-trait = "0.1.89" actix = "0.13.5" diff --git a/server/src/handlers/integration_proxy.rs b/server/src/handlers/integration_proxy.rs deleted file mode 100644 index 34f51bc604..0000000000 --- a/server/src/handlers/integration_proxy.rs +++ /dev/null @@ -1,316 +0,0 @@ -//! LocalThought catalog and actor-bound OAuth handoff. Credentials stay in host storage. -use crate::{appstate::AppState, context::RequestContext, errors::AtomicServerResult as Result}; -use actix_web::{web, HttpRequest, HttpResponse}; -use atomic_lib::{ - db::{ - plugin_secret::{PluginSecret, PluginSecretKey}, - trees::Tree, - }, - Db, -}; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use rand::RngCore; -use serde::{Deserialize, Serialize}; -use serde_json::json; - -pub(super) fn client() -> Result { - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|_| "Could not initialize integration proxy client".into()) -} -fn origin() -> Result { - let value = std::env::var("ATOMIC_INTEGRATION_PROXY_URL") - .unwrap_or_else(|_| "https://localthought.io".into()); - let u = url::Url::parse(&value).map_err(|_| "Invalid integration proxy URL")?; - if u.origin().ascii_serialization() != value - || (u.scheme() != "https" - && !(u.scheme() == "http" && matches!(u.host_str(), Some("localhost" | "127.0.0.1")))) - { - return Err("Integration proxy must be an HTTPS origin (or localhost for testing)".into()); - } - Ok(value) -} -fn random_id() -> String { - let mut b = [0u8; 32]; - rand::rngs::OsRng.fill_bytes(&mut b); - URL_SAFE_NO_PAD.encode(b) -} -fn sign(secret: &str, text: &str) -> String { - URL_SAFE_NO_PAD.encode(ring::hmac::sign( - &ring::hmac::Key::new(ring::hmac::HMAC_SHA256, secret.as_bytes()), - text.as_bytes(), - )) -} -async fn platforms(base: &str) -> Result> { - let response = client()? - .get(format!("{base}/catalog")) - .send() - .await - .map_err(|_| "Could not reach integration catalog")?; - if !response.status().is_success() { - return Err("Integration catalog is unavailable".into()); - } - let names: Vec = response - .json() - .await - .map_err(|_| "Invalid integration catalog")?; - if names.len() > 200 - || names.iter().any(|s| { - s.is_empty() - || s.len() > 80 - || !s - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - }) - { - return Err("Invalid catalog platform identifier".into()); - } - Ok(names) -} -pub async fn catalog() -> Result { - let base = origin()?; - Ok(HttpResponse::Ok().json(json!({"platforms": platforms(&base).await?, "origin": base}))) -} -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Start { - drive: String, - platform: String, - return_url: String, -} -#[derive(Clone, Serialize, Deserialize)] -pub(super) struct Connection { - pub(super) drive: String, - pub(super) actor: String, - pub(super) platform: String, - pub(super) origin: String, - pub(super) expires: i64, - pub(super) ready: bool, -} -fn key(id: &str) -> String { - format!("integration-proxy/v1/{id}") -} -fn put(db: &Db, id: &str, value: &Connection) -> Result<()> { - db.kv.insert( - Tree::PluginMeta, - key(id).as_bytes(), - &serde_json::to_vec(value).map_err(|_| "Could not encode connection")?, - )?; - db.flush()?; - Ok(()) -} -fn get(db: &Db, id: &str) -> Result { - let bytes = db - .kv - .get(Tree::PluginMeta, key(id).as_bytes())? - .ok_or("Connection not found")?; - serde_json::from_slice(&bytes).map_err(|_| "Invalid connection".into()) -} -pub(super) fn secret_key(c: &Connection, id: &str) -> PluginSecretKey { - PluginSecretKey::new(&c.drive, &format!("integration-proxy:{id}"), "connection") -} -fn owned(c: &Connection, drive: &str, actor: &str) -> Result<()> { - if c.drive != drive || c.actor != actor { - return Err("This connection belongs to another drive or agent".into()); - } - Ok(()) -} -fn return_url(raw: &str) -> Result { - let u = url::Url::parse(raw).map_err(|_| "Invalid return URL")?; - let allowed = std::env::var("ATOMIC_INTEGRATION_FRONTEND_ORIGIN") - .map_err(|_| "Configure ATOMIC_INTEGRATION_FRONTEND_ORIGIN on this server")?; - if u.origin().ascii_serialization() != allowed - || u.path() != "/app/integrations" - || !u.username().is_empty() - || u.password().is_some() - || u.fragment().is_some() - || u.query().is_some() - { - return Err("Return URL must be the configured frontend integrations page".into()); - } - Ok(u) -} -pub async fn start( - app: web::Data, - body: web::Json, - req: HttpRequest, - ctx: RequestContext, -) -> Result { - let actor = super::plugin_schedule::authorize(&app, &req, &ctx, &body.drive) - .await? - .to_string(); - let base = origin()?; - if !platforms(&base).await?.contains(&body.platform) { - return Err("Platform is not in the integration catalog".into()); - } - let secret = std::env::var("TENANT_SECRET") - .ok() - .filter(|s| !s.is_empty()) - .ok_or("Configure TENANT_SECRET on this server to connect accounts")?; - let tenant = secret - .split_once('.') - .and_then(|(id, _)| URL_SAFE_NO_PAD.decode(id).ok()) - .and_then(|b| String::from_utf8(b).ok()) - .ok_or("Invalid tenant secret configuration")?; - let mut callback = return_url(&body.return_url)?; - let id = random_id(); - callback - .query_pairs_mut() - .append_pair("integration_state", &id) - .append_pair("platform", &body.platform); - #[derive(Deserialize)] - struct Challenge { - ts: u64, - nonce: String, - challenge: String, - } - let response = client()? - .get(format!("{base}/session")) - .send() - .await - .map_err(|_| "Could not obtain integration challenge")?; - if !response.status().is_success() { - return Err("Integration challenge is unavailable".into()); - } - let challenge: Challenge = response - .json() - .await - .map_err(|_| "Invalid integration challenge")?; - let mut connect = - url::Url::parse(&format!("{base}/connect")).map_err(|_| "Invalid proxy URL")?; - connect - .query_pairs_mut() - .append_pair("redirect_uri", callback.as_str()) - .append_pair("platform", &body.platform) - .append_pair("ts", &challenge.ts.to_string()) - .append_pair("nonce", &challenge.nonce) - .append_pair("challenge", &challenge.challenge) - .append_pair("tenant_id", &tenant) - .append_pair("user_id", &actor) - .append_pair("user_id_sig", &sign(&secret, &actor)) - .append_pair("response", &sign(&secret, &challenge.challenge)); - put( - &app.store, - &id, - &Connection { - drive: body.drive.clone(), - actor, - platform: body.platform.clone(), - origin: base, - expires: atomic_lib::utils::now() + 600_000, - ready: false, - }, - )?; - Ok(HttpResponse::Ok().json(json!({"url":connect.as_str(),"state":id}))) -} -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Finish { - drive: String, - state: String, - connection_code: String, -} -pub async fn finish( - app: web::Data, - body: web::Json, - req: HttpRequest, - ctx: RequestContext, -) -> Result { - let actor = super::plugin_schedule::authorize(&app, &req, &ctx, &body.drive) - .await? - .to_string(); - let _lock = app.store.lock_plugin(&key(&body.state)).await; - let mut c = get(&app.store, &body.state)?; - owned(&c, &body.drive, &actor)?; - if c.ready - || c.expires < atomic_lib::utils::now() - || body.connection_code.is_empty() - || body.connection_code.len() > 4096 - { - return Err("Invalid, expired or already completed connection request".into()); - } - app.store.set_plugin_secret( - &secret_key(&c, &body.state), - &PluginSecret::new( - body.connection_code.clone(), - vec![c.origin.clone()], - atomic_lib::utils::now(), - ), - )?; - c.ready = true; - put(&app.store, &body.state, &c)?; - Ok(HttpResponse::Ok().json(json!({"connection":body.state,"platform":c.platform}))) -} -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct Fetch { - drive: String, - connection: String, - constants: std::collections::BTreeMap, - calendar_range: Option, -} -pub async fn fetch_records( - app: web::Data, - body: web::Json, - req: HttpRequest, - ctx: RequestContext, -) -> Result { - let actor = super::plugin_schedule::authorize(&app, &req, &ctx, &body.drive) - .await? - .to_string(); - let _lock = app.store.lock_plugin(&key(&body.connection)).await; - let c = get(&app.store, &body.connection)?; - owned(&c, &body.drive, &actor)?; - if !c.ready { - return Err("Finish connecting your account first".into()); - } - let data = super::integration_proxy_sync::sync( - app.store.clone(), - c, - body.connection.clone(), - body.constants.clone(), - body.calendar_range.as_ref(), - ) - .await?; - Ok(HttpResponse::Ok().json(data)) -} -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn connection_is_bound_to_actor_and_drive() { - let c = Connection { - drive: "a".into(), - actor: "b".into(), - platform: "github-issues".into(), - origin: "https://localthought.io".into(), - expires: 0, - ready: true, - }; - assert!(owned(&c, "a", "b").is_ok()); - assert!(owned(&c, "other", "b").is_err()); - assert!(owned(&c, "a", "other").is_err()); - } - #[test] - fn signatures_match_proxy_hmac_contract() { - assert_eq!( - sign("key", "The quick brown fox jumps over the lazy dog"), - "97yD9DBThCSxMpjmqm-xQ-9NWaFJRhdZl0edvC0aPNg" - ); - } -} - -#[derive(Deserialize)] -pub struct PlatformQuery { - platform: String, -} -pub async fn platform(query: web::Query) -> Result { - let base = origin()?; - if !platforms(&base).await?.contains(&query.platform) { - return Err("Platform is not in the catalog".into()); - } - Ok(HttpResponse::Ok() - .json(super::integration_proxy_sync::describe(&base, &query.platform).await?)) -} diff --git a/server/src/handlers/integration_proxy_sync.rs b/server/src/handlers/integration_proxy_sync.rs deleted file mode 100644 index 275e81af12..0000000000 --- a/server/src/handlers/integration_proxy_sync.rs +++ /dev/null @@ -1,515 +0,0 @@ -//! Syncables transport and preview storage. No graph writes happen while fetching. -use super::integration_proxy::{client, secret_key, Connection}; -use crate::errors::AtomicServerResult as Result; -use atomic_lib::{db::plugin_secret::PluginSecret, Db}; -use serde_json::{json, Value}; -use std::{ - collections::{BTreeMap, BTreeSet}, - io::Write, - sync::{Arc, Mutex}, -}; -use syncables::{ - client::client::{Fetch, HttpRequest, HttpResponse}, - ontology_shortname, ClientConfig, Credentials, Ontology, Record, Storage, StorageError, - SyncClient, TermKind, -}; - -struct ProxyTransport { - db: Db, - connection: Connection, - id: String, - upstream: url::Url, - requests: Mutex, -} -#[async_trait::async_trait] -impl Fetch for ProxyTransport { - async fn fetch(&self, request: HttpRequest) -> syncables::Result { - let error = |message: &str| syncables::Error::Http(message.into()); - let target = - url::Url::parse(&request.url).map_err(|_| error("Invalid OpenAPI request URL"))?; - if request.method != "GET" - || target.origin() != self.upstream.origin() - || !target.username().is_empty() - || target.password().is_some() - || target.fragment().is_some() - { - return Err(error( - "Only read requests to the catalog API origin are allowed", - )); - } - { - let mut count = self - .requests - .lock() - .map_err(|_| error("Request counter failed"))?; - *count += 1; - if *count > 200 { - return Err(error("Import exceeds 200 requests; narrow its scope")); - } - } - let c = &self.connection; - let mut proxy = url::Url::parse(&format!( - "{}/proxy/{}{}", - c.origin, - c.platform, - target.path() - )) - .map_err(|_| error("Invalid proxy URL"))?; - proxy.set_query(target.query()); - let key = secret_key(c, &self.id); - let request = client() - .map_err(|_| error("Could not initialize proxy client"))? - .get(proxy); - let request = self - .db - .use_plugin_secret(&key, &c.origin, atomic_lib::utils::now(), |code| { - request.bearer_auth(code) - }) - .map_err(|_| error("Could not read connection credential"))? - .ok_or_else(|| error("Reconnect your account before fetching again"))?; - self.db - .delete_plugin_secret(&key) - .map_err(|_| error("Could not consume connection code"))?; - self.db - .flush() - .map_err(|_| error("Could not persist consumed code"))?; - let mut response = request - .send() - .await - .map_err(|_| error("Proxy request failed; reconnect before retrying"))?; - if let Some(next) = response - .headers() - .get("x-connection-code") - .and_then(|v| v.to_str().ok()) - { - self.db - .set_plugin_secret( - &key, - &PluginSecret::new( - next.to_string(), - vec![c.origin.clone()], - atomic_lib::utils::now(), - ), - ) - .map_err(|_| error("Could not save rotated code; reconnect"))?; - self.db - .flush() - .map_err(|_| error("Could not persist rotated code; reconnect"))?; - } - let status = response.status().as_u16(); - // Preserve pagination headers. Never expose the rotated credential to Syncables or the browser. - let headers = response - .headers() - .iter() - .filter(|(name, _)| name.as_str() != "x-connection-code") - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|v| (name.as_str().to_string(), v.to_string())) - }) - .collect(); - let mut body = Vec::new(); - while let Some(chunk) = response - .chunk() - .await - .map_err(|_| error("Could not read provider response"))? - { - if body.len() + chunk.len() > 10 * 1024 * 1024 { - return Err(error("Provider page exceeds 10 MB")); - } - body.extend_from_slice(&chunk); - } - Ok(HttpResponse { - status, - headers, - body, - }) - } -} -#[derive(Default)] -struct PreviewStorage { - records: Mutex>, - ontology: Mutex>, -} -#[async_trait::async_trait] -impl Storage for PreviewStorage { - async fn put(&self, record: &Record) -> std::result::Result<(), StorageError> { - let mut records = self.records.lock().unwrap(); - if records.len() >= 5000 { - return Err(StorageError::new( - "Import exceeds 5000 records; narrow its scope", - )); - } - if record.id.is_empty() - || records.iter().any(|r| { - r.namespace == record.namespace - && r.resource == record.resource - && r.id == record.id - }) - { - return Err(StorageError::new( - "Missing or repeated record identity; pagination may not be forwarded by the proxy", - )); - } - records.push(record.clone()); - Ok(()) - } - async fn get( - &self, - namespace: &str, - resource: &str, - id: &str, - ) -> std::result::Result, StorageError> { - Ok(self - .records - .lock() - .unwrap() - .iter() - .find(|r| r.namespace == namespace && r.resource == resource && r.id == id) - .cloned()) - } - async fn list( - &self, - namespace: &str, - resource: &str, - ) -> std::result::Result, StorageError> { - Ok(self - .records - .lock() - .unwrap() - .iter() - .filter(|r| r.namespace == namespace && r.resource == resource) - .cloned() - .collect()) - } - async fn delete(&self, _: &str, _: &str, _: &str) -> std::result::Result<(), StorageError> { - Err(StorageError::new("Preview storage cannot delete records")) - } - async fn put_ontology(&self, ontology: &Ontology) -> std::result::Result<(), StorageError> { - *self.ontology.lock().unwrap() = Some(ontology.clone()); - Ok(()) - } -} -fn typed_value(value: &Value, datatype: Option<&str>) -> Result> { - if value.is_null() { - return Ok(None); - } - if datatype == Some("https://atomicdata.dev/datatypes/timestamp") { - let text = value.as_str().ok_or("Expected RFC3339 timestamp")?; - return Ok(Some(json!(chrono::DateTime::parse_from_rfc3339(text) - .map_err(|_| "Invalid provider timestamp")? - .timestamp_millis()))); - } - Ok(Some(value.clone())) -} -fn preview(ontology: &Ontology, records: &[Record], platform: &str) -> Result { - let field_terms: BTreeMap<_, _> = ontology - .terms - .iter() - .filter(|t| t.kind == TermKind::Property) - .map(|t| (t.shortname.as_str(), t)) - .collect(); - let terms: Vec<_> = ontology.terms.iter().map(|t| json!({ "path":t.path, "kind":if t.kind == TermKind::Class {"class"} else {"property"}, "shortname":t.shortname, "description":t.description, "datatype":t.datatype.as_deref().unwrap_or("https://atomicdata.dev/datatypes/json"), "requires":t.requires, "recommends":t.recommends })).collect(); - let mut output = Vec::new(); - for record in records { - let mut values = serde_json::Map::new(); - for (key, value) in &record.value { - let short = ontology_shortname(key); - if let Some(term) = field_terms.get(short.as_str()) { - if let Some(value) = typed_value(value, term.datatype.as_deref())? { - values.insert(short, value); - } - } - } - output.push(json!({"resource":ontology_shortname(&record.resource),"namespace":record.namespace,"id":record.id,"values":values,"name":record.value.get("title").or_else(||record.value.get("summary")).or_else(||record.value.get("name")).cloned().unwrap_or(json!(record.id))})); - } - Ok( - json!({"platform":platform,"ontology":{"description":ontology.description,"terms":terms},"records":output}), - ) -} -/// UTC date boundaries; the end is exclusive, matching Calendar's timeMax. -#[derive(serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct CalendarRange { - start: String, - end: String, -} - -fn scope_calendar(document: &mut Value, range: &CalendarRange) -> Result<()> { - let parse = |value: &str| { - chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") - .map_err(|_| "Calendar dates must use YYYY-MM-DD") - }; - let start = parse(&range.start)?; - let end = parse(&range.end)?; - if start >= end { - return Err("Calendar end date must be after its start date".into()); - } - let collection = document - .pointer_mut("/components/crudResources/event/collections/events") - .and_then(Value::as_object_mut) - .ok_or("Calendar catalog has no events collection")?; - if collection.get("urlTemplate").and_then(Value::as_str) - != Some("/calendars/{calendarId}/events") - { - return Err("Calendar catalog events path has changed".into()); - } - let query = collection - .entry("x-list-query") - .or_insert_with(|| json!({})) - .as_object_mut() - .ok_or("Invalid Calendar collection query")?; - query.insert("timeMin".into(), json!(format!("{start}T00:00:00Z"))); - query.insert("timeMax".into(), json!(format!("{end}T00:00:00Z"))); - query.insert("singleEvents".into(), json!(true)); - Ok(()) -} - -pub(super) async fn sync( - db: Db, - c: Connection, - id: String, - constants: BTreeMap, - calendar_range: Option<&CalendarRange>, -) -> Result { - let file = load_document(&c.origin, &c.platform).await?; - let document = syncables::load_open_api_document(file.path()) - .await - .map_err(|e| e.to_string())?; - let upstream = url::Url::parse( - syncables::base_url(&document).ok_or("OpenAPI document has no API server")?, - ) - .map_err(|_| "Invalid OpenAPI server URL")?; - let mut scoped_file = tempfile::NamedTempFile::new()?; - if let Some(range) = calendar_range { - if c.platform != "google-calendar" { - return Err("Calendar date range only applies to Google Calendar".into()); - } - let mut scoped = serde_json::to_value(&document).map_err(|e| e.to_string())?; - scope_calendar(&mut scoped, range)?; - serde_json::to_writer(&mut scoped_file, &scoped).map_err(|e| e.to_string())?; - } - let platform = c.platform.clone(); - let fetch = ProxyTransport { - db, - connection: c, - id, - upstream, - requests: Mutex::new(0), - }; - let engine = SyncClient::new( - ClientConfig { - document: if calendar_range.is_some() { - scoped_file.path() - } else { - file.path() - } - .into(), - overlays: vec![], - credentials: Credentials::Anonymous, - constants, - ontology_base_url: format!("https://atomicdata.dev/integrations/{platform}"), - }, - Arc::new(fetch), - ) - .map_err(|e| e.to_string())?; - let storage = PreviewStorage::default(); - let report = tokio::time::timeout(std::time::Duration::from_secs(120), engine.sync(&storage)) - .await - .map_err(|_| "Import timed out; no partial data was proposed")? - .map_err(|e| e.to_string())?; - if !report.errors.is_empty() { - return Err(format!( - "Import incomplete; no changes proposed: {}", - report - .errors - .iter() - .map(String::as_str) - .collect::>() - .into_iter() - .collect::>() - .join("; ") - ) - .into()); - } - let ontology = storage.ontology.lock().unwrap(); - let records = storage.records.lock().unwrap(); - preview( - ontology - .as_ref() - .ok_or("Syncables did not generate an ontology")?, - &records, - &platform, - ) -} - -async fn load_document(base: &str, platform: &str) -> Result { - let document_url = format!("{}/catalog/{}.yaml", base, platform); - let mut response = client()? - .get(&document_url) - .send() - .await - .map_err(|_| "Could not load platform OpenAPI document")?; - if !response.status().is_success() { - return Err(format!( - "Platform OpenAPI document returned HTTP {} at {document_url}", - response.status().as_u16() - ) - .into()); - } - let mut bytes = Vec::new(); - while let Some(chunk) = response - .chunk() - .await - .map_err(|_| "Could not read OpenAPI document")? - { - if bytes.len() + chunk.len() > 10 * 1024 * 1024 { - return Err("OpenAPI document exceeds 10 MB".into()); - } - bytes.extend_from_slice(&chunk); - } - let mut file = - tempfile::NamedTempFile::new().map_err(|_| "Could not create temporary OpenAPI file")?; - file.write_all(&bytes) - .map_err(|_| "Could not store temporary OpenAPI document")?; - Ok(file) -} - -pub(super) async fn describe(base: &str, platform: &str) -> Result { - let file = load_document(base, platform).await?; - let doc = syncables::load_open_api_document(file.path()) - .await - .map_err(|e| e.to_string())?; - let model = syncables::discover_resource_model(&doc).map_err(|e| e.to_string())?; - let parameters: std::collections::BTreeSet<_> = model - .collections - .iter() - .flat_map(|c| c.context_params.iter()) - .filter(|p| model.provider_for(p).is_none()) - .cloned() - .collect(); - let collections: Vec<_> = model.collections.iter().map(|c| c.name.clone()).collect(); - Ok(json!({"parameters": parameters, "collections": collections})) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn calendar_range_preserves_catalog_query_and_rejects_invalid_bounds() { - let mut doc = json!({"components":{"crudResources":{"event":{"collections":{"events":{ - "urlTemplate":"/calendars/{calendarId}/events", "x-list-query":{"maxResults":2} - }}}}}}); - scope_calendar( - &mut doc, - &CalendarRange { - start: "2026-09-09".into(), - end: "2026-10-09".into(), - }, - ) - .unwrap(); - let query = - &doc["components"]["crudResources"]["event"]["collections"]["events"]["x-list-query"]; - assert_eq!(query["timeMin"], "2026-09-09T00:00:00Z"); - assert_eq!(query["timeMax"], "2026-10-09T00:00:00Z"); - assert_eq!(query["singleEvents"], true); - assert_eq!(query["maxResults"], 2); - for (start, end) in [ - ("2026-09-09", "2026-09-09"), - ("2026-09-10", "2026-09-09"), - ("2026-02-30", "2026-10-09"), - ] { - assert!(scope_calendar( - &mut doc, - &CalendarRange { - start: start.into(), - end: end.into() - } - ) - .is_err()); - } - } - struct Pages(Mutex>); - #[async_trait::async_trait] - impl Fetch for Pages { - async fn fetch(&self, req: HttpRequest) -> syncables::Result { - self.0.lock().unwrap().push(req.url.clone()); - let second = req.url.ends_with("?page=2"); - let headers = if second { - Default::default() - } else { - [( - "link".into(), - "; rel=\"next\"".into(), - )] - .into() - }; - let body = json!([{ "id": if second {2} else {1}, "name": if second {"Whiskers"} else {"Rex"}, "age":3,"vaccinated":true,"weight":2.5,"updated_at":"2026-09-09T00:00:00Z" }]); - Ok(HttpResponse { - status: 200, - headers, - body: serde_json::to_vec(&body).unwrap(), - }) - } - } - #[tokio::test] - async fn syncables_walks_catalog_pages_and_generates_typed_ontology() { - let mut file = tempfile::NamedTempFile::new().unwrap(); - file.write_all(include_bytes!( - "../../../integrations/localthought/mock-document.json" - )) - .unwrap(); - let transport = Arc::new(Pages(Mutex::new(vec![]))); - let client = SyncClient::new( - ClientConfig { - document: file.path().into(), - overlays: vec![], - credentials: Credentials::Anonymous, - constants: BTreeMap::new(), - ontology_base_url: "https://example.com/ontology".into(), - }, - transport.clone(), - ) - .unwrap(); - let storage = PreviewStorage::default(); - let report = client.sync(&storage).await.unwrap(); - assert!(report.errors.is_empty(), "{:?}", report.errors); - assert_eq!(report.read["pet"], 2); - assert_eq!( - transport.0.lock().unwrap().as_slice(), - &[ - "https://pets.example/pets", - "https://pets.example/pets?page=2" - ] - ); - let ontology = storage.ontology.lock().unwrap(); - let records = storage.records.lock().unwrap(); - let output = preview(ontology.as_ref().unwrap(), &records, "pets").unwrap(); - assert_eq!(output["records"][0]["values"]["age"], 3); - assert_eq!(output["records"][0]["values"]["vaccinated"], true); - assert_eq!( - output["records"][0]["values"]["updated-at"], - 1788912000000i64 - ); - let age = output["ontology"]["terms"] - .as_array() - .unwrap() - .iter() - .find(|t| t["shortname"] == "age") - .unwrap(); - assert_eq!(age["datatype"], "https://atomicdata.dev/datatypes/integer"); - } - #[tokio::test] - async fn duplicate_pages_are_an_error_not_a_successful_partial_import() { - let storage = PreviewStorage::default(); - let row = Record { - namespace: "".into(), - resource: "pet".into(), - id: "1".into(), - value: Default::default(), - }; - storage.put(&row).await.unwrap(); - assert!(storage.put(&row).await.is_err()); - } -} diff --git a/server/src/handlers/mod.rs b/server/src/handlers/mod.rs index dba9e243fc..ea549fdc70 100644 --- a/server/src/handlers/mod.rs +++ b/server/src/handlers/mod.rs @@ -44,7 +44,3 @@ pub mod plugin_sync; pub mod integration_action; pub mod integration_oauth; - -pub mod integration_proxy; - -mod integration_proxy_sync; diff --git a/server/src/routes.rs b/server/src/routes.rs index 4f2ac3d6fe..2d020245ee 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -369,26 +369,6 @@ pub fn config_routes(app: &mut actix_web::web::ServiceConfig) { .route(web::get().to(handlers::plugin_secret::handle_list_secrets)) .route(web::delete().to(handlers::plugin_secret::handle_delete_secret)), ) - .service( - web::resource("/integration-proxy/platform") - .route(web::get().to(handlers::integration_proxy::platform)), - ) - .service( - web::resource("/integration-proxy/catalog") - .route(web::get().to(handlers::integration_proxy::catalog)), - ) - .service( - web::resource("/integration-proxy/start") - .route(web::post().to(handlers::integration_proxy::start)), - ) - .service( - web::resource("/integration-proxy/finish") - .route(web::post().to(handlers::integration_proxy::finish)), - ) - .service( - web::resource("/integration-proxy/fetch") - .route(web::post().to(handlers::integration_proxy::fetch_records)), - ) .service( web::resource("/integration-oauth/notion/list") .route(web::post().to(handlers::integration_oauth::list)), diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 151765060e..16fceebaa6 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -42,6 +42,10 @@ serde-wasm-bindgen = "0.6" js-sys = "0.3" console_error_panic_hook = "0.1" +syncables = { path = "../integrations/localthought/syncables" } +async-trait = "0.1" +chrono = { version = "0.4", default-features = false, features = ["std"] } + [dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/wasm/src/integrations.rs b/wasm/src/integrations.rs new file mode 100644 index 0000000000..6baa3ef249 --- /dev/null +++ b/wasm/src/integrations.rs @@ -0,0 +1,260 @@ +//! Browser Syncables bridge: catalog parsing, typed previews, no filesystem or server. +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{Arc, Mutex}, +}; +use syncables::{ + client::client::{Fetch, HttpRequest, HttpResponse}, + ontology_shortname, ClientConfig, Credentials, Ontology, Record, Storage, StorageError, + SyncClient, TermKind, +}; +use wasm_bindgen::prelude::*; +type Result = std::result::Result>; + +struct BrowserFetch(js_sys::Function); +#[async_trait::async_trait(?Send)] +impl Fetch for BrowserFetch { + async fn fetch(&self, request: HttpRequest) -> syncables::Result { + if request.method != "GET" { + return Err(syncables::Error::Http( + "Browser imports only allow GET".into(), + )); + } + let error = + || syncables::Error::Http("Browser proxy request failed; reconnect if needed".into()); + let result = self + .0 + .call1(&JsValue::NULL, &JsValue::from_str(&request.url)) + .map_err(|_| error())?; + let value = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::resolve(&result)) + .await + .map_err(|_| error())?; + #[derive(serde::Deserialize)] + struct Response { + status: u16, + headers: BTreeMap, + body: String, + } + let value: Response = + serde_json::from_str(&value.as_string().ok_or_else(error)?).map_err(|_| error())?; + Ok(HttpResponse { + status: value.status, + headers: value.headers.into_iter().collect(), + body: value.body.into_bytes(), + }) + } +} +fn js_error(e: impl std::fmt::Display) -> JsValue { + js_sys::Error::new(&e.to_string()).into() +} +async fn document(text: &str) -> std::result::Result { + let value = syncables::openapi::load::parse_yaml(text).map_err(js_error)?; + syncables::load_open_api_document(value) + .await + .map_err(js_error) +} +#[wasm_bindgen(js_name = describeIntegration)] +pub async fn describe_integration(text: String) -> std::result::Result { + let doc = document(&text).await?; + let model = syncables::discover_resource_model(&doc).map_err(js_error)?; + let parameters: BTreeSet<_> = model + .collections + .iter() + .flat_map(|c| c.context_params.iter()) + .filter(|p| model.provider_for(p).is_none()) + .cloned() + .collect(); + Ok(json!({"parameters": parameters, "collections": model.collections.iter().map(|c| &c.name).collect::>(), + "upstream": syncables::base_url(&doc)}).to_string()) +} +#[wasm_bindgen(js_name = fetchIntegration)] +pub async fn fetch_integration( + text: String, + platform: String, + constants: String, + range: Option, + fetch: js_sys::Function, +) -> std::result::Result { + let mut doc = document(&text).await?; + if let Some(range) = range { + if platform != "google-calendar" { + return Err(js_error("Calendar range only applies to Google Calendar")); + } + let range: CalendarRange = serde_json::from_str(&range).map_err(js_error)?; + let mut value = serde_json::to_value(&doc).map_err(js_error)?; + scope_calendar(&mut value, &range).map_err(js_error)?; + doc = serde_json::from_value(value).map_err(js_error)?; + } + let engine = SyncClient::new( + ClientConfig { + document: Default::default(), + overlays: vec![], + credentials: Credentials::Anonymous, + constants: serde_json::from_str(&constants).map_err(js_error)?, + ontology_base_url: format!("https://atomicdata.dev/integrations/{platform}"), + }, + Arc::new(BrowserFetch(fetch)), + ) + .map_err(js_error)?; + let storage = PreviewStorage::default(); + let report = engine + .sync_document(&doc, &storage) + .await + .map_err(js_error)?; + if !report.errors.is_empty() { + return Err(js_error(format!( + "Import incomplete; no changes proposed: {}", + report.errors.join("; ") + ))); + } + let ontology = storage.ontology.lock().unwrap(); + let records = storage.records.lock().unwrap(); + preview( + ontology + .as_ref() + .ok_or_else(|| js_error("Missing ontology"))?, + &records, + &platform, + ) + .map(|v| v.to_string()) + .map_err(js_error) +} +#[derive(Default)] +struct PreviewStorage { + records: Mutex>, + ontology: Mutex>, +} +#[async_trait::async_trait(?Send)] +impl Storage for PreviewStorage { + async fn put(&self, record: &Record) -> std::result::Result<(), StorageError> { + let mut records = self.records.lock().unwrap(); + if records.len() >= 5000 { + return Err(StorageError::new( + "Import exceeds 5000 records; narrow its scope", + )); + } + if record.id.is_empty() + || records.iter().any(|r| { + r.namespace == record.namespace + && r.resource == record.resource + && r.id == record.id + }) + { + return Err(StorageError::new( + "Missing or repeated record identity; pagination may not be forwarded by the proxy", + )); + } + records.push(record.clone()); + Ok(()) + } + async fn get( + &self, + namespace: &str, + resource: &str, + id: &str, + ) -> std::result::Result, StorageError> { + Ok(self + .records + .lock() + .unwrap() + .iter() + .find(|r| r.namespace == namespace && r.resource == resource && r.id == id) + .cloned()) + } + async fn list( + &self, + namespace: &str, + resource: &str, + ) -> std::result::Result, StorageError> { + Ok(self + .records + .lock() + .unwrap() + .iter() + .filter(|r| r.namespace == namespace && r.resource == resource) + .cloned() + .collect()) + } + async fn delete(&self, _: &str, _: &str, _: &str) -> std::result::Result<(), StorageError> { + Err(StorageError::new("Preview storage cannot delete records")) + } + async fn put_ontology(&self, ontology: &Ontology) -> std::result::Result<(), StorageError> { + *self.ontology.lock().unwrap() = Some(ontology.clone()); + Ok(()) + } +} +fn typed_value(value: &Value, datatype: Option<&str>) -> Result> { + if value.is_null() { + return Ok(None); + } + if datatype == Some("https://atomicdata.dev/datatypes/timestamp") { + let text = value.as_str().ok_or("Expected RFC3339 timestamp")?; + return Ok(Some(json!(chrono::DateTime::parse_from_rfc3339(text) + .map_err(|_| "Invalid provider timestamp")? + .timestamp_millis()))); + } + Ok(Some(value.clone())) +} +fn preview(ontology: &Ontology, records: &[Record], platform: &str) -> Result { + let field_terms: BTreeMap<_, _> = ontology + .terms + .iter() + .filter(|t| t.kind == TermKind::Property) + .map(|t| (t.shortname.as_str(), t)) + .collect(); + let terms: Vec<_> = ontology.terms.iter().map(|t| json!({ "path":t.path, "kind":if t.kind == TermKind::Class {"class"} else {"property"}, "shortname":t.shortname, "description":t.description, "datatype":t.datatype.as_deref().unwrap_or("https://atomicdata.dev/datatypes/json"), "requires":t.requires, "recommends":t.recommends })).collect(); + let mut output = Vec::new(); + for record in records { + let mut values = serde_json::Map::new(); + for (key, value) in &record.value { + let short = ontology_shortname(key); + if let Some(term) = field_terms.get(short.as_str()) { + if let Some(value) = typed_value(value, term.datatype.as_deref())? { + values.insert(short, value); + } + } + } + output.push(json!({"resource":ontology_shortname(&record.resource),"namespace":record.namespace,"id":record.id,"values":values,"name":record.value.get("title").or_else(||record.value.get("summary")).or_else(||record.value.get("name")).cloned().unwrap_or(json!(record.id))})); + } + Ok( + json!({"platform":platform,"ontology":{"description":ontology.description,"terms":terms},"records":output}), + ) +} +/// UTC date boundaries; the end is exclusive, matching Calendar's timeMax. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct CalendarRange { + start: String, + end: String, +} + +fn scope_calendar(document: &mut Value, range: &CalendarRange) -> Result<()> { + let parse = |value: &str| { + chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") + .map_err(|_| "Calendar dates must use YYYY-MM-DD") + }; + let start = parse(&range.start)?; + let end = parse(&range.end)?; + if start >= end { + return Err("Calendar end date must be after its start date".into()); + } + let collection = document + .pointer_mut("/components/crudResources/event/collections/events") + .and_then(Value::as_object_mut) + .ok_or("Calendar catalog has no events collection")?; + if collection.get("urlTemplate").and_then(Value::as_str) + != Some("/calendars/{calendarId}/events") + { + return Err("Calendar catalog events path has changed".into()); + } + let query = collection + .entry("x-list-query") + .or_insert_with(|| json!({})) + .as_object_mut() + .ok_or("Invalid Calendar collection query")?; + query.insert("timeMin".into(), json!(format!("{start}T00:00:00Z"))); + query.insert("timeMax".into(), json!(format!("{end}T00:00:00Z"))); + query.insert("singleEvents".into(), json!(true)); + Ok(()) +} diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index cd4cf14b77..04b195c56f 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -1125,3 +1125,5 @@ impl ClientDb { .map_err(to_js_err) } } + +mod integrations;