From 3c88ae812bcbfb89bb6219176c7f2826645f6e6e Mon Sep 17 00:00:00 2001 From: bahdotsh Date: Wed, 30 Sep 2026 21:08:35 +0530 Subject: [PATCH 1/3] docs(spec): the local API chapter One server process fronting one engine for several local applications: JSON-RPC 2.0 over a WebSocket, a Unix domain socket by default and TCP on loopback with a per-launch token as the opt-in, a `hello` that declares the client's application id once and stamps it on every send, and the engine's events relayed unchanged as notifications. The chapter partitions the interface definition into the 126 methods a client may call and the 92 platform operations it never can, including the run loop and the drain, which the server owns because the engine delivers and acknowledges a message only from inside the drain. It is the first complete catalogue of the engine's 75 event tags with their fields, vocabularies and routing (by application id, by an identifier the server issued, or to everyone), and it keeps the 25-variant error taxonomy: the JSON-RPC code is the variant's position in the append-only enum. There is no HTTP request path. The pinned WebSocket library's handshake parser accepts only GET and drops any other method without a response, which replaces the program record's H5 wording and is recorded in the chapter so it is not re-added. An optional unauthenticated GET /health through the request hook is allowed. docs/bridges/local-api.md states which shared bridge rules a server inherits (C1, C2, C3, C6, C10, C12) and the new rule that a Rust guard pins the three tables to the definition, the engine and the reference server. Index rows added; docs/README.md corrected to thirteen shared rules and twenty-five ADRs, and given the peer-stream framing row it lacked. --- CHANGELOG.md | 21 + docs/README.md | 7 +- docs/bridges/README.md | 1 + docs/bridges/local-api.md | 121 +++++ docs/spec/README.md | 1 + docs/spec/local-api.md | 945 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 1094 insertions(+), 2 deletions(-) create mode 100644 docs/bridges/local-api.md create mode 100644 docs/spec/local-api.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 551396f87..42f60e319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,27 @@ archived by series under [docs/changelog/](docs/changelog/); see the ### Added +- **The local API chapter.** `docs/spec/local-api.md` specifies how one + server process fronts one engine for several local applications: JSON-RPC + 2.0 over a WebSocket on a Unix domain socket by default (TCP on loopback + with a per-launch token as the opt-in), a `hello` that declares the + client's application id once and stamps it on every send, and the events + relayed unchanged as notifications. It is the first complete catalogue of + the engine's events: all seventy-five tags with their fields and the + vocabularies of the enum-valued ones, and how each is routed (by + application id, by an identifier the server issued, or to everyone). The + method table partitions the interface definition into the 126 methods a + client may call and the 92 platform operations it never can, including the + run loop and the drain, which the server owns because the engine delivers + a message only when something drains. Errors keep the engine's + twenty-five-variant taxonomy: the JSON-RPC code is the variant's position + in the append-only enum. There is no HTTP request path: the pinned + WebSocket library drops any non-`GET` handshake without a response, which + the chapter records so it is not re-added. `docs/bridges/local-api.md` + states which shared bridge rules the server inherits and the new rule that + a Rust guard pins the three tables to the definition, the engine and the + reference server. Nothing ships in this entry but the contract; the + reference server follows it. - **The leaf node builds for ESP32 RISC-V parts and for Cortex-M0.** `offline-protocol-leaf` now compiles, and CI lints it, for `riscv32imac-unknown-none-elf` (ESP32-C6, ESP32-H2), diff --git a/docs/README.md b/docs/README.md index 174db4d38..a1c40a96f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -60,8 +60,10 @@ implementation written against these documents should interoperate. | [Capability negotiation](spec/capability-negotiation.md) | What peers advertise, what it gates, what absence means | | [Leaf node provisioning](spec/leaf-provisioning.md) | What a constrained device owes at pairing, the never-committing profile | | [Bluetooth LE framing](spec/ble-framing.md) | The GATT contract, the fragment header, and what a receiver owes on reassembly | +| [Peer-stream framing](spec/stream-framing.md) | The preamble that proves a stream's peer, the length-prefixed message frame, and the LAN discovery hint | | [Username discovery and invites](spec/username-discovery.md) | The self-certifying invite payload, the username directory, and the signing-domain registry | | [The gateway contract](spec/gateway-contract.md) | What a gateway is, the five verbs, the daemon wire protocol, and the backbone | +| [The local API](spec/local-api.md) | One server, several local applications: JSON-RPC over a WebSocket, the method and event tables, routing, replay, errors | | [Conformance](spec/conformance.md) | The two profiles, what every implementation owes, and the vectors that decide it | ## Security @@ -87,7 +89,7 @@ implementation written against these documents should interoperate. | Document | Scope | |----------|-------| -| [ADR index](adr/README.md) | Twenty-four decisions that are expensive to reverse or easy to undo by accident | +| [ADR index](adr/README.md) | Twenty-five decisions that are expensive to reverse or easy to undo by accident | If something in the codebase looks redundant or over-engineered, check here before simplifying it. @@ -99,11 +101,12 @@ in here fails **silently** when violated. | Document | Scope | |----------|-------| -| [Shared contract](bridges/README.md) | The ten rules every binding shares | +| [Shared contract](bridges/README.md) | The thirteen rules every binding shares | | [Swift](bridges/swift.md) | iOS native and the React Native iOS bridge | | [Kotlin](bridges/kotlin.md) | Android native and the React Native Android bridge | | [Python](bridges/python.md) | Desktop and tooling | | [TypeScript](bridges/typescript.md) | The React Native JavaScript surface | +| [Local API](bridges/local-api.md) | A server fronting one engine for several local clients over a socket | ## Release history diff --git a/docs/bridges/README.md b/docs/bridges/README.md index da19a73c5..e7f5a5c00 100644 --- a/docs/bridges/README.md +++ b/docs/bridges/README.md @@ -11,6 +11,7 @@ binding. | [Kotlin](kotlin.md) | Android native, the Android library and the React Native Android bridge | | [Python](python.md) | Desktop and tooling | | [TypeScript](typescript.md) | The React Native JavaScript surface | +| [Local API](local-api.md) | A server fronting one engine for several local clients over a socket, and the reference server | Integration **guides** live elsewhere: [iOS](../ios-integration.md), [Android](../android-integration.md), diff --git a/docs/bridges/local-api.md b/docs/bridges/local-api.md new file mode 100644 index 000000000..3f8baad01 --- /dev/null +++ b/docs/bridges/local-api.md @@ -0,0 +1,121 @@ +# Local API contract + +Covers a server that fronts one engine for several local clients over the +[local API](../spec/local-api.md), and the reference server in the Python +package. + +Read [the shared contract](README.md) first. A server is a binding one level +up: it is written against a generated binding (the reference one against +Python), so everything that binding owes, the server owes through it, and +this document covers what is specific to sitting between the binding and a +socket. + +## Which shared rules apply verbatim + +| Rule | Why it reaches the server | +|---|---| +| [C1](README.md#c1-regenerate-every-binding-together) | The server calls a generated binding. A partial regeneration fails at the server's first call, at runtime, exactly as it does in an application | +| [C2](README.md#c2-the-error-enum-is-append-only) | The JSON-RPC error code is the variant's position in the enum. Append-only is what makes that number stable; an inserted variant would renumber every client's error handling as well as breaking the bindings | +| [C3](README.md#c3-events-cross-as-opaque-json) | An event notification's `params` is the engine's JSON, unchanged. The server never parses an event for anything but routing (`type`, `app_id`, the correlation identifiers), and never re-encodes one | +| [C6](README.md#c6-config-parsers-must-not-default-to-literals) | A dictionary parameter arrives partial. The server passes the fields the client sent and lets the definition's defaults fill the rest; a literal fallback in the server would reset every field a client did not mention | +| [C10](README.md#c10-lifecycle-rules-for-event-emission) | The per-application-id hold of stamped inbound events is C10's held inbound buffer, one level up: the same 256-entry capacity and the same reason (the engine has already acknowledged the message), over two of C10's three tags (`message_received`, `file_received`) plus `media_resend_required` because it is stamped; `message_decryption_failed` is excluded until it carries an `app_id` | +| [C12](README.md#c12-telemetry-is-callback-free-and-the-binding-fills-the-platform) | The server is the binding that fills the platform fields and owns the session boundary. Telemetry is therefore a server-owned operation, and no client can enable, disable or flush it | + +## L1. The method and event tables are pinned by a Rust guard + +The chapter carries three tables that name things the compiler never sees +together: the methods a client may call, the platform operations it may not, +and the events it may receive. Each rots silently. A method added to the +interface definition and to neither table is unreachable from every client +and unmentioned in the contract; an event added to the engine and not to the +catalogue is one no client is written for; a method the reference server +exposes that the chapter lists as platform-only is a hole in the boundary +the chapter's fifth invariant draws. + +A Rust guard in the FFI crate reads the chapter, the interface definition, +the engine's event enum and the reference server's dispatch table, and +asserts that the exposed set plus the platform set is exactly the set of +declarations, that the dispatch table exposes exactly the exposed set, and +that the catalogue's tags are exactly the enum's variants. It follows the +skip-if-tree-absent idiom the other document-reading guards use, so a +published crate's tests still build without the repository around them. + +The chapter states the shape the guard parses (a row's first cell is the +backticked name; the three tables sit under three named headings). Changing +one of those headings, or moving a row out from under them, is a change to +the guard and fails it, which is the point: the document and the code can +only move together. + +What this prevents: a new engine method quietly reachable by every local +application because the server's dispatcher was generated from the +definition and nobody classified it. The guard makes "unclassified" a +failing test rather than a default. + +## L2. The server owns the run loop, the drain and the lifecycle + +No client calls `process()`, `receive_message()`, `start()`, `stop()`, +`pause()` or `resume()`, and none of them is a method on the wire. The +server runs the loop the mobile bridges run (100 ms), drains on every tick, +and starts and stops the engine on the operator's instruction. + +This is the first invariant of the chapter and it is load-bearing in a way +that is easy to miss: the engine emits `message_received` from inside the +drain, and acknowledges and dedup-marks the message there. A client that +could drain would take messages out from under the server's routing; two +drainers would split one stream; a server that did not drain would leave +every inbound message unacknowledged while the sender retries into silence. +The Python manager's own documentation already says that a second caller of +`receive_message()` always sees `None`; a server makes that impossibility +structural instead of documented. + +**The drain's return value is never relayed.** `receive_message()` returns +the core `Message` as JSON, a different shape from the `message_received` +event the engine emits from inside the same call (`id` rather than +`message_id`, a capitalised `priority`, `forwarded_from` rather than +`forward_info`, no `transport`). The FFI crate discards that return value on +purpose, and the Python manager synthesises a second `message_received` from +it and hands it to the same handler as the engine's event. A server built on +that manager drops the synthesised event and relays only the engine's. The +failure otherwise is two `message_received` notifications per message, one +of them in a shape the catalogue does not describe, and a client that +deduplicates by `message_id` cannot even pair them, because the synthesised +one has no such field. + +## L3. Routing and the hold are the server's, and never reach the wire + +The three routing classes (stamped, correlated, broadcast), the per-application +hold, service ownership, space scoping, method denial and the refusal of an +unlisted id at `hello` once any allow-list or deny is configured are all +rules the server applies between the engine and the socket. Nothing about +them is sent to a peer, a relay or a gateway, and nothing about them changes +an event or a frame. + +The reason to say so here is the temptation to make one of them real: an +ownership claim in the service descriptor, a per-application member in a +space. Both would be a second membership system beside MLS, which +[document replication](../spec/data-sync.md) forbids for the reason it +states there, and both would mean a peer could see which applications sit +behind an identity. The server knows; the mesh never does. + +## L4. Session refusals reuse the engine's taxonomy + +A method before `hello` is `InvalidState`; a bad application id is +`InvalidArgument`; a wrong token, a service another application owns, a +space outside the client's allow-list, a denied method group, or an unlisted +id at `hello` once any allow-list or deny is configured, is +`PermissionDenied`. No error variant exists only on the wire. + +A server that invented its own codes for these would hand every client a +second taxonomy to handle, and would break the property that a client +written against the JSON-RPC surface handles the same variants an embedded +application does. + +## What the reference server owes + +| | | +|---|---| +| Carrier | A Unix domain socket by default, created `0600` in a `0700` directory; TCP on loopback only when enabled, with the per-launch token in a `0600` file | +| Frame limit | Raised from the WebSocket library's 1 MiB default to cover the engine's file size limit plus base64, so a media send is refused by the engine and not by a closed connection | +| Dispatch table | Checked in, classified against the two method tables, and read by the guard in L1 | +| Hold | Per application id, 256 entries, oldest dropped, each drop logged | +| Health | `GET /health` through the library's request hook, with the body the chapter shows; nothing else on HTTP | diff --git a/docs/spec/README.md b/docs/spec/README.md index b77d0f75c..91eb1b928 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -25,6 +25,7 @@ document says which reading is normative for the wire. | [Peer-stream framing](stream-framing.md) | The preamble that proves a stream's peer, the length-prefixed message frame, and the LAN discovery hint | | [Username discovery and invites](username-discovery.md) | The self-certifying invite payload, and the non-authoritative username directory | | [The gateway contract](gateway-contract.md) | What a gateway is, the five verbs it implements, the gateway-daemon wire protocol, and the backbone | +| [The local API](local-api.md) | One server fronting one engine for several local applications: JSON-RPC over a WebSocket, the `hello` handshake, the method and event tables, routing, replay, and the errors | | [Conformance](conformance.md) | The two profiles, what every implementation owes, and how the vectors decide it | ## Conformance vectors diff --git a/docs/spec/local-api.md b/docs/spec/local-api.md new file mode 100644 index 000000000..de63aa164 --- /dev/null +++ b/docs/spec/local-api.md @@ -0,0 +1,945 @@ +# The local API + +## What this chapter is for + +Every binding in this repository puts the engine inside the application's own +process: a Swift or Kotlin bridge, a Python module, a React Native package. +One application, one process, one engine. A long-lived host that serves +several local applications at once, on a desktop, a server, or a headless +box, has no shape to run the engine in, because nothing outside the process +can reach it. + +This chapter specifies that shape: one **server** process owns one engine +instance, and any number of local **clients** reach it over a socket. The +wire between them is JSON-RPC 2.0 over a WebSocket. A client written against +this chapter, in any language, works against any conforming server, and a +server written against it serves any conforming client. The reference server +ships in the Python package; it is one implementation of this contract, not +the contract. + +Three things this chapter deliberately does not do. It does not invent a +second API: every method is one of the interface definition's own methods +under its own name, with its own parameters, returning its own result, and +every event is the engine's own event JSON, relayed unchanged. It does not +change anything on the mesh: nothing here reaches another device, and a peer +cannot tell whether it is talking to an application with an embedded engine +or to a server fronting six of them. And it does not add a second identity: +one server is one identity, and the applications behind it share it by +construction. What separating them requires is stated in the invariants, and +it is the server's job, never the wire's. + +## Invariants + +Six things hold for every server and every client. A server that keeps the +mechanisms below and breaks one of these is not a conforming server. + +- **The server owns the run loop and the drain.** The server calls + `process()` on its timer and drains `receive_message()` on every tick; no + client ever does either, and neither is a method on the wire. This is + load-bearing rather than tidy: the engine emits `message_received` from + inside the drain, so a message is delivered, acknowledged and dedup-marked + only when something drains. Two drainers would split one stream between + them, and no drainer would leave every inbound message unacknowledged + forever while the sender retries into a black hole. +- **A connection sends as the application it declared.** The first request + on a connection is `hello`, which names an application id, and the server + stamps that id on every message and media transfer sent over that + connection, through the per-send application id the send options carry, + so a frame on the mesh names the application it came from and the + receiving side can route it. A connection cannot send as a different + application from the one it declared, and a server that let it would + defeat every routing rule below. The id is self-declared and it is + metadata, never an access control + ([R17](../security/threat-model.md#r17-the-application-id-on-every-frame-is-cleartext-and-unsigned)): + what stops a local process from declaring another application's id is the + socket boundary and the unlisted-id rule below, not the wire. +- **An event reaches the clients it is for.** An event that carries an + application id reaches only the clients that declared that id. An event + that names an identifier the server handed to one client reaches that + client. Everything else is engine-level and reaches every client. A server + never delivers a stamped or correlated event to a client it is not for, + because the alternative is every application reading every other + application's messages through the side door of a shared socket. +- **The socket is the authorization boundary.** Whoever can open the socket + is one of the operator's applications. There is no user database, no + session token that outlives a launch, and no permission model on the + wire. A Unix domain socket with owner-only permissions is that boundary by + construction; a TCP listener on loopback is that boundary only with the + per-launch token, and is never bound to anything but loopback. Inside the + boundary sit root and every process running as the owner's uid, and the + server does not check peer credentials, so nothing below distinguishes + one of those processes from another. Application ids are self-declared, + and the server-side rules (which application owns a service, which spaces + one may open, which ids are listed) separate the applications from each + other's mistakes: a name collision, a space opened by accident, a client + started with the wrong id. They do not separate them from a hostile + local application, which is inside the boundary by definition. Every one + of those rules is applied before a call reaches the engine, and none + appears on the mesh ([data replication](data-sync.md) invariant: a space + is an MLS scope with no second membership system). +- **The platform operations are never on the wire.** The interface + definition has two kinds of method: what an application calls, and what + the platform layer that drives a radio, a relay socket or the run loop + calls. The second kind (every `ble_*`, `internet_*`, `wifi_direct_*`, + `reticulum_*` and `nostr_*` operation, every callback installer, the + lifecycle, the storage attach, the telemetry pipe and the host feeds) is + the server's own and is not reachable from a client under any name. A + client that could inject an inbound frame or announce a peer would hold the + same authority the threat model's boundary 2 exists to keep on the far + side of a dedicated entry point. +- **Nothing here is a second implementation of anything.** Method names, + parameter names, result shapes, enum spellings, event tags and error + variants are the interface definition's. The server maps a JSON request to + the generated binding's call and the binding's result to JSON; it does not + reimplement, re-validate, or re-encode a message. The failure this + prevents is the one every hand-written bridge has had: a field renamed in + one copy and not the other, invisible until a device disagrees. + +## Framing + +### JSON-RPC 2.0 over one WebSocket + +A connection is one WebSocket. Every message on it is one JSON-RPC 2.0 +object as a text frame: a request from the client, a response from the +server, or a notification from the server. A client never sends a +notification, and a server never sends a request. + +``` +--> {"jsonrpc":"2.0","id":1,"method":"hello","params":{"app_id":"notes"}} +<-- {"jsonrpc":"2.0","id":1,"result":{"api_version":1,"server":{"name":"offline-protocol-service","version":"0.28.0"},"state":"Running","local_address":"off1..."}} +--> {"jsonrpc":"2.0","id":2,"method":"send_message","params":{"recipient":"off1...","content":"hi","priority":"Medium","reply_to_msg":null}} +<-- {"jsonrpc":"2.0","id":2,"result":"5f0c..."} +<-- {"jsonrpc":"2.0","method":"event","params":{"type":"message_delivered","message_id":"5f0c...","latency_ms":412,"hop_count":1,"transport":"ble"}} +``` + +- **Requests carry `params` by name**, as a JSON object whose keys are the + method's parameter names. Positional `params` are refused with + `-32602`. A method with no parameters takes `{}` or no `params` member. +- **The request `id`** is a string or a number; the response carries it back + unchanged. Responses correlate by `id` and MAY arrive in a different order + from the requests, so a client that pipelines keeps its own map. A request + with no `id` is a JSON-RPC notification, which a client never sends; the + server drops it without a response. +- **Events are notifications** with `method` set to `event` and `params` + set to the event object itself, exactly as the engine serialised it, with + its `type` tag. Nothing is added, removed or renamed + ([C3](../bridges/README.md#c3-events-cross-as-opaque-json)). +- **Batches** (a JSON array of requests) are not supported in this version. + A server answers one with `-32600`. +- **One request in flight per method call**: the server MAY execute requests + from one connection concurrently or in order, and a client MUST NOT depend + on either. The engine serialises calls on its own lock regardless. +- **A one-shot call is a connection that sends `hello`, one request, and + closes.** Every mainstream runtime carries a WebSocket client, so this is + the whole of the "curl-shaped" use case, and it is why no second protocol + is needed for it. + +### Why there is no HTTP + +An earlier draft of this chapter served plain HTTP `POST` on the same port +for one-shot calls, through the WebSocket library's request hook. That was +verified before this chapter was written against `websockets` 16.1, the +version the Python package's lock file pins (the package itself accepts any +release from 12 up to but not including 17, so 16.1 is the only version the +behaviour has been checked on), and it does not work: the library's +handshake parser accepts only `GET`, and a connection that opens with any +other method is closed with no HTTP response at all, before the hook runs. A +`POST` would fail with zero bytes back rather than with a JSON-RPC error, +which is the worst possible failure for the caller this feature was meant to +serve. So there is no HTTP request path in this contract, and a server MUST +NOT advertise one; a client that wants a single call opens a WebSocket for +it. The reference server pins the fact with a test, so a library upgrade +that changes it is noticed rather than assumed. + +What the hook can do is answer a plain `GET` that does not ask for an +upgrade. A server MAY serve `GET /health` that way, answering `200` with a +small JSON body: + +```json +{"server":{"name":"offline-protocol-service","version":"0.28.0"},"api_version":1,"carrier":"unix"} +``` + +It is unauthenticated because it reveals nothing beyond the fact that a server +is listening, which the open port already reveals. It carries no engine state, +no address and no client count. A server that does not implement it answers +the library's default (`426 Upgrade Required`), and a client MUST NOT depend +on it. + +### Carriers + +**A Unix domain socket is the default.** The server creates it with owner-only +permissions (`0600`) inside a directory only the owner can enter (`0700`), +and removes a stale socket file at the same path before binding. The path is +the operator's choice; the reference server documents its default. On this +carrier `hello` carries no token, because the file permissions are the +credential. + +**TCP on loopback is opt-in and carries a token.** When an operator enables +it, the server binds `127.0.0.1` or `::1` only, generates 32 random bytes at +launch, writes them hex-encoded to a file at a path the operator names, and +requires them in `hello.token` on every connection. The file is created with +mode `0600` in the same call that creates it, never created and then +narrowed, because the moment between the two is exactly the window the mode +exists to close. The token is regenerated at every launch, so a token read +once never outlives the server that issued it. On neither carrier does the +server check the peer's credentials: the socket permissions and the token +are the whole of the check, and root or any process of the owner's uid +passes it. A `hello` without the token, or +with the wrong one, is answered with `PermissionDenied` and the connection is +closed with WebSocket close code `1008`. A server MUST NOT bind a non-loopback +address in this version; a deployment that wants the API across a network +puts a reverse proxy with its own authentication in front, and that proxy is +outside this contract. + +**Frame size.** The WebSocket library's default limit on one message is +1 MiB. `send_media` and `send_file` carry the file bytes inside the request, +so a server MUST raise its inbound limit to at least the engine's file size +limit (100 MiB, and not configurable through the interface today) plus +base64 overhead, or every media send above 1 MiB fails as a closed connection +instead of as the engine's own refusal. A client's outbound limit is its own +business. + +## The session + +### `hello` + +The first request on every connection MUST be `hello`, with one exception: +the three instance-less functions at the end of the method table need no +engine and no application id, and MAY be called before it. Any other method +before `hello` is refused with `InvalidState`, and a second `hello` on the +same connection is refused the same way. A server that answered another +method first would have no application id to stamp on it. + +```json +{"jsonrpc":"2.0","id":1,"method":"hello","params":{ + "app_id": "notes", + "client": "notes-desktop/2.4", + "token": "…" +}} +``` + +| Parameter | | | +|---|---|---| +| `app_id` | required | The application this connection speaks for. Validated by the same rule as the engine's own `app_id`: at most 256 bytes, not empty, not `.` or `..`, no control character, no `/`, `\` or `:` ([wire format](wire-format.md#the-abstract-message)). An id that fails it is `InvalidArgument`. | +| `client` | optional | A free-form name and version for the server's log. Never on the mesh. | +| `token` | required on TCP | The per-launch token. Ignored on a Unix socket. | + +The result: + +| Field | | +|---|---| +| `api_version` | `1`. A client that needs something this version does not have checks here and refuses to proceed rather than probing. | +| `server` | `{"name","version"}` of the server implementation. | +| `state` | The engine's `ProtocolState`: `Stopped`, `Running` or `Paused`. A server MAY accept clients before `start()`; every method that needs a running engine then answers `NotStarted`, which is the same answer an embedded caller gets. | +| `local_address` | The identity's `off1…` address, or `null` before the identity exists. Every client of one server gets the same address, which is the one-identity rule made visible. | + +Several clients MAY declare the same application id, at once or in sequence; +a second process of the same application is the ordinary case. They are one +application to every rule below: an event for that id reaches all of them, +and any of them may unregister a service the other registered. Whether an +id the server has never seen is accepted depends on whether the operator +has configured a space allow-list or a method deny; see the unlisted-id +rule under server-side rules. + +### `subscribe` and `unsubscribe` + +After `hello`, a client receives every event the routing rules deliver to +it. `subscribe` narrows that to a list of tags, and `unsubscribe` widens it +back: + +```json +{"jsonrpc":"2.0","id":3,"method":"subscribe","params":{"types":["message_received","message_delivered"]}} +{"jsonrpc":"2.0","id":4,"method":"unsubscribe","params":{"types":["message_delivered"]}} +{"jsonrpc":"2.0","id":5,"method":"subscribe","params":{"types":"all"}} +``` + +The filter is the client's own convenience and changes nothing about +routing: a tag a client subscribed to still reaches it only if the rules +below deliver it. A tag the server does not know is accepted and never +fires, so a client built against a newer engine keeps working against an +older server. Both return `true`. + +### Session methods + +These three are the server's own and are not in the interface definition. +They are the only such methods; everything else on the wire is one of the +engine's. + +| Method | Params | Result | +|---|---|---| +| `hello` | `app_id`, `client?`, `token?` | the object above | +| `subscribe` | `types` (`[]` or `"all"`) | `true` | +| `unsubscribe` | `types` (`[]` or `"all"`) | `true` | + +## Encoding + +The interface definition's types map to JSON one way, in both directions, +and this is the whole of the mapping: + +| Interface type | JSON | +|---|---| +| `string`, `string?` | string; an optional one may be `null` or omitted | +| `boolean` | `true` / `false` | +| `u8`, `u16`, `u32`, `u64`, `i16`, `i64`, `f32`, `double` | number. A `u64` above 2^53 loses precision in a JavaScript client; none of the engine's counters and timestamps reach it in practice, and a client that must be exact parses the raw text | +| enums (`MessagePriority`, `TransportType`, `ProtocolState`, `ContentType`, …) | the definition's own spelling as a string: `"Medium"`, `"WiFiDirect"`, `"Running"`, `"Image"` | +| `sequence`, `bytes` | a base64 string (RFC 4648, standard alphabet, padded). This is the one place the wire departs from the events, which carry bytes as the engine serialises them (see the catalogue) | +| `sequence` | array of strings | +| `record` | object | +| dictionaries (`SendMessageOptions`, `DorsConfig`, `MlsWelcomeMessage`, …) | object with the dictionary's field names. A field with a default in the definition may be omitted and takes that default; a server passes only the fields the caller sent and never substitutes a literal of its own ([C6](../bridges/README.md#c6-config-parsers-must-not-default-to-literals)) | +| `void` | `null` | + +The legend used in the tables below: a bare name is a string; `#` is a +number; `!` a boolean; `b64` bytes as base64; `[]` an array of strings; +`{Name}` an object shaped as the named dictionary; `` a string from the +named enum; a trailing `?` is optional. + +## Method table + +Every method below is one of the interface definition's own, called with its +own parameters. The three objects of the definition keep their own prefix on +the wire, so the guard that pins this table can map every row back to its +declaration: engine methods are bare, `MeshServices` methods carry +`services.`, and `DataStore` methods carry `data.`. The instance-less +functions of the definition's namespace are bare too, and are the only +methods a client may call before `hello`. + +### Engine: identity and state + +| Method | Params | Result | +|---|---|---| +| `get_state` | | `` | +| `local_address` | | string or `null` | +| `get_identity_public_key` | | `b64` (32 bytes) | +| `derive_user_id_from_public_key` | `public_key b64` | string | +| `sign_data` | `data b64` | `b64` (64 bytes). Signs with the shared identity key, which every client of one server shares by construction; an operator who does not want every application to hold a signing oracle denies it by application id (see server-side rules) | +| `verify_signature` | `public_key b64`, `data b64`, `signature b64` | `!` | +| `create_invite` | `petname?`, `sign !` | string (the invite blob) | +| `resolve_username` | `username` | `!` | + +### Engine: messaging + +| Method | Params | Result | +|---|---|---| +| `send_message` | `recipient`, `content`, `priority `, `reply_to_msg?` | string (message id). Stamped with the connection's application id; the server calls the rich surface with only that option set | +| `send_message_rich` | `recipient`, `content`, `options {SendMessageOptions}` | string (message id). `options.app_id` is set by the server to the connection's id; a value the client sends is refused with `InvalidArgument` | +| `forward_message` | `original_message_json`, `new_recipient`, `priority ?` | string (message id) | +| `send_presence_update` | `recipient`, `status ` | string (message id) | +| `send_typing_indicator` | `recipient`, `conversation_id`, `is_typing !` | string (message id) | +| `send_read_receipt` | `recipient`, `message_ids []` | string (message id) | + +### Engine: connection requests + +| Method | Params | Result | +|---|---|---| +| `send_connection_request` | `recipient`, `sender_name`, `key_package b64?`, `initial_message?` | string (message id) | +| `accept_connection_request` | `recipient`, `accepter_name`, `key_package b64?` | string (message id) | +| `reject_connection_request` | `recipient` | string (message id) | +| `cancel_connection_request` | `recipient` | string (message id) | + +### Engine: media and files + +| Method | Params | Result | +|---|---|---| +| `send_media` | `recipient`, `file_data b64`, `file_name`, `content_type `, `media_metadata {MediaMetadata}?` | string (file id). Stamped like `send_message` | +| `send_media_rich` | `recipient`, `file_data b64`, `file_name`, `content_type `, `options {MediaSendOptions}` | string (file id). `options.app_id` as for `send_message_rich` | +| `send_file` | `recipient`, `file_data b64`, `file_name` | string (file id) | +| `get_file_progress` | `file_id` | `{FileProgress}` or `null` | +| `cancel_file_transfer` | `file_id` | `null` | + +### Engine: secure sessions + +| Method | Params | Result | +|---|---|---| +| `is_mls_initialized` | | `!` | +| `has_pending_key_package` | `peer_id` | `!` | +| `get_establishment_state` | `peer_id` | `` | +| `establish_secure_session` | `peer_id` | `{MlsWelcomeMessage}` or `null` | +| `rekey_session` | `peer_id` | `!` | +| `mls_has_session` | `other_user_id` | `!` | +| `mls_list_sessions` | | `[]` | +| `mls_delete_session` | `other_user_id` | `null` | + +### Engine: manual MLS + +The manual surface operates on the shared identity's sessions, exactly as it +does for an embedded application. A client that drives it is driving every +other client's sessions too. + +| Method | Params | Result | +|---|---|---| +| `mls_generate_key_package` | | `{MlsKeyPackageBundle}` | +| `mls_get_or_create_key_package` | | `{MlsKeyPackageBundle}` | +| `mls_import_key_package` | `user_id`, `key_package_data b64` | `null` | +| `mls_get_pending_key_packages` | | `[{MlsKeyPackageBundle}]` | +| `mls_mark_key_package_synced` | `package_id` | `null` | +| `mls_create_session` | `other_user_id` | `{MlsWelcomeMessage}` | +| `mls_join_session` | `welcome {MlsWelcomeMessage}` | `{MlsGroupInfo}` | +| `mls_encrypt_for_user` | `other_user_id`, `plaintext b64` | `{MlsEncryptedMessage}` | +| `mls_decrypt_from_user` | `encrypted {MlsEncryptedMessage}` | `b64` or `null` | +| `mls_get_pending_welcome` | `other_user_id` | `{MlsWelcomeMessage}` or `null` | +| `mls_clear_pending_welcome` | `other_user_id` | `null` | +| `mls_decrypt` | `encrypted {MlsEncryptedMessage}` | `b64` or `null` | +| `mls_process_welcome` | `welcome {MlsWelcomeMessage}` | `{MlsGroupInfo}` | + +### Engine: groups + +| Method | Params | Result | +|---|---|---| +| `create_group` | `group_name` | `{MlsGroupInfo}` | +| `send_group_message` | `group_id`, `content`, `priority ?`, `reply_to_msg?` | `[]` (message ids). Group sends carry the configured application id: the per-send id is a 1:1 option today | +| `forward_message_to_group` | `original_message_json`, `group_id`, `priority ?` | `[]` (message ids) | +| `invite_to_group` | `group_id`, `invitee_user_id` | `null` | +| `remove_from_group` | `group_id`, `member_id` | `null` | +| `leave_group` | `group_id` | `null` | +| `list_groups` | | `[]` | +| `get_group_info` | `group_id` | `{MlsGroupInfo}` or `null` | +| `group_rich_readiness` | `group_id` | `{GroupRichReadiness}` | +| `group_relay_sync_state` | `group_id` | `` | +| `request_group_relay_registration` | `group_id` | `!` | +| `set_member_role` | `group_id`, `user_id`, `role` | `null` | +| `get_member_role` | `group_id`, `user_id` | string | +| `get_group_roles` | `group_id` | object of user id to role | +| `rename_group` | `group_id`, `new_name` | `null` | + +### Engine: blocking + +| Method | Params | Result | +|---|---|---| +| `block_user` | `user_id` | `null` | +| `unblock_user` | `user_id` | `null` | +| `get_blocked_users` | | `[]` | +| `is_user_blocked` | `user_id` | `!` | + +### Engine: transports and metrics + +Read-only views of the instance. Everything here describes the whole server, +never one client's share of it. + +| Method | Params | Result | +|---|---|---| +| `get_active_transports` | | `[]` | +| `get_transport_metrics` | `transport_type ` | `{TransportMetrics}` or `null` | +| `should_escalate_to_wifi` | | `!` | +| `get_topology` | | `{NetworkTopology}` | +| `get_message_stats` | | `[{MessageStats}]` | +| `get_delivery_success_rate` | | `#` | +| `get_median_latency` | | `#` | +| `get_median_hops` | | `#` | +| `get_battery_level` | | `#` or `null` | +| `get_is_charging` | | `!` | +| `is_relay` | | `!` | +| `get_relay_priority` | | `` | +| `get_relay_config` | | `{RelayConfig}` | +| `get_dors_config` | | `{DorsConfig}` | +| `get_dedup_stats` | | `{DedupStats}` | +| `get_pending_ack_count` | | `#` | +| `get_retry_queue_size` | | `#` | +| `get_mesh_relay_stats` | | `{MeshRelayStats}` | +| `get_mesh_relay_tunables` | | `{MeshRelayTunables}` | + +### Engine: instance-wide tuning + +Every call here changes the engine for every client. They are on the wire +because an embedded application can call them, and because the socket is the +authorization boundary; an operator who does not want every application +retuning the relay denies the group by application id (see server-side +rules). None of them is per client and none is undone when the caller +disconnects. + +| Method | Params | Result | +|---|---|---| +| `set_relay_priority` | `priority ` | `null` | +| `update_relay_config` | `config {RelayConfig}` | `null` | +| `force_transport` | `transport_type ` | `null` | +| `release_transport_lock` | | `null` | +| `update_dors_config` | `config {DorsConfig}` | `null` | +| `update_ack_config` | `config {AckConfig}` | `null` | +| `update_retry_config` | `config {RetryConfig}` | `null` | +| `update_dedup_config` | `config {DedupConfig}` | `null` | + +### Services + +The `MeshServices` object, one per server, under `services.`. Ownership is a +server-side rule stated below: a service is owned by the application id that +registered it. + +| Method | Params | Result | +|---|---|---| +| `services.register_service` | `service_id`, `version`, `capabilities` (object) | `null` | +| `services.unregister_service` | `service_id` | `!` | +| `services.discover_services` | `service_id?` | string (query id) | +| `services.send_service_request` | `provider`, `service_id`, `method`, `body` | string (request id) | +| `services.respond_to_service_request` | `request_id`, `requester`, `service_id`, `status`, `body` | string (message id) | + +### Documents + +The `DataStore` object, one per server, under `data.`. Space scoping is a +server-side rule stated below. + +| Method | Params | Result | +|---|---|---| +| `data.create_doc` | `space_id`, `doc_id` | `null` | +| `data.delete_doc` | `space_id`, `doc_id` | `null` | +| `data.remove_doc` | `space_id`, `doc_id` | `null` | +| `data.remove_space` | `space_id` | `null` | +| `data.set_interest` | `space_id`, `patterns []` | `null` | +| `data.list_docs` | `space_id` | `[]` | +| `data.list_spaces` | | `[]`, filtered to the spaces the client may open | +| `data.map_set` | `space_id`, `doc_id`, `collection`, `key`, `value_json` | `null` | +| `data.map_delete` | `space_id`, `doc_id`, `collection`, `key` | `null` | +| `data.map_get_json` | `space_id`, `doc_id`, `collection`, `key` | string or `null` | +| `data.list_push` | `space_id`, `doc_id`, `collection`, `value_json` | `null` | +| `data.list_delete` | `space_id`, `doc_id`, `collection`, `index #`, `count #` | `null` | +| `data.list_len` | `space_id`, `doc_id`, `collection` | `#` | +| `data.text_insert` | `space_id`, `doc_id`, `collection`, `position #`, `text` | `null` | +| `data.text_delete` | `space_id`, `doc_id`, `collection`, `position #`, `count #` | `null` | +| `data.text_value` | `space_id`, `doc_id`, `collection` | string | +| `data.counter_increment` | `space_id`, `doc_id`, `collection`, `amount #` | `null` | +| `data.counter_value` | `space_id`, `doc_id`, `collection` | `#` | +| `data.doc_json` | `space_id`, `doc_id` | string | +| `data.export_raw` | `space_id`, `doc_id` | `b64` | +| `data.flush` | `space_id`, `doc_id` | `null` | +| `data.flush_all` | | `null` | +| `data.doc_size` | `space_id`, `doc_id` | `#` | +| `data.attachment_hash` | `data b64` | string | +| `data.fetch_attachment` | `space_id`, `hash` | `null` | +| `data.provide_attachment` | `space_id`, `peer_id`, `hash`, `data b64` | `null` | +| `data.decline_attachment` | `space_id`, `peer_id`, `hash` | `null` | +| `data.fetch_attachment_from` | `space_id`, `peer_id`, `hash` | `null` | + +### Instance-less + +The definition's namespace functions that need no engine. They are the one +group a client may call before `hello`, because there is nothing to stamp and +nothing to route. + +| Method | Params | Result | +|---|---|---| +| `derive_address` | `public_key b64` | string | +| `parse_invite` | `blob` | `{InviteInfo}` | +| `verify_identity_assertion` | `assertion b64` | string (the proved address) | + +## Platform operations + +Everything in the interface definition that is not in the method table is a +platform operation and is not on the wire. A request naming one is answered +with `-32601`, the same answer an unknown name gets, so a client cannot +discover the set by probing. The two tables together partition the +definition: every declaration is in exactly one, and the guard that reads +this chapter asserts it. + +| Operation | Why the server owns it | +|---|---| +| `constructor`, `start`, `stop`, `pause`, `resume`, `process`, `receive_message`, `set_event_callback`, `poll_event`, `emit_test_event` | The lifecycle, the run loop and the drain (first invariant) | +| `initialize_mls`, `initialize_mls_with_file_stores`, `close_file_stores` | The identity's storage is attached once, by the process that holds the store key | +| `enable_telemetry`, `disable_telemetry`, `set_telemetry_enabled`, `flush_telemetry`, `flush_telemetry_blocking`, `telemetry_stats`, `end_telemetry_session`, `notify_app_state`, `telemetry_install_id` | The binding fills the platform fields and owns the session boundary ([C12](../bridges/README.md#c12-telemetry-is-callback-free-and-the-binding-fills-the-platform)); the server is the binding | +| `set_ble_transport_callback`, `set_wifi_direct_transport_callback`, `set_reticulum_transport_callback`, `set_nostr_transport_callback` | A callback interface has no JSON form, and each installs the driver of a carrier | +| `ble_peer_discovered`, `ble_peer_lost`, `ble_status_changed`, `ble_fragment_received`, `ble_get_next_fragment`, `ble_return_fragment`, `ble_get_peer_count`, `ble_set_peer_mtu`, `ble_clear_peer_mtu`, `ble_undersized_mtu_reports`, `ble_fragment_fallback_count`, `ble_recipient_not_among_peers_count`, `protocol_lock_diagnostics` | The Bluetooth LE driver's feed and drain, and the host's lock diagnostics | +| `internet_status_changed`, `internet_message_received`, `internet_get_next_message`, `internet_confirm_sent`, `internet_send_failed`, `internet_send_failed_with_reason`, `internet_peer_presence`, `internet_presence_watchlist`, `internet_relay_capabilities`, `internet_group_report_received`, `internet_address_declared`, `internet_address_declaration_refused` | The relay client's feed and drain, including the dedicated entry points the threat model's boundary 2 requires | +| `wifi_direct_status_changed`, `wifi_direct_message_received`, `wifi_direct_get_next_message`, `wifi_direct_peer_connected`, `wifi_direct_peer_disconnected` | The peer-stream driver's feed and drain | +| `reticulum_status_changed`, `reticulum_message_received`, `reticulum_get_next_message`, `reticulum_confirm_sent`, `reticulum_send_failed`, `reticulum_send_failed_with_reason`, `reticulum_address_declared`, `reticulum_address_declaration_refused`, `reticulum_gateway_capabilities`, `reticulum_peer_presence`, `reticulum_presence_watchlist` | The gateway client's feed and drain | +| `nostr_status_changed`, `nostr_message_received`, `nostr_message_received_at`, `nostr_get_next_message`, `nostr_confirm_sent`, `nostr_send_failed`, `nostr_send_failed_with_reason`, `nostr_get_public_key`, `nostr_get_subscription_filter`, `nostr_get_next_query`, `nostr_query_event_received`, `nostr_query_completed` | The Nostr client's feed and drain | +| `update_transport_metrics`, `remove_transport`, `set_battery_level`, `set_battery_state` | Facts about the host that only the host process knows | +| `identity_assertion`, `gateway_address_declaration` | Signatures a carrier driver presents to prove this device to a peer or a gateway; a client that could mint them could impersonate the server's carriers | +| `process_file_chunk`, `finalize_file` | The inbound chunk driver of a platform transport | +| `services.constructor`, `data.constructor`, `data.with_storage` | The server constructs the two objects once, over the engine it owns; `with_storage` takes a callback interface | +| `data.wipe_all` | Erases every client's documents and the identity's key documents at once. That is the operator's logout, taken at the server, never one application's call | +| `run_storage_conformance` | Takes a callback interface; a storage backend is verified by the host that supplies it | + +## Event catalogue + +### Routing + +Every event the engine emits is relayed as a notification to the clients the +rules below select, in the order the engine emitted it. The rules are the +server's; they never change an event and they never reach the mesh. + +1. **Stamped.** An event carrying an `app_id` field reaches the clients that + declared that id, and no other. Three events are stamped today: + `message_received`, `file_received` and `media_resend_required`. On the + last two the field is optional, and the engine does emit `file_received` + without it, for a transfer whose assembly had no metadata entry to read + the id from. An event of a stamped tag whose `app_id` is absent is + broadcast, not held, and logged, because the server has nothing to route + or hold it by; dropping it would lose a message the engine counts as + delivered. +2. **Correlated.** An event naming an identifier the server handed to one + client as a method result (a message id, a file id, a query id, a request + id) reaches that client, and no other. An event naming a service id + reaches the clients whose application owns that service. The server keeps + these tables in memory for the life of the process; an identifier issued + before a restart, or one that came from a peer rather than from a client's + call, is unknown, and an event naming an unknown identifier is broadcast. + A `message_delivered` for a message a client sent therefore reaches that + client and not its neighbours, and a `message_delivered` for a message the + relay re-sent after a restart reaches everyone. + + The engine emits synchronously, on the thread that is inside its call, so + some events fire before the call that caused them has returned: + `message_sent` is emitted inside the send when a transport takes the + frame at once, and the server does not yet hold the id it will correlate + by. An event emitted while the server is executing one client's call is + therefore that client's, whatever it names, and the server records the + identifiers it carries as that client's too. Without this rule the very + first event of every send would be broadcast. +3. **Broadcast.** Everything else is engine-level and reaches every client. + Group, presence, session and transport events are shared state of the one + identity, so every application sees them. + +The `Routing` column of the catalogue names the key: `app_id`, one of the +identifiers, or `all`. + +### The catalogue + +Every event the engine can emit, its fields, and how it is routed. The tag +is the `type` member; fields are the object's other members, using the legend +above with one difference: bytes in an event are an array of numbers, because +the event is relayed as the engine serialises it and the engine has always +written them that way. Sub-object shapes and the enum vocabularies follow the +table. The API reference documents about twenty of these with prose; this +table is read from the event definitions, held to them by the guard below, +and is the one a client is written against. + +| Tag | Fields | Routing | +|---|---|---| +| `message_sent` | `message_id`, `sender`, `recipient`, `content`, `priority`, `requires_ack !`, `timestamp #`, `lamport_clock #`, `forward_info {ForwardInfo}?` | `message_id` | +| `message_received` | `message_id`, `sender`, `recipient`, `content`, `hop_count #`, `transport`, `timestamp #`, `lamport_clock #`, `reply_to_msg?`, `reply_context {ReplyContext}?`, `content_type`, `media_metadata {MediaMetadata}?`, `forward_info {ForwardInfo}?`, `encrypted !`, `app_id` | `app_id` | +| `message_delivered` | `message_id`, `latency_ms #`, `hop_count #`, `transport` | `message_id` | +| `message_failed` | `message_id`, `reason`, `retry_count #` | `message_id` | +| `message_decryption_failed` | `message_id`, `sender`, `code `, `reason` | `all` | +| `transport_switched` | `from?`, `to`, `reason` | `all` | +| `relay_promoted` | `connection_count #`, `battery_level #?` | `all` | +| `relay_demoted` | `reason` | `all` | +| `neighbor_discovered` | `peer_id`, `transport`, `rssi #?` | `all` | +| `neighbor_lost` | `peer_id` | `all` | +| `identity_ready` | `address` | `all` | +| `network_metrics` | `neighbor_count #`, `relay_count #`, `delivery_ratio #`, `avg_latency_ms #` | `all` | +| `file_progress` | `file_id`, `chunks_sent #`, `total_chunks #`, `percentage #` | `file_id` | +| `file_received` | `file_id`, `file_name`, `file_size #`, `sender`, `content_type`, `media_metadata {MediaMetadata}?`, `file_data` (base64), `timestamp #?`, `caption?`, `reply_to_msg?`, `reply_context {ReplyContext}?`, `forward_info {ForwardInfo}?`, `app_id?` | `app_id` | +| `file_receive_failed` | `file_id`, `file_name`, `sender`, `reason` | `all` | +| `media_sent` | `file_id`, `content_type`, `recipient` | `file_id` | +| `media_send_failed` | `file_id`, `recipient`, `reason` | `file_id` | +| `message_deferred` | `message_id`, `recipient`, `reason`, `retry_count #`, `next_retry_at #?` | `message_id` | +| `message_retrying` | `message_id`, `recipient`, `retry_count #`, `next_retry_at #` | `message_id` | +| `message_undeliverable` | `message_id`, `recipient`, `reason`, `file_id?` | `message_id` | +| `media_resend_required` | `file_id`, `recipient`, `file_name`, `file_size #`, `app_id?` | `app_id` | +| `ack_evicted` | `message_id`, `priority`, `reason` | `message_id` | +| `fragment_assembly_evicted` | `message_id`, `completion_percent #`, `reason` | `all` | +| `relay_demoted_battery` | `battery_level #`, `min_required #` | `all` | +| `secure_session_established` | `peer_id`, `group_id`, `is_session !`, `initiated_by_local !` | `all` | +| `secure_session_failed` | `peer_id`, `reason` | `all` | +| `convergence_diag` | `stage`, `peer_id`, `detail` | `all` | +| `welcome_send_attempted` | `peer_id`, `message_id`, `group_id`, `attempt #` | `all` | +| `welcome_send_succeeded` | `peer_id`, `message_id`, `group_id`, `attempt #` | `all` | +| `welcome_send_failed` | `peer_id`, `message_id`, `group_id`, `attempt #`, `reason_code `, `transport_error?`, `retryable !`, `next_retry_at #?` | `all` | +| `welcome_send_expired` | `peer_id`, `message_id`, `attempt #`, `reason_code ` | `all` | +| `connection_request_received` | `sender`, `sender_name`, `timestamp #`, `key_package [#]?`, `initial_message?` | `all` | +| `connection_request_undeliverable` | `recipient`, `message_id`, `reason` | `message_id` | +| `connection_accepted` | `accepted_by`, `accepted_by_name`, `timestamp #`, `key_package [#]?` | `all` | +| `connection_rejected` | `rejected_by` | `all` | +| `connection_request_cancelled` | `cancelled_by` | `all` | +| `group_created` | `group_id`, `name` | `all` | +| `group_message_received` | `group_id`, `sender`, `content`, `timestamp` (a string), `message_id`, `reply_to_msg?`, `forward_info {ForwardInfo}?`, `media_metadata {MediaMetadata}?`, `content_type?` | `all` | +| `group_member_added` | `group_id`, `user_id`, `added_by`, `group_name?`, `authorized !?` | `all` | +| `group_member_removed` | `group_id`, `user_id`, `removed_by`, `authorized !?` | `all` | +| `group_info` | `group_id`, `name`, `created_by`, `created_at`, `members [{GroupInfoMember}]` | `all` | +| `user_groups` | `groups [{UserGroupSummary}]` | `all` | +| `group_error` | `reason`, `group_id?` | `all` | +| `group_relay_sync_changed` | `group_id`, `synced !`, `reason` | `all` | +| `username_resolved` | `username`, `claims [{UsernameClaim}]`, `rejected #`, `truncated #` | `all` | +| `group_message_sent` | `group_id`, `message_ids []`, `member_count #` | `all` | +| `group_message_partial_failure` | `group_id`, `failed_members []`, `succeeded_members []` | `all` | +| `group_message_delivery_report` | `group_id`, `message_id`, `delivered []`, `pushed []`, `missed_reissued []` | `all` | +| `group_rich_extras_dropped` | `group_id`, `unknown_members []` | `all` | +| `group_unauthorized_membership_change` | `group_id`, `committer`, `added []`, `removed []`, `reason`, `enforced !` | `all` | +| `group_epoch_fork_detected` | `group_id`, `local_epoch #?` | `all` | +| `group_epoch_fork_resolved` | `group_id`, `resolved_epoch #`, `failed_members []` | `all` | +| `group_role_changed` | `group_id`, `user_id`, `new_role`, `changed_by` | `all` | +| `group_renamed` | `group_id`, `new_name`, `old_name?`, `renamed_by` | `all` | +| `service_discovered` | `query_id`, `service_id`, `version`, `provider_peer_id`, `capabilities` (object), `hop_count #` | `query_id` | +| `service_request_received` | `request_id`, `service_id`, `method`, `body`, `sender` | `service_id` | +| `service_response_received` | `request_id`, `service_id`, `status`, `body`, `provider_peer_id` | `request_id` | +| `presence_updated` | `peer_id`, `status `, `timestamp #`, `last_seen_ms #?`, `source ` | `all` | +| `typing_indicator_received` | `sender`, `conversation_id`, `is_typing !`, `timestamp #` | `all` | +| `read_receipt_received` | `sender`, `message_ids []`, `timestamp #` | `all` | +| `dors_score_updated` | `scores` (array of `[transport, score #]` pairs) | `all` | +| `dors_transport_selected` | `from?`, `transport`, `reason_code `, `score #` | `all` | +| `dors_transport_switched` | `from?`, `to`, `reason_code `, `reason_detail?` | `all` | +| `dors_escalation_triggered` | `phase `, `from`, `to`, `reason_code `, `reason_detail?` | `all` | +| `security_warning` | `peer_id`, `reason_code `, `reason` | `all` | +| `message_relayed` | `message_id`, `sender`, `recipient`, `hop_count #`, `remaining_ttl #` | `all` | +| `user_blocked` | `user_id` | `all` | +| `user_unblocked` | `user_id` | `all` | +| `data_changed` | `space_id`, `doc_id`, `delta_bytes #` | `all` | +| `data_doc_removed` | `space_id`, `doc_id`, `by ` | `all` | +| `data_doc_size_warning` | `space_id`, `doc_id`, `compacted_bytes #`, `cap_bytes #` | `all` | +| `data_attachment_requested` | `space_id`, `peer_id`, `hash` | `all` | +| `data_attachment_received` | `space_id`, `peer_id`, `hash`, `data` (base64) | `all` | +| `data_attachment_unavailable` | `space_id`, `peer_id`, `hash`, `reason` | `all` | +| `data_doc_unsyncable` | `space_id`, `doc_id`, `bytes #`, `reason` | `all` | + +Two rows deserve a note. `message_decryption_failed` stands in for a message +that could not be read, and the event does not carry the frame's application +id even though the frame did, so it is broadcast; when the engine adds the +id (additive under C3) it becomes stamped. `message_relayed` reports a frame +this device forwarded for others and names another identity's message id, +which is why it is broadcast rather than correlated. + +Document events (`data_*`) are broadcast, and a client MUST apply its own +space filter to them: the server's space allow-list gates calls, and the +events carry the space id for the client to match, so a client that may not +open a space still learns that it changed. Filtering them at the server is a +server-side rule an implementation MAY add, and the reference server does. + +### Shapes + +| Object | Members | +|---|---| +| `ForwardInfo` | `original_sender`, `original_message_id`, `original_timestamp #`, `forward_count #` | +| `ReplyContext` | `sender`, `text`, `timestamp #?`, `reply_media_label?`, `reply_content_type?` | +| `MediaMetadata` | `mime_type`, `file_name`, `file_size #`, `duration_ms #?`, `width #?`, `height #?`, `thumbnail_base64?`, `media_id?`, `download_url?`, `thumbnail_url?`, `encryption_key?`, `iv?`, `ciphertext_hash?`, `sticker_provider?`, `sticker_remote_id?`, `sticker_kind?` | +| `GroupInfoMember` | `user_id`, `role`, `joined_at` | +| `UserGroupSummary` | `group_id`, `name`, `created_at` | +| `UsernameClaim` | `address`, `public_key`, `issued_at_ms #` | + +### Vocabularies + +| Enum | Values | +|---|---| +| `PresenceStatus` | `online`, `away`, `offline` | +| `PresenceSource` | `internet`, `reticulum`, `peer` | +| `DocRemovedBy` | `local`, `peer` | +| `DecryptionFailureCode` | `INVALID_PAYLOAD`, `NOT_INITIALIZED`, `INVALID_CIPHERTEXT`, `IDENTITY_MISMATCH`, `CRYPTO_FAILURE`, `PENDING_QUEUE_DROPPED`, `UNKNOWN` | +| `WelcomeReasonCode` | `TRANSPORT_UNAVAILABLE`, `PEER_UNREACHABLE`, `PEER_DISCONNECTED`, `TIMEOUT`, `INTERNAL_ERROR`, `RETRY_EXHAUSTED` | +| `DorsReasonCode` | `INITIAL_SELECTION`, `PRIMARY_SELECTED`, `PRIMARY_SUCCESS`, `FALLBACK_SUCCESS`, `ESCALATION_APPLIED`, `CURRENT_UNAVAILABLE` | +| `DorsEscalationPhase` | `TRIGGERED`, `APPLIED` | +| `DorsEscalationReasonCode` | `FALLBACK_SUCCESS`, `RETRY_THRESHOLD`, `POOR_SIGNAL`, `CONGESTION`, `LOW_TTL`, `LOW_SUCCESS_RATE` | +| `SecurityWarningCode` | `SENDER_ADDRESS_MISMATCH`, `TRANSPORT_IDENTITY_MISMATCH`, `CONTROL_SIGNATURE_INVALID`, `UNSIGNED_CONTROL_REJECTED`, `MEDIA_SENDER_GROUP_MISMATCH`, `PLAINTEXT_SEND`, `PLAINTEXT_RECEIVE_REJECTED`, `SESSION_SENDER_GROUP_MISMATCH`, `SESSION_REKEY_TRIGGERED`, `NOSTR_KEY_PACKAGE_SLOT_EXHAUSTED`, `PUSH_KEY_PACKAGE_POOL_EXHAUSTED`, `RELAY_ADDRESS_BINDING_MISMATCH`, `RELAY_ADDRESS_DECLARATION_REFUSED`, `GROUP_LEAF_IDENTITY_UNPROVEN`, `STALE_CONTROL_FRAME`, `GATEWAY_ADDRESS_BINDING_MISMATCH`, `GATEWAY_ADDRESS_DECLARATION_REFUSED` | + +The transport name is spelled two ways by the engine, and a client accepts +both. `message_received`, `message_delivered`, the DORS events and a +`transport_switched` the engine emits carry the transport's label: `ble`, +`wifiDirect`, `internet`, `reticulum`, `nostr`, or `delayed` on a +`message_received` for a message that was held and decrypted later. +`neighbor_discovered`, and a `transport_switched` the FFI layer emits when a +carrier's status changes, carry the casing the platform bridge passes as a +string: `BLE`, `WiFiDirect`, `Internet`, `Reticulum`, `Nostr`, and on that +`transport_switched` the literal `None` as `to` when a carrier disconnects +and nothing takes over. Neither spelling is the interface definition's enum +(`Ble`), and `wifi_direct` never occurs. The `priority` string is spelled +two ways by the engine and a client accepts both: lowercase on `message_sent` +(`low`, `medium`, `high`, `critical`) and capitalised on `ack_evicted` (`Low`, +`Medium`, `High`, `Critical`). + +These are the spellings the events carry. A method parameter of an enum type +uses the interface definition's spelling, so a client sends `"Online"` to +`send_presence_update` and reads `"online"` back on `presence_updated`. + +The vocabularies above are the engine's, pinned against the TypeScript +declarations by the guards [T2](../bridges/typescript.md#t2-event-types-are-pinned-from-the-rust-side) +names; this chapter restates them so a client author has one place to read, +and the guard for this chapter (below) holds the tag column to the engine. + +## Replay + +The interface that drives a radio has a rule for events that fire before +anyone is listening ([C10](../bridges/README.md#c10-lifecycle-rules-for-event-emission)), +and a server has the same problem one level up: an application's client may +not be connected when its message arrives. The engine has already +acknowledged that message, dedup-marked its id and dropped its queued copy +by the time `message_received` is emitted, so the sender will not resend and +nothing in the engine will restate the event. A server that dropped it +would lose a message that the mesh counts as delivered. + +So a server holds **stamped inbound events** for an application id with no +connected client: two of C10's three tags, `message_received` and +`file_received`, plus `media_resend_required` because it is stamped too. +C10's third tag, `message_decryption_failed`, is excluded until it carries an +`app_id`, because there is no id to hold it under. Each held event is kept +whole, in arrival order, in a per-application-id FIFO capped at 256 entries +with the oldest dropped past the cap. The cap is the same real capacity the +mobile bridges use, and it is a capacity rather than a backstop: past it, +messages are lost, and a server SHOULD log each drop. When a client next +declares that application id and the server accepts the `hello`, the server +delivers the held events to it, in order, after the `hello` result and +before any newer event for that id. Delivery is **at least once**: a client +that disconnected mid-delivery may see an event twice, and message ids are +the idempotency key. An event of a stamped tag that arrives with no `app_id` +is not held (see routing), because there is nothing to hold it under. + +Nothing else is held. A broadcast event describes state the next event of +its kind restates, and a correlated event names an identifier whose client +was connected when it was issued; a server MAY hold correlated events for a +client that disconnected and reconnected under the same application id, and +MUST NOT hold them past the process's life. When `message_decryption_failed` +becomes stamped it joins the held set. + +The hold is per application id and shared by every client of that id: a +second process of the same application that connects while the first is +connected receives nothing from the hold, because the first already did. + +## Errors + +A failed call is a JSON-RPC error object. The `code` is an integer because +the framing requires one, and the client switches on the variant name, which +is what the engine's own bindings switch on: + +```json +{"jsonrpc":"2.0","id":2,"error":{"code":-32004,"message":"No key package available for recipient: off1...","data":{"variant":"NoKeyPackage"}}} +``` + +- **`error.data.variant`** is the `ProtocolError` variant name from the + interface definition, unchanged. The taxonomy is the engine's, all + twenty-five variants of it, and the distinctions the engine draws survive: + `NotStarted` is an engine that is not running, `InvalidState` is an + operation that is wrong for the engine's current state or for a + resource's, and `InvalidConfiguration` is a value the engine refused, + where retrying with the same value cannot help. +- **`error.message`** is the variant's display string, with the detail the + engine attached. +- **`error.code`** is `-32000 - n`, where `n` is the variant's position in + the definition's error enum, zero-based. The enum is append-only by + contract ([C2](../bridges/README.md#c2-the-error-enum-is-append-only)), + which is the same rule that keeps the generated bindings decoding it, so a + position never changes and a variant appended later takes the next + number without anyone assigning it. The position is zero-based, so + `NotStarted` is `-32000`; the discriminant the generated bindings use for + the same variant is one-based (`NotStarted` is `1`), and the two are not + meant to agree, so nobody should "correct" one to the other. The + server-defined range the JSON-RPC specification reserves runs to + `-32099`; the first fifty numbers are this taxonomy's, and nothing else is + ever assigned in them. +- **The standard codes** keep their standard meaning: `-32700` parse error, + `-32600` invalid request (including a batch), `-32601` method not found + (including every platform operation), `-32602` invalid params (a missing + required parameter, a positional array, a value of the wrong JSON type), + `-32603` an internal failure of the server itself, never of the engine. +- **Session refusals reuse the taxonomy.** A method before `hello`, or a + second `hello`, is `InvalidState`; a bad application id is + `InvalidArgument`; a wrong token, a service another application owns, a + space outside the client's allow-list, and a method group the operator + denied are all `PermissionDenied`. No variant exists only on this wire. + +| Variant | Code | The engine's meaning | +|---|---|---| +| `NotStarted` | `-32000` | The engine is not running. Retry after the server starts it | +| `AlreadyStarted` | `-32001` | Not reachable from a client; `start` is the server's | +| `InvalidConfiguration` | `-32002` | A value was refused. Retrying with the same value cannot help | +| `SendFailed` | `-32003` | The transport refused or lost the send | +| `NoKeyPackage` | `-32004` | No key package for the recipient yet | +| `SessionNotReady` | `-32005` | Establishment in progress; the message names the state | +| `EncryptFailed` | `-32006` | Outbound encryption failed | +| `InvalidState` | `-32007` | Wrong for the current state of the engine or of the named resource | +| `MlsNotInitialized` | `-32008` | The server has not attached storage | +| `MlsError` | `-32009` | An MLS operation failed | +| `UserBlocked` | `-32010` | The target is blocked | +| `MediaTransferLimit` | `-32011` | Too many transfers to one recipient; retry after one completes | +| `LockPoisoned` | `-32012` | A thread panicked inside the engine | +| `Other` | `-32013` | Unclassified | +| `TransportError` | `-32014` | A transport failure outside a send | +| `SerializationError` | `-32015` | A payload did not serialise or parse | +| `ServiceError` | `-32016` | A mesh service registry or request failure | +| `GroupNotFound` | `-32017` | The group does not exist locally | +| `PermissionDenied` | `-32018` | A role the caller lacks, or a server-side rule | +| `InvalidArgument` | `-32019` | A caller-supplied value failed validation | +| `DataDisabled` | `-32020` | The data layer is off in configuration | +| `DataStorageUnavailable` | `-32021` | The data layer has no storage yet | +| `DocTooLarge` | `-32022` | The document is over its cap | +| `DataCorrupted` | `-32023` | A stored document could not be read | +| `TelemetryConfigInvalid` | `-32024` | Not reachable from a client; telemetry is the server's | + +## Server-side rules + +Four rules separate the applications behind one server. Each is applied by +the server before a call reaches the engine, each refuses with +`PermissionDenied`, and none has any representation on the mesh: a peer +sees one identity registering services and syncing spaces, exactly as it +would from one application. They separate applications from each other's +mistakes, not from a hostile process inside the socket boundary (fourth +invariant). + +**Unlisted ids.** With no rule configured at all, a server accepts any +well-formed application id at `hello`. Once any space allow-list or method +deny is configured, the set of ids those entries name is the set of +applications the operator knows about, and a `hello` declaring an id with +no entry is refused with `PermissionDenied`. Service ownership does not +count: it is a runtime shadow the clients' own registrations build, not +something the operator configured, so a first registration never flips an +open server to default-deny. Without this rule the two configured rules are +void: an application whose id is denied a method, or scoped to a space, +reconnects under an id no rule names and is denied nothing and scoped to +nothing. An operator who wants an open server configures no rule; an +operator who configures one has listed every application. + +**Service ownership.** `services.register_service` records the calling +application id as the owner of that service id. `services.unregister_service` +and `services.respond_to_service_request` for a service another application +owns are refused; `service_request_received` for it is routed to the owner's +clients. The engine has no owner on a registration and no way to enumerate +them, so the server's registry is a shadow of the engine's, rebuilt from +the clients' calls, and empty after a restart until they register again. An +application that registers a service id another application already holds is +refused, which is the collision the registry exists to catch: without it the +second registration silently replaces the first in the engine and the first +application's requests start arriving at the second. + +**Space scoping.** The operator's configuration MAY map each application id +to a list of glob patterns over space ids. Every `data.*` call naming a space +outside the caller's patterns is refused, `data.list_spaces` is filtered to +them, and a document event for a space outside them is not delivered. An +application with no entry may open every space. The failure this prevents is +two applications choosing the same space name by accident and merging each +other's documents, which the engine cannot notice because to it there is one +member. + +**Method groups.** The operator's configuration MAY deny a named group of +methods to an application id: `sign_data`, the manual MLS group, the +instance-wide tuning group, or any single method. A denied method is refused +with `PermissionDenied`, not `-32601`, because the method exists and the +refusal is a policy the client should be able to read. Nothing is denied by +default. + +## What a guard pins + +The method table and the platform table together name every declaration in +the interface definition exactly once, and the catalogue names every event +tag the engine can emit. Both claims rot silently: a method added to the +definition and to no table is a method a client cannot find, or worse one the +reference server exposes without this chapter saying so; an event added to +the engine and not here is one no client is written for. + +A Rust guard in the FFI crate reads this chapter, the interface definition, +the engine's event definitions and the reference server's dispatch table, and +asserts: + +1. the set of names in the method table (with `services.` and `data.` + stripped) plus the set in the platform table equals the set of + declarations in the definition, with no name in both; +2. the reference server's dispatch table exposes exactly the method table's + names and no platform name; +3. every variant of the engine's event enum appears as a tag in the + catalogue, and every tag in the catalogue is a variant. + +The tables are written to be read by that guard as well as by a person: a +row's first cell is the backticked name, the method table runs from the +heading `Method table` to the heading `Platform operations`, the platform +table from there to `Event catalogue`, and the catalogue's rows are the ones +whose first cell is a backticked tag between `The catalogue` and `Shapes`. +Rows in the platform table list several names in one cell; the guard reads +every backticked token in the first cell. A change to those headings is a +change to the guard. + +## What this chapter does not cover + +- **The reference server's own configuration**: socket paths, the token + file, the allow-lists, the store key. Those are its guide's, and a second + server may configure them however it likes. +- **Transport drivers.** A server that wants Bluetooth LE or a peer stream + runs the driver in its own process, against the platform operations. This + chapter says only that a client cannot. +- **Authentication of anything but the socket.** There is no account, no + per-application credential, and no way for one application to prove to the + server that it is not another. An operator who needs that runs one server + per identity, which is one server per set of mutually trusting + applications. +- **A network-facing API.** Loopback only, by construction, in this version. From 9c8e9be595cd0dc61d6a10bb0f045ad88dadbce3 Mon Sep 17 00:00:00 2001 From: bahdotsh Date: Wed, 30 Sep 2026 21:38:33 +0530 Subject: [PATCH 2/3] feat(bindings): the Python reference server for the local API --- CHANGELOG.md | 19 + bindings/python/README.md | 41 + .../local_api/__init__.py | 34 + .../offline_protocol_sdk/local_api/authz.py | 171 ++++ .../offline_protocol_sdk/local_api/cli.py | 158 ++++ .../offline_protocol_sdk/local_api/codec.py | 205 +++++ .../local_api/dispatch.py | 506 +++++++++++ .../offline_protocol_sdk/local_api/mux.py | 160 ++++ .../offline_protocol_sdk/local_api/server.py | 452 ++++++++++ .../offline_protocol_sdk/local_api/session.py | 86 ++ .../offline_protocol_sdk/local_api/table.py | 815 ++++++++++++++++++ bindings/python/pyproject.toml | 5 + .../scripts/generate_local_api_table.py | 230 +++++ bindings/python/tests/local_api/__init__.py | 0 bindings/python/tests/local_api/conftest.py | 205 +++++ .../tests/local_api/test_local_api_codec.py | 99 +++ .../tests/local_api/test_local_api_drain.py | 85 ++ .../tests/local_api/test_local_api_e2e.py | 120 +++ .../tests/local_api/test_local_api_router.py | 166 ++++ .../tests/local_api/test_local_api_server.py | 453 ++++++++++ .../tests/local_api/test_local_api_table.py | 121 +++ crates/offline-protocol-uniffi/src/lib.rs | 241 ++++++ docs/bridges/local-api.md | 4 + 23 files changed, 4376 insertions(+) create mode 100644 bindings/python/offline_protocol_sdk/local_api/__init__.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/authz.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/cli.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/codec.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/dispatch.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/mux.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/server.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/session.py create mode 100644 bindings/python/offline_protocol_sdk/local_api/table.py create mode 100644 bindings/python/scripts/generate_local_api_table.py create mode 100644 bindings/python/tests/local_api/__init__.py create mode 100644 bindings/python/tests/local_api/conftest.py create mode 100644 bindings/python/tests/local_api/test_local_api_codec.py create mode 100644 bindings/python/tests/local_api/test_local_api_drain.py create mode 100644 bindings/python/tests/local_api/test_local_api_e2e.py create mode 100644 bindings/python/tests/local_api/test_local_api_router.py create mode 100644 bindings/python/tests/local_api/test_local_api_server.py create mode 100644 bindings/python/tests/local_api/test_local_api_table.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 42f60e319..d6ff8064a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,25 @@ archived by series under [docs/changelog/](docs/changelog/); see the ### Added +- **The Python reference server for the local API.** + `offline_protocol_sdk.local_api` and the `offline-protocol-service` command + front one engine for any number of local applications over JSON-RPC 2.0 on + a WebSocket, on an owner-only Unix domain socket by default or on loopback + TCP with a per-launch token. The server owns the run loop and the drain; + a client declares its application id once in `hello`, every send is + stamped with it, and every event is relayed as the engine serialised it to + the clients the chapter's rules select (by application id, by an + identifier the server issued, or to everyone), with the stamped inbound + events held for an application whose client is away. The method table is + generated from the interface definition and checked in + (`local_api/table.py`), every declaration is classified as exposed or + platform-only in `dispatch.py`, and a Rust guard in the FFI crate holds + the chapter, the definition and that classification to one another, so + an unclassified method is a failing test rather than a method every local + application can reach. Optional rules from one JSON file: a space + allow-list and method denials per application id, which once configured + refuse a `hello` under an id no rule names. A test over two servers on + one host exchanges a message over the peer-stream transport. - **The local API chapter.** `docs/spec/local-api.md` specifies how one server process fronts one engine for several local applications: JSON-RPC 2.0 over a WebSocket on a Unix domain socket by default (TCP on loopback diff --git a/bindings/python/README.md b/bindings/python/README.md index 90fecc561..bfae45e79 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -144,6 +144,47 @@ sealed protocol-state record. Install a real secret service (gnome-keyring, kwallet) for any deployment where that matters, supply your own `MlsStorageProvider`, or use the built-in file stores below. +### Run as a service: the local API + +One process can own the engine and serve several local applications at once, +over JSON-RPC 2.0 on a WebSocket. The contract is +[the local API chapter](../../docs/spec/local-api.md); the reference server +ships in this package as `offline_protocol_sdk.local_api` and as the +`offline-protocol-service` command: + +```bash +export OFFLINE_PROTOCOL_STORE_KEY="$(openssl rand -hex 32)" # once; keep it +offline-protocol-service --config config.json \ + --mls-root /var/lib/example/keys --state-root /var/lib/example/state \ + --socket /run/example/api.sock --listen 0.0.0.0:7878 --peer 10.0.0.2:7878 +``` + +`config.json` holds the `ProtocolConfig` fields by name; the socket is +created owner-only. A client opens the socket, sends `hello` with its +application id, and calls the engine's own methods by name: + +```python +import asyncio, json +from websockets.asyncio.client import unix_connect + +async def main(): + async with unix_connect("/run/example/api.sock", uri="ws://localhost/") as ws: + await ws.send(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "hello", + "params": {"app_id": "notes"}})) + print(json.loads(await ws.recv())["result"]["local_address"]) + +asyncio.run(main()) +``` + +Every message a `notes` client sends is stamped with that id, and a +`message_received` for `notes` reaches only `notes` clients; what the server +holds for an application whose client is away, and what it never puts on the +wire, is the chapter's. `--policy policy.json` adds the optional rules +(`spaces`, `denied`); `--tcp PORT --token-file PATH` serves loopback TCP with +a per-launch token instead of the socket. See +[the bridge contract](../../docs/bridges/local-api.md) for what the server +owes. + ### Headless hosts: the built-in file stores A server or container usually has no secret service at all. Pass a store key diff --git a/bindings/python/offline_protocol_sdk/local_api/__init__.py b/bindings/python/offline_protocol_sdk/local_api/__init__.py new file mode 100644 index 000000000..008e78dec --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/__init__.py @@ -0,0 +1,34 @@ +"""The reference server for the local API (``docs/spec/local-api.md``). + +One process owns one engine and serves any number of local applications over +JSON-RPC 2.0 on a WebSocket, on a Unix domain socket by default. See +:class:`LocalApiServer`, and ``offline-protocol-service`` for the shell entry +point. +""" + +from .authz import METHOD_GROUPS, Policy, ServiceOwnership # noqa: F401 +from .codec import RpcError # noqa: F401 +from .dispatch import BEFORE_HELLO, EXPOSED, ID_RESULTS, PLATFORM, SESSION_METHODS # noqa: F401 +from .mux import HELD_TAGS, HOLD_CAPACITY, EventRouter # noqa: F401 +from .server import API_VERSION, MAX_MESSAGE_SIZE, SERVER_NAME, LocalApiServer # noqa: F401 +from .session import Session # noqa: F401 + +__all__ = [ + "API_VERSION", + "BEFORE_HELLO", + "EXPOSED", + "EventRouter", + "HELD_TAGS", + "HOLD_CAPACITY", + "ID_RESULTS", + "LocalApiServer", + "MAX_MESSAGE_SIZE", + "METHOD_GROUPS", + "PLATFORM", + "Policy", + "RpcError", + "SERVER_NAME", + "SESSION_METHODS", + "ServiceOwnership", + "Session", +] diff --git a/bindings/python/offline_protocol_sdk/local_api/authz.py b/bindings/python/offline_protocol_sdk/local_api/authz.py new file mode 100644 index 000000000..e6c23a91e --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/authz.py @@ -0,0 +1,171 @@ +"""The three server-side rules that separate the applications behind one +identity: service ownership, space scoping and method groups. + +Each is applied before a call reaches the engine, each refuses with the +engine's ``PermissionDenied``, and none has any representation on the mesh +(chapter, "Server-side rules"; bridge rule L3). The engine sees one identity +registering services and syncing spaces, exactly as it would from one +application. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from fnmatch import fnmatchcase +from typing import Any + +from .codec import taxonomy_error + +#: The named method groups an operator may deny to an application id. A +#: policy entry that is not a group name is a single method's wire name. +METHOD_GROUPS: dict[str, frozenset[str]] = { + "sign_data": frozenset({"sign_data"}), + "manual_mls": frozenset( + { + "mls_generate_key_package", + "mls_get_or_create_key_package", + "mls_import_key_package", + "mls_get_pending_key_packages", + "mls_mark_key_package_synced", + "mls_create_session", + "mls_join_session", + "mls_encrypt_for_user", + "mls_decrypt_from_user", + "mls_get_pending_welcome", + "mls_clear_pending_welcome", + "mls_decrypt", + "mls_process_welcome", + } + ), + "tuning": frozenset( + { + "set_relay_priority", + "update_relay_config", + "force_transport", + "release_transport_lock", + "update_dors_config", + "update_ack_config", + "update_retry_config", + "update_dedup_config", + } + ), +} + + +@dataclass +class Policy: + """The operator's configuration of the optional rules. + + ``spaces`` maps an application id to the glob patterns over space ids it + may open; an application with no entry may open every space. ``denied`` + maps an application id to method group names and single method names it + may not call; nothing is denied by default. ``applications`` lists ids + that have no rule of their own but are still admitted once a rule exists; + on its own it configures nothing. + + An application id is self-declared per connection, so a rule keyed by + id separates applications from each other's mistakes, never from a + hostile local process, which is inside the socket boundary already. + What keeps a rule from being stepped around by a reconnect under + another name is the chapter's unlisted-id rule: once a space allow-list + or a method deny is configured, ``hello`` with an id no entry names is + refused. With neither configured, every id is admitted. Service + ownership is a runtime shadow, never a configured rule, and never counts. + """ + + spaces: dict[str, list[str]] = field(default_factory=dict) + denied: dict[str, list[str]] = field(default_factory=dict) + applications: list[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, raw: dict[str, Any] | None) -> "Policy": + raw = raw or {} + unknown = sorted(set(raw) - {"spaces", "denied", "applications"}) + if unknown: + raise ValueError(f"policy has no section {unknown[0]!r}") + spaces = {str(k): [str(p) for p in v] for k, v in (raw.get("spaces") or {}).items()} + denied = {str(k): [str(m) for m in v] for k, v in (raw.get("denied") or {}).items()} + applications = [str(a) for a in (raw.get("applications") or [])] + for app_id, names in denied.items(): + for name in names: + if name not in METHOD_GROUPS and "." not in name and not name.isidentifier(): + raise ValueError(f"policy denies {name!r} for {app_id!r}: not a method or group") + return cls(spaces=spaces, denied=denied, applications=applications) + + def restricts(self) -> bool: + """Whether a space allow-list or a method deny is configured.""" + return bool(self.spaces or self.denied) + + def admits(self, app_id: str) -> bool: + """Whether ``hello`` may declare ``app_id``.""" + if not self.restricts(): + return True + return app_id in self.spaces or app_id in self.denied or app_id in self.applications + + def check_admission(self, app_id: str) -> None: + if not self.admits(app_id): + raise taxonomy_error( + "PermissionDenied", + f"no rule names application {app_id!r}; rules are configured, so unnamed ids are refused", + ) + + def allows_space(self, app_id: str, space_id: str) -> bool: + patterns = self.spaces.get(app_id) + if patterns is None: + return True + return any(fnmatchcase(space_id, pattern) for pattern in patterns) + + def denies(self, app_id: str, method: str) -> bool: + for name in self.denied.get(app_id, ()): + if name == method or method in METHOD_GROUPS.get(name, ()): + return True + return False + + def check_space(self, app_id: str, space_id: str) -> None: + if not self.allows_space(app_id, space_id): + raise taxonomy_error( + "PermissionDenied", + f"space {space_id!r} is outside what {app_id!r} may open", + ) + + def check_method(self, app_id: str, method: str) -> None: + if self.denies(app_id, method): + raise taxonomy_error("PermissionDenied", f"{method} is denied to {app_id!r}") + + +class ServiceOwnership: + """A shadow of the engine's service registry, keyed by application id. + + The engine has no owner on a registration and no way to enumerate them, + so this is rebuilt from the clients' calls and is empty after a restart + until they register again. An id another application already holds is + refused, which is the collision the registry exists to catch: without it + the second registration silently replaces the first in the engine and + the first application's requests start arriving at the second. + """ + + def __init__(self) -> None: + self._owner: dict[str, str] = {} + + def owner(self, service_id: str) -> str | None: + return self._owner.get(service_id) + + def claim(self, service_id: str, app_id: str) -> None: + holder = self._owner.get(service_id) + if holder is not None and holder != app_id: + raise taxonomy_error( + "PermissionDenied", + f"service {service_id!r} is registered by another application", + ) + self._owner[service_id] = app_id + + def release(self, service_id: str) -> None: + self._owner.pop(service_id, None) + + def check(self, service_id: str, app_id: str) -> None: + holder = self._owner.get(service_id) + if holder is not None and holder != app_id: + raise taxonomy_error( + "PermissionDenied", + f"service {service_id!r} belongs to another application", + ) diff --git a/bindings/python/offline_protocol_sdk/local_api/cli.py b/bindings/python/offline_protocol_sdk/local_api/cli.py new file mode 100644 index 000000000..4f337524d --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/cli.py @@ -0,0 +1,158 @@ +"""``offline-protocol-service``: run the reference server from the shell. + +The process owns one engine over the built-in file stores (or the platform +keyring with ``--keyring``), serves the local API on a Unix domain socket by +default, and runs until it is told to stop. + +Importing this module loads the native library, because the package's own +``__init__`` does; ``--help`` therefore needs the library present. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import signal +import sys +from pathlib import Path +from typing import Any + +from ..protocol_manager import ProtocolManager +from . import codec +from .authz import Policy +from .server import LocalApiServer + +DEFAULT_STORE_KEY_ENV = "OFFLINE_PROTOCOL_STORE_KEY" + + +def default_socket_path() -> Path: + runtime = os.environ.get("XDG_RUNTIME_DIR") + base = Path(runtime) if runtime else Path.home() / ".offline-protocol" + return base / "offline-protocol" / "api.sock" if runtime else base / "api.sock" + + +def _load_json(path: str, what: str) -> Any: + try: + with open(path, encoding="utf-8") as handle: + return json.load(handle) + except OSError as exc: + raise SystemExit(f"cannot read the {what} at {path}: {exc}") from None + except ValueError as exc: + raise SystemExit(f"the {what} at {path} is not JSON: {exc}") from None + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="offline-protocol-service", + description="Serve one Offline Protocol engine to local applications over the local API.", + ) + parser.add_argument("--config", required=True, help="JSON file with the ProtocolConfig fields") + carrier = parser.add_mutually_exclusive_group() + carrier.add_argument("--socket", help=f"Unix socket path (default: {default_socket_path()})") + carrier.add_argument("--tcp", type=int, metavar="PORT", help="serve on loopback TCP instead, with a token") + parser.add_argument("--tcp-host", default="127.0.0.1", help="loopback address for --tcp (127.0.0.1 or ::1)") + parser.add_argument("--token-file", help="where --tcp writes the per-launch token (mode 0600)") + stores = parser.add_argument_group("storage") + stores.add_argument("--mls-root", help="the sealed MLS store's directory (or OFFLINE_PROTOCOL_MLS_ROOT)") + stores.add_argument("--state-root", help="the protocol-state directory (or OFFLINE_PROTOCOL_STATE_ROOT)") + stores.add_argument( + "--store-key-env", + default=DEFAULT_STORE_KEY_ENV, + help=f"environment variable holding the 32-byte store key (default: {DEFAULT_STORE_KEY_ENV})", + ) + stores.add_argument( + "--keyring", + action="store_true", + help="use the platform keyring and the application state store instead of the file stores", + ) + mesh = parser.add_argument_group("peer stream") + mesh.add_argument("--listen", metavar="HOST:PORT", help="where the peer-stream transport accepts streams") + mesh.add_argument("--peer", action="append", default=[], metavar="ENTRY", help="a peer to keep a stream to: host:port or off1...@host:port") + parser.add_argument("--policy", help="JSON file with the space allow-lists and method denials") + parser.add_argument("--no-health", action="store_true", help="do not answer GET /health") + parser.add_argument("--log-level", default="INFO") + return parser + + +def build_manager(args: argparse.Namespace) -> ProtocolManager: + raw = _load_json(args.config, "config") + try: + config = codec.decode("ProtocolConfig", raw, "config") + except codec.RpcError as exc: + raise SystemExit(f"config: {exc.message}") from None + if args.keyring: + manager = ProtocolManager(config, state_root=args.state_root) + else: + try: + manager = ProtocolManager( + config, + mls_root=args.mls_root, + state_root=args.state_root, + store_key_env=args.store_key_env, + ) + except ValueError as exc: + raise SystemExit(str(exc)) from None + if manager.peer_stream is not None and (args.listen or args.peer): + listen_host, listen_port = None, ... + if args.listen: + host, _, port = args.listen.rpartition(":") + if not host or not port.isdigit(): + raise SystemExit("--listen takes HOST:PORT") + listen_host, listen_port = host.strip("[]"), int(port) + manager.peer_stream.configure(listen_host=listen_host, listen_port=listen_port, peers=args.peer) + return manager + + +def build_server(args: argparse.Namespace, manager: ProtocolManager) -> LocalApiServer: + policy = Policy.from_dict(_load_json(args.policy, "policy")) if args.policy else Policy() + if args.tcp is not None: + if not args.token_file: + raise SystemExit("--tcp requires --token-file") + return LocalApiServer( + manager, + policy=policy, + tcp_port=args.tcp, + tcp_host=args.tcp_host, + token_path=args.token_file, + health=not args.no_health, + ) + return LocalApiServer( + manager, + policy=policy, + socket_path=args.socket or default_socket_path(), + health=not args.no_health, + ) + + +async def run(server: LocalApiServer) -> None: + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(sig, stop.set) + except (NotImplementedError, RuntimeError): + signal.signal(sig, lambda *_: stop.set()) + await server.start() + try: + await stop.wait() + finally: + await server.stop() + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + logging.basicConfig(level=args.log_level.upper(), format="%(asctime)s %(levelname)s %(name)s: %(message)s") + manager = build_manager(args) + try: + server = build_server(args, manager) + except ValueError as exc: + raise SystemExit(str(exc)) from None + asyncio.run(run(server)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bindings/python/offline_protocol_sdk/local_api/codec.py b/bindings/python/offline_protocol_sdk/local_api/codec.py new file mode 100644 index 000000000..8504e42c9 --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/codec.py @@ -0,0 +1,205 @@ +"""The one mapping between the interface definition's types and JSON. + +The chapter's encoding table, applied in both directions from the generated +table rather than from a hand-written list per method: a parameter is decoded +by its declared type, a result is encoded by its declared type, and a +dictionary's fields are decoded by the dictionary's declaration. Nothing here +knows a method by name. + +Errors are the engine's own taxonomy. A ``ProtocolError`` becomes a JSON-RPC +error whose ``code`` is ``-32000 - position`` in the definition's error enum +and whose ``data.variant`` is the variant name; the session refusals the +chapter defines reuse the same variants, so no error exists only on this +wire. +""" + +from __future__ import annotations + +import base64 +import binascii +import enum +from typing import Any + +from .. import offline_protocol as generated +from .table import TABLE + +PARSE_ERROR = -32700 +INVALID_REQUEST = -32600 +METHOD_NOT_FOUND = -32601 +INVALID_PARAMS = -32602 +INTERNAL_ERROR = -32603 + +#: The base of the taxonomy's range: variant ``n`` is ``TAXONOMY_BASE - n``. +TAXONOMY_BASE = -32000 + +_INT_RANGES = { + "u8": (0, 2**8 - 1), + "u16": (0, 2**16 - 1), + "u32": (0, 2**32 - 1), + "u64": (0, 2**64 - 1), + "i16": (-(2**15), 2**15 - 1), + "i32": (-(2**31), 2**31 - 1), + "i64": (-(2**63), 2**63 - 1), +} +_FLOATS = ("f32", "double") + +ERROR_VARIANTS: tuple[str, ...] = TABLE["errors"]["ProtocolError"] + + +class RpcError(Exception): + """A JSON-RPC error object, raised to end a call.""" + + def __init__(self, code: int, message: str, data: dict[str, Any] | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.data = data + + def to_json(self) -> dict[str, Any]: + error: dict[str, Any] = {"code": self.code, "message": self.message} + if self.data is not None: + error["data"] = self.data + return error + + +def taxonomy_error(variant: str, message: str) -> RpcError: + """A refusal spelled with one of the engine's variants.""" + position = ERROR_VARIANTS.index(variant) + return RpcError(TAXONOMY_BASE - position, message, {"variant": variant}) + + +def error_from_protocol(exc: BaseException) -> RpcError: + """The engine's exception as the wire's error, variant and all.""" + variant = type(exc).__name__ + if variant not in ERROR_VARIANTS: + # A generated error class the table does not know: the table is + # stale against the library, which a test refuses before this can + # happen in a checked-in tree. + return RpcError(INTERNAL_ERROR, f"unmapped engine error {variant}: {exc}") + return taxonomy_error(variant, str(exc)) + + +def invalid_params(message: str) -> RpcError: + return RpcError(INVALID_PARAMS, message) + + +# -- decoding (JSON -> the generated binding's values) ------------------------ + + +def _b64decode(value: Any, where: str) -> bytes: + if not isinstance(value, str): + raise invalid_params(f"{where}: bytes are a base64 string") + try: + return base64.b64decode(value, validate=True) + except (binascii.Error, ValueError) as exc: + raise invalid_params(f"{where}: not valid base64 ({exc})") from None + + +def decode(type_name: str, value: Any, where: str) -> Any: + """``value`` (parsed JSON) as the Python value the binding takes for + ``type_name`` (a type as the definition spells it, spaces removed).""" + if type_name.endswith("?"): + if value is None: + return None + return decode(type_name[:-1], value, where) + if type_name == "string": + if not isinstance(value, str): + raise invalid_params(f"{where}: expected a string") + return value + if type_name == "boolean": + if not isinstance(value, bool): + raise invalid_params(f"{where}: expected true or false") + return value + if type_name in _INT_RANGES: + if isinstance(value, bool) or not isinstance(value, int): + raise invalid_params(f"{where}: expected an integer") + low, high = _INT_RANGES[type_name] + if not low <= value <= high: + raise invalid_params(f"{where}: {value} is outside {type_name}") + return value + if type_name in _FLOATS: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise invalid_params(f"{where}: expected a number") + return float(value) + if type_name == "bytes": + return _b64decode(value, where) + if type_name == "sequence": + return list(_b64decode(value, where)) + if type_name.startswith("sequence<"): + inner = type_name[len("sequence<") : -1] + if not isinstance(value, list): + raise invalid_params(f"{where}: expected an array") + return [decode(inner, item, f"{where}[{i}]") for i, item in enumerate(value)] + if type_name.startswith("record Any: + if not isinstance(value, dict): + raise invalid_params(f"{where}: expected an object shaped as {type_name}") + fields = TABLE["records"][type_name] + known = {name for name, _, _ in fields} + unknown = sorted(set(value) - known) + if unknown: + raise invalid_params(f"{where}: {type_name} has no field {unknown[0]!r}") + kwargs: dict[str, Any] = {} + for name, field_type, has_default in fields: + if name in value: + kwargs[name] = decode(field_type, value[name], f"{where}.{name}") + elif has_default: + # Left to the definition's own default (C6): the server never + # substitutes a literal for a field the client did not send. + continue + elif field_type.endswith("?"): + kwargs[name] = None + else: + raise invalid_params(f"{where}: {type_name} requires {name}") + return getattr(generated, type_name)(**kwargs) + + +# -- encoding (the binding's values -> JSON) ---------------------------------- + + +def encode(type_name: str, value: Any) -> Any: + """``value`` as returned by the binding for ``type_name``, as JSON.""" + if value is None: + return None + if type_name.endswith("?"): + return encode(type_name[:-1], value) + if type_name == "void": + return None + if type_name in ("string", "boolean") or type_name in _INT_RANGES or type_name in _FLOATS: + return value + if type_name == "bytes": + return base64.b64encode(bytes(value)).decode("ascii") + if type_name == "sequence": + return base64.b64encode(bytes(value)).decode("ascii") + if type_name.startswith("sequence<"): + inner = type_name[len("sequence<") : -1] + return [encode(inner, item) for item in value] + if type_name.startswith("record tuple[str, str]: + """``(object, declaration)`` for a wire name.""" + if method.startswith("services."): + return "MeshServices", method[len("services.") :] + if method.startswith("data."): + return "DataStore", method[len("data.") :] + if method in TABLE["methods"]["namespace"]: + return "namespace", method + return "OfflineProtocol", method + + +def all_wire_names() -> frozenset[str]: + """Every declaration in the table by its wire name.""" + prefixes = {"OfflineProtocol": "", "MeshServices": "services.", "DataStore": "data.", "namespace": ""} + return frozenset( + prefix + name for obj, prefix in prefixes.items() for name in TABLE["methods"][obj] + ) + + +class Dispatcher: + """Executes exposed methods against the binding for one server.""" + + def __init__( + self, + engine: Any, + services: Any, + data: Any, + router: EventRouter, + policy: Policy, + ownership: ServiceOwnership, + ) -> None: + self._engine = engine + self._services = services + #: The ``DataStore``, or the exception its construction raised, so + #: every ``data.*`` call answers with the engine's own refusal. + self._data = data + self._router = router + self._policy = policy + self._ownership = ownership + + # -- entry ---------------------------------------------------------------- + + def call(self, session: Session, method: str, params: Any) -> Any: + if method not in EXPOSED: + raise RpcError(codec.METHOD_NOT_FOUND, f"unknown method {method}") + if session.app_id is None: + if method not in BEFORE_HELLO: + raise taxonomy_error("InvalidState", "hello has not been sent on this connection") + else: + self._policy.check_method(session.app_id, method) + obj, declaration = resolve(method) + spec_params, result_type = TABLE["methods"][obj][declaration] + args = self._decode_params(method, spec_params, params) + target, declaration, args, result_type = self._prepare( + session, method, obj, declaration, args, result_type + ) + fn: Callable[..., Any] = getattr(target, declaration) + self._router.current_caller = session + try: + result = fn(**args) + except generated.ProtocolError as exc: + self._after_failure(method, args) + raise error_from_protocol(exc) from None + except RpcError: + raise + except Exception as exc: # the binding itself failed + self._after_failure(method, args) + raise RpcError(codec.INTERNAL_ERROR, f"{method}: {exc}") from None + finally: + self._router.current_caller = None + result = self._after_success(session, method, args, result) + return codec.encode(result_type, result) + + # -- parameters ----------------------------------------------------------- + + @staticmethod + def _decode_params( + method: str, spec: tuple[tuple[str, str], ...], params: Any + ) -> dict[str, Any]: + if params is None: + params = {} + if not isinstance(params, dict): + raise codec.invalid_params(f"{method}: params must be an object of named parameters") + known = {name for name, _ in spec} + unknown = sorted(set(params) - known) + if unknown: + raise codec.invalid_params(f"{method} has no parameter {unknown[0]!r}") + args: dict[str, Any] = {} + for name, type_name in spec: + if name in params: + args[name] = codec.decode(type_name, params[name], f"{method}.{name}") + elif type_name.endswith("?"): + args[name] = None + else: + raise codec.invalid_params(f"{method} requires {name}") + return args + + # -- the rules and the stamping, before the call ------------------------- + + def _prepare( + self, + session: Session, + method: str, + obj: str, + declaration: str, + args: dict[str, Any], + result_type: str, + ) -> tuple[Any, str, dict[str, Any], str]: + app_id = session.app_id + if obj == "namespace": + return generated, declaration, args, result_type + if obj == "MeshServices": + assert app_id is not None + service_id = args.get("service_id") + if declaration == "register_service": + self._ownership.claim(service_id, app_id) + elif declaration in ("unregister_service", "respond_to_service_request"): + self._ownership.check(service_id, app_id) + return self._services, declaration, args, result_type + if obj == "DataStore": + assert app_id is not None + if isinstance(self._data, BaseException): + raise error_from_protocol(self._data) + space_id = args.get("space_id") + if isinstance(space_id, str): + self._policy.check_space(app_id, space_id) + return self._data, declaration, args, result_type + # OfflineProtocol: the two stamped methods and their rich twins. + assert app_id is not None + if method == "send_message": + options = generated.SendMessageOptions( + priority=args["priority"], reply_to_msg=args["reply_to_msg"], app_id=app_id + ) + return ( + self._engine, + "send_message_rich", + {"recipient": args["recipient"], "content": args["content"], "options": options}, + "string", + ) + if method == "send_media": + options = generated.MediaSendOptions( + media_metadata=args["media_metadata"], app_id=app_id + ) + return ( + self._engine, + "send_media_rich", + { + "recipient": args["recipient"], + "file_data": args["file_data"], + "file_name": args["file_name"], + "content_type": args["content_type"], + "options": options, + }, + "string", + ) + if method in ("send_message_rich", "send_media_rich"): + options = args["options"] + if options.app_id is not None: + raise taxonomy_error( + "InvalidArgument", + "options.app_id is set by the server from the connection's hello", + ) + options.app_id = app_id + return self._engine, declaration, args, result_type + + # -- after the call ------------------------------------------------------- + + def _after_failure(self, method: str, args: dict[str, Any]) -> None: + if method == "services.register_service": + # The engine refused the registration; a claim nothing backs + # would block the next attempt by another application forever. + self._ownership.release(args["service_id"]) + + def _after_success(self, session: Session, method: str, args: dict[str, Any], result: Any) -> Any: + app_id = session.app_id + if method == "services.unregister_service" and result: + self._ownership.release(args["service_id"]) + if method == "data.list_spaces" and app_id is not None: + result = [space for space in result if self._policy.allows_space(app_id, space)] + if method in ID_RESULTS and app_id is not None: + self._router.note_ids(app_id, result if isinstance(result, list) else [result]) + return result diff --git a/bindings/python/offline_protocol_sdk/local_api/mux.py b/bindings/python/offline_protocol_sdk/local_api/mux.py new file mode 100644 index 000000000..94946455e --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/mux.py @@ -0,0 +1,160 @@ +"""Event routing and the per-application hold. + +Implements the chapter's routing classes (stamped, then the caller of the +call an event was emitted inside, then correlated, then broadcast) and its +replay rule. The event object the engine serialised is pushed unchanged +(C3); this module reads only ``type``, ``app_id``, the correlation +identifiers and ``space_id``. +""" + +from __future__ import annotations + +import logging +from collections import deque +from typing import Any, Callable, Iterable + +from .authz import Policy, ServiceOwnership +from .session import Session + +logger = logging.getLogger(__name__) + +#: The stamped inbound tags held for an application with no connected client +#: (chapter, "Replay"). The engine has already acknowledged and dedup-marked +#: the message by the time these fire, so nothing will restate them. +HELD_TAGS: frozenset[str] = frozenset({"message_received", "file_received", "media_resend_required"}) + +#: The hold's capacity per application id: the same real capacity the +#: mobile bridges keep for inbound tags before a subscriber exists (C10). +HOLD_CAPACITY = 256 + +#: Identifier fields the server correlates by, in the order they are read. +CORRELATION_KEYS: tuple[str, ...] = ("message_id", "file_id", "query_id", "request_id") + + +class EventRouter: + """Selects the sessions an event reaches and pushes it to them.""" + + def __init__( + self, + policy: Policy, + ownership: ServiceOwnership, + *, + on_drop: Callable[[str, dict[str, Any]], None] | None = None, + ) -> None: + self._policy = policy + self._ownership = ownership + self._sessions: dict[str, list[Session]] = {} + self._issued: dict[str, str] = {} + self._held: dict[str, deque[dict[str, Any]]] = {} + self._on_drop = on_drop + #: The session whose call the server is executing right now, if any. + self.current_caller: Session | None = None + self.dropped = 0 + + # -- sessions ------------------------------------------------------------- + + def attach(self, session: Session) -> list[dict[str, Any]]: + """Registers a session under its application id and returns the + events held for that id, oldest first, emptying the hold.""" + assert session.app_id is not None + self._sessions.setdefault(session.app_id, []).append(session) + held = self._held.pop(session.app_id, None) + return list(held) if held else [] + + def detach(self, session: Session) -> None: + if session.app_id is None: + return + sessions = self._sessions.get(session.app_id) + if sessions and session in sessions: + sessions.remove(session) + if not sessions: + del self._sessions[session.app_id] + + def sessions_for(self, app_id: str) -> list[Session]: + return list(self._sessions.get(app_id, ())) + + def all_sessions(self) -> list[Session]: + return [s for sessions in self._sessions.values() for s in sessions] + + def held_count(self, app_id: str) -> int: + return len(self._held.get(app_id, ())) + + # -- identifiers ---------------------------------------------------------- + + def note_ids(self, app_id: str, ids: Iterable[Any]) -> None: + """Records identifiers the server handed to ``app_id`` as results.""" + for value in ids: + if isinstance(value, str) and value: + self._issued[value] = app_id + + def _record_from_event(self, app_id: str, event: dict[str, Any]) -> None: + for key in CORRELATION_KEYS: + value = event.get(key) + if isinstance(value, str) and value: + self._issued.setdefault(value, app_id) + + def _correlated_app(self, event: dict[str, Any]) -> str | None: + for key in CORRELATION_KEYS: + value = event.get(key) + if isinstance(value, str) and value in self._issued: + return self._issued[value] + service_id = event.get("service_id") + if isinstance(service_id, str): + return self._ownership.owner(service_id) + return None + + # -- routing -------------------------------------------------------------- + + def route(self, event: dict[str, Any]) -> None: + """Pushes ``event`` to the sessions the rules select.""" + tag = event.get("type") + app_id = event.get("app_id") + if isinstance(app_id, str): + targets = self.sessions_for(app_id) + if not targets and tag in HELD_TAGS: + self._hold(app_id, event) + return + elif tag in HELD_TAGS: + # A stamped tag whose id is absent (a `file_received` assembled + # with no metadata entry, a `media_resend_required` without one) + # has nothing to route or hold by, so it is broadcast, and said. + logger.info("%s carries no app_id: broadcast, not held", tag) + targets = self.all_sessions() + elif self.current_caller is not None and self.current_caller.app_id is not None: + caller = self.current_caller + self._record_from_event(caller.app_id, event) + targets = self.sessions_for(caller.app_id) + else: + owner = self._correlated_app(event) + targets = self.sessions_for(owner) if owner is not None else self.all_sessions() + for session in targets: + self._deliver(session, tag, event) + + def _deliver(self, session: Session, tag: Any, event: dict[str, Any]) -> None: + if not session.wants(tag): + return + if isinstance(tag, str) and tag.startswith("data_"): + # The reference server filters document events by the space + # allow-list, which the chapter allows a server to add. + space_id = event.get("space_id") + if isinstance(space_id, str) and session.app_id is not None: + if not self._policy.allows_space(session.app_id, space_id): + return + session.push_event(event) + + def _hold(self, app_id: str, event: dict[str, Any]) -> None: + held = self._held.get(app_id) + if held is None: + held = self._held[app_id] = deque() + if len(held) >= HOLD_CAPACITY: + dropped = held.popleft() + self.dropped += 1 + logger.warning( + "hold for %r is full (%d): dropping the oldest %s", + app_id, + HOLD_CAPACITY, + dropped.get("type"), + ) + if self._on_drop is not None: + self._on_drop(app_id, dropped) + held.append(event) diff --git a/bindings/python/offline_protocol_sdk/local_api/server.py b/bindings/python/offline_protocol_sdk/local_api/server.py new file mode 100644 index 000000000..53033cb9b --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/server.py @@ -0,0 +1,452 @@ +"""The reference server: one engine, any number of local clients, JSON-RPC +2.0 over a WebSocket on a Unix domain socket or a loopback TCP port. + +The server owns the :class:`~offline_protocol_sdk.ProtocolManager` (its run +loop and its drain), the ``MeshServices`` and ``DataStore`` handles, and the +routing tables. A connection runs one request at a time; every engine call +runs on the event loop's thread, so an event the engine emits inside a call +is attributed to that call's connection without any locking. + +There is no HTTP request path. The pinned WebSocket library's handshake +parser accepts only ``GET`` and drops any other method before the request +hook runs (verified against ``websockets`` 16.1), so a ``POST`` would fail +with zero bytes back rather than with a JSON-RPC error. The request hook +serves one thing: ``GET /health``. +""" + +from __future__ import annotations + +import asyncio +import hmac +import http +import json +import logging +import os +import secrets +import stat +from pathlib import Path +from typing import Any + +from websockets.asyncio.server import Server, ServerConnection, serve, unix_serve +from websockets.exceptions import ConnectionClosed +from websockets.http11 import Request, Response + +from .. import offline_protocol as generated +from ..protocol_manager import ProtocolManager +from . import codec +from .authz import Policy, ServiceOwnership +from .codec import RpcError, taxonomy_error +from .dispatch import SESSION_METHODS, Dispatcher +from .mux import EventRouter +from .session import Session + +logger = logging.getLogger(__name__) + +API_VERSION = 1 +SERVER_NAME = "offline-protocol-service" + +#: The engine's file size limit (``file_transfer.rs``, ``max_file_size``), +#: which is not configurable through the interface. +FILE_SIZE_LIMIT = 100 * 1024 * 1024 +#: The inbound frame limit: the file limit as base64, plus room for the +#: request around it. Below this a media send fails as a closed connection +#: instead of as the engine's own refusal. +MAX_MESSAGE_SIZE = FILE_SIZE_LIMIT * 4 // 3 + 1024 * 1024 +TOKEN_BYTES = 32 +#: WebSocket close code for a connection refused by policy. +POLICY_VIOLATION = 1008 +#: The engine's own rule for an application id (``ProtocolConfig`` validation). +APP_ID_MAX_BYTES = 256 +LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1"}) + + +def _package_version() -> str: + try: + from importlib.metadata import version + + return version("offline-protocol-sdk") + except Exception: # not installed as a distribution + return "0" + + +def validate_app_id(value: Any) -> str: + """The engine's rule for an application id, applied to ``hello``.""" + if not isinstance(value, str): + raise taxonomy_error("InvalidArgument", "app_id must be a string") + if not value or value in (".", ".."): + raise taxonomy_error("InvalidArgument", "app_id must not be empty, '.' or '..'") + if len(value.encode("utf-8")) > APP_ID_MAX_BYTES: + raise taxonomy_error("InvalidArgument", f"app_id is over {APP_ID_MAX_BYTES} bytes") + if any(ord(ch) < 0x20 or ch == "\x7f" for ch in value): + raise taxonomy_error("InvalidArgument", "app_id contains a control character") + if any(ch in value for ch in "/\\:"): + raise taxonomy_error("InvalidArgument", "app_id must not contain '/', '\\' or ':'") + return value + + +class _CloseAfter(RpcError): + """An error after which the connection is closed with a policy code.""" + + +class LocalApiServer: + """Fronts one :class:`ProtocolManager` for local clients. + + Parameters + ---------- + manager: + A manager that has not been started. The server registers itself as + its event handler, starts it, and starts its peer-stream transport + when the configuration enables one. + policy: + The space allow-lists and method denials; empty by default. + socket_path: + The Unix domain socket to serve on (the default carrier). Created + ``0600`` in a directory made ``0700``; a stale file at the path is + removed first. + tcp_port, tcp_host, token_path: + The loopback TCP alternative: ``tcp_port`` (``0`` picks a free port, + readable as :attr:`port`), a loopback ``tcp_host``, and the file the + per-launch token is written to, ``0600``. A client presents the + token in ``hello``. + health: + Serve ``GET /health`` through the request hook. + max_size: + The inbound frame limit. + """ + + def __init__( + self, + manager: ProtocolManager, + *, + policy: Policy | None = None, + socket_path: str | Path | None = None, + tcp_port: int | None = None, + tcp_host: str = "127.0.0.1", + token_path: str | Path | None = None, + health: bool = True, + max_size: int = MAX_MESSAGE_SIZE, + ) -> None: + if (socket_path is None) == (tcp_port is None): + raise ValueError("pass exactly one of socket_path or tcp_port") + if tcp_port is not None: + if token_path is None: + raise ValueError("tcp_port requires token_path: TCP is served only with the token") + if tcp_host not in LOOPBACK_HOSTS: + raise ValueError(f"tcp_host must be loopback ({sorted(LOOPBACK_HOSTS)}), not {tcp_host!r}") + self._manager = manager + self._policy = policy or Policy() + self._ownership = ServiceOwnership() + self._router = EventRouter(self._policy, self._ownership) + self.socket_path = Path(socket_path) if socket_path is not None else None + self._tcp_port = tcp_port + self._tcp_host = tcp_host + self.token_path = Path(token_path) if token_path is not None else None + self._health = health + self._max_size = max_size + self._loop: asyncio.AbstractEventLoop | None = None + self._server: Server | None = None + self._dispatcher: Dispatcher | None = None + self._sessions: set[Session] = set() + self.token: str | None = None + self.port: int | None = None + self.version = _package_version() + + # -- lifecycle ------------------------------------------------------------ + + @property + def carrier(self) -> str: + return "unix" if self.socket_path is not None else "tcp" + + @property + def manager(self) -> ProtocolManager: + return self._manager + + @property + def router(self) -> EventRouter: + return self._router + + async def start(self) -> None: + self._loop = asyncio.get_running_loop() + self._manager.on_event(self._on_engine_event) + await self._manager.start() + try: + if self._manager.peer_stream is not None: + await self._manager.peer_stream.start() + engine = self._manager.protocol + services = generated.MeshServices(engine) + data: Any + try: + data = generated.DataStore(engine) + except generated.ProtocolError as exc: + # The data layer is off or has no storage; every `data.*` + # call answers with this same refusal. + data = exc + self._dispatcher = Dispatcher( + engine, services, data, self._router, self._policy, self._ownership + ) + if self.socket_path is not None: + self._server = await self._serve_unix(self.socket_path) + else: + self._server = await self._serve_tcp() + except BaseException: + # The two handles hold the engine, and the engine holds the + # storage callbacks; freed here, before the interpreter is on + # its way out, rather than from a finaliser at exit. + self._dispatcher = None + await self._stop_manager() + raise + logger.info("local API serving on %s", self.socket_path or f"{self._tcp_host}:{self.port}") + + async def _serve_unix(self, path: Path) -> Server: + path.parent.mkdir(parents=True, exist_ok=True) + os.chmod(path.parent, stat.S_IRWXU) + if path.exists() or path.is_symlink(): + path.unlink() + server = await unix_serve( + self._handle, + path=str(path), + process_request=self._process_request, + max_size=self._max_size, + ) + # The credential on this carrier is the file's mode: whoever can + # open the socket is one of the operator's applications. The peer's + # credentials (its uid, as the kernel could report them) are not + # read, and nothing here tells one local process from another. + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + return server + + async def _serve_tcp(self) -> Server: + assert self.token_path is not None and self._tcp_port is not None + self.token = secrets.token_hex(TOKEN_BYTES) + self.token_path.parent.mkdir(parents=True, exist_ok=True) + # Created with its mode in one step, never created and then + # narrowed: a file that exists world-readable for even a moment has + # already published the token. A stale file from an earlier launch + # is removed first; its token died with that process. + try: + self.token_path.unlink() + except FileNotFoundError: + pass + fd = os.open( + self.token_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + stat.S_IRUSR | stat.S_IWUSR, + ) + with os.fdopen(fd, "w", encoding="ascii") as handle: + handle.write(self.token + "\n") + server = await serve( + self._handle, + self._tcp_host, + self._tcp_port, + process_request=self._process_request, + max_size=self._max_size, + ) + self.port = server.sockets[0].getsockname()[1] + return server + + async def stop(self) -> None: + """Closes every connection, then the engine and its transports.""" + server, self._server = self._server, None + if server is not None: + server.close() + await server.wait_closed() + for session in list(self._sessions): + self._router.detach(session) + self._sessions.clear() + self._dispatcher = None + await self._stop_manager() + if self.socket_path is not None: + try: + self.socket_path.unlink() + except FileNotFoundError: + pass + if self.token_path is not None: + try: + self.token_path.unlink() + except FileNotFoundError: + pass + self.token = None + + async def _stop_manager(self) -> None: + if self._manager._file_stores is not None: + await self._manager.close() + else: + await self._manager.stop() + + # -- events --------------------------------------------------------------- + + def _on_engine_event(self, event: dict[str, Any]) -> None: + """The manager's one event handler, on whatever thread emits.""" + loop = self._loop + if loop is None or loop.is_closed(): + return + try: + on_loop = asyncio.get_running_loop() is loop + except RuntimeError: + on_loop = False + if on_loop: + self._route(event) + else: + loop.call_soon_threadsafe(self._route, event) + + def _route(self, event: dict[str, Any]) -> None: + if not isinstance(event, dict): + return + if event.get("type") == "message_received" and "message_id" not in event: + # The manager's drain wraps the pull-style `receive_message()` + # JSON as a second `message_received` (keyed `id`). The engine + # emitted the real event inside that same `receive_message()` + # call, so this copy is dropped rather than delivered twice. + return + self._router.route(event) + + # -- the request hook ----------------------------------------------------- + + def _process_request(self, connection: ServerConnection, request: Request) -> Response | None: + if not self._health or request.path != "/health": + return None + if "upgrade" in request.headers.get("Connection", "").lower(): + return None + body = json.dumps( + { + "server": {"name": SERVER_NAME, "version": self.version}, + "api_version": API_VERSION, + "carrier": self.carrier, + } + ) + response = connection.respond(http.HTTPStatus.OK, body + "\n") + response.headers["Content-Type"] = "application/json" + return response + + # -- connections ---------------------------------------------------------- + + async def _handle(self, websocket: ServerConnection) -> None: + session = Session(self.carrier) + self._sessions.add(session) + sender = asyncio.ensure_future(session.sender(websocket)) + try: + async for raw in websocket: + self._handle_frame(session, raw) + except ConnectionClosed: + pass + finally: + self._router.detach(session) + self._sessions.discard(session) + session.closed = True + sender.cancel() + await asyncio.gather(sender, return_exceptions=True) + + def _handle_frame(self, session: Session, raw: str | bytes) -> None: + if isinstance(raw, bytes): + session.push(self._error(None, RpcError(codec.INVALID_REQUEST, "text frames only"))) + return + try: + request = json.loads(raw) + except ValueError: + session.push(self._error(None, RpcError(codec.PARSE_ERROR, "parse error"))) + return + if isinstance(request, list): + session.push( + self._error(None, RpcError(codec.INVALID_REQUEST, "batches are not supported")) + ) + return + if not isinstance(request, dict) or request.get("jsonrpc") != "2.0": + session.push(self._error(None, RpcError(codec.INVALID_REQUEST, "not a JSON-RPC 2.0 request"))) + return + if "id" not in request: + # A notification: a client never sends one, and it gets no answer. + return + request_id = request["id"] + if not (request_id is None or isinstance(request_id, (str, int))) or isinstance(request_id, bool): + session.push(self._error(None, RpcError(codec.INVALID_REQUEST, "id must be a string or a number"))) + return + method = request.get("method") + if not isinstance(method, str): + session.push(self._error(request_id, RpcError(codec.INVALID_REQUEST, "method must be a string"))) + return + params = request.get("params") + if params is not None and not isinstance(params, dict): + session.push( + self._error(request_id, RpcError(codec.INVALID_PARAMS, "params must be an object of named parameters")) + ) + return + try: + if method == "hello": + result = self._hello(session, params or {}) + session.push({"jsonrpc": "2.0", "id": request_id, "result": result}) + # Registered only now: the hello result is queued ahead of + # the held events, and the held events ahead of anything the + # router delivers from here on. + for event in self._router.attach(session): + session.push_event(event) + return + if method in SESSION_METHODS: + result = self._subscription(session, method, params or {}) + else: + assert self._dispatcher is not None + result = self._dispatcher.call(session, method, params) + except _CloseAfter as exc: + session.push(self._error(request_id, exc)) + session.push_close(POLICY_VIOLATION, "policy violation") + return + except RpcError as exc: + session.push(self._error(request_id, exc)) + return + session.push({"jsonrpc": "2.0", "id": request_id, "result": result}) + + @staticmethod + def _error(request_id: Any, error: RpcError) -> dict[str, Any]: + return {"jsonrpc": "2.0", "id": request_id, "error": error.to_json()} + + # -- the session methods -------------------------------------------------- + + def _hello(self, session: Session, params: dict[str, Any]) -> dict[str, Any]: + if session.app_id is not None: + raise taxonomy_error("InvalidState", "hello was already sent on this connection") + unknown = sorted(set(params) - {"app_id", "client", "token"}) + if unknown: + raise codec.invalid_params(f"hello has no parameter {unknown[0]!r}") + if "app_id" not in params: + raise codec.invalid_params("hello requires app_id") + app_id = validate_app_id(params["app_id"]) + client = params.get("client") + if client is not None and not isinstance(client, str): + raise codec.invalid_params("hello.client must be a string") + if self.carrier == "tcp": + token = params.get("token") + if not isinstance(token, str) or self.token is None or not hmac.compare_digest(token, self.token): + raise _CloseAfter(*_permission_denied("hello.token is missing or wrong")) + # Once any rule is configured, an id no rule names is refused, so a + # rule cannot be stepped around by reconnecting under another name. + self._policy.check_admission(app_id) + session.app_id = app_id + session.client = client + engine = self._manager.protocol + return { + "api_version": API_VERSION, + "server": {"name": SERVER_NAME, "version": self.version}, + "state": codec.encode("ProtocolState", engine.get_state()), + "local_address": engine.local_address(), + } + + @staticmethod + def _subscription(session: Session, method: str, params: dict[str, Any]) -> bool: + if session.app_id is None: + raise taxonomy_error("InvalidState", "hello has not been sent on this connection") + unknown = sorted(set(params) - {"types"}) + if unknown: + raise codec.invalid_params(f"{method} has no parameter {unknown[0]!r}") + types = params.get("types") + if types != "all": + if not isinstance(types, list) or not all(isinstance(t, str) for t in types): + raise codec.invalid_params(f"{method}.types is a list of tags or \"all\"") + if method == "subscribe": + session.subscribe(types) + else: + session.unsubscribe(types) + return True + + +def _permission_denied(message: str) -> tuple[int, str, dict[str, Any]]: + error = taxonomy_error("PermissionDenied", message) + return error.code, error.message, error.data or {} diff --git a/bindings/python/offline_protocol_sdk/local_api/session.py b/bindings/python/offline_protocol_sdk/local_api/session.py new file mode 100644 index 000000000..c5ff5801b --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/session.py @@ -0,0 +1,86 @@ +"""One connection's state: the application id it declared, its event +filter, and the outbound queue everything it receives goes through. + +Every response and every notification a connection receives is pushed onto +one queue and written by one task, in push order. That is what makes the +chapter's ordering rules cheap to keep: the ``hello`` result is pushed, then +the held events, then whatever the router delivers later, and nothing can +overtake anything. +""" + +from __future__ import annotations + +import asyncio +import itertools +import json +from typing import Any + +_ids = itertools.count(1) + + +class _Close: + """A close request queued behind everything pushed before it.""" + + def __init__(self, code: int, reason: str) -> None: + self.code = code + self.reason = reason + + +class Session: + """Per-connection state. ``app_id`` is ``None`` until ``hello``.""" + + def __init__(self, carrier: str) -> None: + self.id: int = next(_ids) + self.carrier = carrier + self.app_id: str | None = None + self.client: str | None = None + #: ``None`` delivers every event the routing rules select; a set + #: narrows delivery to those tags. + self.subscriptions: set[str] | None = None + self._queue: asyncio.Queue[str | _Close] = asyncio.Queue() + self.closed = False + + # -- the filter --------------------------------------------------------- + + def wants(self, tag: str | None) -> bool: + if self.subscriptions is None: + return True + return tag is not None and tag in self.subscriptions + + def subscribe(self, types: Any) -> None: + if types == "all": + self.subscriptions = None + return + if self.subscriptions is None: + self.subscriptions = set() + self.subscriptions.update(types) + + def unsubscribe(self, types: Any) -> None: + if types == "all": + self.subscriptions = set() + return + if self.subscriptions is not None: + self.subscriptions.difference_update(types) + + # -- the outbound queue ------------------------------------------------- + + def push(self, payload: dict[str, Any]) -> None: + if not self.closed: + self._queue.put_nowait(json.dumps(payload, separators=(",", ":"))) + + def push_event(self, event: dict[str, Any]) -> None: + self.push({"jsonrpc": "2.0", "method": "event", "params": event}) + + def push_close(self, code: int, reason: str) -> None: + self._queue.put_nowait(_Close(code, reason)) + + async def sender(self, websocket: Any) -> None: + """Writes the queue to the socket until a close is queued or the + socket goes away. Runs as one task per connection.""" + while True: + item = await self._queue.get() + if isinstance(item, _Close): + self.closed = True + await websocket.close(item.code, item.reason) + return + await websocket.send(item) diff --git a/bindings/python/offline_protocol_sdk/local_api/table.py b/bindings/python/offline_protocol_sdk/local_api/table.py new file mode 100644 index 000000000..56dd28f7e --- /dev/null +++ b/bindings/python/offline_protocol_sdk/local_api/table.py @@ -0,0 +1,815 @@ +"""The interface definition as data, generated. Do not edit. + +Written by ``bindings/python/scripts/generate_local_api_table.py`` from +``crates/offline-protocol-uniffi/src/offline_protocol.udl``. A test +regenerates it in memory and fails when this file is stale, so every +change to the definition reaches the reference server as a diff here +and a classification in ``dispatch.py``. +""" + +from __future__ import annotations + +UDL_SHA256 = "6e46f0053aa0b08d7b1b5e31cb2e3802fc126351fed72a97dee28558f70ee424" + +TABLE = {'callbacks': ('MlsStorageProvider', + 'ProtocolStateStorageProvider', + 'EventCallback', + 'BleTransportCallback', + 'WifiDirectTransportCallback', + 'ReticulumTransportCallback', + 'NostrTransportCallback'), + 'enums': {'AppState': ('Active', 'Background', 'Inactive'), + 'ContentType': ('Text', + 'Image', + 'Video', + 'Audio', + 'VoiceNote', + 'VideoNote', + 'File', + 'FileChunk', + 'Poll'), + 'EstablishmentState': ('NoKeyPackage', + 'HaveKeyPackage', + 'SessionPending', + 'SessionConfirmed'), + 'MessagePriority': ('Low', 'Medium', 'High', 'Critical'), + 'MlsVerbosity': ('Off', 'Lifecycle', 'Diagnostic'), + 'OverflowPolicy': ('DropOldest', 'DropNewest'), + 'PresenceStatus': ('Online', 'Away', 'Offline'), + 'ProtocolState': ('Stopped', 'Running', 'Paused'), + 'RelayPriority': ('Never', 'Auto', 'Always'), + 'RelaySyncState': ('Synced', 'Pending', 'Unsynced'), + 'TelemetryOs': ('Ios', 'Android', 'Linux', 'Macos', 'Windows', 'Other'), + 'TransportType': ('Internet', 'Ble', 'WiFiDirect', 'Reticulum', 'Nostr')}, + 'errors': {'MlsStorageError': ('StoreFailed', + 'LoadFailed', + 'DeleteFailed', + 'KeyNotFound', + 'CorruptedData'), + 'ProtocolError': ('NotStarted', + 'AlreadyStarted', + 'InvalidConfiguration', + 'SendFailed', + 'NoKeyPackage', + 'SessionNotReady', + 'EncryptFailed', + 'InvalidState', + 'MlsNotInitialized', + 'MlsError', + 'UserBlocked', + 'MediaTransferLimit', + 'LockPoisoned', + 'Other', + 'TransportError', + 'SerializationError', + 'ServiceError', + 'GroupNotFound', + 'PermissionDenied', + 'InvalidArgument', + 'DataDisabled', + 'DataStorageUnavailable', + 'DocTooLarge', + 'DataCorrupted', + 'TelemetryConfigInvalid')}, + 'methods': {'DataStore': {'attachment_hash': ((('data', 'bytes'),), 'string'), + 'constructor': ((('protocol', 'OfflineProtocol'),), 'object'), + 'counter_increment': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('amount', 'double')), + 'void'), + 'counter_value': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string')), + 'double'), + 'create_doc': ((('space_id', 'string'), ('doc_id', 'string')), 'void'), + 'decline_attachment': ((('space_id', 'string'), + ('peer_id', 'string'), + ('hash', 'string')), + 'void'), + 'delete_doc': ((('space_id', 'string'), ('doc_id', 'string')), 'void'), + 'doc_json': ((('space_id', 'string'), ('doc_id', 'string')), 'string'), + 'doc_size': ((('space_id', 'string'), ('doc_id', 'string')), 'u64'), + 'export_raw': ((('space_id', 'string'), ('doc_id', 'string')), 'bytes'), + 'fetch_attachment': ((('space_id', 'string'), ('hash', 'string')), + 'void'), + 'fetch_attachment_from': ((('space_id', 'string'), + ('peer_id', 'string'), + ('hash', 'string')), + 'void'), + 'flush': ((('space_id', 'string'), ('doc_id', 'string')), 'void'), + 'flush_all': ((), 'void'), + 'list_delete': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('index', 'u32'), + ('count', 'u32')), + 'void'), + 'list_docs': ((('space_id', 'string'),), 'sequence'), + 'list_len': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string')), + 'u32'), + 'list_push': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('value_json', 'string')), + 'void'), + 'list_spaces': ((), 'sequence'), + 'map_delete': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('key', 'string')), + 'void'), + 'map_get_json': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('key', 'string')), + 'string?'), + 'map_set': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('key', 'string'), + ('value_json', 'string')), + 'void'), + 'provide_attachment': ((('space_id', 'string'), + ('peer_id', 'string'), + ('hash', 'string'), + ('data', 'bytes')), + 'void'), + 'remove_doc': ((('space_id', 'string'), ('doc_id', 'string')), 'void'), + 'remove_space': ((('space_id', 'string'),), 'void'), + 'set_interest': ((('space_id', 'string'), + ('patterns', 'sequence')), + 'void'), + 'text_delete': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('position', 'u32'), + ('count', 'u32')), + 'void'), + 'text_insert': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string'), + ('position', 'u32'), + ('text', 'string')), + 'void'), + 'text_value': ((('space_id', 'string'), + ('doc_id', 'string'), + ('collection', 'string')), + 'string'), + 'wipe_all': ((), 'void'), + 'with_storage': ((('protocol', 'OfflineProtocol'), + ('storage', 'ProtocolStateStorageProvider')), + 'object')}, + 'MeshServices': {'constructor': ((('protocol', 'OfflineProtocol'),), 'object'), + 'discover_services': ((('service_id', 'string?'),), 'string'), + 'register_service': ((('service_id', 'string'), + ('version', 'string'), + ('capabilities', 'record')), + 'void'), + 'respond_to_service_request': ((('request_id', 'string'), + ('requester', 'string'), + ('service_id', 'string'), + ('status', 'string'), + ('body', 'string')), + 'string'), + 'send_service_request': ((('provider', 'string'), + ('service_id', 'string'), + ('method', 'string'), + ('body', 'string')), + 'string'), + 'unregister_service': ((('service_id', 'string'),), 'boolean')}, + 'OfflineProtocol': {'accept_connection_request': ((('recipient', 'string'), + ('accepter_name', 'string'), + ('key_package', 'sequence?')), + 'string'), + 'ble_clear_peer_mtu': ((('peer_id', 'string'),), 'void'), + 'ble_fragment_fallback_count': ((), 'u64'), + 'ble_fragment_received': ((('sender_id', 'string'), + ('fragment', 'sequence')), + 'void'), + 'ble_get_next_fragment': ((), 'BleFragment?'), + 'ble_get_peer_count': ((), 'u32'), + 'ble_peer_discovered': ((('peer_id', 'string'), ('rssi', 'i16')), + 'void'), + 'ble_peer_lost': ((('peer_id', 'string'),), 'void'), + 'ble_recipient_not_among_peers_count': ((), 'u64'), + 'ble_return_fragment': ((), 'void'), + 'ble_set_peer_mtu': ((('peer_id', 'string'), + ('max_payload', 'u32')), + 'void'), + 'ble_status_changed': ((('is_available', 'boolean'),), 'void'), + 'ble_undersized_mtu_reports': ((), 'u64'), + 'block_user': ((('user_id', 'string'),), 'void'), + 'cancel_connection_request': ((('recipient', 'string'),), + 'string'), + 'cancel_file_transfer': ((('file_id', 'string'),), 'void'), + 'close_file_stores': ((), 'void'), + 'constructor': ((('config', 'ProtocolConfig'),), 'object'), + 'create_group': ((('group_name', 'string'),), 'MlsGroupInfo'), + 'create_invite': ((('petname', 'string?'), ('sign', 'boolean')), + 'string'), + 'derive_user_id_from_public_key': ((('public_key', + 'sequence'),), + 'string'), + 'disable_telemetry': ((), 'void'), + 'emit_test_event': ((), 'void'), + 'enable_telemetry': ((('config', 'TelemetryConfig'), + ('app_state', 'AppState')), + 'void'), + 'end_telemetry_session': ((), 'void'), + 'establish_secure_session': ((('peer_id', 'string'),), + 'MlsWelcomeMessage?'), + 'finalize_file': ((('file_id', 'string'),), 'void'), + 'flush_telemetry': ((), 'void'), + 'flush_telemetry_blocking': ((('deadline_ms', 'u64'),), 'boolean'), + 'force_transport': ((('transport_type', 'TransportType'),), + 'void'), + 'forward_message': ((('original_message_json', 'string'), + ('new_recipient', 'string'), + ('priority', 'MessagePriority?')), + 'string'), + 'forward_message_to_group': ((('original_message_json', 'string'), + ('group_id', 'string'), + ('priority', 'MessagePriority?')), + 'sequence'), + 'gateway_address_declaration': ((('challenge', 'sequence'),), + 'GatewayAddressDeclaration'), + 'get_active_transports': ((), 'sequence'), + 'get_battery_level': ((), 'u8?'), + 'get_blocked_users': ((), 'sequence'), + 'get_dedup_stats': ((), 'DedupStats'), + 'get_delivery_success_rate': ((), 'f32'), + 'get_dors_config': ((), 'DorsConfig'), + 'get_establishment_state': ((('peer_id', 'string'),), + 'EstablishmentState'), + 'get_file_progress': ((('file_id', 'string'),), 'FileProgress?'), + 'get_group_info': ((('group_id', 'string'),), 'MlsGroupInfo?'), + 'get_group_roles': ((('group_id', 'string'),), + 'record'), + 'get_identity_public_key': ((), 'sequence'), + 'get_is_charging': ((), 'boolean'), + 'get_median_hops': ((), 'u8'), + 'get_median_latency': ((), 'u64'), + 'get_member_role': ((('group_id', 'string'), + ('user_id', 'string')), + 'string'), + 'get_mesh_relay_stats': ((), 'MeshRelayStats'), + 'get_mesh_relay_tunables': ((), 'MeshRelayTunables'), + 'get_message_stats': ((), 'sequence'), + 'get_pending_ack_count': ((), 'u64'), + 'get_relay_config': ((), 'RelayConfig'), + 'get_relay_priority': ((), 'RelayPriority'), + 'get_retry_queue_size': ((), 'u64'), + 'get_state': ((), 'ProtocolState'), + 'get_topology': ((), 'NetworkTopology'), + 'get_transport_metrics': ((('transport_type', 'TransportType'),), + 'TransportMetrics?'), + 'group_relay_sync_state': ((('group_id', 'string'),), + 'RelaySyncState'), + 'group_rich_readiness': ((('group_id', 'string'),), + 'GroupRichReadiness'), + 'has_pending_key_package': ((('peer_id', 'string'),), 'boolean'), + 'identity_assertion': ((('signed_data', 'sequence'),), + 'sequence'), + 'initialize_mls': ((('secure_storage', 'MlsStorageProvider'), + ('protocol_state_storage', + 'ProtocolStateStorageProvider')), + 'void'), + 'initialize_mls_with_file_stores': ((('mls_root', 'string'), + ('state_root', 'string'), + ('store_key', 'bytes')), + 'void'), + 'internet_address_declaration_refused': ((('reason', 'string'),), + 'void'), + 'internet_address_declared': ((('address', 'string'),), 'void'), + 'internet_confirm_sent': ((('message_id', 'string'),), 'void'), + 'internet_get_next_message': ((), 'InternetMessage?'), + 'internet_group_report_received': ((('report_json', 'string'),), + 'void'), + 'internet_message_received': ((('sender_id', 'string'), + ('data', 'sequence')), + 'void'), + 'internet_peer_presence': ((('peer_id', 'string'), + ('online', 'boolean'), + ('last_seen_ms', 'i64?')), + 'void'), + 'internet_presence_watchlist': ((), 'sequence'), + 'internet_relay_capabilities': ((('capabilities', + 'sequence'),), + 'void'), + 'internet_send_failed': ((('message_id', 'string'),), 'void'), + 'internet_send_failed_with_reason': ((('message_id', 'string'), + ('reason', 'string?')), + 'void'), + 'internet_status_changed': ((('is_connected', 'boolean'),), + 'void'), + 'invite_to_group': ((('group_id', 'string'), + ('invitee_user_id', 'string')), + 'void'), + 'is_mls_initialized': ((), 'boolean'), + 'is_relay': ((), 'boolean'), + 'is_user_blocked': ((('user_id', 'string'),), 'boolean'), + 'leave_group': ((('group_id', 'string'),), 'void'), + 'list_groups': ((), 'sequence'), + 'local_address': ((), 'string?'), + 'mls_clear_pending_welcome': ((('other_user_id', 'string'),), + 'void'), + 'mls_create_session': ((('other_user_id', 'string'),), + 'MlsWelcomeMessage'), + 'mls_decrypt': ((('encrypted', 'MlsEncryptedMessage'),), + 'sequence?'), + 'mls_decrypt_from_user': ((('encrypted', 'MlsEncryptedMessage'),), + 'sequence?'), + 'mls_delete_session': ((('other_user_id', 'string'),), 'void'), + 'mls_encrypt_for_user': ((('other_user_id', 'string'), + ('plaintext', 'sequence')), + 'MlsEncryptedMessage'), + 'mls_generate_key_package': ((), 'MlsKeyPackageBundle'), + 'mls_get_or_create_key_package': ((), 'MlsKeyPackageBundle'), + 'mls_get_pending_key_packages': ((), + 'sequence'), + 'mls_get_pending_welcome': ((('other_user_id', 'string'),), + 'MlsWelcomeMessage?'), + 'mls_has_session': ((('other_user_id', 'string'),), 'boolean'), + 'mls_import_key_package': ((('user_id', 'string'), + ('key_package_data', 'sequence')), + 'void'), + 'mls_join_session': ((('welcome', 'MlsWelcomeMessage'),), + 'MlsGroupInfo'), + 'mls_list_sessions': ((), 'sequence'), + 'mls_mark_key_package_synced': ((('package_id', 'string'),), + 'void'), + 'mls_process_welcome': ((('welcome', 'MlsWelcomeMessage'),), + 'MlsGroupInfo'), + 'nostr_confirm_sent': ((('message_id', 'string'),), 'void'), + 'nostr_get_next_message': ((), 'NostrMessage?'), + 'nostr_get_next_query': ((), 'NostrQuery?'), + 'nostr_get_public_key': ((), 'string?'), + 'nostr_get_subscription_filter': ((('subscription_id', 'string'),), + 'string?'), + 'nostr_message_received': ((('sender_id', 'string'), + ('data', 'sequence')), + 'void'), + 'nostr_message_received_at': ((('sender_id', 'string'), + ('data', 'sequence'), + ('created_at', 'i64')), + 'void'), + 'nostr_query_completed': ((('query_id', 'string'),), 'void'), + 'nostr_query_event_received': ((('query_id', 'string'), + ('event_json', 'string')), + 'void'), + 'nostr_send_failed': ((('message_id', 'string'),), 'void'), + 'nostr_send_failed_with_reason': ((('message_id', 'string'), + ('reason', 'string?')), + 'void'), + 'nostr_status_changed': ((('is_connected', 'boolean'),), 'void'), + 'notify_app_state': ((('state', 'AppState'),), 'void'), + 'pause': ((), 'void'), + 'poll_event': ((), 'string?'), + 'process': ((), 'void'), + 'process_file_chunk': ((('file_id', 'string'), + ('chunk_index', 'u32'), + ('total_chunks', 'u32'), + ('file_size', 'u64'), + ('file_name', 'string'), + ('file_checksum', 'string'), + ('data', 'sequence')), + 'void'), + 'protocol_lock_diagnostics': ((), 'ProtocolLockDiagnostics'), + 'receive_message': ((), 'string?'), + 'reject_connection_request': ((('recipient', 'string'),), + 'string'), + 'rekey_session': ((('peer_id', 'string'),), 'boolean'), + 'release_transport_lock': ((), 'void'), + 'remove_from_group': ((('group_id', 'string'), + ('member_id', 'string')), + 'void'), + 'remove_transport': ((('transport_type', 'TransportType'),), + 'void'), + 'rename_group': ((('group_id', 'string'), ('new_name', 'string')), + 'void'), + 'request_group_relay_registration': ((('group_id', 'string'),), + 'boolean'), + 'resolve_username': ((('username', 'string'),), 'boolean'), + 'resume': ((), 'void'), + 'reticulum_address_declaration_refused': ((('reason', 'string'),), + 'void'), + 'reticulum_address_declared': ((('address', 'string'),), 'void'), + 'reticulum_confirm_sent': ((('message_id', 'string'),), 'void'), + 'reticulum_gateway_capabilities': ((('capabilities', + 'sequence'),), + 'void'), + 'reticulum_get_next_message': ((), 'ReticulumMessage?'), + 'reticulum_message_received': ((('sender_id', 'string'), + ('data', 'sequence')), + 'void'), + 'reticulum_peer_presence': ((('peer_id', 'string'), + ('online', 'boolean'), + ('last_seen_ms', 'i64?')), + 'void'), + 'reticulum_presence_watchlist': ((), 'sequence'), + 'reticulum_send_failed': ((('message_id', 'string'),), 'void'), + 'reticulum_send_failed_with_reason': ((('message_id', 'string'), + ('reason', 'string?')), + 'void'), + 'reticulum_status_changed': ((('is_connected', 'boolean'),), + 'void'), + 'send_connection_request': ((('recipient', 'string'), + ('sender_name', 'string'), + ('key_package', 'sequence?'), + ('initial_message', 'string?')), + 'string'), + 'send_file': ((('recipient', 'string'), + ('file_data', 'sequence'), + ('file_name', 'string')), + 'string'), + 'send_group_message': ((('group_id', 'string'), + ('content', 'string'), + ('priority', 'MessagePriority?'), + ('reply_to_msg', 'string?')), + 'sequence'), + 'send_media': ((('recipient', 'string'), + ('file_data', 'sequence'), + ('file_name', 'string'), + ('content_type', 'ContentType'), + ('media_metadata', 'MediaMetadata?')), + 'string'), + 'send_media_rich': ((('recipient', 'string'), + ('file_data', 'sequence'), + ('file_name', 'string'), + ('content_type', 'ContentType'), + ('options', 'MediaSendOptions')), + 'string'), + 'send_message': ((('recipient', 'string'), + ('content', 'string'), + ('priority', 'MessagePriority'), + ('reply_to_msg', 'string?')), + 'string'), + 'send_message_rich': ((('recipient', 'string'), + ('content', 'string'), + ('options', 'SendMessageOptions')), + 'string'), + 'send_presence_update': ((('recipient', 'string'), + ('status', 'PresenceStatus')), + 'string'), + 'send_read_receipt': ((('recipient', 'string'), + ('message_ids', 'sequence')), + 'string'), + 'send_typing_indicator': ((('recipient', 'string'), + ('conversation_id', 'string'), + ('is_typing', 'boolean')), + 'string'), + 'set_battery_level': ((('level', 'u8'),), 'void'), + 'set_battery_state': ((('level', 'u8'), + ('is_charging', 'boolean')), + 'void'), + 'set_ble_transport_callback': ((('callback', + 'BleTransportCallback'),), + 'void'), + 'set_event_callback': ((('callback', 'EventCallback'),), 'void'), + 'set_member_role': ((('group_id', 'string'), + ('user_id', 'string'), + ('role', 'string')), + 'void'), + 'set_nostr_transport_callback': ((('callback', + 'NostrTransportCallback'),), + 'void'), + 'set_relay_priority': ((('priority', 'RelayPriority'),), 'void'), + 'set_reticulum_transport_callback': ((('callback', + 'ReticulumTransportCallback'),), + 'void'), + 'set_telemetry_enabled': ((('enabled', 'boolean'),), 'void'), + 'set_wifi_direct_transport_callback': ((('callback', + 'WifiDirectTransportCallback'),), + 'void'), + 'should_escalate_to_wifi': ((), 'boolean'), + 'sign_data': ((('data', 'sequence'),), 'sequence'), + 'start': ((), 'void'), + 'stop': ((), 'void'), + 'telemetry_install_id': ((), 'string?'), + 'telemetry_stats': ((), 'TelemetryStats?'), + 'unblock_user': ((('user_id', 'string'),), 'void'), + 'update_ack_config': ((('config', 'AckConfig'),), 'void'), + 'update_dedup_config': ((('config', 'DedupConfig'),), 'void'), + 'update_dors_config': ((('config', 'DorsConfig'),), 'void'), + 'update_relay_config': ((('config', 'RelayConfig'),), 'void'), + 'update_retry_config': ((('config', 'RetryConfig'),), 'void'), + 'update_transport_metrics': ((('transport_type', 'TransportType'), + ('metrics', 'TransportMetrics')), + 'void'), + 'verify_signature': ((('public_key', 'sequence'), + ('data', 'sequence'), + ('signature', 'sequence')), + 'boolean'), + 'wifi_direct_get_next_message': ((), 'WifiDirectMessage?'), + 'wifi_direct_message_received': ((('sender_id', 'string'), + ('data', 'sequence')), + 'void'), + 'wifi_direct_peer_connected': ((('peer_id', 'string'),), 'void'), + 'wifi_direct_peer_disconnected': ((('peer_id', 'string'),), + 'void'), + 'wifi_direct_status_changed': ((('is_connected', 'boolean'),), + 'void')}, + 'namespace': {'derive_address': ((('public_key', 'sequence'),), 'string'), + 'parse_invite': ((('blob', 'string'),), 'InviteInfo'), + 'run_storage_conformance': ((('storage', + 'ProtocolStateStorageProvider'),), + 'string'), + 'verify_identity_assertion': ((('assertion', 'sequence'),), + 'string')}}, + 'records': {'AckConfig': (('default_timeout_ms', 'u64', False), + ('max_pending_acks', 'u64', False)), + 'BleFragment': (('recipient_id', 'string', False), ('data', 'sequence', False)), + 'DedupConfig': (('max_tracked_messages', 'u64', False), + ('retention_time_secs', 'u64', False)), + 'DedupStats': (('total_tracked', 'u64', False), + ('recent_tracked', 'u64', False), + ('capacity_used_percent', 'u8', False), + ('mode', 'string', False)), + 'DorsConfig': (('prefer_online', 'boolean', False), + ('switch_hysteresis', 'f32', False), + ('switch_cooldown_secs', 'u64', False), + ('ble_to_wifi_retry_threshold', 'u32', False), + ('min_success_rate_before_escalation', 'f32', False), + ('min_ble_samples_before_success_rate_escalation', 'u64', False), + ('rssi_switch_threshold', 'i16', False), + ('congestion_queue_threshold', 'u64', False), + ('stability_window_secs', 'u64', False), + ('poor_signal_duration_secs', 'u64', False), + ('ttl_escalation_threshold', 'u8', False), + ('congestion_duration_secs', 'u64', False), + ('ttl_escalation_hold_secs', 'u64', False), + ('history_window_size', 'u64', False), + ('queue_recovery_ratio', 'f32', False), + ('low_battery_threshold', 'u8', False), + ('relay_min_battery_level', 'u8', False), + ('relay_optimal_connection_count', 'u8', False)), + 'EncryptionConfig': (('enabled', 'boolean', False), + ('auto_key_exchange', 'boolean', False), + ('store_pending', 'boolean', False), + ('require_encryption', 'boolean', True), + ('pending_queue', 'PendingQueueConfig', False)), + 'FileProgress': (('file_id', 'string', False), + ('chunks_sent', 'u32', False), + ('total_chunks', 'u32', False), + ('percentage', 'u8', False)), + 'ForwardInfo': (('original_sender', 'string', False), + ('original_message_id', 'string', False), + ('original_timestamp', 'i64', False), + ('forward_count', 'u32', False)), + 'GatewayAddressDeclaration': (('address', 'string', False), + ('public_key', 'sequence', False), + ('signature', 'sequence', False)), + 'GroupRichReadiness': (('ready', 'boolean', False), + ('unknown_members', 'sequence', False)), + 'InternetMessage': (('message_id', 'string', False), + ('recipient_id', 'string', False), + ('data', 'sequence', False), + ('reply_to_msg', 'string?', False), + ('control_op', 'string?', False), + ('control_payload', 'string?', False)), + 'InviteInfo': (('address', 'string', False), + ('public_key', 'sequence', False), + ('petname', 'string?', False), + ('signed', 'boolean', False)), + 'MediaMetadata': (('mime_type', 'string', False), + ('file_name', 'string', False), + ('file_size', 'u64', False), + ('duration_ms', 'u64?', False), + ('width', 'u32?', False), + ('height', 'u32?', False), + ('thumbnail_base64', 'string?', False), + ('media_id', 'string?', True), + ('download_url', 'string?', True), + ('thumbnail_url', 'string?', True), + ('encryption_key', 'string?', True), + ('iv', 'string?', True), + ('ciphertext_hash', 'string?', True), + ('sticker_provider', 'string?', True), + ('sticker_remote_id', 'string?', True), + ('sticker_kind', 'string?', True)), + 'MediaSendOptions': (('media_metadata', 'MediaMetadata?', True), + ('caption', 'string?', True), + ('reply_to_msg', 'string?', True), + ('reply_context', 'ReplyContext?', True), + ('forward_info', 'ForwardInfo?', True), + ('file_id', 'string?', True), + ('app_id', 'string?', True)), + 'MeshRelayConfig': (('max_ttl', 'u8?', True), + ('dense_max_ttl', 'u8?', True), + ('dense_degree', 'u64?', True), + ('fanout', 'u64?', True), + ('jitter_min_ms', 'u64?', True), + ('jitter_max_ms', 'u64?', True), + ('rate_per_sec', 'f32?', True), + ('burst', 'f32?', True), + ('peer_rate_per_sec', 'f32?', True), + ('peer_burst', 'f32?', True), + ('queue_capacity', 'u64?', True), + ('bias_min_scale', 'f32?', True), + ('bias_max_handicap_ms', 'u64?', True), + ('activity_window_ms', 'u64?', True), + ('activity_min_forwards', 'u64?', True), + ('activity_idle_windows', 'u32?', True)), + 'MeshRelayStats': (('forwarded', 'u64', False), + ('transmissions', 'u64', False), + ('queued', 'u64', False), + ('awaiting_transmission', 'u64', False), + ('duplicates_suppressed', 'u64', False), + ('covered_by_a_neighbor', 'u64', False), + ('peer_rate_limited', 'u64', False), + ('refused_queue_full', 'u64', False), + ('rate_deferred', 'u64', False), + ('abandoned_overdue', 'u64', False), + ('hop_limit_reached', 'u64', False), + ('reach_clamped', 'u64', False), + ('dropped_for_capacity', 'u64', False)), + 'MeshRelayTunables': (('max_ttl', 'u8', False), + ('dense_max_ttl', 'u8', False), + ('dense_degree', 'u64', False), + ('fanout', 'u64', False), + ('jitter_min_ms', 'u64', False), + ('jitter_max_ms', 'u64', False), + ('rate_per_sec', 'f32', False), + ('burst', 'f32', False), + ('peer_rate_per_sec', 'f32', False), + ('peer_burst', 'f32', False), + ('queue_capacity', 'u64', False), + ('bias_min_scale', 'f32', False), + ('bias_max_handicap_ms', 'u64', False), + ('activity_window_ms', 'u64', False), + ('activity_min_forwards', 'u64', False), + ('activity_idle_windows', 'u32', False)), + 'MessageStats': (('message_id', 'string', False), + ('sent_at_ms', 'u64', False), + ('delivered_at_ms', 'u64?', False), + ('hop_count', 'u8', False), + ('status', 'string', False)), + 'MlsAddMemberResult': (('welcome', 'MlsWelcomeMessage', False), + ('commit', 'MlsEncryptedMessage', False)), + 'MlsEncryptedMessage': (('group_id', 'string', False), + ('message_type', 'string', False), + ('epoch', 'u64', False), + ('ciphertext', 'sequence', False), + ('sender_id', 'string', False), + ('timestamp_ms', 'u64', False)), + 'MlsGroupInfo': (('group_id', 'string', False), + ('name', 'string?', False), + ('members', 'sequence', False), + ('epoch', 'u64', False), + ('is_session', 'boolean', False), + ('created_at_ms', 'u64', False), + ('last_activity_ms', 'u64', False)), + 'MlsKeyPackageBundle': (('package_id', 'string', False), + ('user_id', 'string', False), + ('key_package_data', 'sequence', False), + ('created_at_ms', 'u64', False), + ('expires_at_ms', 'u64', False), + ('synced', 'boolean', False)), + 'MlsWelcomeMessage': (('group_id', 'string', False), + ('welcome_data', 'sequence', False), + ('inviter_id', 'string', False), + ('group_name', 'string?', False), + ('timestamp_ms', 'u64', False)), + 'NetworkLink': (('source_id', 'string', False), + ('target_id', 'string', False), + ('transport', 'string', False), + ('quality', 'f32', False)), + 'NetworkNode': (('node_id', 'string', False), + ('role', 'string', False), + ('rssi', 'i16?', False), + ('battery_level', 'u8?', False), + ('connection_count', 'u32', False), + ('last_seen_ms', 'u64', False)), + 'NetworkTopology': (('nodes', 'sequence', False), + ('links', 'sequence', False), + ('message_stats', 'sequence', False)), + 'NostrMessage': (('message_id', 'string', False), + ('event_id', 'string', False), + ('event_json', 'string', False)), + 'NostrQuery': (('query_id', 'string', False), ('req_json', 'string', False)), + 'PeerDevice': (('peer_id', 'string', False), + ('rssi', 'i16', False), + ('last_seen_ms', 'u64', False)), + 'PendingQueueConfig': (('max_pending_per_peer', 'u64', False), + ('max_pending_global', 'u64', False), + ('pending_ttl_ms', 'u64', False), + ('overflow_policy', 'OverflowPolicy', False)), + 'ProtocolConfig': (('app_id', 'string', False), + ('profile', 'string', False), + ('ble_enabled', 'boolean', False), + ('wifi_direct_enabled', 'boolean', False), + ('internet_enabled', 'boolean', False), + ('reticulum_enabled', 'boolean', False), + ('nostr_enabled', 'boolean', False), + ('prefer_online', 'boolean', False), + ('initial_ttl', 'u8', False), + ('encryption_enabled', 'boolean', False), + ('auto_key_exchange', 'boolean', False), + ('store_pending', 'boolean', False), + ('require_encryption', 'boolean', True), + ('max_pending_per_peer', 'u64', False), + ('max_pending_global', 'u64', False), + ('pending_ttl_ms', 'u64', False), + ('overflow_policy', 'OverflowPolicy', False), + ('edge_driven_unreachable_dm', 'boolean', True), + ('max_group_members', 'u32', True), + ('group_relay_enabled', 'boolean', True), + ('group_relay_broadcast_enabled', 'boolean', True), + ('group_enforce_admin_commits', 'boolean', True), + ('require_transport_identity', 'boolean', True), + ('binary_wire_enabled', 'boolean', True), + ('nostr_sealing_enabled', 'boolean', True), + ('nostr_cold_contact_enabled', 'boolean', True), + ('nostr_username_discovery_enabled', 'boolean', True), + ('compact_envelope_enabled', 'boolean', True), + ('rich_payload_enabled', 'boolean', True), + ('crypto_recovery_enabled', 'boolean', True), + ('mesh_relay', 'MeshRelayConfig?', True), + ('data_enabled', 'boolean', True), + ('control_freshness_enforced', 'boolean', True)), + 'ProtocolLockDiagnostics': (('held', 'boolean', False), + ('holder_location', 'string', False), + ('holder_thread', 'string', False), + ('held_for_ms', 'u64', False)), + 'RelayConfig': (('min_battery_for_relay', 'u8', False), + ('allow_relay', 'boolean', False), + ('relay_priority', 'RelayPriority', False)), + 'ReliabilityConfig': (('ack', 'AckConfig', False), + ('retry', 'RetryConfig', False), + ('dedup', 'DedupConfig', False)), + 'ReplyContext': (('sender', 'string', False), + ('text', 'string', False), + ('timestamp', 'i64?', True), + ('reply_media_label', 'string?', True), + ('reply_content_type', 'string?', True)), + 'ReticulumMessage': (('message_id', 'string', False), + ('recipient_id', 'string', False), + ('data', 'sequence', False), + ('reply_to_msg', 'string?', False)), + 'RetryConfig': (('max_retries', 'u32', False), + ('initial_delay_ms', 'u64', False), + ('max_delay_ms', 'u64', False), + ('backoff_multiplier', 'f32', False), + ('outbox_max_lifetime_ms', 'u64', False), + ('pending_message_max_lifetime_ms', 'u64', False), + ('edge_driven_unreachable_dm', 'boolean', True)), + 'SendMessageOptions': (('priority', 'MessagePriority?', True), + ('reply_to_msg', 'string?', True), + ('content_type', 'ContentType?', True), + ('reply_context', 'ReplyContext?', True), + ('media_metadata', 'MediaMetadata?', True), + ('forward_info', 'ForwardInfo?', True), + ('app_id', 'string?', True)), + 'TelemetryConfig': (('api_key', 'string', False), + ('app_id', 'string', False), + ('os', 'TelemetryOs', False), + ('os_major', 'u16', False), + ('app_version', 'string?', False), + ('debug', 'boolean?', False), + ('flush_interval_ms', 'u64?', False), + ('max_batch_bytes', 'u64?', False), + ('max_buffered_records', 'u32?', False), + ('include_device_id', 'boolean?', False), + ('scrub_ids', 'boolean?', False), + ('mls_verbosity', 'MlsVerbosity?', False), + ('metrics_cadence_ms', 'u64?', False), + ('routing_diagnostic', 'boolean?', False), + ('mls_sampling_bypass', 'boolean?', False)), + 'TelemetryStats': (('buffered', 'u64', False), + ('sent_events', 'u64', False), + ('accepted_events', 'u64', False), + ('dropped', 'u64', False), + ('session_id', 'string', False), + ('last_error', 'string?', False), + ('last_flush_at_ms', 'i64?', False)), + 'TransportConfig': (('ble_enabled', 'boolean', False), + ('wifi_direct_enabled', 'boolean', False), + ('internet_enabled', 'boolean', False), + ('reticulum_enabled', 'boolean', False), + ('nostr_enabled', 'boolean', False), + ('nostr_sealing_enabled', 'boolean', True), + ('nostr_cold_contact_enabled', 'boolean', True), + ('nostr_username_discovery_enabled', 'boolean', True)), + 'TransportMetrics': (('packets_sent', 'u32', False), + ('packets_received', 'u32', False), + ('bytes_sent', 'u32', False), + ('bytes_received', 'u32', False), + ('error_rate', 'f32', False), + ('avg_latency_ms', 'u32', False), + ('rssi', 'i16?', False), + ('bandwidth_bps', 'u64?', False), + ('congestion', 'f32?', False), + ('queue_depth', 'u32?', False), + ('battery_level', 'u8?', False), + ('is_charging', 'boolean?', False), + ('relay_connection_count', 'u8?', False), + ('is_active_relay', 'boolean?', False), + ('delivery_ratio', 'f32?', False), + ('drop_rate', 'f32?', False), + ('average_hop_count', 'f32?', False), + ('energy_cost', 'f32?', False)), + 'WifiDirectMessage': (('recipient_id', 'string', False), + ('data', 'sequence', False))}} diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 716fbf2e5..a86a0cdf2 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -57,6 +57,11 @@ lan = [ "ifaddr>=0.2,<1", ] +[project.scripts] +# The reference server for the local API (docs/spec/local-api.md): one +# engine, served to local applications over a Unix domain socket. +offline-protocol-service = "offline_protocol_sdk.local_api.cli:main" + [project.urls] Homepage = "https://github.com/Offline-Protocol/offline-protocol-sdk" Repository = "https://github.com/Offline-Protocol/offline-protocol-sdk" diff --git a/bindings/python/scripts/generate_local_api_table.py b/bindings/python/scripts/generate_local_api_table.py new file mode 100644 index 000000000..51c247c8a --- /dev/null +++ b/bindings/python/scripts/generate_local_api_table.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Generates the local API's method table from the interface definition. + +Reads ``crates/offline-protocol-uniffi/src/offline_protocol.udl`` and writes +``offline_protocol_sdk/local_api/table.py``: every declaration of the three +objects and the namespace with its parameters and result type, every enum +with its variants in declaration order, every dictionary with its fields, and +the error enum in order. The reference server dispatches from that table and +classifies each declaration as exposed or platform-only in ``dispatch.py``. + +The table is checked in rather than generated at import so that a change to +the definition shows up as a diff, and so that a declaration nobody has +classified fails a test rather than being exposed or hidden by default. A +test regenerates the table in memory and compares; a Rust guard in the FFI +crate reads the definition, the chapter and the classification and asserts +the three agree. + +Usage:: + + python bindings/python/scripts/generate_local_api_table.py [--check] + +``--check`` exits non-zero when the checked-in table is stale. +""" + +from __future__ import annotations + +import argparse +import hashlib +import pprint +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +PACKAGE = HERE.parent / "offline_protocol_sdk" +REPO = HERE.parents[2] +UDL = REPO / "crates" / "offline-protocol-uniffi" / "src" / "offline_protocol.udl" +OUTPUT = PACKAGE / "local_api" / "table.py" + +#: The wire prefix of each object's methods. The chapter keeps each object's +#: own prefix so the guard can map every row back to its declaration. +PREFIXES = { + "OfflineProtocol": "", + "MeshServices": "services.", + "DataStore": "data.", + "namespace": "", +} + +_BLOCK = re.compile( + r"(?:\[(?P[^\]]*)\]\s*)?" + r"(?Pcallback interface|namespace|interface|dictionary|enum)\s+" + r"(?P\w+)\s*\{(?P.*?)\};", + re.DOTALL, +) +_METHOD = re.compile( + r"^(?:\[(?P[^\]]*)\]\s*)?(?P[\w<>?, ]+?)\s+(?P\w+)\s*\((?P.*)\)$", + re.DOTALL, +) +_CTOR = re.compile( + r"^(?:\[(?P[^\]]*)\]\s*)?constructor\s*\((?P.*)\)$", + re.DOTALL, +) +_FIELD = re.compile(r"^(?P[\w<>?, ]+?)\s+(?P\w+)\s*(?:=\s*(?P.+))?$") + + +def _strip_comments(text: str) -> str: + return "\n".join(line.split("//", 1)[0] for line in text.splitlines()) + + +def _norm_type(text: str) -> str: + return re.sub(r"\s+", "", text) + + +def _split_params(text: str) -> list[str]: + """Splits a parameter list on the commas at angle-bracket depth zero.""" + out: list[str] = [] + depth = 0 + current = "" + for ch in text: + if ch == "<": + depth += 1 + elif ch == ">": + depth -= 1 + if ch == "," and depth == 0: + out.append(current) + current = "" + else: + current += ch + if current.strip(): + out.append(current) + return [p.strip() for p in out if p.strip()] + + +def _parse_params(text: str) -> tuple[tuple[str, str], ...]: + params = [] + for item in _split_params(text): + head, _, name = item.rpartition(" ") + if not head: + raise ValueError(f"cannot parse parameter {item!r}") + params.append((name.strip(), _norm_type(head))) + return tuple(params) + + +def _parse_methods(body: str) -> dict[str, tuple[tuple[tuple[str, str], ...], str]]: + methods: dict[str, tuple[tuple[tuple[str, str], ...], str]] = {} + for statement in body.split(";"): + statement = statement.strip() + if not statement: + continue + ctor = _CTOR.match(statement) + if ctor: + attrs = ctor.group("attrs") or "" + named = re.search(r"Name\s*=\s*(\w+)", attrs) + name = named.group(1) if named else "constructor" + methods[name] = (_parse_params(ctor.group("params")), "object") + continue + method = _METHOD.match(statement) + if not method: + raise ValueError(f"cannot parse declaration {statement!r}") + methods[method.group("name")] = ( + _parse_params(method.group("params")), + _norm_type(method.group("ret")), + ) + return methods + + +def _parse_fields(body: str) -> tuple[tuple[str, str, bool], ...]: + fields = [] + for statement in body.split(";"): + statement = statement.strip() + if not statement: + continue + field = _FIELD.match(statement) + if not field: + raise ValueError(f"cannot parse field {statement!r}") + fields.append( + (field.group("name"), _norm_type(field.group("type")), field.group("default") is not None) + ) + return tuple(fields) + + +def parse_udl(text: str) -> dict: + """The table as a plain dict, from the definition's text.""" + clean = _strip_comments(text) + enums: dict[str, tuple[str, ...]] = {} + records: dict[str, tuple[tuple[str, str, bool], ...]] = {} + callbacks: list[str] = [] + methods: dict[str, dict] = {} + errors: dict[str, tuple[str, ...]] = {} + for block in _BLOCK.finditer(clean): + kind, name, body = block.group("kind"), block.group("name"), block.group("body") + attrs = block.group("attrs") or "" + if kind == "enum": + variants = tuple(re.findall(r'"(\w+)"', body)) + if "Error" in attrs: + errors[name] = variants + else: + enums[name] = variants + elif kind == "dictionary": + records[name] = _parse_fields(body) + elif kind == "callback interface": + callbacks.append(name) + elif kind in ("interface", "namespace"): + key = "namespace" if kind == "namespace" else name + methods[key] = _parse_methods(body) + for expected in ("OfflineProtocol", "MeshServices", "DataStore", "namespace"): + if expected not in methods: + raise ValueError(f"the definition has no {expected} block") + return { + "enums": enums, + "errors": errors, + "records": records, + "callbacks": tuple(callbacks), + "methods": methods, + } + + +def wire_names(table: dict) -> dict[str, tuple[str, str]]: + """Every declaration by its wire name: ``name -> (object, declaration)``.""" + names: dict[str, tuple[str, str]] = {} + for obj, prefix in PREFIXES.items(): + for declaration in table["methods"][obj]: + names[prefix + declaration] = (obj, declaration) + return names + + +def render(table: dict, udl_text: str) -> str: + digest = hashlib.sha256(udl_text.encode("utf-8")).hexdigest() + body = pprint.pformat(table, width=100, sort_dicts=True) + return ( + '"""The interface definition as data, generated. Do not edit.\n' + "\n" + "Written by ``bindings/python/scripts/generate_local_api_table.py`` from\n" + "``crates/offline-protocol-uniffi/src/offline_protocol.udl``. A test\n" + "regenerates it in memory and fails when this file is stale, so every\n" + "change to the definition reaches the reference server as a diff here\n" + "and a classification in ``dispatch.py``.\n" + '"""\n' + "\n" + "from __future__ import annotations\n" + "\n" + f'UDL_SHA256 = "{digest}"\n' + "\n" + f"TABLE = {body}\n" + ) + + +def generate() -> str: + text = UDL.read_text(encoding="utf-8") + return render(parse_udl(text), text) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true", help="fail when the table is stale") + args = parser.parse_args(argv) + rendered = generate() + if args.check: + current = OUTPUT.read_text(encoding="utf-8") if OUTPUT.exists() else "" + if current != rendered: + print(f"{OUTPUT} is stale; run {Path(__file__).name}", file=sys.stderr) + return 1 + return 0 + OUTPUT.write_text(rendered, encoding="utf-8") + print(f"wrote {OUTPUT}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bindings/python/tests/local_api/__init__.py b/bindings/python/tests/local_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bindings/python/tests/local_api/conftest.py b/bindings/python/tests/local_api/conftest.py new file mode 100644 index 000000000..e5b645672 --- /dev/null +++ b/bindings/python/tests/local_api/conftest.py @@ -0,0 +1,205 @@ +"""Fixtures for the local API: a server over a Unix socket in the test's +temporary directory, and a small JSON-RPC client that keeps responses and +notifications apart.""" + +from __future__ import annotations + +import asyncio +import itertools +import json +import os +import shutil +import tempfile +from pathlib import Path +from typing import Any, Callable + +import pytest +from websockets.asyncio.client import connect, unix_connect + +from offline_protocol_sdk.local_api import LocalApiServer, Policy +from offline_protocol_sdk.offline_protocol import OverflowPolicy, ProtocolConfig +from offline_protocol_sdk.protocol_manager import ProtocolManager + + +def make_config(**overrides: Any) -> ProtocolConfig: + """A config with the Internet transport on (the validator refuses a + config with every transport off) and encryption off, so a test never + needs a key exchange.""" + fields: dict[str, Any] = dict( + app_id="server-app", + profile="server-user", + ble_enabled=False, + wifi_direct_enabled=False, + internet_enabled=True, + reticulum_enabled=False, + nostr_enabled=False, + prefer_online=True, + initial_ttl=3, + encryption_enabled=False, + auto_key_exchange=False, + store_pending=True, + require_encryption=False, + max_pending_per_peer=100, + max_pending_global=1000, + pending_ttl_ms=60000, + overflow_policy=OverflowPolicy.DROP_OLDEST, + ) + fields.update(overrides) + return ProtocolConfig(**fields) + + +class RpcClient: + """One connection: ``call`` awaits the matching response; notifications + land in :attr:`events` and can be awaited by tag.""" + + def __init__(self, websocket: Any) -> None: + self._ws = websocket + self._ids = itertools.count(1) + self._pending: dict[Any, asyncio.Future[dict[str, Any]]] = {} + self.events: list[dict[str, Any]] = [] + self._event_waiters: list[tuple[Callable[[dict[str, Any]], bool], asyncio.Future[dict[str, Any]]]] = [] + self.closed: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + self._reader = asyncio.ensure_future(self._read()) + + async def _read(self) -> None: + try: + async for raw in self._ws: + message = json.loads(raw) + if "id" in message: + waiter = self._pending.pop(message["id"], None) + if waiter is not None and not waiter.done(): + waiter.set_result(message) + continue + event = message["params"] + self.events.append(event) + for predicate, future in list(self._event_waiters): + if not future.done() and predicate(event): + future.set_result(event) + self._event_waiters.remove((predicate, future)) + except Exception as exc: # the socket closed + if not self.closed.done(): + self.closed.set_result(exc) + else: + if not self.closed.done(): + self.closed.set_result(None) + + async def send_raw(self, text: str) -> None: + await self._ws.send(text) + + async def call_raw(self, request: dict[str, Any]) -> dict[str, Any]: + future: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future() + self._pending[request["id"]] = future + await self._ws.send(json.dumps(request)) + return await asyncio.wait_for(future, 10) + + async def call(self, method: str, params: dict[str, Any] | None = None) -> Any: + request: dict[str, Any] = {"jsonrpc": "2.0", "id": next(self._ids), "method": method} + if params is not None: + request["params"] = params + response = await self.call_raw(request) + if "error" in response: + raise RpcFailure(response["error"]) + return response["result"] + + async def hello(self, app_id: str, **extra: Any) -> dict[str, Any]: + return await self.call("hello", {"app_id": app_id, **extra}) + + def event_of(self, tag: str, **fields: Any) -> asyncio.Future[dict[str, Any]]: + def matches(event: dict[str, Any]) -> bool: + return event.get("type") == tag and all(event.get(k) == v for k, v in fields.items()) + + future: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future() + for event in self.events: + if matches(event): + future.set_result(event) + return future + self._event_waiters.append((matches, future)) + return future + + async def wait_event(self, tag: str, timeout: float = 10, **fields: Any) -> dict[str, Any]: + return await asyncio.wait_for(self.event_of(tag, **fields), timeout) + + def events_of(self, tag: str) -> list[dict[str, Any]]: + return [e for e in self.events if e.get("type") == tag] + + async def close(self) -> None: + await self._ws.close() + await asyncio.gather(self._reader, return_exceptions=True) + + +class RpcFailure(Exception): + def __init__(self, error: dict[str, Any]) -> None: + super().__init__(error.get("message")) + self.code = error["code"] + self.message = error.get("message") + self.variant = (error.get("data") or {}).get("variant") + + +class ServerHarness: + """Starts servers under a short temporary directory and closes them after. + + Short on purpose: a Unix socket path is limited to about a hundred bytes, + and pytest's own ``tmp_path`` is longer than that on macOS. + """ + + def __init__(self, tmp_path: Any) -> None: + self._tmp = Path(tempfile.mkdtemp(prefix="opsvc-", dir="/tmp" if os.path.isdir("/tmp") else None)) + self._servers: list[LocalApiServer] = [] + self._clients: list[RpcClient] = [] + self._count = itertools.count(1) + + async def server( + self, + *, + config: ProtocolConfig | None = None, + policy: Policy | None = None, + tcp: bool = False, + health: bool = True, + manager: ProtocolManager | None = None, + ) -> LocalApiServer: + n = next(self._count) + if manager is None: + manager = ProtocolManager(config or make_config(profile=f"user-{n}")) + if tcp: + server = LocalApiServer( + manager, + policy=policy, + tcp_port=0, + token_path=self._tmp / f"token-{n}", + health=health, + ) + else: + server = LocalApiServer( + manager, + policy=policy, + socket_path=self._tmp / f"run-{n}" / "api.sock", + health=health, + ) + await server.start() + self._servers.append(server) + return server + + async def client(self, server: LocalApiServer) -> RpcClient: + if server.socket_path is not None: + websocket = await unix_connect(str(server.socket_path), uri="ws://localhost/") + else: + websocket = await connect(f"ws://127.0.0.1:{server.port}/") + client = RpcClient(websocket) + self._clients.append(client) + return client + + async def close(self) -> None: + for client in self._clients: + await client.close() + for server in self._servers: + await server.stop() + shutil.rmtree(self._tmp, ignore_errors=True) + + +@pytest.fixture +async def harness(tmp_path): + h = ServerHarness(tmp_path) + try: + yield h + finally: + await h.close() diff --git a/bindings/python/tests/local_api/test_local_api_codec.py b/bindings/python/tests/local_api/test_local_api_codec.py new file mode 100644 index 000000000..bc21b25ae --- /dev/null +++ b/bindings/python/tests/local_api/test_local_api_codec.py @@ -0,0 +1,99 @@ +"""The encoding table, both directions.""" + +from __future__ import annotations + +import base64 + +import pytest + +from offline_protocol_sdk import offline_protocol as generated +from offline_protocol_sdk.local_api import codec +from offline_protocol_sdk.local_api.codec import RpcError + + +def _refused(type_name, value): + with pytest.raises(RpcError) as err: + codec.decode(type_name, value, "p") + assert err.value.code == codec.INVALID_PARAMS + return err.value.message + + +def test_enums_use_the_definition_spelling_both_ways(): + assert codec.decode("MessagePriority", "Medium", "p") is generated.MessagePriority.MEDIUM + assert codec.decode("TransportType", "WiFiDirect", "p") is generated.TransportType.WI_FI_DIRECT + assert codec.encode("TransportType", generated.TransportType.WI_FI_DIRECT) == "WiFiDirect" + assert codec.encode("ProtocolState", generated.ProtocolState.RUNNING) == "Running" + assert "Low, Medium, High, Critical" in _refused("MessagePriority", "medium") + assert codec.decode("MessagePriority?", None, "p") is None + + +def test_bytes_are_base64_and_sequences_of_u8_become_lists(): + raw = bytes(range(8)) + text = base64.b64encode(raw).decode() + assert codec.decode("bytes", text, "p") == raw + assert codec.decode("sequence", text, "p") == list(raw) + assert codec.encode("sequence", list(raw)) == text + assert codec.encode("bytes", raw) == text + _refused("sequence", "not base64!") + _refused("sequence", [1, 2, 3]) + + +def test_integers_are_checked_for_range_and_kind(): + assert codec.decode("u8", 255, "p") == 255 + _refused("u8", 256) + _refused("u32", -1) + _refused("u64", True) + _refused("i16", 40000) + _refused("string", 5) + _refused("boolean", 1) + assert codec.decode("double", 3, "p") == 3.0 + + +def test_records_take_the_definition_defaults_for_omitted_fields(): + options = codec.decode("SendMessageOptions", {"priority": "High"}, "p") + assert isinstance(options, generated.SendMessageOptions) + assert options.priority is generated.MessagePriority.HIGH + assert options.app_id is None + assert options.reply_to_msg is None + # An optional field with no default is passed as null. + metadata = codec.decode( + "MediaMetadata", + {"mime_type": "image/png", "file_name": "a.png", "file_size": 3}, + "p", + ) + assert metadata.duration_ms is None and metadata.width is None + assert "requires file_size" in _refused( + "MediaMetadata", {"mime_type": "image/png", "file_name": "a.png"} + ) + assert "no field" in _refused("SendMessageOptions", {"priorty": "High"}) + _refused("SendMessageOptions", "not an object") + + +def test_records_encode_field_by_field_with_nested_types(): + info = generated.ForwardInfo( + original_sender="a", original_message_id="m", original_timestamp=5, forward_count=1 + ) + assert codec.encode("ForwardInfo?", info) == { + "original_sender": "a", + "original_message_id": "m", + "original_timestamp": 5, + "forward_count": 1, + } + assert codec.encode("ForwardInfo?", None) is None + assert codec.encode("void", None) is None + assert codec.encode("sequence", ["x"]) == ["x"] + assert codec.encode("record", {"k": "v"}) == {"k": "v"} + + +def test_records_and_maps_decode_recursively(): + assert codec.decode("record", {"k": "v"}, "p") == {"k": "v"} + _refused("record", {"k": 1}) + assert codec.decode("sequence", ["a"], "p") == ["a"] + _refused("sequence", "a") + + +def test_values_without_a_json_form_are_refused_not_crashed(): + assert "no JSON form" in _refused("EventCallback", {}) + with pytest.raises(RpcError) as err: + codec.encode("OfflineProtocol", object()) + assert err.value.code == codec.INTERNAL_ERROR diff --git a/bindings/python/tests/local_api/test_local_api_drain.py b/bindings/python/tests/local_api/test_local_api_drain.py new file mode 100644 index 000000000..365242e3b --- /dev/null +++ b/bindings/python/tests/local_api/test_local_api_drain.py @@ -0,0 +1,85 @@ +"""The drain's return value is never relayed (bridge rule L2). + +``receive_message()`` returns the core ``Message`` as JSON, and the Python +manager synthesises a second ``message_received`` from it in the drain and +hands it to the same handler as the engine's event. On four of the five +carriers the FFI drains inside the inbound entry point, so the manager's +drain sees ``None`` and no copy is made; Nostr, and a message the engine +releases on a later ``process()`` tick, do reach the drain. The server must +drop the copy on every path, so this test feeds the seam directly rather +than depending on which carrier a test happens to run over. +""" + +from __future__ import annotations + +import asyncio +import json +from unittest.mock import MagicMock + +#: What the FFI's `receive_message()` returns for one message: the core +#: `Message` keyed `id`, a capitalised priority, no `transport`. +DRAIN_JSON = json.dumps( + { + "id": "8d1f0a2c-5a5e-4b8e-9c6a-0f1e2d3c4b5a", + "sender": "off1qsender", + "recipient": "off1qrecipient", + "app_id": "notes", + "content": "from the drain", + "timestamp": 1700000000000, + "lamport_clock": 7, + "ttl": 3, + "hop_count": 0, + "priority": "Medium", + "content_type": "text", + } +) + +#: The engine's own event for the same message, as emitted inside the drain. +ENGINE_EVENT = { + "type": "message_received", + "message_id": "8d1f0a2c-5a5e-4b8e-9c6a-0f1e2d3c4b5a", + "sender": "off1qsender", + "recipient": "off1qrecipient", + "content": "from the drain", + "hop_count": 0, + "transport": "nostr", + "timestamp": 1700000000000, + "lamport_clock": 7, + "reply_to_msg": None, + "reply_context": None, + "content_type": "text", + "media_metadata": None, + "forward_info": None, + "encrypted": False, + "app_id": "notes", +} + + +async def test_the_drains_copy_is_dropped_and_the_engines_event_relayed_once(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + manager = server.manager + + # The engine's event, then the drain's copy of the same message, in the + # order the manager produces them on a tick that finds a message. + manager._protocol.receive_message = MagicMock(side_effect=[DRAIN_JSON, None]) + server._on_engine_event(dict(ENGINE_EVENT)) + manager._drain_incoming_messages() + await asyncio.sleep(0.2) + + received = client.events_of("message_received") + assert len(received) == 1, received + assert received[0]["message_id"] == ENGINE_EVENT["message_id"] + assert received[0]["transport"] == "nostr" + assert "id" not in received[0] + + +async def test_the_drains_copy_is_not_held_either(harness): + server = await harness.server() + manager = server.manager + manager._protocol.receive_message = MagicMock(side_effect=[DRAIN_JSON, None]) + manager._drain_incoming_messages() + assert server.router.held_count("notes") == 0 + server._on_engine_event(dict(ENGINE_EVENT)) + assert server.router.held_count("notes") == 1 diff --git a/bindings/python/tests/local_api/test_local_api_e2e.py b/bindings/python/tests/local_api/test_local_api_e2e.py new file mode 100644 index 000000000..6c0659471 --- /dev/null +++ b/bindings/python/tests/local_api/test_local_api_e2e.py @@ -0,0 +1,120 @@ +"""Two servers on one host over the peer-stream transport: a client of one +sends, a client of the other receives, and the routing rules hold across +the mesh.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from offline_protocol_sdk.protocol_manager import ProtocolManager + +from .conftest import RpcFailure, make_config + + +async def until(predicate, timeout: float = 15.0) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while not predicate(): + if loop.time() > deadline: + raise AssertionError("condition not met in time") + await asyncio.sleep(0.05) + + +async def two_servers(harness): + """Two engines over loopback streams: B keeps a stream to A.""" + config_a = make_config(profile="alice", app_id="server-app", wifi_direct_enabled=True, internet_enabled=False) + manager_a = ProtocolManager(config_a) + manager_a.peer_stream.configure(listen_host="127.0.0.1", listen_port=0) + server_a = await harness.server(manager=manager_a) + port_a = manager_a.peer_stream.listen_port + config_b = make_config(profile="bob", app_id="server-app", wifi_direct_enabled=True, internet_enabled=False) + manager_b = ProtocolManager(config_b) + manager_b.peer_stream.configure(listen_host="127.0.0.1", listen_port=0, peers=[f"127.0.0.1:{port_a}"]) + server_b = await harness.server(manager=manager_b) + await until(lambda: manager_a.local_address in manager_b.peer_stream.connected_peers()) + await until(lambda: manager_b.local_address in manager_a.peer_stream.connected_peers()) + return server_a, server_b + + +async def test_a_message_reaches_only_the_application_it_was_sent_from(harness): + server_a, server_b = await two_servers(harness) + notes_b = await harness.client(server_b) + other_b = await harness.client(server_b) + notes_a = await harness.client(server_a) + other_a = await harness.client(server_a) + await notes_b.hello("notes") + await other_b.hello("other") + await notes_a.hello("notes") + await other_a.hello("other") + + message_id = await notes_a.call( + "send_message", + {"recipient": server_b.manager.local_address, "content": "hello over the stream", "priority": "Medium"}, + ) + received = await notes_b.wait_event("message_received", message_id=message_id) + # Stamped with the sending client's application id, not the server's. + assert received["app_id"] == "notes" + assert received["content"] == "hello over the stream" + assert received["sender"] == server_a.manager.local_address + assert received["message_id"] == message_id + assert "transport" in received + # Exactly one `message_received` per message: the drain's copy of the + # same message is never relayed. + await asyncio.sleep(0.5) + assert len(notes_b.events_of("message_received")) == 1 + assert other_b.events_of("message_received") == [] + + # The sender's own events correlate to the sender's client only. + sent = await notes_a.wait_event("message_sent", message_id=message_id) + assert sent["message_id"] == message_id + assert other_a.events_of("message_sent") == [] + delivered = await notes_a.wait_event("message_delivered", message_id=message_id) + assert delivered["message_id"] == message_id + assert other_a.events_of("message_delivered") == [] + + +async def test_a_message_for_an_application_with_no_client_is_held_and_replayed(harness): + server_a, server_b = await two_servers(harness) + other_b = await harness.client(server_b) + await other_b.hello("other") + notes_a = await harness.client(server_a) + await notes_a.hello("notes") + first = await notes_a.call( + "send_message", {"recipient": server_b.manager.local_address, "content": "one", "priority": "High"} + ) + second = await notes_a.call( + "send_message", {"recipient": server_b.manager.local_address, "content": "two", "priority": "High"} + ) + await until(lambda: server_b.router.held_count("notes") == 2) + assert other_b.events_of("message_received") == [] + late = await harness.client(server_b) + hello = await late.hello("notes") + assert hello["state"] == "Running" + replayed = [await late.wait_event("message_received", message_id=m) for m in (first, second)] + assert [e["content"] for e in replayed] == ["one", "two"] + # Held events come after the hello result and before anything newer; + # the client's log shows them first. + assert [e["message_id"] for e in late.events_of("message_received")] == [first, second] + assert server_b.router.held_count("notes") == 0 + + +async def test_send_message_rich_is_stamped_too(harness): + server_a, server_b = await two_servers(harness) + receiver = await harness.client(server_b) + await receiver.hello("mail") + sender = await harness.client(server_a) + await sender.hello("mail") + message_id = await sender.call( + "send_message_rich", + {"recipient": server_b.manager.local_address, "content": "rich", "options": {"priority": "Low"}}, + ) + received = await receiver.wait_event("message_received", message_id=message_id) + assert received["app_id"] == "mail" + with pytest.raises(RpcFailure) as err: + await sender.call( + "send_message_rich", + {"recipient": server_b.manager.local_address, "content": "x", "options": {"app_id": "mail"}}, + ) + assert err.value.variant == "InvalidArgument" diff --git a/bindings/python/tests/local_api/test_local_api_router.py b/bindings/python/tests/local_api/test_local_api_router.py new file mode 100644 index 000000000..15fdf3813 --- /dev/null +++ b/bindings/python/tests/local_api/test_local_api_router.py @@ -0,0 +1,166 @@ +"""The routing rules and the hold, against the router alone.""" + +from __future__ import annotations + +import json + +import pytest + +from offline_protocol_sdk.local_api import ( + HELD_TAGS, + HOLD_CAPACITY, + EventRouter, + Policy, + ServiceOwnership, + Session, +) + + +def drain(session: Session) -> list[dict]: + """Everything queued on a session, as the event objects.""" + out = [] + while not session._queue.empty(): + item = session._queue.get_nowait() + out.append(json.loads(item)["params"]) + return out + + +def attached(router: EventRouter, app_id: str) -> Session: + session = Session("unix") + session.app_id = app_id + router.attach(session) + return session + + +@pytest.fixture +def router(): + return EventRouter(Policy(), ServiceOwnership()) + + +async def test_a_stamped_event_reaches_only_its_application(router): + notes = attached(router, "notes") + other = attached(router, "other") + router.route({"type": "message_received", "message_id": "m1", "app_id": "notes"}) + assert [e["message_id"] for e in drain(notes)] == ["m1"] + assert drain(other) == [] + + +async def test_a_stamped_event_with_no_client_is_held_and_replayed_in_order(router): + other = attached(router, "other") + for i in range(3): + router.route({"type": "message_received", "message_id": f"m{i}", "app_id": "notes"}) + router.route({"type": "file_received", "file_id": "f1", "app_id": "notes"}) + assert drain(other) == [] + assert router.held_count("notes") == 4 + late = Session("unix") + late.app_id = "notes" + held = router.attach(late) + assert [e.get("message_id") or e.get("file_id") for e in held] == ["m0", "m1", "m2", "f1"] + assert router.held_count("notes") == 0 + # A second client of the same application gets nothing from the hold. + second = attached(router, "notes") + assert router.attach(second) == [] + + +async def test_the_hold_drops_the_oldest_past_its_capacity(router): + dropped = [] + router = EventRouter(Policy(), ServiceOwnership(), on_drop=lambda app, e: dropped.append(e["message_id"])) + for i in range(HOLD_CAPACITY + 2): + router.route({"type": "message_received", "message_id": f"m{i}", "app_id": "notes"}) + assert router.held_count("notes") == HOLD_CAPACITY + assert dropped == ["m0", "m1"] + assert router.dropped == 2 + late = Session("unix") + late.app_id = "notes" + held = router.attach(late) + assert held[0]["message_id"] == "m2" + assert held[-1]["message_id"] == f"m{HOLD_CAPACITY + 1}" + + +async def test_only_the_stamped_inbound_tags_are_held(router): + assert HELD_TAGS == {"message_received", "file_received", "media_resend_required"} + # No client at all: a broadcast event is dropped, not held. + router.route({"type": "neighbor_discovered", "peer_id": "p", "transport": "BLE"}) + router.route({"type": "message_decryption_failed", "message_id": "x", "sender": "s"}) + late = attached(router, "notes") + assert router.attach(late) == [] + + +async def test_a_stamped_tag_with_no_app_id_is_broadcast_not_held(router, caplog): + notes = attached(router, "notes") + other = attached(router, "other") + with caplog.at_level("INFO", logger="offline_protocol_sdk.local_api.mux"): + router.route({"type": "file_received", "file_id": "f", "app_id": None}) + assert [e["file_id"] for e in drain(notes)] == ["f"] + assert [e["file_id"] for e in drain(other)] == ["f"] + assert router.held_count("notes") == 0 + assert any("carries no app_id" in r.message for r in caplog.records) + + +async def test_an_event_emitted_inside_a_call_belongs_to_the_caller(router): + notes = attached(router, "notes") + other = attached(router, "other") + router.current_caller = notes + router.route({"type": "message_sent", "message_id": "m9", "sender": "a", "recipient": "b"}) + router.current_caller = None + assert [e["type"] for e in drain(notes)] == ["message_sent"] + assert drain(other) == [] + # The id it named is now the caller's: a later event correlates. + router.route({"type": "message_delivered", "message_id": "m9"}) + assert [e["type"] for e in drain(notes)] == ["message_delivered"] + assert drain(other) == [] + + +async def test_results_the_server_handed_out_correlate_later_events(router): + notes = attached(router, "notes") + other = attached(router, "other") + router.note_ids("notes", ["m1"]) + router.route({"type": "message_failed", "message_id": "m1", "reason": "x", "retry_count": 1}) + router.route({"type": "file_progress", "file_id": "unknown", "chunks_sent": 1}) + assert [e["type"] for e in drain(notes)] == ["message_failed", "file_progress"] + assert [e["type"] for e in drain(other)] == ["file_progress"] + + +async def test_a_service_request_reaches_the_owning_application(router): + ownership = ServiceOwnership() + router = EventRouter(Policy(), ownership) + notes = attached(router, "notes") + other = attached(router, "other") + ownership.claim("svc", "notes") + router.route({"type": "service_request_received", "request_id": "r", "service_id": "svc"}) + router.route({"type": "service_request_received", "request_id": "r2", "service_id": "nobodys"}) + assert [e["request_id"] for e in drain(notes)] == ["r", "r2"] + assert [e["request_id"] for e in drain(other)] == ["r2"] + + +async def test_the_subscription_filter_narrows_delivery_without_changing_routing(router): + notes = attached(router, "notes") + notes.subscribe(["message_delivered"]) + router.route({"type": "message_received", "message_id": "m1", "app_id": "notes"}) + router.route({"type": "neighbor_lost", "peer_id": "p"}) + assert drain(notes) == [] + notes.subscribe("all") + router.route({"type": "neighbor_lost", "peer_id": "p"}) + assert [e["type"] for e in drain(notes)] == ["neighbor_lost"] + notes.unsubscribe("all") + router.route({"type": "neighbor_lost", "peer_id": "p"}) + assert drain(notes) == [] + + +async def test_document_events_are_filtered_by_the_space_allow_list(): + policy = Policy(spaces={"notes": ["notes-*"]}) + router = EventRouter(policy, ServiceOwnership()) + notes = attached(router, "notes") + other = attached(router, "other") + router.route({"type": "data_changed", "space_id": "notes-1", "doc_id": "d", "delta_bytes": 1}) + router.route({"type": "data_changed", "space_id": "mail-1", "doc_id": "d", "delta_bytes": 1}) + assert [e["space_id"] for e in drain(notes)] == ["notes-1"] + assert [e["space_id"] for e in drain(other)] == ["notes-1", "mail-1"] + + +async def test_detach_stops_delivery(router): + notes = attached(router, "notes") + router.detach(notes) + router.route({"type": "message_received", "message_id": "m1", "app_id": "notes"}) + assert drain(notes) == [] + assert router.held_count("notes") == 1 diff --git a/bindings/python/tests/local_api/test_local_api_server.py b/bindings/python/tests/local_api/test_local_api_server.py new file mode 100644 index 000000000..be3be217c --- /dev/null +++ b/bindings/python/tests/local_api/test_local_api_server.py @@ -0,0 +1,453 @@ +"""The server over a real engine: the session, the framing, the refusals, +the carriers and the three server-side rules.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import stat + +import pytest +import websockets +from websockets.asyncio.client import connect + +from offline_protocol_sdk.local_api import ( + API_VERSION, + MAX_MESSAGE_SIZE, + SERVER_NAME, + LocalApiServer, + Policy, +) +from offline_protocol_sdk.local_api import codec +from offline_protocol_sdk.local_api.server import FILE_SIZE_LIMIT +from offline_protocol_sdk.protocol_manager import ProtocolManager + +from .conftest import RpcFailure, make_config + + +def code_of(variant: str) -> int: + return codec.taxonomy_error(variant, "").code + + +# -- the session --------------------------------------------------------------- + + +async def test_hello_returns_the_server_and_the_shared_identity(harness): + server = await harness.server() + client = await harness.client(server) + result = await client.hello("notes", client="tests/1") + assert result["api_version"] == API_VERSION + assert result["server"]["name"] == SERVER_NAME + assert result["state"] == "Running" + assert result["local_address"] == server.manager.local_address + assert result["local_address"].startswith("off1") + second = await harness.client(server) + assert (await second.hello("other"))["local_address"] == result["local_address"] + + +async def test_methods_before_hello_are_invalid_state_except_the_instance_less_ones(harness): + server = await harness.server() + client = await harness.client(server) + with pytest.raises(RpcFailure) as err: + await client.call("get_state") + assert err.value.variant == "InvalidState" and err.value.code == code_of("InvalidState") + with pytest.raises(RpcFailure) as err: + await client.call("subscribe", {"types": "all"}) + assert err.value.variant == "InvalidState" + key = base64.b64encode(bytes(32)).decode() + # Reached the engine, not the session gate. + assert (await client.call("derive_address", {"public_key": key})).startswith("off1") + with pytest.raises(RpcFailure) as err: + await client.call("parse_invite", {"blob": "not an invite"}) + assert err.value.variant == "MlsError" + with pytest.raises(RpcFailure) as err: + await client.call("verify_identity_assertion", {"assertion": base64.b64encode(b"abc").decode()}) + assert err.value.variant == "MlsError" + + +async def test_a_second_hello_and_a_bad_app_id_are_refused(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + with pytest.raises(RpcFailure) as err: + await client.hello("notes") + assert err.value.variant == "InvalidState" + for bad in ("", ".", "..", "a/b", "a:b", "a\\b", "a\x01b", "x" * 257): + fresh = await harness.client(server) + with pytest.raises(RpcFailure) as err: + await fresh.hello(bad) + assert err.value.variant == "InvalidArgument", bad + with pytest.raises(RpcFailure) as err: + await (await harness.client(server)).call("hello", {"app_id": "x", "extra": 1}) + assert err.value.code == codec.INVALID_PARAMS + + +# -- framing ------------------------------------------------------------------- + + +async def test_framing_refusals_use_the_standard_codes(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + positional = await client.call_raw({"jsonrpc": "2.0", "id": 7, "method": "get_state", "params": []}) + assert positional["error"]["code"] == codec.INVALID_PARAMS + missing = await client.call_raw({"jsonrpc": "2.0", "id": 8, "method": "send_message", "params": {"recipient": "x"}}) + assert missing["error"]["code"] == codec.INVALID_PARAMS and "requires content" in missing["error"]["message"] + unknown_param = await client.call_raw({"jsonrpc": "2.0", "id": 9, "method": "get_state", "params": {"x": 1}}) + assert unknown_param["error"]["code"] == codec.INVALID_PARAMS + wrong_type = await client.call_raw( + {"jsonrpc": "2.0", "id": 10, "method": "send_message", "params": {"recipient": "x", "content": "c", "priority": 3}} + ) + assert wrong_type["error"]["code"] == codec.INVALID_PARAMS + with pytest.raises(RpcFailure) as err: + await client.call("no_such_method") + assert err.value.code == codec.METHOD_NOT_FOUND + # A platform operation gets the same answer as an unknown name. + for platform_op in ("process", "receive_message", "stop", "data.wipe_all", "ble_peer_lost"): + with pytest.raises(RpcFailure) as err: + await client.call(platform_op) + assert err.value.code == codec.METHOD_NOT_FOUND, platform_op + + +async def test_parse_errors_batches_and_notifications(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + waiter = client.call_raw({"jsonrpc": "2.0", "id": "after", "method": "get_state"}) + await client.send_raw("{not json") + await client.send_raw(json.dumps([{"jsonrpc": "2.0", "id": 1, "method": "get_state"}])) + await client.send_raw(json.dumps({"jsonrpc": "1.0", "id": 2, "method": "get_state"})) + await client.send_raw(json.dumps({"jsonrpc": "2.0", "method": "get_state"})) # a notification: no answer + assert (await waiter)["result"] == "Running" + # The three answers with a null id arrived as events did not: they are + # responses, so the client saw them as responses with id null. + # (The harness client keys responses by id; a null id has no waiter, so + # they are simply dropped there.) Read them directly instead: + raw = await connect_raw(server) + try: + await raw.send("{") + assert json.loads(await raw.recv())["error"]["code"] == codec.PARSE_ERROR + await raw.send("[]") + assert json.loads(await raw.recv())["error"]["code"] == codec.INVALID_REQUEST + await raw.send(json.dumps({"jsonrpc": "2.0", "id": True, "method": "x"})) + assert json.loads(await raw.recv())["error"]["code"] == codec.INVALID_REQUEST + await raw.send(json.dumps({"jsonrpc": "2.0", "id": 1, "method": 5})) + assert json.loads(await raw.recv())["error"]["code"] == codec.INVALID_REQUEST + finally: + await raw.close() + + +async def connect_raw(server: LocalApiServer): + from websockets.asyncio.client import unix_connect + + return await unix_connect(str(server.socket_path), uri="ws://localhost/") + + +async def test_a_one_shot_call_is_one_connection(harness): + server = await harness.server() + for _ in range(3): + raw = await connect_raw(server) + await raw.send(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "hello", "params": {"app_id": "shot"}})) + assert "result" in json.loads(await raw.recv()) + await raw.send(json.dumps({"jsonrpc": "2.0", "id": 2, "method": "local_address"})) + assert json.loads(await raw.recv())["result"].startswith("off1") + await raw.close() + # The server's side of a closed connection is detached on its next turn. + for _ in range(100): + if not server.router.sessions_for("shot"): + break + await asyncio.sleep(0.02) + assert server.router.sessions_for("shot") == [] + + +async def test_the_frame_limit_covers_the_engine_file_limit_as_base64(): + assert FILE_SIZE_LIMIT == 100 * 1024 * 1024 + assert MAX_MESSAGE_SIZE >= FILE_SIZE_LIMIT * 4 // 3 + + +# -- errors -------------------------------------------------------------------- + + +async def test_engine_errors_carry_the_variant_and_the_positional_code(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + with pytest.raises(RpcFailure) as err: + await client.call("get_member_role", {"group_id": "nope", "user_id": "x"}) + assert err.value.variant == "GroupNotFound" + assert err.value.code == code_of("GroupNotFound") == -32017 + assert "nope" in err.value.message + # An optional result is null, not an error. + assert await client.call("get_group_info", {"group_id": "nope"}) is None + + +async def test_send_message_rich_refuses_a_client_supplied_app_id(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + with pytest.raises(RpcFailure) as err: + await client.call( + "send_message_rich", + {"recipient": "off1zzzz", "content": "hi", "options": {"app_id": "other"}}, + ) + assert err.value.variant == "InvalidArgument" + + +# -- subscriptions ------------------------------------------------------------- + + +async def test_subscribe_and_unsubscribe(harness): + server = await harness.server() + client = await harness.client(server) + await client.hello("notes") + assert await client.call("subscribe", {"types": ["message_received", "not_a_tag_yet"]}) is True + assert await client.call("unsubscribe", {"types": ["message_received"]}) is True + assert await client.call("subscribe", {"types": "all"}) is True + with pytest.raises(RpcFailure) as err: + await client.call("subscribe", {"types": "some"}) + assert err.value.code == codec.INVALID_PARAMS + + +# -- the carriers -------------------------------------------------------------- + + +async def test_the_unix_socket_is_owner_only(harness): + server = await harness.server() + mode = stat.S_IMODE(os.stat(server.socket_path).st_mode) + assert mode == 0o600 + assert stat.S_IMODE(os.stat(server.socket_path.parent).st_mode) == 0o700 + + +async def test_tcp_requires_the_per_launch_token(harness): + server = await harness.server(tcp=True) + assert server.token and len(server.token) == 64 + assert stat.S_IMODE(os.stat(server.token_path).st_mode) == 0o600 + assert server.token_path.read_text().strip() == server.token + # Without the token: PermissionDenied, then the connection closes 1008. + ws = await connect(f"ws://127.0.0.1:{server.port}/") + await ws.send(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "hello", "params": {"app_id": "a"}})) + reply = json.loads(await ws.recv()) + assert reply["error"]["data"]["variant"] == "PermissionDenied" + with pytest.raises(websockets.exceptions.ConnectionClosed) as closed: + await ws.recv() + assert closed.value.rcvd.code == 1008 + # A wrong token, the same. + ws = await connect(f"ws://127.0.0.1:{server.port}/") + await ws.send(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "hello", "params": {"app_id": "a", "token": "0" * 64}})) + assert json.loads(await ws.recv())["error"]["data"]["variant"] == "PermissionDenied" + with pytest.raises(websockets.exceptions.ConnectionClosed): + await ws.recv() + # The token from the file works. + client = await harness.client(server) + result = await client.hello("a", token=server.token_path.read_text().strip()) + assert result["state"] == "Running" + + +async def test_tcp_binds_loopback_only(): + manager = ProtocolManager(make_config(profile="tcp-only")) + with pytest.raises(ValueError): + LocalApiServer(manager, tcp_port=0, tcp_host="0.0.0.0", token_path="/tmp/x") + with pytest.raises(ValueError): + LocalApiServer(manager, tcp_port=0) + with pytest.raises(ValueError): + LocalApiServer(manager) + + +async def test_health_is_served_on_a_plain_get_and_nothing_else_is_http(harness): + server = await harness.server(tcp=True) + reader, writer = await asyncio.open_connection("127.0.0.1", server.port) + writer.write(b"GET /health HTTP/1.1\r\nHost: localhost\r\n\r\n") + await writer.drain() + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5) + assert head.startswith(b"HTTP/1.1 200") + assert b"application/json" in head.lower() + body = json.loads(await asyncio.wait_for(reader.readline(), 5)) + assert body == {"server": {"name": SERVER_NAME, "version": server.version}, "api_version": API_VERSION, "carrier": "tcp"} + writer.close() + # Another path: the library's default answer. + reader, writer = await asyncio.open_connection("127.0.0.1", server.port) + writer.write(b"GET /other HTTP/1.1\r\nHost: localhost\r\n\r\n") + await writer.drain() + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5) + assert head.startswith(b"HTTP/1.1 426") + writer.close() + + +async def test_a_post_receives_nothing_at_all(harness): + """Pins the library fact the chapter's "Why there is no HTTP" rests on, + verified on websockets 16.1 (the lock; pyproject accepts >=12,<17). A + version that starts answering a POST turns this red, which is the + moment to revisit the chapter rather than to relax this test.""" + assert websockets.__version__.split(".")[0] == "16", websockets.__version__ + server = await harness.server(tcp=True) + reader, writer = await asyncio.open_connection("127.0.0.1", server.port) + body = b'{"jsonrpc":"2.0","id":1,"method":"local_address"}' + writer.write( + b"POST /rpc HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\n" + + b"Content-Length: %d\r\n\r\n" % len(body) + + body + ) + await writer.drain() + data = await asyncio.wait_for(reader.read(), 5) + assert data == b"" + writer.close() + + +async def test_health_can_be_turned_off(harness): + server = await harness.server(tcp=True, health=False) + reader, writer = await asyncio.open_connection("127.0.0.1", server.port) + writer.write(b"GET /health HTTP/1.1\r\nHost: localhost\r\n\r\n") + await writer.drain() + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 5) + assert head.startswith(b"HTTP/1.1 426") + writer.close() + + +# -- the server-side rules ----------------------------------------------------- + + +async def test_service_ownership_is_by_application_id(harness): + server = await harness.server() + a = await harness.client(server) + b = await harness.client(server) + a2 = await harness.client(server) + await a.hello("app-a") + await b.hello("app-b") + await a2.hello("app-a") + await a.call("services.register_service", {"service_id": "printer", "version": "1", "capabilities": {"k": "v"}}) + with pytest.raises(RpcFailure) as err: + await b.call("services.unregister_service", {"service_id": "printer"}) + assert err.value.variant == "PermissionDenied" + with pytest.raises(RpcFailure) as err: + await b.call("services.register_service", {"service_id": "printer", "version": "2", "capabilities": {}}) + assert err.value.variant == "PermissionDenied" + with pytest.raises(RpcFailure) as err: + await b.call( + "services.respond_to_service_request", + {"request_id": "r", "requester": "off1x", "service_id": "printer", "status": "ok", "body": "{}"}, + ) + assert err.value.variant == "PermissionDenied" + # Another client of the same application may release it, and a rival + # may claim the id once it is free. + assert await a2.call("services.unregister_service", {"service_id": "printer"}) is True + await b.call("services.register_service", {"service_id": "printer", "version": "2", "capabilities": {}}) + + +async def test_a_refused_registration_leaves_no_claim(harness): + server = await harness.server() + a = await harness.client(server) + b = await harness.client(server) + await a.hello("app-a") + await b.hello("app-b") + with pytest.raises(RpcFailure) as err: + await a.call("services.register_service", {"service_id": "", "version": "1", "capabilities": {}}) + assert err.value.variant == "InvalidConfiguration" + # `b` is not blocked by a claim the engine never backed. + with pytest.raises(RpcFailure) as err: + await b.call("services.register_service", {"service_id": "", "version": "1", "capabilities": {}}) + assert err.value.variant == "InvalidConfiguration" + + +async def test_service_ownership_never_turns_admission_into_default_deny(harness): + server = await harness.server() + for app_id in ("app-a", "app-b", "app-c"): + client = await harness.client(server) + await client.hello(app_id) + await client.call( + "services.register_service", {"service_id": f"svc-{app_id}", "version": "1", "capabilities": {}} + ) + # Ownership is a runtime shadow, not a configured rule: with no rules + # configured, an id nobody has seen is still admitted. + newcomer = await harness.client(server) + assert (await newcomer.hello("never-seen"))["api_version"] == API_VERSION + with pytest.raises(RpcFailure) as err: + await newcomer.call("services.unregister_service", {"service_id": "svc-app-a"}) + assert err.value.variant == "PermissionDenied" + + +async def test_space_scoping_gates_calls_and_filters_the_listing(harness): + policy = Policy(spaces={"notes": ["notes-*"]}, applications=["mail"]) + server = await harness.server(config=make_config(profile="scoped", data_enabled=True), policy=policy) + notes = await harness.client(server) + mail = await harness.client(server) + await notes.hello("notes") + await mail.hello("mail") + await notes.call("data.create_doc", {"space_id": "notes-1", "doc_id": "todo"}) + with pytest.raises(RpcFailure) as err: + await notes.call("data.create_doc", {"space_id": "mail-1", "doc_id": "inbox"}) + assert err.value.variant == "PermissionDenied" + await mail.call("data.create_doc", {"space_id": "mail-1", "doc_id": "inbox"}) + assert await notes.call("data.list_spaces") == ["notes-1"] + assert sorted(await mail.call("data.list_spaces")) == ["mail-1", "notes-1"] + with pytest.raises(RpcFailure) as err: + await notes.call("data.doc_json", {"space_id": "mail-1", "doc_id": "inbox"}) + assert err.value.variant == "PermissionDenied" + + +async def test_data_calls_answer_with_the_engine_refusal_when_the_layer_is_off(harness): + server = await harness.server(config=make_config(profile="nodata", data_enabled=False)) + client = await harness.client(server) + await client.hello("notes") + with pytest.raises(RpcFailure) as err: + await client.call("data.list_spaces") + assert err.value.variant == "DataDisabled" + + +async def test_method_groups_are_denied_by_application_id(harness): + policy = Policy(denied={"kiosk": ["sign_data", "tuning", "data.flush_all"]}, applications=["admin"]) + server = await harness.server(policy=policy) + kiosk = await harness.client(server) + admin = await harness.client(server) + await kiosk.hello("kiosk") + await admin.hello("admin") + for method, params in ( + ("sign_data", {"data": base64.b64encode(b"x").decode()}), + ("release_transport_lock", None), + ("data.flush_all", None), + ): + with pytest.raises(RpcFailure) as err: + await kiosk.call(method, params) + assert err.value.variant == "PermissionDenied", method + assert err.value.code != codec.METHOD_NOT_FOUND + # Not denied elsewhere: the admin signs. + signature = await admin.call("sign_data", {"data": base64.b64encode(b"x").decode()}) + assert len(base64.b64decode(signature)) == 64 + assert await admin.call("release_transport_lock") is None + + +async def test_once_rules_exist_an_unnamed_application_id_is_refused(harness): + policy = Policy(denied={"kiosk": ["sign_data"]}) + server = await harness.server(policy=policy) + kiosk = await harness.client(server) + await kiosk.hello("kiosk") + with pytest.raises(RpcFailure) as err: + await kiosk.call("sign_data", {"data": base64.b64encode(b"x").decode()}) + assert err.value.variant == "PermissionDenied" + # Reconnecting under a name no rule mentions does not step around it. + again = await harness.client(server) + with pytest.raises(RpcFailure) as err: + await again.hello("kiosk-2") + assert err.value.variant == "PermissionDenied" + assert server.router.sessions_for("kiosk-2") == [] + + +async def test_with_no_rules_every_application_id_is_admitted(harness): + server = await harness.server() + for app_id in ("anything", "at-all", "x" * 256): + client = await harness.client(server) + assert (await client.hello(app_id))["api_version"] == API_VERSION + + +async def test_policy_from_dict_refuses_unknown_sections_and_names(): + with pytest.raises(ValueError): + Policy.from_dict({"nope": {}}) + with pytest.raises(ValueError): + Policy.from_dict({"denied": {"a": ["not a method!"]}}) + policy = Policy.from_dict({"spaces": {"a": ["a-*"]}, "denied": {"b": ["tuning"]}, "applications": ["c"]}) + assert policy.admits("a") and policy.admits("b") and policy.admits("c") and not policy.admits("d") + assert Policy.from_dict(None).admits("anyone") + # A list of applications on its own configures no rule, so it restricts nothing. + assert Policy.from_dict({"applications": ["c"]}).admits("anyone") + assert policy.denies("b", "force_transport") and not policy.denies("a", "force_transport") diff --git a/bindings/python/tests/local_api/test_local_api_table.py b/bindings/python/tests/local_api/test_local_api_table.py new file mode 100644 index 000000000..9451744ec --- /dev/null +++ b/bindings/python/tests/local_api/test_local_api_table.py @@ -0,0 +1,121 @@ +"""The method table: fresh against the definition, classified in full, and +in agreement with the chapter. + +The Rust guard in the FFI crate holds the same three claims from the other +side; this is the copy that runs with the Python suite alone. +""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path + +import pytest + +from offline_protocol_sdk.local_api import ( + BEFORE_HELLO, + EXPOSED, + ID_RESULTS, + METHOD_GROUPS, + PLATFORM, + SESSION_METHODS, +) +from offline_protocol_sdk.local_api import codec, dispatch +from offline_protocol_sdk.local_api.table import TABLE, UDL_SHA256 +from offline_protocol_sdk.offline_protocol import ProtocolError + +PACKAGE = Path(__file__).resolve().parents[2] +REPO = PACKAGE.parents[1] +GENERATOR = PACKAGE / "scripts" / "generate_local_api_table.py" +CHAPTER = REPO / "docs" / "spec" / "local-api.md" + + +def _generator(): + spec = importlib.util.spec_from_file_location("generate_local_api_table", GENERATOR) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +@pytest.mark.skipif(not GENERATOR.exists(), reason="the generator ships with the repository only") +def test_table_is_fresh_against_the_definition(): + generator = _generator() + if not generator.UDL.exists(): + pytest.skip("the interface definition ships with the repository only") + rendered = generator.generate() + current = generator.OUTPUT.read_text(encoding="utf-8") + assert rendered == current, "table.py is stale: run bindings/python/scripts/generate_local_api_table.py" + assert UDL_SHA256 in rendered + + +def test_every_declaration_is_classified_exactly_once(): + names = dispatch.all_wire_names() + assert EXPOSED | PLATFORM == names, { + "unclassified": sorted(names - EXPOSED - PLATFORM), + "not in the definition": sorted((EXPOSED | PLATFORM) - names), + } + assert not (EXPOSED & PLATFORM) + assert len(EXPOSED) + len(PLATFORM) == len(names) + + +def test_session_methods_are_not_declarations(): + assert not (SESSION_METHODS & dispatch.all_wire_names()) + assert BEFORE_HELLO <= EXPOSED + assert ID_RESULTS <= EXPOSED + for group in METHOD_GROUPS.values(): + assert group <= EXPOSED + + +def test_run_loop_and_drain_are_never_on_the_wire(): + for name in ("process", "receive_message", "start", "stop", "pause", "resume", "constructor"): + assert name in PLATFORM + assert name not in EXPOSED + + +def test_error_codes_are_zero_based_positions_in_the_definition(): + variants = TABLE["errors"]["ProtocolError"] + assert variants[0] == "NotStarted" + assert codec.taxonomy_error("NotStarted", "x").code == -32000 + assert codec.taxonomy_error("PermissionDenied", "x").code == -32000 - variants.index("PermissionDenied") + assert codec.taxonomy_error(variants[-1], "x").code == -32000 - (len(variants) - 1) + # Every generated error class maps, and by name, not by the binding's + # own (one-based) discriminant. + for position, variant in enumerate(variants): + exc = getattr(ProtocolError, variant)("detail") + error = codec.error_from_protocol(exc) + assert error.code == -32000 - position + assert error.data == {"variant": variant} + assert error.message == "detail" + + +def _chapter_sets() -> tuple[set[str], set[str], set[str]]: + text = CHAPTER.read_text(encoding="utf-8") + + def section(start: str, end: str) -> str: + return text.split(start, 1)[1].split(end, 1)[0] + + exposed = { + line.split("`")[1] + for line in section("## Method table", "## Platform operations").splitlines() + if line.startswith("| `") + } + platform: set[str] = set() + for line in section("## Platform operations", "## Event catalogue").splitlines(): + if line.startswith("| `"): + platform.update(re.findall(r"`([^`]+)`", line.split("|")[1])) + tags = { + line.split("`")[1] + for line in section("### The catalogue", "### Shapes").splitlines() + if line.startswith("| `") + } + return exposed, platform, tags + + +@pytest.mark.skipif(not CHAPTER.exists(), reason="the chapter ships with the repository only") +def test_dispatch_agrees_with_the_chapter(): + exposed, platform, tags = _chapter_sets() + assert exposed == EXPOSED, {"chapter only": sorted(exposed - EXPOSED), "server only": sorted(EXPOSED - exposed)} + assert platform == PLATFORM, {"chapter only": sorted(platform - PLATFORM), "server only": sorted(PLATFORM - platform)} + assert len(tags) >= 60 diff --git a/crates/offline-protocol-uniffi/src/lib.rs b/crates/offline-protocol-uniffi/src/lib.rs index e0b5d276a..5f5370d3f 100644 --- a/crates/offline-protocol-uniffi/src/lib.rs +++ b/crates/offline-protocol-uniffi/src/lib.rs @@ -18970,4 +18970,245 @@ mod tests { ); } } + + /// The local API chapter, the interface definition and the reference + /// server's dispatch table name every declaration exactly once, and the + /// chapter's catalogue names every event tag the engine can emit + /// (docs/bridges/local-api.md, rule L1). + /// + /// Three claims that rot silently: a method added to the definition and + /// to neither table is one no client can find, or one the reference + /// server exposes without the contract saying so; an event added to the + /// engine and not to the catalogue is one no client is written for. + /// The chapter states the row shapes this reads; changing one of its + /// headings is a change to this guard. + #[test] + fn local_api_tables_partition_the_definition() { + use std::collections::BTreeSet; + + let manifest = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let sources = [ + manifest.join("../../docs/spec/local-api.md"), + manifest.join("../../bindings/python/offline_protocol_sdk/local_api/dispatch.py"), + manifest.join("../offline-protocol/src/events.rs"), + ]; + let mut texts = Vec::new(); + for path in &sources { + // The guard applies in the repo checkout; skip when the docs or + // bindings tree is not present (a vendored crate). + let Ok(text) = std::fs::read_to_string(path) else { + eprintln!( + "repository tree not present, skipping the local API guard for {}", + path.display() + ); + return; + }; + texts.push(text); + } + let (chapter, dispatch, events_rs) = (&texts[0], &texts[1], &texts[2]); + let udl = include_str!("offline_protocol.udl"); + + // The definition, by wire name: engine and namespace methods bare, + // `services.` and `data.` for the two objects. + let stripped: String = udl + .lines() + .map(|l| l.split("//").next().unwrap_or("")) + .collect::>() + .join("\n"); + let mut udl_names = BTreeSet::new(); + for (opener, prefix) in [ + ("interface OfflineProtocol {", ""), + ("interface MeshServices {", "services."), + ("interface DataStore {", "data."), + ("namespace offline_protocol {", ""), + ] { + let body = stripped + .split_once(opener) + .unwrap_or_else(|| panic!("{opener} missing from the UDL")) + .1 + .split_once("\n};") + .expect("unterminated block") + .0; + for statement in body.split(';') { + let mut rest = statement.trim(); + if rest.is_empty() { + continue; + } + let mut named = None; + if let Some(after) = rest.strip_prefix('[') { + let (attrs, tail) = after.split_once(']').expect("unterminated attribute"); + if let Some(name) = attrs.split("Name=").nth(1) { + named = Some(name.trim().split(',').next().unwrap().trim().to_string()); + } + rest = tail.trim(); + } + let head = rest.split('(').next().expect("a declaration").trim(); + let name = if head == "constructor" { + named.unwrap_or_else(|| "constructor".to_string()) + } else { + head.rsplit(char::is_whitespace).next().unwrap().to_string() + }; + assert!( + udl_names.insert(format!("{prefix}{name}")), + "{prefix}{name} declared twice" + ); + } + } + assert!( + udl_names.len() > 200, + "UDL scan looks broken: {}", + udl_names.len() + ); + + // The chapter's two method tables and its catalogue. + let section = |start: &str, end: &str| -> &str { + chapter + .split_once(start) + .unwrap_or_else(|| panic!("heading {start:?} missing from the chapter")) + .1 + .split_once(end) + .unwrap_or_else(|| panic!("heading {end:?} missing from the chapter")) + .0 + }; + let first_cell = |line: &str| -> String { + line.trim_start_matches('|') + .split('|') + .next() + .unwrap() + .to_string() + }; + let backticked = |cell: &str| -> Vec { + cell.split('`') + .skip(1) + .step_by(2) + .map(str::to_string) + .collect() + }; + let chapter_exposed: BTreeSet = + section("## Method table", "## Platform operations") + .lines() + .filter(|l| l.starts_with("| `")) + .map(|l| backticked(&first_cell(l))[0].clone()) + .collect(); + let chapter_platform: BTreeSet = + section("## Platform operations", "## Event catalogue") + .lines() + .filter(|l| l.starts_with("| `")) + .flat_map(|l| backticked(&first_cell(l))) + .collect(); + let catalogue: BTreeSet = section("### The catalogue", "### Shapes") + .lines() + .filter(|l| l.starts_with("| `")) + .map(|l| backticked(&first_cell(l))[0].clone()) + .collect(); + + let both: Vec<_> = chapter_exposed.intersection(&chapter_platform).collect(); + assert!(both.is_empty(), "in both chapter tables: {both:?}"); + let classified: BTreeSet = + chapter_exposed.union(&chapter_platform).cloned().collect(); + let unclassified: Vec<_> = udl_names.difference(&classified).collect(); + assert!( + unclassified.is_empty(), + "declarations in neither chapter table (classify each as exposed or platform): {unclassified:?}" + ); + let phantom: Vec<_> = classified.difference(&udl_names).collect(); + assert!( + phantom.is_empty(), + "chapter names not in the definition: {phantom:?}" + ); + + // The reference server's classification is the chapter's. + let quoted = |marker: &str| -> BTreeSet { + dispatch + .split_once(marker) + .unwrap_or_else(|| panic!("{marker:?} missing from dispatch.py")) + .1 + .split_once("\n)") + .expect("unterminated set") + .0 + .split('"') + .skip(1) + .step_by(2) + .map(str::to_string) + .collect() + }; + let dispatch_exposed = quoted("EXPOSED: frozenset[str] = frozenset("); + let dispatch_platform = quoted("PLATFORM: frozenset[str] = frozenset("); + assert_eq!( + dispatch_exposed, chapter_exposed, + "dispatch.py EXPOSED differs from the chapter's method table" + ); + assert_eq!( + dispatch_platform, chapter_platform, + "dispatch.py PLATFORM differs from the chapter's platform table" + ); + + // Every `Event` variant is a catalogue row, and every row a variant. + // The scan is the one `react_native_types_cover_all_event_variants` + // uses: variant names at 4-space indent inside `pub enum Event`. + let snake = |name: &str| -> String { + let mut out = String::new(); + for (i, ch) in name.chars().enumerate() { + if ch.is_ascii_uppercase() { + if i > 0 { + out.push('_'); + } + out.push(ch.to_ascii_lowercase()); + } else { + out.push(ch); + } + } + out + }; + let mut rust_tags = BTreeSet::new(); + let mut in_enum = false; + let mut depth = 0usize; + for line in events_rs.lines() { + if !in_enum { + if line.starts_with("pub enum Event {") { + in_enum = true; + depth = 1; + } + continue; + } + let trimmed = line.trim(); + if trimmed.starts_with("//") || trimmed.starts_with("#[") { + continue; + } + if depth == 1 + && line.starts_with(" ") + && !line.starts_with(" ") + && trimmed + .chars() + .next() + .is_some_and(|c| c.is_ascii_uppercase()) + { + let name: String = trimmed + .chars() + .take_while(|c| c.is_ascii_alphanumeric()) + .collect(); + rust_tags.insert(snake(&name)); + } + depth += line.matches('{').count(); + depth = depth.saturating_sub(line.matches('}').count()); + if depth == 0 { + break; + } + } + assert!( + rust_tags.len() >= 60, + "enum scan looks broken: {}", + rust_tags.len() + ); + let missing: Vec<_> = rust_tags.difference(&catalogue).collect(); + assert!( + missing.is_empty(), + "event tags missing from the chapter's catalogue: {missing:?}" + ); + let extra: Vec<_> = catalogue.difference(&rust_tags).collect(); + assert!( + extra.is_empty(), + "catalogue rows that are not Event variants: {extra:?}" + ); + } } diff --git a/docs/bridges/local-api.md b/docs/bridges/local-api.md index 3f8baad01..ac48d311b 100644 --- a/docs/bridges/local-api.md +++ b/docs/bridges/local-api.md @@ -119,3 +119,7 @@ application does. | Dispatch table | Checked in, classified against the two method tables, and read by the guard in L1 | | Hold | Per application id, 256 entries, oldest dropped, each drop logged | | Health | `GET /health` through the library's request hook, with the body the chapter shows; nothing else on HTTP | +| Policy | One JSON file (`--policy`): `spaces` (application id to glob patterns), `denied` (application id to method groups `sign_data`, `manual_mls`, `tuning`, or single wire names), and `applications`, ids with no rule of their own that are still admitted once a rule exists. Either of the first two turns on the unlisted-id rule; the third alone configures nothing, and service ownership never counts | +| The drain | The Python manager hands the server both the engine's `message_received` event and the drain's synthesised copy of the same message; the server drops the copy by its shape (no `message_id`, because the drain's JSON keys the id `id`) and relays the engine's. On four of the five carriers the FFI drains inside its inbound entry point, so the manager's drain sees `None` and no copy is made; Nostr, and a message the engine releases on a later `process()` tick, do reach the drain. The seam is therefore pinned directly: a test feeds the server the engine's event and then the drain's copy of the same message and asserts one `message_received` reaches the client, and a test over two servers asserts the same over the peer stream | +| Token file | Written with its mode in one `open` (`O_CREAT` and `O_EXCL`, `0600`), never created and then narrowed, so the token is never readable by anyone else for an instant; a stale file from an earlier launch is removed first | +| Calls | Executed one at a time per connection, on the event loop's thread. An event the engine emits inside a call is attributed to that connection without locking because nothing else runs until the call returns; the price is that a large media send stalls the loop for its decode | From a9df28c870b1042ea4f7afac3abf7d3f9a153a4e Mon Sep 17 00:00:00 2001 From: bahdotsh Date: Wed, 30 Sep 2026 23:08:23 +0530 Subject: [PATCH 3/3] docs(bindings): local API clients and guide A guide to running the SDK as a service several local applications share (docs/local-api.md), a Python client example over the Unix socket or TCP, a Node client example over TCP with the built-in WebSocket and no dependencies, and a test that runs both against in-process servers so the examples cannot drift from the wire. The Node built-in client dials only ws: and wss: URLs, so that example uses the token-carrying TCP carrier. Two facts found by running the examples and now stated in the guide: data.map_set takes a tagged value while data.doc_json reads back plain JSON, and data.create_doc on an existing document succeeds. --- CHANGELOG.md | 7 + bindings/python/README.md | 7 + bindings/python/examples/local_api_client.py | 189 ++++++++++++ .../python/tests/local_api/test_examples.py | 268 ++++++++++++++++++ docs/README.md | 1 + docs/local-api.md | 196 +++++++++++++ examples/local-api/client.mjs | 169 +++++++++++ 7 files changed, 837 insertions(+) create mode 100644 bindings/python/examples/local_api_client.py create mode 100644 bindings/python/tests/local_api/test_examples.py create mode 100644 docs/local-api.md create mode 100644 examples/local-api/client.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index d6ff8064a..2e362131b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ archived by series under [docs/changelog/](docs/changelog/); see the ### Added +- **A local API guide and two client examples.** `docs/local-api.md` is + the guide to running the SDK as a service several local applications + share; `bindings/python/examples/local_api_client.py` (Python) and + `examples/local-api/client.mjs` (Node 22, no dependencies) each show + `hello`, `subscribe`, a send with its delivery event, a document edit read + back, and the one-shot idiom. The Python suite runs both against an + in-process server, the Node one when `node` is on the path. - **The Python reference server for the local API.** `offline_protocol_sdk.local_api` and the `offline-protocol-service` command front one engine for any number of local applications over JSON-RPC 2.0 on diff --git a/bindings/python/README.md b/bindings/python/README.md index bfae45e79..f0a3626f2 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -185,6 +185,13 @@ a per-launch token instead of the socket. See [the bridge contract](../../docs/bridges/local-api.md) for what the server owes. +The guide is [docs/local-api.md](../../docs/local-api.md). Two clients ship +as examples and are run by the test suite against an in-process server: +[`examples/local_api_client.py`](examples/local_api_client.py) (this package's +`websockets` dependency, Unix socket or TCP) and +[`examples/local-api/client.mjs`](../../examples/local-api/client.mjs) at the +repository root (Node 22 or later, no dependencies, TCP with the token). + ### Headless hosts: the built-in file stores A server or container usually has no secret service at all. Pass a store key diff --git a/bindings/python/examples/local_api_client.py b/bindings/python/examples/local_api_client.py new file mode 100644 index 000000000..d51ad77ab --- /dev/null +++ b/bindings/python/examples/local_api_client.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""A client of the local API: one application talking to a running +``offline-protocol-service``. In order: ``hello`` with an application id, +``subscribe``, ``send_message`` and its ``message_delivered`` event, a +document edit read back, the one-shot idiom (a fresh connection: ``hello``, +one request, close), and, with ``--wait``, one inbound ``message_received``. +Errors print as the JSON-RPC code and the engine's variant name, which is +what a client switches on. Needs only the ``websockets`` package the SDK +depends on. Contract: docs/spec/local-api.md; guide: docs/local-api.md. + +Usage: + python examples/local_api_client.py --socket /run/example/api.sock --app-id notes + python examples/local_api_client.py --tcp 7800 --token-file /run/example/token \ + --app-id notes --to off1... --wait 30 +""" + +from __future__ import annotations + +import argparse +import asyncio +import itertools +import json +import sys +from pathlib import Path +from typing import Any + +from websockets.asyncio.client import connect, unix_connect +from websockets.exceptions import WebSocketException + + +class RpcFailure(Exception): + """A JSON-RPC error: ``code`` is the number, ``variant`` the engine's name.""" + + def __init__(self, error: dict[str, Any]) -> None: + super().__init__(error.get("message")) + self.code = error["code"] + self.variant = (error.get("data") or {}).get("variant") + + +class Client: + """One connection: ``call`` awaits the matching response, events queue up.""" + + def __init__(self, websocket: Any) -> None: + self._ws = websocket + self._ids = itertools.count(1) + self._pending: dict[int, asyncio.Future[dict[str, Any]]] = {} + self.events: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + self._reader = asyncio.ensure_future(self._read()) + + async def _read(self) -> None: + reason = "connection closed" + try: + async for raw in self._ws: + message = json.loads(raw) + if "id" in message: # a response; events carry no id + waiter = self._pending.pop(message["id"], None) + if waiter is not None: + waiter.set_result(message) + else: + self.events.put_nowait(message["params"]) + except Exception as exc: # the socket closed without a normal close frame + reason = f"connection closed: {exc}" + finally: + # Nothing pending will be answered now: fail it rather than hang. + for waiter in self._pending.values(): + if not waiter.done(): + waiter.set_exception(ConnectionError(reason)) + self._pending.clear() + + async def call(self, method: str, params: dict[str, Any] | None = None) -> Any: + request_id = next(self._ids) + future: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future() + self._pending[request_id] = future + await self._ws.send(json.dumps({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params or {}})) + response = await future + if "error" in response: + raise RpcFailure(response["error"]) + return response["result"] + + async def next_event(self, tag: str | tuple[str, ...], timeout: float = 30.0, **fields: Any) -> dict[str, Any]: + """The next event with one of these tags whose fields match; others are skipped.""" + tags = (tag,) if isinstance(tag, str) else tag + deadline = asyncio.get_running_loop().time() + timeout + while True: + remaining = deadline - asyncio.get_running_loop().time() + event = await asyncio.wait_for(self.events.get(), max(remaining, 0.01)) + if event.get("type") in tags and all(event.get(k) == v for k, v in fields.items()): + return event + + async def close(self) -> None: + await self._ws.close() + await asyncio.gather(self._reader, return_exceptions=True) + + +async def open_client(socket: str | None, tcp: int | None, token_file: str | None) -> tuple[Client, str | None]: + """Connects on the carrier the service uses; returns the client and the token.""" + if socket: + return Client(await unix_connect(socket, uri="ws://localhost/")), None + token = Path(token_file or "").read_text(encoding="ascii").strip() # written 0600 at launch + return Client(await connect(f"ws://127.0.0.1:{tcp}/")), token + + +async def hello(client: Client, app_id: str, token: str | None) -> dict[str, Any]: + params: dict[str, Any] = {"app_id": app_id, "client": "local_api_client.py"} + if token is not None: + params["token"] = token + return await client.call("hello", params) + + +async def send_and_confirm(client: Client, recipient: str, content: str) -> dict[str, Any]: + """Sends, then waits for the terminal event naming the returned id: + ``message_delivered``, or ``message_undeliverable`` when the engine gives up.""" + message_id = await client.call("send_message", {"recipient": recipient, "content": content, "priority": "Medium"}) + return await client.next_event(("message_delivered", "message_undeliverable"), message_id=message_id) + + +async def edit_document(client: Client, space_id: str, doc_id: str, key: str, text: str) -> dict[str, Any]: + """Creates the document (a repeat is a no-op), sets one text key, reads it back. + + A written value is tagged with its kind (`{"kind": "text", "value": ...}`); + `doc_json` reads back plain JSON.""" + await client.call("data.create_doc", {"space_id": space_id, "doc_id": doc_id}) + value_json = json.dumps({"kind": "text", "value": text}) + await client.call( + "data.map_set", + {"space_id": space_id, "doc_id": doc_id, "collection": "fields", "key": key, "value_json": value_json}, + ) + return json.loads(await client.call("data.doc_json", {"space_id": space_id, "doc_id": doc_id})) + + +async def one_shot(socket: str | None, tcp: int | None, token_file: str | None, app_id: str, method: str) -> Any: + """The one-shot idiom: a connection that sends hello, one request, and closes.""" + client, token = await open_client(socket, tcp, token_file) + try: + await hello(client, app_id, token) + return await client.call(method) + finally: + await client.close() + + +async def run(args: argparse.Namespace) -> int: + client: Client | None = None + try: + client, token = await open_client(args.socket, args.tcp, args.token_file) + greeting = await hello(client, args.app_id, token) + print(json.dumps({"hello": greeting})) + await client.call("subscribe", {"types": ["message_received", "message_delivered", "message_undeliverable"]}) + if args.to: + outcome = await send_and_confirm(client, args.to, args.content) + print(json.dumps({outcome["type"].removeprefix("message_"): outcome})) + document = await edit_document(client, args.space, args.doc, "edited_by", args.app_id) + print(json.dumps({"document": document})) + print(json.dumps({"one_shot": await one_shot(args.socket, args.tcp, args.token_file, args.app_id, "local_address")})) + if args.wait: + received = await client.next_event("message_received", timeout=args.wait) + print(json.dumps({"received": received})) + except RpcFailure as exc: + print(json.dumps({"error": {"code": exc.code, "variant": exc.variant, "message": str(exc)}}), file=sys.stderr) + return 1 + except (OSError, TimeoutError, asyncio.TimeoutError, WebSocketException) as exc: + # Not a refusal from the engine: the socket, the token file, or a wait ran out. + print(json.dumps({"error": {"message": str(exc) or type(exc).__name__}}), file=sys.stderr) + return 1 + finally: + if client is not None: + await client.close() + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="A local API client.") + carrier = parser.add_mutually_exclusive_group(required=True) + carrier.add_argument("--socket", help="the service's Unix socket path") + carrier.add_argument("--tcp", type=int, metavar="PORT", help="the service's loopback TCP port") + parser.add_argument("--token-file", help="the token file the service wrote (with --tcp)") + parser.add_argument("--app-id", default="notes") + parser.add_argument("--to", help="an off1... address to send one message to") + parser.add_argument("--content", default="hello from the local API") + parser.add_argument("--space", default="notes-1") + parser.add_argument("--doc", default="todo") + parser.add_argument("--wait", type=float, default=0, help="seconds to wait for one inbound message") + args = parser.parse_args(argv) + if args.tcp is not None and not args.token_file: + parser.error("--tcp needs --token-file") + return asyncio.run(run(args)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bindings/python/tests/local_api/test_examples.py b/bindings/python/tests/local_api/test_examples.py new file mode 100644 index 000000000..580d22249 --- /dev/null +++ b/bindings/python/tests/local_api/test_examples.py @@ -0,0 +1,268 @@ +"""The shipped client examples, run against in-process servers, so a change +to the wire that breaks either is caught here rather than by a reader. + +The Python example is imported as a module and driven function by function, +then run as a subprocess the way a reader runs it. The Node example runs +end to end when ``node`` is on the path, over the TCP carrier with a token +file, because Node's built-in WebSocket dials only ``ws:`` and ``wss:``.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from offline_protocol_sdk.protocol_manager import ProtocolManager + +from .conftest import make_config + +PYTHON_EXAMPLE = Path(__file__).resolve().parents[2] / "examples" / "local_api_client.py" +NODE_EXAMPLE = Path(__file__).resolve().parents[4] / "examples" / "local-api" / "client.mjs" + + +def load_example(): + spec = importlib.util.spec_from_file_location("local_api_client", PYTHON_EXAMPLE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +async def until(predicate, timeout: float = 15.0) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while not predicate(): + if loop.time() > deadline: + raise AssertionError("condition not met in time") + await asyncio.sleep(0.05) + + +async def finish(process, timeout: float = 60.0) -> tuple[bytes, bytes]: + """``communicate`` with a deadline that does not leave the child running.""" + try: + return await asyncio.wait_for(process.communicate(), timeout) + except (TimeoutError, asyncio.TimeoutError): + process.kill() + await process.wait() + raise + + +def output_lines(stdout: bytes) -> list[dict]: + return [json.loads(line) for line in stdout.decode().splitlines() if line.strip()] + + +async def two_servers(harness, *, tcp_a: bool = False): + """Two engines with the data layer on, over loopback streams: B keeps a + stream to A. Server A serves TCP with a token when ``tcp_a``.""" + config_a = make_config(profile="alice", wifi_direct_enabled=True, internet_enabled=False, data_enabled=True) + manager_a = ProtocolManager(config_a) + manager_a.peer_stream.configure(listen_host="127.0.0.1", listen_port=0) + server_a = await harness.server(manager=manager_a, tcp=tcp_a) + port_a = manager_a.peer_stream.listen_port + config_b = make_config(profile="bob", wifi_direct_enabled=True, internet_enabled=False, data_enabled=True) + manager_b = ProtocolManager(config_b) + manager_b.peer_stream.configure(listen_host="127.0.0.1", listen_port=0, peers=[f"127.0.0.1:{port_a}"]) + server_b = await harness.server(manager=manager_b) + await until(lambda: manager_a.local_address in manager_b.peer_stream.connected_peers()) + await until(lambda: manager_b.local_address in manager_a.peer_stream.connected_peers()) + return server_a, server_b + + +async def test_the_python_example_round_trips_a_message_and_a_document(harness): + example = load_example() + server_a, server_b = await two_servers(harness) + receiver, _ = await example.open_client(str(server_b.socket_path), None, None) + sender, _ = await example.open_client(str(server_a.socket_path), None, None) + try: + greeting = await example.hello(receiver, "notes", None) + assert greeting["api_version"] == 1 + assert greeting["local_address"] == server_b.manager.local_address + await example.hello(sender, "notes", None) + + delivered = await example.send_and_confirm(sender, server_b.manager.local_address, "from the example") + received = await receiver.next_event("message_received", message_id=delivered["message_id"]) + assert received["content"] == "from the example" + assert received["app_id"] == "notes" + + document = await example.edit_document(sender, "notes-1", "todo", "edited_by", "notes") + assert document == {"fields": {"edited_by": "notes"}} + # A second edit through the same path: create_doc is a no-op on an + # existing document, and the read shows the newer value. + document = await example.edit_document(sender, "notes-1", "todo", "edited_by", "notes again") + assert document == {"fields": {"edited_by": "notes again"}} + + one_shot = await example.one_shot(str(server_a.socket_path), None, None, "notes", "local_address") + assert one_shot == server_a.manager.local_address + finally: + await sender.close() + await receiver.close() + + +async def test_the_python_example_reports_an_engine_refusal_by_variant(harness): + example = load_example() + server = await harness.server(config=make_config(profile="solo", data_enabled=False)) + client, _ = await example.open_client(str(server.socket_path), None, None) + try: + await example.hello(client, "notes", None) + with pytest.raises(example.RpcFailure) as err: + await example.edit_document(client, "notes-1", "todo", "k", "v") + assert err.value.variant == "DataDisabled" + assert err.value.code <= -32000 + finally: + await client.close() + + +async def test_the_python_example_runs_from_the_shell(harness): + """The example as a reader runs it: a subprocess against server A, sending + to server B, whose client (this test) receives the message.""" + example = load_example() + server_a, server_b = await two_servers(harness) + receiver, _ = await example.open_client(str(server_b.socket_path), None, None) + try: + await example.hello(receiver, "notes", None) + env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} + process = await asyncio.create_subprocess_exec( + sys.executable, + str(PYTHON_EXAMPLE), + "--socket", + str(server_a.socket_path), + "--app-id", + "notes", + "--to", + server_b.manager.local_address, + "--content", + "from the shell", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + stdout, stderr = await finish(process) + assert process.returncode == 0, stderr.decode() + lines = output_lines(stdout) + keys = [next(iter(line)) for line in lines] + assert keys == ["hello", "delivered", "document", "one_shot"] + assert lines[0]["hello"]["local_address"] == server_a.manager.local_address + assert lines[2]["document"] == {"fields": {"edited_by": "notes"}} + assert lines[3]["one_shot"] == server_a.manager.local_address + received = await receiver.next_event("message_received", message_id=lines[1]["delivered"]["message_id"]) + assert received["content"] == "from the shell" + finally: + await receiver.close() + + +def test_the_node_example_parses(): + node = shutil.which("node") + if node is None: + pytest.skip("node is not on the path") + subprocess.run([node, "--check", str(NODE_EXAMPLE)], check=True) + + +async def test_the_node_example_runs_end_to_end_over_tcp(harness): + node = shutil.which("node") + if node is None: + pytest.skip("node is not on the path") + example = load_example() + server_a, server_b = await two_servers(harness, tcp_a=True) + assert server_a.token_path is not None and server_a.port is not None + receiver, _ = await example.open_client(str(server_b.socket_path), None, None) + try: + await example.hello(receiver, "notes", None) + process = await asyncio.create_subprocess_exec( + node, + str(NODE_EXAMPLE), + "--port", + str(server_a.port), + "--token-file", + str(server_a.token_path), + "--app-id", + "notes", + "--to", + server_b.manager.local_address, + "--content", + "from Node", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await finish(process) + assert process.returncode == 0, stderr.decode() + lines = output_lines(stdout) + keys = [next(iter(line)) for line in lines] + assert keys == ["hello", "delivered", "document", "one_shot"] + assert lines[0]["hello"]["local_address"] == server_a.manager.local_address + assert lines[2]["document"] == {"fields": {"edited_by": "notes"}} + assert lines[3]["one_shot"] == server_a.manager.local_address + received = await receiver.next_event("message_received", message_id=lines[1]["delivered"]["message_id"]) + assert received["content"] == "from Node" + finally: + await receiver.close() + + +async def test_the_python_example_fails_instead_of_hanging_when_the_server_closes(tmp_path): + """A server that reads one frame and closes without answering: the pending + ``hello`` must fail with ``ConnectionError``, not wait forever.""" + from websockets.asyncio.server import serve + + example = load_example() + + async def read_one_then_close(websocket): + await websocket.recv() + await websocket.close(1011, "gone") + + token_file = tmp_path / "token" + token_file.write_text("not-checked\n", encoding="ascii") + async with serve(read_one_then_close, "127.0.0.1", 0) as server: + port = server.sockets[0].getsockname()[1] + client, token = await example.open_client(None, port, str(token_file)) + try: + with pytest.raises(ConnectionError): + await asyncio.wait_for(example.hello(client, "notes", token), 5) + finally: + await client.close() + + +async def test_the_examples_report_a_connection_failure_on_one_line(tmp_path): + """No traceback and no unsettled await: a socket that is not there, and a + port nothing listens on, each cost one JSON line on stderr and exit 1.""" + env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} + process = await asyncio.create_subprocess_exec( + sys.executable, + str(PYTHON_EXAMPLE), + "--socket", + str(tmp_path / "nowhere.sock"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + stdout, stderr = await finish(process) + assert process.returncode == 1 + assert stdout == b"" + (line,) = stderr.decode().splitlines() + assert "message" in json.loads(line)["error"] + + node = shutil.which("node") + if node is None: + pytest.skip("node is not on the path") + token_file = tmp_path / "token" + token_file.write_text("unused\n", encoding="ascii") + process = await asyncio.create_subprocess_exec( + node, + str(NODE_EXAMPLE), + "--port", + "1", + "--token-file", + str(token_file), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = await finish(process) + assert process.returncode == 1, stderr.decode() + assert stdout == b"" + (line,) = stderr.decode().splitlines() + assert "message" in json.loads(line)["error"] diff --git a/docs/README.md b/docs/README.md index a1c40a96f..c172495b8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ before changing behaviour, not before using the SDK. | [React Native Integration](react-native-integration.md) | Full SDK integration guide with complete API reference | | [iOS Integration](ios-integration.md) | Native iOS (Swift) setup and usage | | [Android Integration](android-integration.md) | Native Android (Kotlin) setup and usage | +| [The local API](local-api.md) | Run the SDK as a service several local applications share: the service, the policy file, and the two client examples | ## Core Concepts diff --git a/docs/local-api.md b/docs/local-api.md new file mode 100644 index 000000000..1dde6ec51 --- /dev/null +++ b/docs/local-api.md @@ -0,0 +1,196 @@ +# The local API: running the SDK as a service + +The SDK is usually embedded: one application, one process, one engine. The +local API is the other shape. One process owns the engine and serves any +number of local applications over a socket, so a machine with several +programs that all want the same identity, the same peers and the same +documents runs one node instead of several. Each application talks JSON-RPC +2.0 over a WebSocket, declares which application it is, and gets the +engine's own methods and events by name. + +This page is the guide: how to start the service, how a client talks to it, +and what the two shipped examples show. The contract, with every method, +event and error, is [the local API chapter](spec/local-api.md); what the +reference server owes is in [the bridge contract](bridges/local-api.md). + +## What holds, whatever the client does + +Four things are true of every service, and a client written against them +does not break when the server changes: + +- **The server owns the engine.** It runs the loop, drains inbound messages, + starts and stops. No client can call `process()`, `receive_message()` or + `stop()`, because two drainers would split one inbound stream and a + stopped engine would leave every other application's messages + unacknowledged. +- **A connection speaks for one application.** The first request is + `hello` with an `app_id`. Every message that connection sends is stamped + with it, and an inbound message stamped with it reaches only connections + that declared it. Two processes of the same application declare the same + id and are one application to every rule. +- **The socket is the authorization boundary.** Whoever can open the Unix + socket, or holds the launch token on TCP, is one of the operator's + applications. The server does not tell one local process from another, + and the rules below separate applications from each other's mistakes, not + from a hostile process on the same account. +- **Events are the engine's JSON, unchanged.** A client that handles + `message_received` from the embedded SDK handles the same object here. + +## Starting the service + +The Python package ships the reference server as the +`offline-protocol-service` command. It runs over the built-in file stores, +so a host with no secret service needs only a directory and a key: + +```bash +export OFFLINE_PROTOCOL_STORE_KEY="$(openssl rand -hex 32)" # once; keep it +offline-protocol-service --config config.json \ + --mls-root /var/lib/example/keys --state-root /var/lib/example/state \ + --socket /run/example/api.sock \ + --listen 0.0.0.0:7878 --peer 10.0.0.2:7878 +``` + +`config.json` holds the `ProtocolConfig` fields by name (see +[Configuration](configuration.md)); `--listen` and `--peer` drive the +peer-stream transport when the configuration enables it. Two carriers: + +| Carrier | How | Credential | +|---|---|---| +| Unix domain socket (default) | `--socket PATH` | The socket file is `0600` in a `0700` directory. `hello` carries no token. | +| Loopback TCP (opt-in) | `--tcp PORT --token-file PATH` | A 32-byte token, new at every launch, written to the file with mode `0600` in the same call that creates it. `hello` carries it. | + +The server never binds a non-loopback address. A deployment that wants the +API across a network puts a reverse proxy with its own authentication in +front. `GET /health` on either carrier answers with the server's name and +version, the API version and the carrier, and nothing else is served over +plain HTTP: the +pinned WebSocket library accepts only `GET` and drops any other method +before the server sees it, so a `POST` would fail silently rather than with +an error. + +## The policy file + +With no policy, any well-formed application id is accepted and nothing is +denied. `--policy policy.json` adds rules per application id: + +```json +{ + "spaces": {"notes": ["notes-*"], "mail": ["mail-*", "shared"]}, + "denied": {"kiosk": ["sign_data", "manual_mls", "tuning"]}, + "applications": ["admin"] +} +``` + +| Section | What it does | The failure it prevents | +|---|---|---| +| `spaces` | Glob patterns over document space ids. A `data.*` call outside them is refused, `data.list_spaces` is filtered, and document events for other spaces are not delivered. | Two applications picking the same space name by accident and merging each other's documents; the engine cannot notice, because to it there is one member. | +| `denied` | Method groups (`sign_data`, `manual_mls`, `tuning`) or single wire names an application may not call. Refused with `PermissionDenied`, not "method not found", so the client can read the refusal. | Every application holding a signing oracle over the shared identity key. | +| `applications` | Ids with no rule of their own that are still admitted. | An application locked out by the rule below because it needs no restriction. | + +**Once `spaces` or `denied` names anyone, an id no section names is refused +at `hello`.** Without that, an application scoped or denied under one id +would reconnect under another and be scoped and denied nothing. Service +ownership (which application registered which service) is a runtime record +the server rebuilds from the clients' own calls, and it never turns this +rule on. + +## Talking to it + +Every request is a JSON-RPC 2.0 object with an `id`; every event is a +notification with `method` set to `event` and `params` set to the event +object itself. + +**`hello` first.** It declares the application, and on TCP carries the +token. The result has the API version, the server's name and version, the +engine's state and the node's `off1…` address, which every client of one +server shares. + +```json +{"jsonrpc":"2.0","id":1,"method":"hello","params":{"app_id":"notes","client":"notes-desktop/2.4"}} +{"jsonrpc":"2.0","id":1,"result":{"api_version":1,"server":{"name":"offline-protocol-service","version":"0.28.0"},"state":"Running","local_address":"off1…"}} +``` + +**Sending.** The engine's methods by name, with the parameters the interface +definition gives them. `send_message` returns the message id; the +`message_sent` and `message_delivered` events that name it come back to the +connection that sent it and to no other, while the server that issued the +id is running. A message the engine gives up on is reported the same way, +as `message_undeliverable`, so a client watches for both. + +```json +{"jsonrpc":"2.0","id":2,"method":"send_message","params":{"recipient":"off1…","content":"hi","priority":"Medium"}} +``` + +**Subscribing and replay.** After `hello` a connection receives every event +the routing rules deliver to its application. `subscribe` with a list of +tags narrows that, `unsubscribe` widens it back, and `subscribe` with +`"all"` resets it (`unsubscribe` with `"all"` mutes everything). An +inbound message for an application with no connected client is held, up to +256 per application id with the oldest dropped past that, and delivered +right after the next `hello` under that id. Delivery is at least once, so +`message_id` is the key a client deduplicates on. The engine has already +acknowledged a held message, which is why the server holds it rather than +dropping it: the sender will not send it again. + +**Documents.** The replicated document store is reachable as `data.*` +methods, and the mesh services as `services.*`. A write carries a tagged +value; a read returns plain JSON: + +```json +{"jsonrpc":"2.0","id":3,"method":"data.create_doc","params":{"space_id":"notes-1","doc_id":"todo"}} +{"jsonrpc":"2.0","id":4,"method":"data.map_set","params":{"space_id":"notes-1","doc_id":"todo","collection":"fields","key":"title","value_json":"{\"kind\":\"text\",\"value\":\"groceries\"}"}} +{"jsonrpc":"2.0","id":5,"method":"data.doc_json","params":{"space_id":"notes-1","doc_id":"todo"}} +{"jsonrpc":"2.0","id":5,"result":"{\"fields\":{\"title\":\"groceries\"}}"} +``` + +`create_doc` on a document that exists is a no-op, so a client need not +check first. When the data layer is off in the configuration, every +`data.*` call answers `DataDisabled`. + +**One-shot calls.** There is no HTTP request path (see above). A single +call is a connection that sends `hello`, one request, and closes; every +runtime has a WebSocket client, so this is the whole "curl-shaped" case. + +**Errors.** A failed call is a JSON-RPC error whose `data.variant` is the +engine's own `ProtocolError` variant name. Switch on that, not on the +number: the number is the variant's position in the append-only error enum, +stable but not meaningful. + +```json +{"jsonrpc":"2.0","id":6,"error":{"code":-32004,"message":"No key package available for recipient: off1…","data":{"variant":"NoKeyPackage"}}} +``` + +A method before `hello` is `InvalidState`; a bad application id is +`InvalidArgument`; a wrong token, another application's service, a space +outside the allow-list, a denied method, or an unlisted id once rules exist +is `PermissionDenied`. There is no error that exists only on the wire. + +## What is not exposed + +The interface definition has two kinds of method. The ones an application +calls are on the wire. The ones a platform bridge calls to drive a radio, +attach storage, feed the engine a frame, run the loop, or own telemetry are +not, because they belong to whoever owns the engine, and here that is the +server. The chapter lists both sets in full, and a test in the Rust +workspace holds the lists, the interface definition and the server's +dispatch table to one another, so a method added to the engine is a failing +test until someone says which side of the line it is on. + +## The examples + +Both examples do the same thing, in the same order: `hello`, `subscribe`, +`send_message` and the `message_delivered` event for it (when given an +address), a document edit read back, and a one-shot call on a fresh +connection. + +- [`bindings/python/examples/local_api_client.py`](../bindings/python/examples/local_api_client.py): + Python over the Unix socket or TCP, with `--wait` to sit and receive one + inbound message. Needs only the `websockets` package the SDK depends on. +- [`examples/local-api/client.mjs`](../examples/local-api/client.mjs): Node + 22 or later with the built-in `WebSocket`, no dependencies. The built-in + client dials only `ws:` and `wss:` URLs, so it uses the TCP carrier and + reads the token file. + +A test in the Python suite runs the Python example against an in-process +server, and runs the Node example end to end when `node` is on the path, so +a change to the wire that breaks either is caught before it ships. diff --git a/examples/local-api/client.mjs b/examples/local-api/client.mjs new file mode 100644 index 000000000..b7dd5001a --- /dev/null +++ b/examples/local-api/client.mjs @@ -0,0 +1,169 @@ +#!/usr/bin/env node +// A client of the local API in Node, with no dependencies: the built-in +// WebSocket (Node 22 or later; it is global there) and the file system. +// +// The built-in client accepts only ws: and wss: URLs, so this example uses +// the service's loopback TCP carrier with its per-launch token; start the +// service with `--tcp PORT --token-file PATH`. A client that must use the +// Unix socket needs a WebSocket library that can dial one. +// +// What it shows, in order: hello with an application id and the token, +// subscribe, send_message and the terminal event for it (message_delivered, +// or message_undeliverable when the engine gives up), a document edit +// (data.map_set, then data.doc_json to read it back), and the one-shot +// idiom (a fresh connection: hello, one request, close). An engine refusal +// is printed as the JSON-RPC code and the engine's variant name; any other +// failure as its message. +// +// Usage: +// node examples/local-api/client.mjs --port 7800 --token-file /run/example/token \ +// --app-id notes [--to off1... --content "hi"] [--space notes-1 --doc todo] + +import { readFileSync } from "node:fs"; + +if (typeof WebSocket === "undefined") { + console.error("Node 22 or later is required: this example uses the built-in WebSocket"); + process.exit(1); +} + +function parseArgs(argv) { + const args = { appId: "notes", content: "hello from Node", space: "notes-1", doc: "todo" }; + for (let i = 0; i < argv.length; i += 2) { + const [flag, value] = [argv[i], argv[i + 1]]; + if (flag === "--port") args.port = Number(value); + else if (flag === "--token-file") args.tokenFile = value; + else if (flag === "--app-id") args.appId = value; + else if (flag === "--to") args.to = value; + else if (flag === "--content") args.content = value; + else if (flag === "--space") args.space = value; + else if (flag === "--doc") args.doc = value; + else throw new Error(`unknown flag ${flag}`); + } + if (!args.port || !args.tokenFile) throw new Error("--port and --token-file are required"); + return args; +} + +class RpcFailure extends Error { + constructor(error) { + super(error.message); + this.code = error.code; + this.variant = error.data?.variant ?? null; + } +} + +// One connection: call() resolves with the matching response; events queue +// up. When the socket closes, every pending call and event wait is rejected +// rather than left hanging. +class Client { + constructor(socket) { + this.socket = socket; + this.nextId = 1; + this.pending = new Map(); // id -> { resolve, reject } + this.events = []; + this.waiters = []; // { matches, resolve, reject } + socket.addEventListener("message", (frame) => { + const message = JSON.parse(frame.data); + if ("id" in message) { + const waiter = this.pending.get(message.id); + this.pending.delete(message.id); + if (waiter) message.error ? waiter.reject(new RpcFailure(message.error)) : waiter.resolve(message.result); + return; + } + this.events.push(message.params); // a notification: the event object itself + this.waiters = this.waiters.filter((w) => !(w.matches(message.params) && (w.resolve(message.params), true))); + }); + const fail = (reason) => { + for (const waiter of [...this.pending.values(), ...this.waiters]) waiter.reject(new Error(reason)); + this.pending.clear(); + this.waiters = []; + }; + socket.addEventListener("close", (event) => fail(`connection closed (${event.code})`), { once: true }); + socket.addEventListener("error", () => fail("connection error"), { once: true }); + } + + static async open(port) { + const socket = new WebSocket(`ws://127.0.0.1:${port}/`); + await new Promise((resolve, reject) => { + socket.addEventListener("open", resolve, { once: true }); + socket.addEventListener("error", () => reject(new Error(`cannot connect to port ${port}`)), { once: true }); + }); + return new Client(socket); + } + + call(method, params = {}) { + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.socket.send(JSON.stringify({ jsonrpc: "2.0", id, method, params })); + }); + } + + // The next event with one of these types whose fields match; earlier ones count too. + nextEvent(types, fields = {}, timeoutMs = 30000) { + const wanted = Array.isArray(types) ? types : [types]; + const matches = (e) => wanted.includes(e.type) && Object.entries(fields).every(([k, v]) => e[k] === v); + const found = this.events.find(matches); + if (found) return Promise.resolve(found); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`no ${wanted.join("/")} within ${timeoutMs} ms`)), timeoutMs); + const settle = (fn) => (value) => (clearTimeout(timer), fn(value)); + this.waiters.push({ matches, resolve: settle(resolve), reject: settle(reject) }); + }); + } + + close() { + this.socket.close(); + } +} + +async function hello(client, appId, token) { + return client.call("hello", { app_id: appId, client: "client.mjs", token }); +} + +async function oneShot(port, token, appId, method) { + const client = await Client.open(port); + try { + await hello(client, appId, token); + return await client.call(method); + } finally { + client.close(); + } +} + +async function main() { + let client = null; + try { + const args = parseArgs(process.argv.slice(2)); + const token = readFileSync(args.tokenFile, "ascii").trim(); // written 0600 at launch + client = await Client.open(args.port); + console.log(JSON.stringify({ hello: await hello(client, args.appId, token) })); + await client.call("subscribe", { types: ["message_received", "message_delivered", "message_undeliverable"] }); + if (args.to) { + const messageId = await client.call("send_message", { recipient: args.to, content: args.content, priority: "Medium" }); + const outcome = await client.nextEvent(["message_delivered", "message_undeliverable"], { message_id: messageId }); + console.log(JSON.stringify({ [outcome.type.replace(/^message_/, "")]: outcome })); + } + // create_doc is a no-op for a document that exists. A written value is + // tagged with its kind; doc_json reads back plain JSON. + await client.call("data.create_doc", { space_id: args.space, doc_id: args.doc }); + await client.call("data.map_set", { + space_id: args.space, doc_id: args.doc, collection: "fields", key: "edited_by", + value_json: JSON.stringify({ kind: "text", value: args.appId }), + }); + const document = JSON.parse(await client.call("data.doc_json", { space_id: args.space, doc_id: args.doc })); + console.log(JSON.stringify({ document })); + console.log(JSON.stringify({ one_shot: await oneShot(args.port, token, args.appId, "local_address") })); + } catch (error) { + if (error instanceof RpcFailure) { + console.error(JSON.stringify({ error: { code: error.code, variant: error.variant, message: error.message } })); + } else { + console.error(JSON.stringify({ error: { message: error.message } })); + } + return 1; + } finally { + client?.close(); + } + return 0; +} + +process.exitCode = await main();