Budget sync chunks in escaped UTF-8 bytes and never cut a surrogate pair

src/js/syncChunks.js

@@ -14,13 +14,22 @@
  */
 export const SYNC_CHUNK_CHARS = 150 * 1024;
 
+/**
+ * UTF-8 bytes a sync chunk's `data` may take once JSON-escaped, quotes
+ * included. The wire grows with bytes, not characters: sealing adds base64's
+ * third and a few hundred bytes on top, and a test holds the sealed size
+ * under the budget.
+ */
+export const SYNC_CHUNK_BYTES = 150 * 1024;
+
 /**
  * The row types and fields of one framed protocol. `carry` names the payload
  * fields every row repeats; `keepPairs` never cuts between the two halves of
- * a surrogate pair, which a UTF-8 encoder downstream would turn into '?'.
+ * a surrogate pair, which a UTF-8 encoder downstream would turn into '?';
+ * `bytes` measures the limit in escaped UTF-8 bytes instead of characters.
  */
 const SYNC_FRAME = Object.freeze({
-    chunk: 'sync_chunk', manifest: 'sync_manifest', id: 'syncId', carry: ['ts'], keepPairs: false
+    chunk: 'sync_chunk', manifest: 'sync_manifest', id: 'syncId', carry: ['ts'], keepPairs: true, bytes: true
 });
 
 /**
@@ -29,7 +38,7 @@ const SYNC_FRAME = Object.freeze({
  * every row so a run can be ranked before it is assembled.
  */
 export const ADMIN_FRAME = Object.freeze({
-    chunk: 'admin_chunk', manifest: 'admin_manifest', id: 'runId', carry: ['rev', 'ts'], keepPairs: true
+    chunk: 'admin_chunk', manifest: 'admin_manifest', id: 'runId', carry: ['rev', 'ts'], keepPairs: true, bytes: false
 });
 
 /**
@@ -38,23 +47,18 @@ export const ADMIN_FRAME = Object.freeze({
  * @param {Object} payload - The whole snapshot
  * @param {string} runId - Ties the chunks to their manifest
  * @param {Object} frame - Row types and fields (SYNC_FRAME, ADMIN_FRAME)
- * @param {number} limit - Characters per chunk
+ * @param {number} limit - Per chunk: characters, or escaped UTF-8 bytes of
+ *   its `data` when the frame counts bytes
  * @returns {Object[]} - The payload itself when it fits, else chunks + manifest
  */
 export function splitFramed(payload, runId, frame, limit) {
     const serialised = JSON.stringify(payload);
-    if (serialised.length <= limit) return [payload];
+    const size = frame.bytes ? new TextEncoder().encode(serialised).length : serialised.length;
+    if (size <= limit) return [payload];
 
-    const slices = [];
-    for (let start = 0; start < serialised.length;) {
-        let end = Math.min(start + limit, serialised.length);
-        if (frame.keepPairs && end < serialised.length && end - 1 > start
-                && isHighSurrogate(serialised.charCodeAt(end - 1))) {
-            end -= 1;
-        }
-        slices.push(serialised.slice(start, end));
-        start = end;
-    }
+    const slices = frame.bytes
+        ? sliceByBytes(serialised, limit)
+        : sliceByChars(serialised, limit, frame.keepPairs);
 
     const header = {};
     for (const field of frame.carry) header[field] = payload[field];
@@ -117,19 +121,67 @@ export function joinFramed(messages, frame, onDropped) {
     return out;
 }
 
+function sliceByChars(s, limit, keepPairs) {
+    const slices = [];
+    for (let start = 0; start < s.length;) {
+        let end = Math.min(start + limit, s.length);
+        if (keepPairs && end < s.length && end - 1 > start && isHighSurrogate(s.charCodeAt(end - 1))) {
+            end -= 1;
+        }
+        slices.push(s.slice(start, end));
+        start = end;
+    }
+    return slices;
+}
+
+/** Slices whose JSON.stringify takes at most `limit` UTF-8 bytes; a pair is one unit. */
+function sliceByBytes(s, limit) {
+    const slices = [];
+    for (let start = 0; start < s.length;) {
+        let end = start;
+        let bytes = 2;
+        while (end < s.length) {
+            const pair = isHighSurrogate(s.charCodeAt(end)) && end + 1 < s.length
+                && isLowSurrogate(s.charCodeAt(end + 1));
+            const cost = pair ? 4 : escapedBytes(s.charCodeAt(end));
+            if (bytes + cost > limit) break;
+            bytes += cost;
+            end += pair ? 2 : 1;
+        }
+        if (end === start) throw new Error(`chunk budget of ${limit} B holds no character`);
+        slices.push(s.slice(start, end));
+        start = end;
+    }
+    return slices;
+}
+
+/** UTF-8 bytes JSON.stringify writes for one UTF-16 unit that is not half of a pair. */
+function escapedBytes(code) {
+    if (code === 0x22 || code === 0x5C) return 2;
+    if (code < 0x20) return [0x08, 0x09, 0x0A, 0x0C, 0x0D].includes(code) ? 2 : 6;
+    if (code < 0x80) return 1;
+    if (code < 0x800) return 2;
+    if (code >= 0xD800 && code <= 0xDFFF) return 6;
+    return 3;
+}
+
 function isHighSurrogate(code) {
     return code >= 0xD800 && code <= 0xDBFF;
 }
 
+function isLowSurrogate(code) {
+    return code >= 0xDC00 && code <= 0xDFFF;
+}
+
 /**
  * Frame a sync snapshot for the wire.
  *
  * @param {Object} payload - The whole `{ type:'sync', v, ts, data }` snapshot
  * @param {string} syncId - Run id, tying the chunks to their manifest
- * @param {number} [limit] - Characters per chunk
+ * @param {number} [limit] - Escaped UTF-8 bytes of each chunk's data
  * @returns {Object[]} - The payload itself when it fits, else chunks + manifest
  */
-export function splitSyncPayload(payload, syncId, limit = SYNC_CHUNK_CHARS) {
+export function splitSyncPayload(payload, syncId, limit = SYNC_CHUNK_BYTES) {
     return splitFramed(payload, syncId, SYNC_FRAME, limit);
 }
 

tests/unit/syncChunks.test.js

@@ -7,7 +7,14 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { ethers } from 'ethers';
 import { splitSyncPayload, reassembleSyncPayloads, SYNC_CHUNK_CHARS } from '../../src/js/syncChunks.js';
+import { dmCrypto } from '../../src/js/dmCrypto.js';
+import { CONFIG } from '../../src/js/config.js';
+
+globalThis.ethers = ethers;
+
+const bytes = (s) => new TextEncoder().encode(s).length;
 
 const snapshot = (fill, ts = 1700000000000) => ({
     type: 'sync', v: 1, ts,
@@ -39,12 +46,30 @@ describe('splitting', () => {
         expect(out.every(m => m.ts === 42)).toBe(true);
     });
 
-    it('never puts more than the budget in one message', () => {
-        const out = splitSyncPayload(snapshot('z'.repeat(5000)), 'run1', 300);
+    it('never cuts an emoji in half, which a UTF-8 encoder would turn into ?', () => {
+        const payload = snapshot('🐦'.repeat(3000));
+        const out = splitSyncPayload(payload, 'run1', 301);
         for (const chunk of out.filter(m => m.type === 'sync_chunk')) {
-            expect(chunk.data.length).toBeLessThanOrEqual(300);
+            expect(chunk.data).toBe(chunk.data.toWellFormed());
+        }
+        expect(reassembleSyncPayloads(out)[0]).toEqual(payload);
+    });
+
+    it('never puts more than the budget in one message, counted as JSON writes it', () => {
+        for (const fill of ['z'.repeat(5000), '"\\'.repeat(2000), 'ação coração '.repeat(400), '🐦'.repeat(2000), '\u0001\n'.repeat(1500)]) {
+            const payload = snapshot(fill);
+            const out = splitSyncPayload(payload, 'run1', 300);
+            for (const chunk of out.filter(m => m.type === 'sync_chunk')) {
+                expect(bytes(JSON.stringify(chunk.data))).toBeLessThanOrEqual(300);
+            }
+            expect(reassembleSyncPayloads(out)[0]).toEqual(payload);
         }
     });
+
+    it('cuts accented text into more, shorter chunks than the same length of ASCII', () => {
+        const count = (fill) => splitSyncPayload(snapshot(fill), 'run1', 300).length;
+        expect(count('ç'.repeat(3000))).toBeGreaterThan(count('c'.repeat(3000)));
+    });
 });
 
 describe('reassembling', () => {
@@ -116,4 +141,22 @@ describe('the budget', () => {
     it('is the measured one — a 150 KB slice reaches the wire near 227 KB', () => {
         expect(SYNC_CHUNK_CHARS).toBe(150 * 1024);
     });
+
+    it('keeps every sealed chunk under the wire budget, whatever the text is made of', async () => {
+        const me = new ethers.Wallet('0x' + '11'.repeat(32));
+        const seal = (message) => dmCrypto.seal(message, {
+            senderPrivateKey: me.privateKey,
+            recipientAddress: me.address,
+            recipientPublicKey: new ethers.SigningKey(me.privateKey).compressedPublicKey
+        });
+        const wireBudget = CONFIG.media.imagePayloadMaxBytes - CONFIG.media.imagePayloadSafetyMarginBytes;
+        for (const fill of ['🐦'.repeat(90000), '"'.repeat(200000), 'ç'.repeat(160000), '\u0001'.repeat(60000)]) {
+            const chunks = splitSyncPayload(snapshot(fill), 'run1').filter(m => m.type === 'sync_chunk');
+            expect(chunks.length).toBeGreaterThan(1);
+            for (const chunk of chunks) {
+                const { envelope } = await seal(chunk);
+                expect(bytes(JSON.stringify(envelope))).toBeLessThanOrEqual(wireBudget);
+            }
+        }
+    });
 });

tests/unit/syncChunks.vectors.test.js

@@ -0,0 +1,26 @@
+// The sync framing vectors are the shared spec: this suite and
+// SyncChunksTest.kt read the same JSON, so a push split on one client
+// reassembles on the other.
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { splitSyncPayload, reassembleSyncPayloads } from '../../src/js/syncChunks.js';
+
+const vectors = JSON.parse(readFileSync(
+    join(dirname(fileURLToPath(import.meta.url)), '..', '..',
+        'docs', 'SYNC-chunk-vectors.json'), 'utf8'));
+
+describe('sync framing parity vectors', () => {
+    for (const v of vectors.split) {
+        it(v.what, () => {
+            expect(splitSyncPayload(v.payload, 'runA', vectors.limit)).toEqual(v.messages);
+        });
+    }
+
+    for (const v of vectors.reassemble) {
+        it(v.what, () => {
+            expect(reassembleSyncPayloads(v.messages)).toEqual(v.payloads);
+        });
+    }
+});

tests/vectors/gen_sync_chunk_vectors.mjs

@@ -8,7 +8,8 @@
 // JSON into the account's state.
 //
 // The vectors fix the split (how a payload becomes messages, at a small
-// budget so the fixtures stay readable) and the reassembly rules (order does
+// budget in escaped UTF-8 bytes so the fixtures stay readable, and never
+// between the halves of a surrogate pair) and the reassembly rules (order does
 // not matter, runs are kept apart, an incomplete or unparseable run is
 // dropped whole).
 import { splitSyncPayload, reassembleSyncPayloads } from '../../src/js/syncChunks.js';
@@ -24,6 +25,9 @@ const big = snapshot('x'.repeat(700), 1789000000000);
 const other = snapshot('y'.repeat(700), 1789000009000);
 const small = snapshot('fits', 1789000005000);
 
+// Text made only of emoji: every cut lands next to a surrogate pair.
+const straddling = snapshot('\u{1F426}'.repeat(150), 1789000029000);
+
 const bigRun = splitSyncPayload(big, 'runA', LIMIT);
 const otherRun = splitSyncPayload(other, 'runB', LIMIT);
 
@@ -39,6 +43,11 @@ console.log(JSON.stringify({
             what: 'a big snapshot becomes chunks numbered from zero, then its manifest',
             payload: big,
             messages: bigRun
+        },
+        {
+            what: 'a cut never falls between the two halves of a surrogate pair',
+            payload: straddling,
+            messages: splitSyncPayload(straddling, 'runA', LIMIT)
         }
     ],
     reassemble: [

tests/vectors/regen.mjs

@@ -35,6 +35,7 @@ const GENERATORS = {
     'gen_storage_purge_vectors.mjs': 'STORAGE-purge-vectors.json',
     'gen_storage_stored_vectors.mjs': 'STORAGE-stored-vectors.json',
     'gen_sync_merge_vectors.mjs': 'SYNC-merge-vectors.json',
+    'gen_sync_chunk_vectors.mjs': 'SYNC-chunk-vectors.json',
     'gen_admin_chunk_vectors.mjs': 'ADMIN-chunk-vectors.json'
 };