From 3ccf54d9084bee52ce3d2b3fff84ebcd98c7ad0a Mon Sep 17 00:00:00 2001 From: Ocnrb Date: Fri, 25 Sep 2026 11:50:07 +0100 Subject: [PATCH 1/3] Generate the sync chunk vectors with the others and check them on the web too Co-authored-by: Claude Opus 5.5 --- tests/unit/syncChunks.vectors.test.js | 26 ++++++++++++++++++++++++++ tests/vectors/regen.mjs | 1 + 2 files changed, 27 insertions(+) create mode 100644 tests/unit/syncChunks.vectors.test.js diff --git a/tests/unit/syncChunks.vectors.test.js b/tests/unit/syncChunks.vectors.test.js new file mode 100644 index 0000000..66f086a --- /dev/null +++ b/tests/unit/syncChunks.vectors.test.js @@ -0,0 +1,26 @@ +// The sync framing vectors are the shared spec: this suite and +// SyncChunksTest.kt read the same JSON, so a push split on one client +// reassembles on the other. +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { splitSyncPayload, reassembleSyncPayloads } from '../../src/js/syncChunks.js'; + +const vectors = JSON.parse(readFileSync( + join(dirname(fileURLToPath(import.meta.url)), '..', '..', + 'docs', 'SYNC-chunk-vectors.json'), 'utf8')); + +describe('sync framing parity vectors', () => { + for (const v of vectors.split) { + it(v.what, () => { + expect(splitSyncPayload(v.payload, 'runA', vectors.limit)).toEqual(v.messages); + }); + } + + for (const v of vectors.reassemble) { + it(v.what, () => { + expect(reassembleSyncPayloads(v.messages)).toEqual(v.payloads); + }); + } +}); diff --git a/tests/vectors/regen.mjs b/tests/vectors/regen.mjs index 79bf204..e59e950 100644 --- a/tests/vectors/regen.mjs +++ b/tests/vectors/regen.mjs @@ -35,6 +35,7 @@ const GENERATORS = { 'gen_storage_purge_vectors.mjs': 'STORAGE-purge-vectors.json', 'gen_storage_stored_vectors.mjs': 'STORAGE-stored-vectors.json', 'gen_sync_merge_vectors.mjs': 'SYNC-merge-vectors.json', + 'gen_sync_chunk_vectors.mjs': 'SYNC-chunk-vectors.json', 'gen_admin_chunk_vectors.mjs': 'ADMIN-chunk-vectors.json' }; From 873cd33e1e3b0e2d195f1aaa1729e275ee8ba360 Mon Sep 17 00:00:00 2001 From: Ocnrb Date: Fri, 25 Sep 2026 11:50:50 +0100 Subject: [PATCH 2/3] Never cut a sync chunk between the two halves of a surrogate pair Co-authored-by: Claude Opus 5.5 --- docs/SYNC-chunk-vectors.json | 52 ++++++++++++++++++++++++ src/js/syncChunks.js | 2 +- tests/unit/syncChunks.test.js | 9 ++++ tests/vectors/gen_sync_chunk_vectors.mjs | 11 +++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/SYNC-chunk-vectors.json b/docs/SYNC-chunk-vectors.json index a234618..8a71d28 100644 --- a/docs/SYNC-chunk-vectors.json +++ b/docs/SYNC-chunk-vectors.json @@ -101,6 +101,58 @@ "chunkCount": 5 } ] + }, + { + "what": "a cut never falls between the two halves of a surrogate pair", + "payload": { + "type": "sync", + "v": 1, + "ts": 1789000029000, + "data": { + "channels": [ + { + "messageStreamId": "0xowner/chan-1", + "name": "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp🐦qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq" + } + ] + } + }, + "messages": [ + { + "type": "sync_chunk", + "v": 1, + "ts": 1789000029000, + "syncId": "runA", + "chunkIndex": 0, + "chunkCount": 3, + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000029000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp" + }, + { + "type": "sync_chunk", + "v": 1, + "ts": 1789000029000, + "syncId": "runA", + "chunkIndex": 1, + "chunkCount": 3, + "data": "🐦qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq" + }, + { + "type": "sync_chunk", + "v": 1, + "ts": 1789000029000, + "syncId": "runA", + "chunkIndex": 2, + "chunkCount": 3, + "data": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq\"}]}}" + }, + { + "type": "sync_manifest", + "v": 1, + "ts": 1789000029000, + "syncId": "runA", + "chunkCount": 3 + } + ] } ], "reassemble": [ diff --git a/src/js/syncChunks.js b/src/js/syncChunks.js index 046516d..90e30be 100644 --- a/src/js/syncChunks.js +++ b/src/js/syncChunks.js @@ -20,7 +20,7 @@ export const SYNC_CHUNK_CHARS = 150 * 1024; * a surrogate pair, which a UTF-8 encoder downstream would turn into '?'. */ const SYNC_FRAME = Object.freeze({ - chunk: 'sync_chunk', manifest: 'sync_manifest', id: 'syncId', carry: ['ts'], keepPairs: false + chunk: 'sync_chunk', manifest: 'sync_manifest', id: 'syncId', carry: ['ts'], keepPairs: true }); /** diff --git a/tests/unit/syncChunks.test.js b/tests/unit/syncChunks.test.js index 5f1c306..2812aab 100644 --- a/tests/unit/syncChunks.test.js +++ b/tests/unit/syncChunks.test.js @@ -39,6 +39,15 @@ describe('splitting', () => { expect(out.every(m => m.ts === 42)).toBe(true); }); + it('never cuts an emoji in half, which a UTF-8 encoder would turn into ?', () => { + const payload = snapshot('🐦'.repeat(3000)); + const out = splitSyncPayload(payload, 'run1', 301); + for (const chunk of out.filter(m => m.type === 'sync_chunk')) { + expect(chunk.data).toBe(chunk.data.toWellFormed()); + } + expect(reassembleSyncPayloads(out)[0]).toEqual(payload); + }); + it('never puts more than the budget in one message', () => { const out = splitSyncPayload(snapshot('z'.repeat(5000)), 'run1', 300); for (const chunk of out.filter(m => m.type === 'sync_chunk')) { diff --git a/tests/vectors/gen_sync_chunk_vectors.mjs b/tests/vectors/gen_sync_chunk_vectors.mjs index 0c8de07..a88298d 100644 --- a/tests/vectors/gen_sync_chunk_vectors.mjs +++ b/tests/vectors/gen_sync_chunk_vectors.mjs @@ -24,6 +24,12 @@ const big = snapshot('x'.repeat(700), 1789000000000); const other = snapshot('y'.repeat(700), 1789000009000); const small = snapshot('fits', 1789000005000); +// An emoji placed so a cut at a multiple of the budget would fall between its +// two halves. +const prefix = JSON.stringify(snapshot('', 1789000029000)).indexOf('"name":""') + '"name":"'.length; +const cut = Math.ceil((prefix + 1) / LIMIT) * LIMIT; +const straddling = snapshot('p'.repeat(cut - 1 - prefix) + '\u{1F426}' + 'q'.repeat(300), 1789000029000); + const bigRun = splitSyncPayload(big, 'runA', LIMIT); const otherRun = splitSyncPayload(other, 'runB', LIMIT); @@ -39,6 +45,11 @@ console.log(JSON.stringify({ what: 'a big snapshot becomes chunks numbered from zero, then its manifest', payload: big, messages: bigRun + }, + { + what: 'a cut never falls between the two halves of a surrogate pair', + payload: straddling, + messages: splitSyncPayload(straddling, 'runA', LIMIT) } ], reassemble: [ From 78f47081ea65c2f34b9ac69cf5d6c03768ad1712 Mon Sep 17 00:00:00 2001 From: Ocnrb Date: Fri, 25 Sep 2026 11:58:05 +0100 Subject: [PATCH 3/3] Budget sync chunks in escaped UTF-8 bytes rather than characters Co-authored-by: Claude Opus 5.5 --- docs/SYNC-chunk-vectors.json | 93 +++++++++++++----------- src/js/syncChunks.js | 86 +++++++++++++++++----- tests/unit/syncChunks.test.js | 42 ++++++++++- tests/vectors/gen_sync_chunk_vectors.mjs | 10 +-- 4 files changed, 162 insertions(+), 69 deletions(-) diff --git a/docs/SYNC-chunk-vectors.json b/docs/SYNC-chunk-vectors.json index 8a71d28..a1a32b6 100644 --- a/docs/SYNC-chunk-vectors.json +++ b/docs/SYNC-chunk-vectors.json @@ -55,7 +55,7 @@ "syncId": "runA", "chunkIndex": 0, "chunkCount": 5, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -64,7 +64,7 @@ "syncId": "runA", "chunkIndex": 1, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -73,7 +73,7 @@ "syncId": "runA", "chunkIndex": 2, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -82,7 +82,7 @@ "syncId": "runA", "chunkIndex": 3, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -91,7 +91,7 @@ "syncId": "runA", "chunkIndex": 4, "chunkCount": 5, - "data": "xxxx\"}]}}" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}]}}" }, { "type": "sync_manifest", @@ -112,7 +112,7 @@ "channels": [ { "messageStreamId": "0xowner/chan-1", - "name": "ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp🐦qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq" + "name": "🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦" } ] } @@ -124,8 +124,8 @@ "ts": 1789000029000, "syncId": "runA", "chunkIndex": 0, - "chunkCount": 3, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000029000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp" + "chunkCount": 4, + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000029000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦" }, { "type": "sync_chunk", @@ -133,8 +133,8 @@ "ts": 1789000029000, "syncId": "runA", "chunkIndex": 1, - "chunkCount": 3, - "data": "🐦qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq" + "chunkCount": 4, + "data": "🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦" }, { "type": "sync_chunk", @@ -142,15 +142,24 @@ "ts": 1789000029000, "syncId": "runA", "chunkIndex": 2, - "chunkCount": 3, - "data": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq\"}]}}" + "chunkCount": 4, + "data": "🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦" + }, + { + "type": "sync_chunk", + "v": 1, + "ts": 1789000029000, + "syncId": "runA", + "chunkIndex": 3, + "chunkCount": 4, + "data": "🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦🐦\"}]}}" }, { "type": "sync_manifest", "v": 1, "ts": 1789000029000, "syncId": "runA", - "chunkCount": 3 + "chunkCount": 4 } ] } @@ -166,7 +175,7 @@ "syncId": "runA", "chunkIndex": 0, "chunkCount": 5, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -175,7 +184,7 @@ "syncId": "runA", "chunkIndex": 1, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -184,7 +193,7 @@ "syncId": "runA", "chunkIndex": 2, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -193,7 +202,7 @@ "syncId": "runA", "chunkIndex": 3, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -202,7 +211,7 @@ "syncId": "runA", "chunkIndex": 4, "chunkCount": 5, - "data": "xxxx\"}]}}" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}]}}" }, { "type": "sync_manifest", @@ -245,7 +254,7 @@ "syncId": "runA", "chunkIndex": 4, "chunkCount": 5, - "data": "xxxx\"}]}}" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}]}}" }, { "type": "sync_chunk", @@ -254,7 +263,7 @@ "syncId": "runA", "chunkIndex": 3, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -263,7 +272,7 @@ "syncId": "runA", "chunkIndex": 2, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -272,7 +281,7 @@ "syncId": "runA", "chunkIndex": 1, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -281,7 +290,7 @@ "syncId": "runA", "chunkIndex": 0, "chunkCount": 5, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" } ], "payloads": [ @@ -310,7 +319,7 @@ "syncId": "runA", "chunkIndex": 0, "chunkCount": 5, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -319,7 +328,7 @@ "syncId": "runA", "chunkIndex": 1, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -328,7 +337,7 @@ "syncId": "runA", "chunkIndex": 2, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -337,7 +346,7 @@ "syncId": "runA", "chunkIndex": 3, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -346,7 +355,7 @@ "syncId": "runA", "chunkIndex": 4, "chunkCount": 5, - "data": "xxxx\"}]}}" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}]}}" }, { "type": "sync_manifest", @@ -375,7 +384,7 @@ "syncId": "runB", "chunkIndex": 0, "chunkCount": 5, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000009000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000009000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" }, { "type": "sync_chunk", @@ -384,7 +393,7 @@ "syncId": "runB", "chunkIndex": 1, "chunkCount": 5, - "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" + "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" }, { "type": "sync_chunk", @@ -393,7 +402,7 @@ "syncId": "runB", "chunkIndex": 2, "chunkCount": 5, - "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" + "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" }, { "type": "sync_chunk", @@ -402,7 +411,7 @@ "syncId": "runB", "chunkIndex": 3, "chunkCount": 5, - "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" + "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" }, { "type": "sync_chunk", @@ -411,7 +420,7 @@ "syncId": "runB", "chunkIndex": 4, "chunkCount": 5, - "data": "yyyy\"}]}}" + "data": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\"}]}}" }, { "type": "sync_manifest", @@ -473,7 +482,7 @@ "syncId": "runA", "chunkIndex": 1, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -482,7 +491,7 @@ "syncId": "runA", "chunkIndex": 2, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -491,7 +500,7 @@ "syncId": "runA", "chunkIndex": 3, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -500,7 +509,7 @@ "syncId": "runA", "chunkIndex": 4, "chunkCount": 5, - "data": "xxxx\"}]}}" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}]}}" }, { "type": "sync_manifest", @@ -522,7 +531,7 @@ "syncId": "runA", "chunkIndex": 0, "chunkCount": 5, - "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "{\"type\":\"sync\",\"v\":1,\"ts\":1789000000000,\"data\":{\"channels\":[{\"messageStreamId\":\"0xowner/chan-1\",\"name\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -531,7 +540,7 @@ "syncId": "runA", "chunkIndex": 1, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -540,7 +549,7 @@ "syncId": "runA", "chunkIndex": 2, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -549,7 +558,7 @@ "syncId": "runA", "chunkIndex": 3, "chunkCount": 5, - "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "type": "sync_chunk", @@ -558,7 +567,7 @@ "syncId": "runA", "chunkIndex": 4, "chunkCount": 5, - "data": "xxxx\"}]}}" + "data": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}]}}" } ], "payloads": [] diff --git a/src/js/syncChunks.js b/src/js/syncChunks.js index 90e30be..8b6049f 100644 --- a/src/js/syncChunks.js +++ b/src/js/syncChunks.js @@ -14,13 +14,22 @@ */ export const SYNC_CHUNK_CHARS = 150 * 1024; +/** + * UTF-8 bytes a sync chunk's `data` may take once JSON-escaped, quotes + * included. The wire grows with bytes, not characters: sealing adds base64's + * third and a few hundred bytes on top, and a test holds the sealed size + * under the budget. + */ +export const SYNC_CHUNK_BYTES = 150 * 1024; + /** * The row types and fields of one framed protocol. `carry` names the payload * fields every row repeats; `keepPairs` never cuts between the two halves of - * a surrogate pair, which a UTF-8 encoder downstream would turn into '?'. + * a surrogate pair, which a UTF-8 encoder downstream would turn into '?'; + * `bytes` measures the limit in escaped UTF-8 bytes instead of characters. */ const SYNC_FRAME = Object.freeze({ - chunk: 'sync_chunk', manifest: 'sync_manifest', id: 'syncId', carry: ['ts'], keepPairs: true + chunk: 'sync_chunk', manifest: 'sync_manifest', id: 'syncId', carry: ['ts'], keepPairs: true, bytes: true }); /** @@ -29,7 +38,7 @@ const SYNC_FRAME = Object.freeze({ * every row so a run can be ranked before it is assembled. */ export const ADMIN_FRAME = Object.freeze({ - chunk: 'admin_chunk', manifest: 'admin_manifest', id: 'runId', carry: ['rev', 'ts'], keepPairs: true + chunk: 'admin_chunk', manifest: 'admin_manifest', id: 'runId', carry: ['rev', 'ts'], keepPairs: true, bytes: false }); /** @@ -38,23 +47,18 @@ export const ADMIN_FRAME = Object.freeze({ * @param {Object} payload - The whole snapshot * @param {string} runId - Ties the chunks to their manifest * @param {Object} frame - Row types and fields (SYNC_FRAME, ADMIN_FRAME) - * @param {number} limit - Characters per chunk + * @param {number} limit - Per chunk: characters, or escaped UTF-8 bytes of + * its `data` when the frame counts bytes * @returns {Object[]} - The payload itself when it fits, else chunks + manifest */ export function splitFramed(payload, runId, frame, limit) { const serialised = JSON.stringify(payload); - if (serialised.length <= limit) return [payload]; + const size = frame.bytes ? new TextEncoder().encode(serialised).length : serialised.length; + if (size <= limit) return [payload]; - const slices = []; - for (let start = 0; start < serialised.length;) { - let end = Math.min(start + limit, serialised.length); - if (frame.keepPairs && end < serialised.length && end - 1 > start - && isHighSurrogate(serialised.charCodeAt(end - 1))) { - end -= 1; - } - slices.push(serialised.slice(start, end)); - start = end; - } + const slices = frame.bytes + ? sliceByBytes(serialised, limit) + : sliceByChars(serialised, limit, frame.keepPairs); const header = {}; for (const field of frame.carry) header[field] = payload[field]; @@ -117,19 +121,67 @@ export function joinFramed(messages, frame, onDropped) { return out; } +function sliceByChars(s, limit, keepPairs) { + const slices = []; + for (let start = 0; start < s.length;) { + let end = Math.min(start + limit, s.length); + if (keepPairs && end < s.length && end - 1 > start && isHighSurrogate(s.charCodeAt(end - 1))) { + end -= 1; + } + slices.push(s.slice(start, end)); + start = end; + } + return slices; +} + +/** Slices whose JSON.stringify takes at most `limit` UTF-8 bytes; a pair is one unit. */ +function sliceByBytes(s, limit) { + const slices = []; + for (let start = 0; start < s.length;) { + let end = start; + let bytes = 2; + while (end < s.length) { + const pair = isHighSurrogate(s.charCodeAt(end)) && end + 1 < s.length + && isLowSurrogate(s.charCodeAt(end + 1)); + const cost = pair ? 4 : escapedBytes(s.charCodeAt(end)); + if (bytes + cost > limit) break; + bytes += cost; + end += pair ? 2 : 1; + } + if (end === start) throw new Error(`chunk budget of ${limit} B holds no character`); + slices.push(s.slice(start, end)); + start = end; + } + return slices; +} + +/** UTF-8 bytes JSON.stringify writes for one UTF-16 unit that is not half of a pair. */ +function escapedBytes(code) { + if (code === 0x22 || code === 0x5C) return 2; + if (code < 0x20) return [0x08, 0x09, 0x0A, 0x0C, 0x0D].includes(code) ? 2 : 6; + if (code < 0x80) return 1; + if (code < 0x800) return 2; + if (code >= 0xD800 && code <= 0xDFFF) return 6; + return 3; +} + function isHighSurrogate(code) { return code >= 0xD800 && code <= 0xDBFF; } +function isLowSurrogate(code) { + return code >= 0xDC00 && code <= 0xDFFF; +} + /** * Frame a sync snapshot for the wire. * * @param {Object} payload - The whole `{ type:'sync', v, ts, data }` snapshot * @param {string} syncId - Run id, tying the chunks to their manifest - * @param {number} [limit] - Characters per chunk + * @param {number} [limit] - Escaped UTF-8 bytes of each chunk's data * @returns {Object[]} - The payload itself when it fits, else chunks + manifest */ -export function splitSyncPayload(payload, syncId, limit = SYNC_CHUNK_CHARS) { +export function splitSyncPayload(payload, syncId, limit = SYNC_CHUNK_BYTES) { return splitFramed(payload, syncId, SYNC_FRAME, limit); } diff --git a/tests/unit/syncChunks.test.js b/tests/unit/syncChunks.test.js index 2812aab..d40d770 100644 --- a/tests/unit/syncChunks.test.js +++ b/tests/unit/syncChunks.test.js @@ -7,7 +7,14 @@ */ import { describe, it, expect } from 'vitest'; +import { ethers } from 'ethers'; import { splitSyncPayload, reassembleSyncPayloads, SYNC_CHUNK_CHARS } from '../../src/js/syncChunks.js'; +import { dmCrypto } from '../../src/js/dmCrypto.js'; +import { CONFIG } from '../../src/js/config.js'; + +globalThis.ethers = ethers; + +const bytes = (s) => new TextEncoder().encode(s).length; const snapshot = (fill, ts = 1700000000000) => ({ type: 'sync', v: 1, ts, @@ -48,12 +55,21 @@ describe('splitting', () => { expect(reassembleSyncPayloads(out)[0]).toEqual(payload); }); - it('never puts more than the budget in one message', () => { - const out = splitSyncPayload(snapshot('z'.repeat(5000)), 'run1', 300); - for (const chunk of out.filter(m => m.type === 'sync_chunk')) { - expect(chunk.data.length).toBeLessThanOrEqual(300); + it('never puts more than the budget in one message, counted as JSON writes it', () => { + for (const fill of ['z'.repeat(5000), '"\\'.repeat(2000), 'ação coração '.repeat(400), '🐦'.repeat(2000), '\u0001\n'.repeat(1500)]) { + const payload = snapshot(fill); + const out = splitSyncPayload(payload, 'run1', 300); + for (const chunk of out.filter(m => m.type === 'sync_chunk')) { + expect(bytes(JSON.stringify(chunk.data))).toBeLessThanOrEqual(300); + } + expect(reassembleSyncPayloads(out)[0]).toEqual(payload); } }); + + it('cuts accented text into more, shorter chunks than the same length of ASCII', () => { + const count = (fill) => splitSyncPayload(snapshot(fill), 'run1', 300).length; + expect(count('ç'.repeat(3000))).toBeGreaterThan(count('c'.repeat(3000))); + }); }); describe('reassembling', () => { @@ -125,4 +141,22 @@ describe('the budget', () => { it('is the measured one — a 150 KB slice reaches the wire near 227 KB', () => { expect(SYNC_CHUNK_CHARS).toBe(150 * 1024); }); + + it('keeps every sealed chunk under the wire budget, whatever the text is made of', async () => { + const me = new ethers.Wallet('0x' + '11'.repeat(32)); + const seal = (message) => dmCrypto.seal(message, { + senderPrivateKey: me.privateKey, + recipientAddress: me.address, + recipientPublicKey: new ethers.SigningKey(me.privateKey).compressedPublicKey + }); + const wireBudget = CONFIG.media.imagePayloadMaxBytes - CONFIG.media.imagePayloadSafetyMarginBytes; + for (const fill of ['🐦'.repeat(90000), '"'.repeat(200000), 'ç'.repeat(160000), '\u0001'.repeat(60000)]) { + const chunks = splitSyncPayload(snapshot(fill), 'run1').filter(m => m.type === 'sync_chunk'); + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + const { envelope } = await seal(chunk); + expect(bytes(JSON.stringify(envelope))).toBeLessThanOrEqual(wireBudget); + } + } + }); }); diff --git a/tests/vectors/gen_sync_chunk_vectors.mjs b/tests/vectors/gen_sync_chunk_vectors.mjs index a88298d..dfd8f24 100644 --- a/tests/vectors/gen_sync_chunk_vectors.mjs +++ b/tests/vectors/gen_sync_chunk_vectors.mjs @@ -8,7 +8,8 @@ // JSON into the account's state. // // The vectors fix the split (how a payload becomes messages, at a small -// budget so the fixtures stay readable) and the reassembly rules (order does +// budget in escaped UTF-8 bytes so the fixtures stay readable, and never +// between the halves of a surrogate pair) and the reassembly rules (order does // not matter, runs are kept apart, an incomplete or unparseable run is // dropped whole). import { splitSyncPayload, reassembleSyncPayloads } from '../../src/js/syncChunks.js'; @@ -24,11 +25,8 @@ const big = snapshot('x'.repeat(700), 1789000000000); const other = snapshot('y'.repeat(700), 1789000009000); const small = snapshot('fits', 1789000005000); -// An emoji placed so a cut at a multiple of the budget would fall between its -// two halves. -const prefix = JSON.stringify(snapshot('', 1789000029000)).indexOf('"name":""') + '"name":"'.length; -const cut = Math.ceil((prefix + 1) / LIMIT) * LIMIT; -const straddling = snapshot('p'.repeat(cut - 1 - prefix) + '\u{1F426}' + 'q'.repeat(300), 1789000029000); +// Text made only of emoji: every cut lands next to a surrogate pair. +const straddling = snapshot('\u{1F426}'.repeat(150), 1789000029000); const bigRun = splitSyncPayload(big, 'runA', LIMIT); const otherRun = splitSyncPayload(other, 'runB', LIMIT);