From ff73f373fa57ec1d4850cb385bdb158954577f19 Mon Sep 17 00:00:00 2001 From: Maximilian Schneider Date: Tue, 14 Sep 2021 14:47:03 +0200 Subject: [PATCH 1/2] add specs for event queue parsing --- packages/serum/src/queue.test.ts | 81 ++++++++++++++++++++++++++++++++ packages/serum/src/queue.ts | 63 +++++++++++++++---------- 2 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 packages/serum/src/queue.test.ts diff --git a/packages/serum/src/queue.test.ts b/packages/serum/src/queue.test.ts new file mode 100644 index 00000000..6a93b6e2 --- /dev/null +++ b/packages/serum/src/queue.test.ts @@ -0,0 +1,81 @@ +import BN from 'bn.js'; +import { + decodeEventQueue, + decodeEventsSince, + EVENT_QUEUE_LAYOUT, +} from './queue'; + +// convenience definitions for readability +const QueueHeader = EVENT_QUEUE_LAYOUT.HEADER; +const Event = EVENT_QUEUE_LAYOUT.NODE; +const seqNum = (e) => e.seqNum; +const orderId = (e) => e.orderId.toNumber(); + +function queueWith(head, count, seqNum) { + const size = 4; + const b = Buffer.alloc(QueueHeader.span + size * Event.span); + QueueHeader.encode( + { + accountFlags: { initialized: true, eventQueue: true }, + head, + count, + seqNum, + }, + b, + 0, + ); + + // use order id to encode the offset from the beginning for easy tests + for (let i = 0; i < size; i++) { + Event.encode({ orderId: new BN(0) }, b, QueueHeader.span + i * Event.span); + } + + return b; +} + +test('should decode empty queue', () => { + const q = queueWith(0, 0, 0); + // [0, 1, 2, 3] + // ^ + // 0 + + expect(decodeEventQueue(q)).toEqual([]); + expect(decodeEventQueue(q, 4).map(orderId)).toEqual([3, 2, 1, 0]); + expect(decodeEventQueue(q, 4).map(seqNum)).toEqual([-1, -2, -3, -4]); + expect(decodeEventsSince(q, 0)).toEqual([]); + expect(decodeEventsSince(q, 3)).toEqual([]); +}); + +test('should decode empty queue from mid', () => { + const q = queueWith(2, 0, 2); + // [0, 1, 2, 3] + // ^ + // 2 + + expect(decodeEventQueue(q).map(orderId)).toEqual([]); + expect(decodeEventQueue(q).map(seqNum)).toEqual([]); + expect(decodeEventQueue(q, 4).map(orderId)).toEqual([1, 0, 3, 2]); + expect(decodeEventQueue(q, 4).map(seqNum)).toEqual([1, 0, -1, -2]); + expect(decodeEventsSince(q, 0).map(orderId)).toEqual([0, 1]); + expect(decodeEventsSince(q, 0).map(seqNum)).toEqual([0, 1]); +}); + +test('should decode full event queue after roll-over', () => { + const q = queueWith(2, 4, 6); + // [0, 1, 2, 3] + // ^ + // 6 + + expect(decodeEventQueue(q).map(orderId)).toEqual([2, 3, 0, 1]); + expect(decodeEventQueue(q).map(seqNum)).toEqual([2, 3, 4, 5]); + expect(decodeEventQueue(q, 4).map(orderId)).toEqual([2, 3, 0, 1].reverse()); + expect(decodeEventQueue(q, 4).map(seqNum)).toEqual([2, 3, 4, 5].reverse()); + expect(decodeEventsSince(q, 0).map(seqNum)).toEqual([2, 3, 4, 5]); + expect(decodeEventsSince(q, 1).map(seqNum)).toEqual([2, 3, 4, 5]); + expect(decodeEventsSince(q, 2).map(seqNum)).toEqual([2, 3, 4, 5]); + expect(decodeEventsSince(q, 3).map(seqNum)).toEqual([3, 4, 5]); + expect(decodeEventsSince(q, 4).map(seqNum)).toEqual([4, 5]); + expect(decodeEventsSince(q, 5).map(seqNum)).toEqual([5]); + expect(decodeEventsSince(q, 6).map(seqNum)).toEqual([]); + expect(decodeEventsSince(q, 7).map(seqNum)).toEqual([]); +}); diff --git a/packages/serum/src/queue.ts b/packages/serum/src/queue.ts index 1482789b..7b36ddd2 100644 --- a/packages/serum/src/queue.ts +++ b/packages/serum/src/queue.ts @@ -107,12 +107,16 @@ function decodeQueue( for (let i = 0; i < Math.min(history, allocLen); ++i) { const nodeIndex = (header.head + header.count + allocLen - 1 - i) % allocLen; - nodes.push(decodeQueueItem(headerLayout, nodeLayout, buffer, nodeIndex)); + const item = decodeQueueItem(headerLayout, nodeLayout, buffer, nodeIndex); + item.seqNum = header.seqNum - 1 - i; + nodes.push(item); } } else { for (let i = 0; i < header.count; ++i) { const nodeIndex = (header.head + i) % allocLen; - nodes.push(decodeQueueItem(headerLayout, nodeLayout, buffer, nodeIndex)); + const item = decodeQueueItem(headerLayout, nodeLayout, buffer, nodeIndex); + item.seqNum = header.seqNum - header.count + i; + nodes.push(item); } } return { header, nodes }; @@ -120,29 +124,40 @@ function decodeQueue( export function decodeEventsSince(buffer: Buffer, lastSeqNum: number): Event[] { const header = EVENT_QUEUE_HEADER.decode(buffer); - const allocLen = Math.floor( - (buffer.length - EVENT_QUEUE_HEADER.span) / EVENT.span, - ); - - // calculate number of missed events - // account for u32 & ringbuffer overflows - const modulo32Uint = 0x100000000; - let missedEvents = (header.seqNum - lastSeqNum + modulo32Uint) % modulo32Uint; - if (missedEvents > allocLen) { - missedEvents = allocLen - 1; - } - const startSeq = (header.seqNum - missedEvents + modulo32Uint) % modulo32Uint; - - // define boundary indexes in ring buffer [start;end) - const endIndex = (header.head + header.count) % allocLen; - const startIndex = (endIndex - missedEvents + allocLen) % allocLen; - const results: Event[] = []; - for (let i = 0; i < missedEvents; ++i) { - const nodeIndex = (startIndex + i) % allocLen; - const event = decodeQueueItem(EVENT_QUEUE_HEADER, EVENT, buffer, nodeIndex); - event.seqNum = (startSeq + i) % modulo32Uint; - results.push(event); + if (lastSeqNum < header.seqNum) { + const allocLen = Math.floor( + (buffer.length - EVENT_QUEUE_HEADER.span) / EVENT.span, + ); + // calculate number of missed events + // account for u32 & ringbuffer overflows + const modulo32Uint = 0x100000000; + let missedEvents = + (header.seqNum - lastSeqNum + modulo32Uint) % modulo32Uint; + + // console.log({ allocLen, missedEvents, lastSeqNum, head: header.seqNum }); + if (missedEvents > allocLen) { + missedEvents = allocLen; + } + const startSeq = + (header.seqNum - missedEvents + modulo32Uint) % modulo32Uint; + + // define boundary indexes in ring buffer [start;end) + const endIndex = (header.head + header.count) % allocLen; + const startIndex = (endIndex - missedEvents + allocLen) % allocLen; + + // console.log({ allocLen, missedEvents, startSeq, endIndex, startIndex }); + for (let i = 0; i < missedEvents; ++i) { + const nodeIndex = (startIndex + i) % allocLen; + const event = decodeQueueItem( + EVENT_QUEUE_HEADER, + EVENT, + buffer, + nodeIndex, + ); + event.seqNum = (startSeq + i) % modulo32Uint; + results.push(event); + } } return results; } From 479829ca3552e6b72f75ea5c72ae6718a5f4cb30 Mon Sep 17 00:00:00 2001 From: Maximilian Schneider Date: Tue, 14 Sep 2021 17:31:10 +0200 Subject: [PATCH 2/2] fix typo --- packages/serum/src/queue.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/serum/src/queue.test.ts b/packages/serum/src/queue.test.ts index 6a93b6e2..87976c69 100644 --- a/packages/serum/src/queue.test.ts +++ b/packages/serum/src/queue.test.ts @@ -27,7 +27,7 @@ function queueWith(head, count, seqNum) { // use order id to encode the offset from the beginning for easy tests for (let i = 0; i < size; i++) { - Event.encode({ orderId: new BN(0) }, b, QueueHeader.span + i * Event.span); + Event.encode({ orderId: new BN(i) }, b, QueueHeader.span + i * Event.span); } return b;