diff --git a/README.md b/README.md index 9edde5f..46e7793 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,11 @@ import "github.com/luxfi/zap" ``` +```python +# Python — pure stdlib, no deps. See python/README.md +from zap_py import Builder, parse +``` + ZAP is a high-performance binary protocol for AI agent communication and inter-process messaging. It provides **17x faster serialization** and **11x less memory** than MCP JSON-RPC, while maintaining full compatibility with existing MCP tools. ## Features diff --git a/go.sum b/go.sum index eec014f..68f7bbe 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE= github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs= +github.com/luxfi/mdns v0.1.0 h1:VB3mQcETc9j5SY1S6lAgFtuGr/rjWuDgPYnxS+OKWMQ= +github.com/luxfi/mdns v0.1.0/go.mod h1:/3dheKVjUk2yiS/ocH1IDzeLXOIe+kpVsErIGDOZdiQ= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000..76180b2 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +*.egg-info/ diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..17daeff --- /dev/null +++ b/python/README.md @@ -0,0 +1,117 @@ +# zap_py — Python client for ZAP + +Pure-stdlib Python reader and builder for the ZAP wire format. Lets non-Go +peers (AI agents, ops scripts, FHE clients, kcolbchain switchboard/monsoon +agents) speak ZAP without leaving Python. + +Wire-compatible with `github.com/luxfi/zap`. Tested in both directions. + +## Install + +No package install — drop the `zap_py/` directory next to your code, or add +`python/` to `PYTHONPATH`. Python ≥ 3.8, no third-party dependencies. + +## Read + +```python +from zap_py import parse + +msg = parse(wire_bytes) +root = msg.root() +print(root.uint32(0)) # 42 +print(root.text(16)) # "from go" +print(root.bytes(24)) # b"\x01\x02\x03\x04" + +inner = root.object(32) +print(inner.uint32(0)) # 7 + +lst = root.list(40) +for i in range(len(lst)): + print(lst.uint32(i)) +``` + +`parse()` accepts `bytes`, `bytearray`, or `memoryview`. The returned +`Message` keeps a `memoryview` over the original buffer — slicing and +`.bytes_view()` stay zero-copy. + +## Build + +```python +from zap_py import Builder + +b = Builder(256) + +inner = b.start_object(24) +inner.set_uint32(0, 7) +inner.set_text(8, "nested") +inner_offset = inner.finish() + +lb = b.start_list(4) +for v in (10, 20, 30, 40): + lb.add_uint32(v) +list_offset, list_len = lb.finish() + +root = b.start_object(56) +root.set_uint32(0, 42) +root.set_uint64(8, 0xDEADBEEFCAFEBABE) +root.set_text(16, "from py") +root.set_bytes(24, b"\x01\x02\x03\x04") +root.set_object(32, inner_offset) +root.set_list(40, list_offset, list_len) +root.finish_as_root() + +wire = b.finish() # bytes, ready to send +wire = b.finish_with_flags(0) # explicit flag word +``` + +Field offsets and data sizes mirror the Go schema exactly — same constants +work on both sides. + +## Tests + +```bash +cd python +python -m venv .venv && .venv/bin/pip install pytest +.venv/bin/pytest tests +``` + +12 tests cover: header validation, scalar/text/bytes roundtrip, lists, +nested objects, null pointers, flag bits, invalid magic/version, and +zero-copy `memoryview` access. + +## Cross-language interop + +Two parity tests confirm wire compatibility against the Go reference: + +**Go-built → Python-parsed:** + +```bash +go run ./python/testdata/gen_fixture.go > /tmp/zap_fixture.bin +ZAP_GO_FIXTURE=/tmp/zap_fixture.bin python -m pytest python/tests/test_roundtrip.py::test_go_fixture_interop +``` + +**Python-built → Go-parsed:** + +```bash +python python/testdata/gen_python_fixture.py > /tmp/zap_python_fixture.bin +ZAP_PYTHON_FIXTURE=/tmp/zap_python_fixture.bin go test -run TestPythonFixture +``` + +Both fixtures use identical schemas; the resulting binaries differ only in +their embedded text payload, byte-for-byte. + +## Coverage + +| Wire feature | Reader | Builder | +|---|---|---| +| Header (magic/version/flags/size) | ✓ | ✓ | +| `bool`, `uint8/16/32/64`, `int8/16/32/64`, `float32/64` | ✓ | ✓ | +| `text`, `bytes` (zero-copy view available) | ✓ | ✓ | +| Nested objects (relative offsets) | ✓ | ✓ | +| Lists of `uint8`/`uint32`/`uint64`/objects/raw bytes | ✓ | ✓ | +| Null object / null list | ✓ | ✓ | + +Not yet ported: EVM helpers (`Address`, `Hash`, `Signature`), MCP bridge, +mDNS node, schema DSL. The reader/builder is enough to interop with any +Go ZAP service that publishes a fixed schema. Higher-level helpers can +follow as Python use cases land. diff --git a/python/testdata/gen_fixture.go b/python/testdata/gen_fixture.go new file mode 100644 index 0000000..a03a284 --- /dev/null +++ b/python/testdata/gen_fixture.go @@ -0,0 +1,43 @@ +// Build a deterministic ZAP message used by the Python interop test. +// +// go run ./python/testdata/gen_fixture.go > /tmp/zap_fixture.bin +// ZAP_GO_FIXTURE=/tmp/zap_fixture.bin pytest python/tests +// +//go:build ignore + +package main + +import ( + "os" + + "github.com/luxfi/zap" +) + +func main() { + b := zap.NewBuilder(512) + + inner := b.StartObject(24) + inner.SetUint32(0, 7) + inner.SetText(8, "nested") + innerOffset := inner.Finish() + + lb := b.StartList(4) + lb.AddUint32(10) + lb.AddUint32(20) + lb.AddUint32(30) + lb.AddUint32(40) + listOffset, listLen := lb.Finish() + + root := b.StartObject(56) + root.SetUint32(0, 42) + root.SetUint64(8, 0xDEADBEEFCAFEBABE) + root.SetText(16, "from go") + root.SetBytes(24, []byte{0x01, 0x02, 0x03, 0x04}) + root.SetObject(32, innerOffset) + root.SetList(40, listOffset, listLen) + root.FinishAsRoot() + + if _, err := os.Stdout.Write(b.Finish()); err != nil { + panic(err) + } +} diff --git a/python/testdata/gen_python_fixture.py b/python/testdata/gen_python_fixture.py new file mode 100644 index 0000000..acce672 --- /dev/null +++ b/python/testdata/gen_python_fixture.py @@ -0,0 +1,44 @@ +"""Build a deterministic ZAP message from Python. + +Used by python_interop_test.go to confirm the Go reader accepts what +zap_py emits. + + python python/testdata/gen_python_fixture.py > /tmp/zap_python_fixture.bin + ZAP_PYTHON_FIXTURE=/tmp/zap_python_fixture.bin go test -run TestPythonFixture +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from zap_py import Builder + + +def build() -> bytes: + b = Builder(512) + + inner = b.start_object(24) + inner.set_uint32(0, 7) + inner.set_text(8, "nested") + inner_offset = inner.finish() + + lb = b.start_list(4) + for v in (10, 20, 30, 40): + lb.add_uint32(v) + list_offset, list_len = lb.finish() + + root = b.start_object(56) + root.set_uint32(0, 42) + root.set_uint64(8, 0xDEADBEEFCAFEBABE) + root.set_text(16, "from py") + root.set_bytes(24, b"\x01\x02\x03\x04") + root.set_object(32, inner_offset) + root.set_list(40, list_offset, list_len) + root.finish_as_root() + + return b.finish() + + +if __name__ == "__main__": + sys.stdout.buffer.write(build()) diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/tests/test_roundtrip.py b/python/tests/test_roundtrip.py new file mode 100644 index 0000000..ba42b1a --- /dev/null +++ b/python/tests/test_roundtrip.py @@ -0,0 +1,221 @@ +"""Python-only roundtrip tests for zap_py. + +The Go-vs-Python interop test lives in test_go_fixture.py and consumes +a binary fixture emitted by `python/testdata/gen_fixture.go`. +""" + +import math +import os + +import pytest + +from zap_py import ( + Builder, + FLAG_COMPRESSED, + FLAG_SIGNED, + HEADER_SIZE, + InvalidMagic, + InvalidVersion, + MAGIC, + parse, +) + + +def test_header_basics(): + b = Builder() + ob = b.start_object(8) + ob.set_uint32(0, 42) + ob.set_uint32(4, 0xDEADBEEF) + ob.finish_as_root() + data = b.finish() + + assert data[:4] == MAGIC + assert len(data) >= HEADER_SIZE + msg = parse(data) + assert msg.size() == len(data) + assert msg.flags() == 0 + + +def test_scalar_roundtrip(): + b = Builder() + ob = b.start_object(64) + ob.set_bool(0, True) + ob.set_uint8(1, 0xAB) + ob.set_uint16(2, 0xBEEF) + ob.set_uint32(4, 0xDEADBEEF) + ob.set_uint64(8, 0x0123456789ABCDEF) + ob.set_int32(16, -12345) + ob.set_int64(20, -987654321) + ob.set_float32(28, 1.5) + ob.set_float64(32, math.pi) + ob.finish_as_root() + + msg = parse(b.finish()) + root = msg.root() + assert root.bool(0) is True + assert root.uint8(1) == 0xAB + assert root.uint16(2) == 0xBEEF + assert root.uint32(4) == 0xDEADBEEF + assert root.uint64(8) == 0x0123456789ABCDEF + assert root.int32(16) == -12345 + assert root.int64(20) == -987654321 + assert root.float32(28) == 1.5 + assert math.isclose(root.float64(32), math.pi) + + +def test_text_and_bytes(): + b = Builder() + ob = b.start_object(24) + ob.set_uint32(0, 7) + ob.set_text(8, "héllo, ZAP") + ob.set_bytes(16, b"\x00\x01\x02\xff") + ob.finish_as_root() + + msg = parse(b.finish()) + root = msg.root() + assert root.uint32(0) == 7 + assert root.text(8) == "héllo, ZAP" + assert root.bytes(16) == b"\x00\x01\x02\xff" + + +def test_empty_text_returns_null(): + b = Builder() + ob = b.start_object(16) + ob.set_text(0, "") + ob.set_text(8, "non-empty") + ob.finish_as_root() + root = parse(b.finish()).root() + assert root.text(0) == "" + assert root.text(8) == "non-empty" + + +def test_uint32_list(): + b = Builder() + lb = b.start_list(4) + for v in (10, 20, 30, 40, 50): + lb.add_uint32(v) + list_offset, length = lb.finish() + assert length == 5 + + ob = b.start_object(8) + ob.set_list(0, list_offset, length) + ob.finish_as_root() + + root = parse(b.finish()).root() + lst = root.list(0) + assert lst.len() == 5 + assert [lst.uint32(i) for i in range(lst.len())] == [10, 20, 30, 40, 50] + + +def test_uint64_list(): + b = Builder() + lb = b.start_list(8) + for v in (1, 1 << 32, (1 << 63) | 1): + lb.add_uint64(v) + list_offset, length = lb.finish() + ob = b.start_object(8) + ob.set_list(0, list_offset, length) + ob.finish_as_root() + root = parse(b.finish()).root() + lst = root.list(0) + assert lst.len() == 3 + assert lst.uint64(0) == 1 + assert lst.uint64(1) == 1 << 32 + assert lst.uint64(2) == (1 << 63) | 1 + + +def test_nested_object(): + b = Builder() + inner = b.start_object(8) + inner.set_uint32(0, 111) + inner.set_uint32(4, 222) + inner_offset = inner.finish() + + outer = b.start_object(16) + outer.set_uint32(0, 999) + outer.set_object(8, inner_offset) + outer.finish_as_root() + + root = parse(b.finish()).root() + assert root.uint32(0) == 999 + inner_view = root.object(8) + assert not inner_view.is_null() + assert inner_view.uint32(0) == 111 + assert inner_view.uint32(4) == 222 + + +def test_null_object_and_list(): + b = Builder() + ob = b.start_object(24) + ob.set_uint32(0, 1) + ob.set_object(8, 0) + ob.set_list(12, 0, 0) + ob.finish_as_root() + root = parse(b.finish()).root() + assert root.object(8).is_null() + assert root.list(12).is_null() + + +def test_flags(): + b = Builder() + ob = b.start_object(8) + ob.set_uint32(0, 1) + ob.finish_as_root() + data = b.finish_with_flags(FLAG_COMPRESSED | FLAG_SIGNED) + msg = parse(data) + assert msg.flags() == FLAG_COMPRESSED | FLAG_SIGNED + + +def test_invalid_magic(): + bad = b"BAD\x00" + bytes(HEADER_SIZE - 4) + with pytest.raises(InvalidMagic): + parse(bad) + + +def test_invalid_version(): + buf = bytearray(HEADER_SIZE) + buf[0:4] = MAGIC + # version = 99 + buf[4] = 99 + buf[5] = 0 + # size = 16 + buf[12] = 16 + with pytest.raises(InvalidVersion): + parse(bytes(buf)) + + +def test_memoryview_zero_copy(): + b = Builder() + ob = b.start_object(16) + ob.set_text(0, "zero-copy view") + ob.finish_as_root() + data = b.finish() + msg = parse(memoryview(data)) + view = msg.root().bytes_view(0) + assert isinstance(view, memoryview) + assert bytes(view) == b"zero-copy view" + + +@pytest.mark.skipif( + not os.environ.get("ZAP_GO_FIXTURE"), + reason="set ZAP_GO_FIXTURE=path/to/fixture.bin (built with gen_fixture.go)", +) +def test_go_fixture_interop(): + """End-to-end: parse a binary built by the Go reference implementation.""" + path = os.environ["ZAP_GO_FIXTURE"] + with open(path, "rb") as f: + data = f.read() + msg = parse(data) + root = msg.root() + # Layout matches gen_fixture.go. + assert root.uint32(0) == 42 + assert root.uint64(8) == 0xDEADBEEFCAFEBABE + assert root.text(16) == "from go" + assert root.bytes(24) == b"\x01\x02\x03\x04" + inner = root.object(32) + assert not inner.is_null() + assert inner.uint32(0) == 7 + assert inner.text(8) == "nested" + lst = root.list(40) + assert lst.len() == 4 + assert [lst.uint32(i) for i in range(lst.len())] == [10, 20, 30, 40] diff --git a/python/zap_py/__init__.py b/python/zap_py/__init__.py new file mode 100644 index 0000000..62afbad --- /dev/null +++ b/python/zap_py/__init__.py @@ -0,0 +1,49 @@ +"""Python client for the ZAP zero-copy application protocol. + +Mirrors the Go implementation at github.com/luxfi/zap. Lets non-Go peers +(AI agents, ops scripts, FHE clients) read and build ZAP messages +without leaving Python. +""" + +from .wire import ( + HEADER_SIZE, + MAGIC, + VERSION, + DEFAULT_PORT, + ALIGNMENT, + FLAG_NONE, + FLAG_COMPRESSED, + FLAG_ENCRYPTED, + FLAG_SIGNED, + InvalidMagic, + InvalidVersion, + BufferTooSmall, + OutOfBounds, + InvalidOffset, +) +from .reader import Message, Object, List, parse +from .builder import Builder, ObjectBuilder, ListBuilder + +__all__ = [ + "HEADER_SIZE", + "MAGIC", + "VERSION", + "DEFAULT_PORT", + "ALIGNMENT", + "FLAG_NONE", + "FLAG_COMPRESSED", + "FLAG_ENCRYPTED", + "FLAG_SIGNED", + "InvalidMagic", + "InvalidVersion", + "BufferTooSmall", + "OutOfBounds", + "InvalidOffset", + "Message", + "Object", + "List", + "parse", + "Builder", + "ObjectBuilder", + "ListBuilder", +] diff --git a/python/zap_py/builder.py b/python/zap_py/builder.py new file mode 100644 index 0000000..1670f8c --- /dev/null +++ b/python/zap_py/builder.py @@ -0,0 +1,248 @@ +"""Builder for ZAP messages — wire-compatible with the Go implementation.""" + +from __future__ import annotations + +import struct +from dataclasses import dataclass, field +from typing import List as _PyList, Optional + +from .wire import ALIGNMENT, HEADER_SIZE, MAGIC, VERSION + +_U16 = struct.Struct(" None: + self._pos = HEADER_SIZE + self._root_offset = 0 + + @property + def pos(self) -> int: + return self._pos + + def _grow(self, n: int) -> None: + needed = self._pos + n + if needed <= len(self._buf): + return + new_cap = len(self._buf) * 2 + if new_cap < needed: + new_cap = needed + self._buf.extend(bytes(new_cap - len(self._buf))) + + def _align(self, alignment: int) -> None: + padding = (alignment - (self._pos % alignment)) % alignment + if padding == 0: + return + self._grow(padding) + for i in range(padding): + self._buf[self._pos + i] = 0 + self._pos += padding + + def finish(self) -> bytes: + _U32.pack_into(self._buf, 8, self._root_offset) + _U32.pack_into(self._buf, 12, self._pos) + return bytes(self._buf[:self._pos]) + + def finish_with_flags(self, flags: int) -> bytes: + _U16.pack_into(self._buf, 6, flags) + return self.finish() + + def start_object(self, data_size: int) -> "ObjectBuilder": + self._align(ALIGNMENT) + return ObjectBuilder(self, self._pos, data_size) + + def start_list(self, elem_size: int) -> "ListBuilder": + self._align(ALIGNMENT) + return ListBuilder(self, self._pos, elem_size) + + def write_bytes(self, data: bytes) -> int: + if not data: + return 0 + self._align(ALIGNMENT) + offset = self._pos + self._grow(len(data)) + self._buf[self._pos:self._pos + len(data)] = data + self._pos += len(data) + return offset + + def write_text(self, s: str) -> int: + return self.write_bytes(s.encode("utf-8")) + + +@dataclass +class _OffsetEntry: + field_offset: int + data: bytes + + +class ObjectBuilder: + """Builds a single ZAP object inside a Builder.""" + + __slots__ = ("_b", "_start_pos", "_data_size", "_offsets") + + def __init__(self, b: Builder, start_pos: int, data_size: int): + self._b = b + self._start_pos = start_pos + self._data_size = data_size + self._offsets: _PyList[_OffsetEntry] = [] + + def _ensure_field(self, end_offset: int) -> None: + needed = self._start_pos + end_offset + b = self._b + if needed > b._pos: + b._grow(needed - b._pos) + for i in range(b._pos, needed): + b._buf[i] = 0 + b._pos = needed + + def set_bool(self, field_offset: int, v: bool) -> None: + self.set_uint8(field_offset, 1 if v else 0) + + def set_uint8(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 1) + self._b._buf[self._start_pos + field_offset] = v & 0xFF + + def set_uint16(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 2) + _U16.pack_into(self._b._buf, self._start_pos + field_offset, v & 0xFFFF) + + def set_uint32(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 4) + _U32.pack_into(self._b._buf, self._start_pos + field_offset, v & 0xFFFFFFFF) + + def set_uint64(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 8) + _U64.pack_into(self._b._buf, self._start_pos + field_offset, v & 0xFFFFFFFFFFFFFFFF) + + def set_int8(self, field_offset: int, v: int) -> None: + self.set_uint8(field_offset, v & 0xFF) + + def set_int16(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 2) + _I16.pack_into(self._b._buf, self._start_pos + field_offset, v) + + def set_int32(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 4) + _I32.pack_into(self._b._buf, self._start_pos + field_offset, v) + + def set_int64(self, field_offset: int, v: int) -> None: + self._ensure_field(field_offset + 8) + _I64.pack_into(self._b._buf, self._start_pos + field_offset, v) + + def set_float32(self, field_offset: int, v: float) -> None: + self._ensure_field(field_offset + 4) + _F32.pack_into(self._b._buf, self._start_pos + field_offset, v) + + def set_float64(self, field_offset: int, v: float) -> None: + self._ensure_field(field_offset + 8) + _F64.pack_into(self._b._buf, self._start_pos + field_offset, v) + + def set_text(self, field_offset: int, v: str) -> None: + self.set_bytes(field_offset, v.encode("utf-8")) + + def set_bytes(self, field_offset: int, v: bytes) -> None: + self._ensure_field(field_offset + 8) + if len(v) == 0: + _U32.pack_into(self._b._buf, self._start_pos + field_offset, 0) + _U32.pack_into(self._b._buf, self._start_pos + field_offset + 4, 0) + return + self._offsets.append(_OffsetEntry(field_offset=field_offset, data=bytes(v))) + _U32.pack_into(self._b._buf, self._start_pos + field_offset + 4, len(v)) + + def set_object(self, field_offset: int, obj_offset: int) -> None: + self._ensure_field(field_offset + 4) + if obj_offset == 0: + _U32.pack_into(self._b._buf, self._start_pos + field_offset, 0) + return + rel_offset = obj_offset - (self._start_pos + field_offset) + _I32.pack_into(self._b._buf, self._start_pos + field_offset, rel_offset) + + def set_list(self, field_offset: int, list_offset: int, length: int) -> None: + self._ensure_field(field_offset + 8) + if list_offset == 0 or length == 0: + _U32.pack_into(self._b._buf, self._start_pos + field_offset, 0) + _U32.pack_into(self._b._buf, self._start_pos + field_offset + 4, 0) + return + rel_offset = list_offset - (self._start_pos + field_offset) + _I32.pack_into(self._b._buf, self._start_pos + field_offset, rel_offset) + _U32.pack_into(self._b._buf, self._start_pos + field_offset + 4, length) + + def finish(self) -> int: + self._ensure_field(self._data_size) + b = self._b + for entry in self._offsets: + data_pos = b._pos + b._grow(len(entry.data)) + b._buf[b._pos:b._pos + len(entry.data)] = entry.data + b._pos += len(entry.data) + field_abs_pos = self._start_pos + entry.field_offset + rel_offset = data_pos - field_abs_pos + _I32.pack_into(b._buf, field_abs_pos, rel_offset) + return self._start_pos + + def finish_as_root(self) -> int: + offset = self.finish() + self._b._root_offset = offset + return offset + + +class ListBuilder: + """Builds a contiguous list of fixed-width elements.""" + + __slots__ = ("_b", "_start_pos", "_elem_size", "_count") + + def __init__(self, b: Builder, start_pos: int, elem_size: int): + self._b = b + self._start_pos = start_pos + self._elem_size = elem_size + self._count = 0 + + def add_uint8(self, v: int) -> None: + self._b._grow(1) + self._b._buf[self._b._pos] = v & 0xFF + self._b._pos += 1 + self._count += 1 + + def add_uint32(self, v: int) -> None: + self._b._grow(4) + _U32.pack_into(self._b._buf, self._b._pos, v & 0xFFFFFFFF) + self._b._pos += 4 + self._count += 1 + + def add_uint64(self, v: int) -> None: + self._b._grow(8) + _U64.pack_into(self._b._buf, self._b._pos, v & 0xFFFFFFFFFFFFFFFF) + self._b._pos += 8 + self._count += 1 + + def add_bytes(self, data: bytes) -> None: + self._b._grow(len(data)) + self._b._buf[self._b._pos:self._b._pos + len(data)] = data + self._b._pos += len(data) + self._count += len(data) + + def finish(self) -> tuple[int, int]: + return self._start_pos, self._count diff --git a/python/zap_py/reader.py b/python/zap_py/reader.py new file mode 100644 index 0000000..8a52585 --- /dev/null +++ b/python/zap_py/reader.py @@ -0,0 +1,265 @@ +"""Zero-copy reader for ZAP messages. + +Mirrors zap.go. Backing storage is a memoryview so slicing does not +copy — the same read-side guarantees as the Go side. +""" + +from __future__ import annotations + +import struct +from typing import Optional, Union + +from .wire import ( + HEADER_SIZE, + MAGIC, + VERSION, + BufferTooSmall, + InvalidMagic, + InvalidVersion, +) + +_U16 = struct.Struct(" "Message": + """Parse a ZAP message without copying the backing bytes.""" + if len(data) < HEADER_SIZE: + raise BufferTooSmall(f"need {HEADER_SIZE} bytes, got {len(data)}") + view = memoryview(data) if not isinstance(data, memoryview) else data + if bytes(view[0:4]) != MAGIC: + raise InvalidMagic(f"got {bytes(view[0:4])!r}, expected {MAGIC!r}") + version, = _U16.unpack_from(view, 4) + if version != VERSION: + raise InvalidVersion(f"version {version} not supported") + size, = _U32.unpack_from(view, 12) + if size > len(view): + raise BufferTooSmall(f"declared size {size} > buffer {len(view)}") + return Message(view[:size]) + + +class Message: + """A parsed ZAP message. Backed by a memoryview — no copy.""" + + __slots__ = ("_data",) + + def __init__(self, data: memoryview): + self._data = data + + @property + def data(self) -> memoryview: + return self._data + + def size(self) -> int: + return len(self._data) + + def flags(self) -> int: + return _U16.unpack_from(self._data, 6)[0] + + def root(self) -> "Object": + offset, = _U32.unpack_from(self._data, 8) + return Object(self, int(offset)) + + def __len__(self) -> int: + return len(self._data) + + +class Object: + """Zero-copy view of a ZAP struct.""" + + __slots__ = ("_msg", "_offset") + + def __init__(self, msg: Optional[Message], offset: int): + self._msg = msg + self._offset = offset + + @property + def offset(self) -> int: + return self._offset + + def is_null(self) -> bool: + return self._msg is None or self._offset == 0 + + def _abs(self, field_offset: int) -> int: + return self._offset + field_offset + + def bool(self, field_offset: int) -> bool: + return self.uint8(field_offset) != 0 + + def uint8(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos >= len(self._msg._data): + return 0 + return self._msg._data[pos] + + def uint16(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos + 2 > len(self._msg._data): + return 0 + return _U16.unpack_from(self._msg._data, pos)[0] + + def uint32(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos + 4 > len(self._msg._data): + return 0 + return _U32.unpack_from(self._msg._data, pos)[0] + + def uint64(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos + 8 > len(self._msg._data): + return 0 + return _U64.unpack_from(self._msg._data, pos)[0] + + def int8(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos >= len(self._msg._data): + return 0 + return _I8.unpack_from(self._msg._data, pos)[0] + + def int16(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos + 2 > len(self._msg._data): + return 0 + return _I16.unpack_from(self._msg._data, pos)[0] + + def int32(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos + 4 > len(self._msg._data): + return 0 + return _I32.unpack_from(self._msg._data, pos)[0] + + def int64(self, field_offset: int) -> int: + pos = self._abs(field_offset) + if pos + 8 > len(self._msg._data): + return 0 + return _I64.unpack_from(self._msg._data, pos)[0] + + def float32(self, field_offset: int) -> float: + pos = self._abs(field_offset) + if pos + 4 > len(self._msg._data): + return 0.0 + return _F32.unpack_from(self._msg._data, pos)[0] + + def float64(self, field_offset: int) -> float: + pos = self._abs(field_offset) + if pos + 8 > len(self._msg._data): + return 0.0 + return _F64.unpack_from(self._msg._data, pos)[0] + + def bytes(self, field_offset: int) -> bytes: + view = self.bytes_view(field_offset) + return bytes(view) if view is not None else b"" + + def bytes_view(self, field_offset: int) -> Optional[memoryview]: + """Like .bytes() but returns a zero-copy memoryview slice.""" + pos = self._abs(field_offset) + data = self._msg._data + if pos + 4 > len(data): + return None + rel_offset = _I32.unpack_from(data, pos)[0] + if rel_offset == 0: + return None + if pos + 8 > len(data): + return None + length = _U32.unpack_from(data, pos + 4)[0] + abs_pos = pos + rel_offset + if abs_pos < 0 or abs_pos + length > len(data): + return None + return data[abs_pos:abs_pos + length] + + def text(self, field_offset: int) -> str: + view = self.bytes_view(field_offset) + if view is None: + return "" + return bytes(view).decode("utf-8") + + def object(self, field_offset: int) -> "Object": + pos = self._abs(field_offset) + data = self._msg._data + if pos + 4 > len(data): + return Object(None, 0) + rel_offset = _I32.unpack_from(data, pos)[0] + if rel_offset == 0: + return Object(None, 0) + abs_offset = pos + rel_offset + if abs_offset < 0 or abs_offset >= len(data): + return Object(None, 0) + return Object(self._msg, abs_offset) + + def list(self, field_offset: int) -> "List": + pos = self._abs(field_offset) + data = self._msg._data + if pos + 8 > len(data): + return List(None, 0, 0) + rel_offset = _I32.unpack_from(data, pos)[0] + if rel_offset == 0: + return List(None, 0, 0) + length = _U32.unpack_from(data, pos + 4)[0] + abs_offset = pos + rel_offset + if abs_offset < 0 or abs_offset >= len(data): + return List(None, 0, 0) + return List(self._msg, abs_offset, int(length)) + + +class List: + """Zero-copy view of a ZAP list.""" + + __slots__ = ("_msg", "_offset", "_length") + + def __init__(self, msg: Optional[Message], offset: int, length: int): + self._msg = msg + self._offset = offset + self._length = length + + def is_null(self) -> bool: + return self._msg is None + + def __len__(self) -> int: + return self._length + + def len(self) -> int: + return self._length + + def uint8(self, i: int) -> int: + if i < 0 or i >= self._length: + return 0 + pos = self._offset + i + if pos >= len(self._msg._data): + return 0 + return self._msg._data[pos] + + def uint32(self, i: int) -> int: + if i < 0 or i >= self._length: + return 0 + pos = self._offset + i * 4 + if pos + 4 > len(self._msg._data): + return 0 + return _U32.unpack_from(self._msg._data, pos)[0] + + def uint64(self, i: int) -> int: + if i < 0 or i >= self._length: + return 0 + pos = self._offset + i * 8 + if pos + 8 > len(self._msg._data): + return 0 + return _U64.unpack_from(self._msg._data, pos)[0] + + def object(self, i: int, elem_size: int) -> Object: + if i < 0 or i >= self._length: + return Object(None, 0) + return Object(self._msg, self._offset + i * elem_size) + + def bytes(self) -> bytes: + if self._msg is None: + return b"" + end = self._offset + self._length + if end > len(self._msg._data): + return b"" + return bytes(self._msg._data[self._offset:end]) diff --git a/python/zap_py/wire.py b/python/zap_py/wire.py new file mode 100644 index 0000000..ef7ed9a --- /dev/null +++ b/python/zap_py/wire.py @@ -0,0 +1,36 @@ +"""Wire-format constants and exceptions shared by reader and builder.""" + +HEADER_SIZE = 16 +MAGIC = b"ZAP\x00" +VERSION = 1 +DEFAULT_PORT = 9999 +ALIGNMENT = 8 + +FLAG_NONE = 0 +FLAG_COMPRESSED = 1 << 0 +FLAG_ENCRYPTED = 1 << 1 +FLAG_SIGNED = 1 << 2 + + +class ZapError(Exception): + pass + + +class InvalidMagic(ZapError): + pass + + +class InvalidVersion(ZapError): + pass + + +class BufferTooSmall(ZapError): + pass + + +class OutOfBounds(ZapError): + pass + + +class InvalidOffset(ZapError): + pass diff --git a/python_interop_test.go b/python_interop_test.go new file mode 100644 index 0000000..4af1a22 --- /dev/null +++ b/python_interop_test.go @@ -0,0 +1,69 @@ +// Copyright (C) 2025, Lux Industries Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package zap_test + +import ( + "os" + "testing" + + "github.com/luxfi/zap" +) + +// TestPythonFixture verifies the Go reader accepts a message built by +// python/zap_py. Generate the fixture with: +// +// python python/testdata/gen_python_fixture.py > /tmp/zap_python_fixture.bin +// ZAP_PYTHON_FIXTURE=/tmp/zap_python_fixture.bin go test -run TestPythonFixture +// +// Skipped when the env var is unset so the suite stays Go-only by default. +func TestPythonFixture(t *testing.T) { + path := os.Getenv("ZAP_PYTHON_FIXTURE") + if path == "" { + t.Skip("set ZAP_PYTHON_FIXTURE=path/to/fixture.bin (built with gen_python_fixture.py)") + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + msg, err := zap.Parse(data) + if err != nil { + t.Fatalf("parse: %v", err) + } + root := msg.Root() + + if got := root.Uint32(0); got != 42 { + t.Errorf("root.Uint32(0): got %d, want 42", got) + } + if got := root.Uint64(8); got != 0xDEADBEEFCAFEBABE { + t.Errorf("root.Uint64(8): got %#x, want 0xDEADBEEFCAFEBABE", got) + } + if got := root.Text(16); got != "from py" { + t.Errorf("root.Text(16): got %q, want %q", got, "from py") + } + if got := string(root.Bytes(24)); got != "\x01\x02\x03\x04" { + t.Errorf("root.Bytes(24): got %x, want 01020304", got) + } + + inner := root.Object(32) + if inner.IsNull() { + t.Fatal("root.Object(32) is null") + } + if got := inner.Uint32(0); got != 7 { + t.Errorf("inner.Uint32(0): got %d, want 7", got) + } + if got := inner.Text(8); got != "nested" { + t.Errorf("inner.Text(8): got %q, want %q", got, "nested") + } + + list := root.List(40) + if list.Len() != 4 { + t.Fatalf("list.Len(): got %d, want 4", list.Len()) + } + want := []uint32{10, 20, 30, 40} + for i, w := range want { + if got := list.Uint32(i); got != w { + t.Errorf("list[%d]: got %d, want %d", i, got, w) + } + } +}