Feature/message deserialization

docs/messages.md

@@ -92,4 +92,39 @@ Header3:
 	opcode:	UInt32(2)
 Body3:
 	data3:	UInt64(40)
-```
+```
+
+Once you have an `OpcodeField` in your message, you can also construct a message straight from bytes data.<br>
+Using the same example as above:
+
+```pycon
+>>> print(bytes(Header3() / Body2()))
+b'\x01\x00\x00\x00\x14\x00\x00\x00'
+>>> print(Message.from_bytes(Header3, b'\x01\x00\x00\x00\x14\x00\x00\x00'))
+Header3:
+    opcode: UInt32(1)
+Body2:
+    data: UInt32(20)
+```
+
+You can also deserialize more structs than just the header using the `Message.from_bytes` function, for example:
+
+```python
+class Footer(Struct):
+    x = UInt32()
+```
+```pycon
+>>> print(bytes(Header3() / Body2() / Footer()))
+b'\x01\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00'
+>>> print(Message.from_bytes(Header3, b'\x01\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00\x00', Footer))
+Header3:
+    opcode: UInt32(1)
+Body2:
+    data: UInt32(20)
+Footer:
+    x: UInt32(0)
+```
+
+* You can also use the `from_stream` function instead `from_bytes`.
+* You can pass more than one footer to the `Message.from_bytes` or `Message.from_stream` functions.
+* You can pass footers that also has an `OpcodeField` in them and it will deserialize it as a message.

hydration/base.py

@@ -6,9 +6,9 @@
 from pyhooks import Hook, precall_register, postcall_register
 from typing import Callable, List, Iterable, Optional
 
-from .helpers import as_obj, assert_no_property_override, as_type
+from .helpers import as_obj, assert_no_property_override, as_type, as_stream
 from .scalars import Scalar, Enum
-from .fields import Field, VLA, FieldPlaceholder
+from .fields import Field, VLA
 from .endianness import Endianness
 
 illegal_field_names = ['value', 'validate', '_fields']
@@ -220,31 +220,7 @@ def from_bytes(cls, data: bytes, *args):
         :return The deserialized struct
         """
 
-        obj = cls(*args)
-
-        for field_name in obj._field_names:
-
-            # Get field for current field name
-            field = getattr(obj, field_name)
-
-            obj.invoke_from_bytes_hooks(field)
-
-            # Bytes hooks can change the field object, so get it again by name
-            field = getattr(obj, field_name)
-
-            if isinstance(field, VLA):
-                field.length = int(getattr(obj, field.length_field_name))
-                field.from_bytes(data)
-                data = data[len(bytes(field)):]
-            else:
-                split_index = field.size
-
-                field_data, data = data[:split_index], data[split_index:]
-                field.value = field.from_bytes(field_data).value
-                with suppress(AttributeError):
-                    field.validator.validate(field.value)
-
-        return obj
+        return cls.from_stream(as_stream(data), *args)
 
     @classmethod
     def from_stream(cls, read_func: Callable[[int], bytes], *args):
@@ -260,19 +236,22 @@ def from_stream(cls, read_func: Callable[[int], bytes], *args):
 
         obj = cls(*args)
 
-        for field in obj._fields:
+        for field_name in obj._field_names:
+
+            # Get field for current field name
+            field = getattr(obj, field_name)
 
             obj.invoke_from_bytes_hooks(field)
 
+            # Bytes hooks can change the field object, so get it again by name
+            field = getattr(obj, field_name)
+
             if isinstance(field, VLA):
                 field.length = int(getattr(obj, field.length_field_name))
-                data = read_func(field.length)
-                field.from_bytes(data)
+                field.from_stream(read_func)
             else:
-                read_size = field.size
+                field.from_stream(read_func)
 
-                data = read_func(read_size)
-                field.value = field.from_bytes(data).value
                 with suppress(AttributeError):
                     field.validator.validate(field.value)
 

hydration/fields.py

@@ -1,8 +1,9 @@
 import abc
 from abc import ABC
-from typing import Union
+from typing import Union, Callable
 
 from .validators import ValidatorABC
+from .helpers import as_stream
 
 
 class Field(ABC):
@@ -47,8 +48,11 @@ def size(self):
     def __bytes__(self) -> bytes:
         raise NotImplementedError
 
-    @abc.abstractmethod
     def from_bytes(self, data: bytes):
+        return self.from_stream(as_stream(data))
+
+    @abc.abstractmethod
+    def from_stream(self, read_func: Callable[[int], bytes]):
         raise NotImplementedError
 
     def __eq__(self, other):
@@ -118,3 +122,6 @@ def __bytes__(self) -> bytes:
 
     def from_bytes(self, data: bytes):
         raise AttributeError('Placeholders cannot be deserialized')
+
+    def from_stream(self, read_func: Callable[[int], bytes]):
+        raise AttributeError('Placeholders cannot be deserialized')

hydration/helpers.py

@@ -1,4 +1,5 @@
 import inspect
+from typing import Callable
 
 
 def as_type(obj):
@@ -9,6 +10,18 @@ def as_obj(obj):
     return obj if not inspect.isclass(obj) else obj()
 
 
+def as_stream(data: bytes) -> Callable[[int], bytes]:
+    class _StreamReader:
+        def __init__(self, _data: bytes):
+            self._data = _data
+
+        def read(self, size: int) -> bytes:
+            user_data, self._data = self._data[:size], self._data[size:]
+            return user_data
+
+    return _StreamReader(data).read
+
+
 def assert_no_property_override(obj, base_class):
     """
     Use this to ensure that a Struct doesn't override properties of Field when using it as one.

hydration/message.py

@@ -1,9 +1,10 @@
 import inspect
 from abc import ABC, abstractmethod
 from contextlib import suppress
-from typing import List, Union, Type, Mapping
+from typing import List, Union, Type, Mapping, Callable
+from bidict import bidict, ValueDuplicationError
 
-from hydration.helpers import as_obj
+from .helpers import as_obj, as_stream
 from .base import Struct
 from .fields import Field
 from .validators import ValidatorABC, as_validator
@@ -137,6 +138,68 @@ def __contains__(self, item):
     def __len__(self):
         return len(self.layers)
 
+    @classmethod
+    def from_bytes(cls, data: bytes, header_class: Type[Struct], *layers: Type[Struct]):
+        """
+        Create a message from bytes data, using a header with an OpcodeField.
+
+        :param data: Data containing the message (in bytes)
+        :param header_class: The header class of the message
+        :param layers: The struct classes that represent the layers of the message
+        :return: A message created from `data`, based on `header_class` and `layers`
+        """
+
+        return cls.from_stream(as_stream(data), header_class, *layers)
+
+    @classmethod
+    def from_stream(cls, read_func: Callable[[int], bytes], header_class: Type[Struct], *layers: Type[Struct]):
+        """
+        Create a message from bytes data, using a header with an OpcodeField and the layers that represent the message.
+
+        :param read_func: The stream's reader function
+        The function needs to receive an int as a positional parameter and return a bytes object.
+        :param header_class: The header class of the message
+        :param layers: The struct classes that represent the layers of the message
+        :return: A message created from `read_func`, based on `header_class` and `layers`
+        """
+
+        # Find the opcode field in the header
+        for opcode_name, opcode_field in as_obj(header_class):
+            if isinstance(opcode_field, OpcodeField):
+                break
+        else:
+            raise AttributeError(f'Header {header_class.__name__} '
+                                 f'must have an opcode field in order to deserialize a message')
+
+        # Create the header object
+        header = header_class.from_stream(read_func)
+
+        # Extract body class from header's opcode field
+        header_opcode_value = getattr(header, opcode_name).value
+        body_class: Type[Struct] = bidict(opcode_field.opcode_dictionary).inverse[header_opcode_value]
+
+        # Create the body
+        try:
+            # Try to treat the body as a message in case it's also contains an OpcodeField
+            body = Message.from_stream(read_func, body_class)
+        except AttributeError:
+            # If it doesn't contain an OpcodeField treat it like a normal struct
+            body = body_class.from_stream(read_func)
+
+        additional_layers = []
+        for layer in layers:
+            try:
+                # Try to treat the body as a message in case it's also contains an OpcodeField
+                msg = cls.from_stream(read_func, layer)
+                additional_layers.extend(msg.layers)
+            except AttributeError:
+                # If it doesn't contain an OpcodeField treat it like a normal struct
+                obj = layer.from_stream(read_func)
+                additional_layers.append(obj)
+
+        print(*additional_layers)
+        return cls(header, body, *additional_layers, update_metadata=False)
+
     @property
     def size(self):
         # layers are structs or bytes, so use len instead of size
@@ -186,6 +249,9 @@ def __bytes__(self) -> bytes:
     def from_bytes(self, data: bytes):
         return self.data_field.from_bytes(data)
 
+    def from_stream(self, read_func: Callable[[int], bytes]):
+        return self.data_field.from_stream(read_func)
+
     @abstractmethod
     def update(self, message: Message, struct: Struct, struct_index: int):
         raise NotImplementedError
@@ -207,6 +273,12 @@ def __init__(self, data_field: FieldType, opcode_dictionary: Mapping):
         super().__init__(data_field)
         self.opcode_dictionary = opcode_dictionary
 
+        try:
+            # Validate that there are no duplicate opcodes
+            bidict(opcode_dictionary)
+        except ValueDuplicationError:
+            raise ValueError("Opcode values must be unique")
+
     def update(self, message: Message, struct: Struct, struct_index: int):
         with suppress(IndexError):
             if not self.validator:

hydration/scalars.py

@@ -122,10 +122,15 @@ def __int__(self) -> int:
     def __float__(self) -> float:
         return float(self.value)
 
-    def from_bytes(self, data: bytes):
+    def from_stream(self, read_func: Callable[[int], bytes]):
         format_string = '{}{}'.format(self.endianness_format, self.scalar_format)
-        # noinspection PyAttributeOutsideInit
-        self.value = struct.unpack(format_string, data)[0]
+
+        try:
+            # noinspection PyAttributeOutsideInit
+            self.value = struct.unpack(format_string, read_func(len(self)))[0]
+        except struct.error:
+            raise ValueError(f'Not enough bytes to unpack {self.__class__.__name__}')
+
         return self
 
     def __trunc__(self):
@@ -317,6 +322,9 @@ def from_bytes(self, data: bytes):
         self.value = self.type.value
         return self
 
+    def from_stream(self, read_func: Callable[[int], bytes]):
+        return self.from_bytes(read_func(len(self)))
+
     @property
     def name(self):
         return self.enum_class(self.value).name

hydration/vectors.py

@@ -1,7 +1,7 @@
 import copy
 from abc import ABC
 from collections import UserList
-from typing import Sequence, Optional, Any, Union, Iterable
+from typing import Sequence, Optional, Any, Union, Iterable, Callable
 from itertools import islice
 
 from .base import Struct
@@ -50,11 +50,11 @@ def __bytes__(self) -> bytes:
 
         return bytes(result)
 
-    def from_bytes(self, data: bytes):
+    def from_stream(self, read_func: Callable[[int], bytes]):
         field_type = copy.deepcopy(self.type)
-        self.value = tuple(field_type.from_bytes(chunk).value for chunk in byte_chunks(data, len(field_type)))
+        self.value = tuple(field_type.from_stream(read_func).value for _ in range(len(self)))
         return self
-
+    
     def __str__(self):
         return '{}{}'.format(self.__class__.__qualname__, self.value)
 
@@ -111,7 +111,7 @@ def extend(self, other: Iterable) -> None:
         self.value = self.data
 
 
-class Array(_Sequence):
+class Array(_Sequence, ABC):
     def __init__(self, length: int,
                  field_type: FieldType = UInt8,
                  value: Optional[Sequence[Any]] = (),
@@ -172,17 +172,24 @@ def value(self, value):
         # This assumes that the Struct will update the length field's value
         self.length = len(value)
 
-    def from_bytes(self, data: bytes):
+    # def from_bytes(self, data: bytes):
+    #     if isinstance(self.type, Field):
+    #         return super().from_bytes(data[:len(self) * len(self.type)])
+    #     else:
+    #         val = []
+    #         for _ in range(len(self)):
+    #             next_obj = self.type.from_bytes(data)
+    #             val.append(next_obj)
+    #             data = data[len(bytes(next_obj)):]
+    #         self.value = val
+    #         return self
+
+    def from_stream(self, read_func: Callable[[int], bytes]):
         if isinstance(self.type, Field):
-            return super().from_bytes(data[:len(self) * len(self.type)])
-        else:
-            val = []
-            for _ in range(len(self)):
-                next_obj = self.type.from_bytes(data)
-                val.append(next_obj)
-                data = data[len(bytes(next_obj)):]
-            self.value = val
-            return self
+            return super().from_stream(read_func)
+
+        self.value = [self.type.from_stream(read_func) for _ in range(len(self))]
+        return self
 
     def __len__(self) -> int:
         return VLA.__len__(self)

requirements.txt

@@ -1,2 +1,3 @@
 pytest
-pyhooks>=1.0.3
+pyhooks>=1.0.3
+bidict