From daaccf86c1328defe0ba08fff9579740ccad2579 Mon Sep 17 00:00:00 2001 From: alonalexander Date: Fri, 14 May 2021 15:45:22 +0300 Subject: [PATCH 1/4] Combine from_bytes and from_stream Add `from_stream` to all fields. Create `as_stream` utility Call `from_stream` from `from_bytes` --- hydration/base.py | 45 +++++++++++--------------------------------- hydration/fields.py | 7 +++++++ hydration/helpers.py | 11 +++++++++++ hydration/message.py | 3 +++ hydration/scalars.py | 15 ++++++++++++++- hydration/vectors.py | 14 ++++++++++++++ 6 files changed, 60 insertions(+), 35 deletions(-) diff --git a/hydration/base.py b/hydration/base.py index 0e849a6..e19f963 100644 --- a/hydration/base.py +++ b/hydration/base.py @@ -6,7 +6,7 @@ from pyhooks import Hook, precall_register, postcall_register from typing import Callable, List, Iterable, Optional -from .helpers import as_obj, assert_no_property_override, as_type +from .helpers import as_obj, assert_no_property_override, as_type, as_stream from .scalars import Scalar, Enum from .fields import Field, VLA, FieldPlaceholder from .endianness import Endianness @@ -219,32 +219,7 @@ def from_bytes(cls, data: bytes, *args): :param args: Arguments for the __init__ of the Struct, if there's any :return The deserialized struct """ - - obj = cls(*args) - - for field_name in obj._field_names: - - # Get field for current field name - field = getattr(obj, field_name) - - obj.invoke_from_bytes_hooks(field) - - # Bytes hooks can change the field object, so get it again by name - field = getattr(obj, field_name) - - if isinstance(field, VLA): - field.length = int(getattr(obj, field.length_field_name)) - field.from_bytes(data) - data = data[len(bytes(field)):] - else: - split_index = field.size - - field_data, data = data[:split_index], data[split_index:] - field.value = field.from_bytes(field_data).value - with suppress(AttributeError): - field.validator.validate(field.value) - - return obj + return cls.from_stream(as_stream(data), *args) @classmethod def from_stream(cls, read_func: Callable[[int], bytes], *args): @@ -260,19 +235,21 @@ def from_stream(cls, read_func: Callable[[int], bytes], *args): obj = cls(*args) - for field in obj._fields: + for field_name in obj._field_names: + + # Get field for current field name + field = getattr(obj, field_name) obj.invoke_from_bytes_hooks(field) + # Bytes hooks can change the field object, so get it again by name + field = getattr(obj, field_name) + if isinstance(field, VLA): field.length = int(getattr(obj, field.length_field_name)) - data = read_func(field.length) - field.from_bytes(data) + field.from_stream(read_func) else: - read_size = field.size - - data = read_func(read_size) - field.value = field.from_bytes(data).value + field.value = field.from_stream(read_func).value with suppress(AttributeError): field.validator.validate(field.value) diff --git a/hydration/fields.py b/hydration/fields.py index 3f22eb2..3782350 100644 --- a/hydration/fields.py +++ b/hydration/fields.py @@ -51,6 +51,10 @@ def __bytes__(self) -> bytes: def from_bytes(self, data: bytes): raise NotImplementedError + @abc.abstractmethod + def from_stream(self, reader): + raise NotImplementedError + def __eq__(self, other): if isinstance(other, Field): return self.value == other.value and len(self) == len(other) @@ -118,3 +122,6 @@ def __bytes__(self) -> bytes: def from_bytes(self, data: bytes): raise AttributeError('Placeholders cannot be deserialized') + + def from_stream(self, reader): + raise AttributeError('Placeholders cannot be deserialized') diff --git a/hydration/helpers.py b/hydration/helpers.py index 928d6b3..20cea6b 100644 --- a/hydration/helpers.py +++ b/hydration/helpers.py @@ -20,3 +20,14 @@ def assert_no_property_override(obj, base_class): if (isinstance(getattr(base_class, attr_name), property) and not isinstance(getattr(type(obj), attr_name), property)): raise NameError(f"'{attr_name}' is an invalid name for an attribute in a sequenced or nested struct") + +def as_stream(data: bytes): + class Reader: + def __init__(self, content: bytes): + self._data = content + + def read(self, size=0): + user_data, self._data = self._data[:size], self._data[size:] + return user_data + + return Reader(data).read \ No newline at end of file diff --git a/hydration/message.py b/hydration/message.py index d54cc83..02229cc 100644 --- a/hydration/message.py +++ b/hydration/message.py @@ -182,6 +182,9 @@ def size(self): def __bytes__(self) -> bytes: return bytes(self.data_field) + + def from_stream(self, reader): + return self.data_field.from_stream(reader) def from_bytes(self, data: bytes): return self.data_field.from_bytes(data) diff --git a/hydration/scalars.py b/hydration/scalars.py index 6b0af12..5c1300d 100644 --- a/hydration/scalars.py +++ b/hydration/scalars.py @@ -122,8 +122,16 @@ def __int__(self) -> int: def __float__(self) -> float: return float(self.value) + def __get_format_string(self): + return '{}{}'.format(self.endianness_format, self.scalar_format) + + def from_stream(self, reader): + format_string = self.__get_format_string() + data = reader(struct.calcsize(format_string)) + return self.from_bytes(data) + def from_bytes(self, data: bytes): - format_string = '{}{}'.format(self.endianness_format, self.scalar_format) + format_string = self.__get_format_string() # noinspection PyAttributeOutsideInit self.value = struct.unpack(format_string, data)[0] return self @@ -311,6 +319,11 @@ def __bytes__(self) -> bytes: return bytes(self.type) except ValueError as e: raise ValueError(f'Error serializing {repr(self)}:\n{str(e)}') + + def from_stream(self, reader): + self.type.from_stream(reader) + self.value = self.type.value + return self def from_bytes(self, data: bytes): self.type.from_bytes(data) diff --git a/hydration/vectors.py b/hydration/vectors.py index 343d953..3aba15a 100644 --- a/hydration/vectors.py +++ b/hydration/vectors.py @@ -153,6 +153,9 @@ def __delitem__(self, key) -> None: def size(self): return len(self) * len(self.type) + def from_stream(self, reader): + return self.from_bytes(reader(self.size)) + class Vector(_Sequence, VLA): @@ -172,6 +175,17 @@ def value(self, value): # This assumes that the Struct will update the length field's value self.length = len(value) + def from_stream(self, reader): + if isinstance(self.type, Field): + return super().from_bytes(reader(len(self) * len(self.type))) + else: + val = [] + for _ in range(len(self)): + next_obj = self.type.from_stream(reader) + val.append(next_obj) + self.value = val + return self + def from_bytes(self, data: bytes): if isinstance(self.type, Field): return super().from_bytes(data[:len(self) * len(self.type)]) From 94dcdf8320d3e801d79ee9e98ec21cb0e418ee15 Mon Sep 17 00:00:00 2001 From: alonalexander Date: Fri, 14 May 2021 15:50:41 +0300 Subject: [PATCH 2/4] Rename from_bytes_hook --- hydration/__init__.py | 5 +++-- hydration/base.py | 26 +++++++++++++++----------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/hydration/__init__.py b/hydration/__init__.py index 97e3c8b..fd88f1b 100644 --- a/hydration/__init__.py +++ b/hydration/__init__.py @@ -10,7 +10,8 @@ pre_bytes_hook = Struct.pre_bytes_hook post_bytes_hook = Struct.post_bytes_hook -from_bytes_hook = Struct.from_bytes_hook +from_bytes_hook = Struct.pre_deserialization_hook +pre_deserialization_hook = Struct.pre_deserialization_hook LittleEndian = Endianness.LittleEndian BigEndian = Endianness.BigEndian @@ -25,5 +26,5 @@ 'Array', 'Vector', 'IPv4', 'FieldPlaceholder', 'ExactValueValidator', 'RangeValidator', 'FunctionValidator', 'SetValidator', 'Message', 'InclusiveLengthField', 'ExclusiveLengthField', 'OpcodeField', - 'pre_bytes_hook', 'post_bytes_hook', 'from_bytes_hook', + 'pre_bytes_hook', 'post_bytes_hook', 'pre_deserialization_hook', 'from_bytes_hook', 'LittleEndian', 'BigEndian', 'NativeEndian', 'NetworkEndian'] diff --git a/hydration/base.py b/hydration/base.py index e19f963..697cf73 100644 --- a/hydration/base.py +++ b/hydration/base.py @@ -101,7 +101,7 @@ def __prepare__(mcs, name, bases, *args, **kwargs): class Struct(metaclass=StructMeta): __frozen = False _field_names: List[str] - _from_bytes_hooks = {} + _pre_deserialization_hooks = {} @property def value(self): @@ -144,7 +144,7 @@ def __init__(self, *args, **kwargs): self.from_bytes = lambda data: self._from_bytes(data, *args) self.from_stream = lambda data: self._from_stream(data, *args) - self._from_bytes_hooks = {} + self._pre_deserialization_hooks = {} # Deepcopy the fields so different instances of Struct have unique fields for name, field in self: @@ -240,7 +240,7 @@ def from_stream(cls, read_func: Callable[[int], bytes], *args): # Get field for current field name field = getattr(obj, field_name) - obj.invoke_from_bytes_hooks(field) + obj.invoke_pre_deserialization_hooks(field) # Bytes hooks can change the field object, so get it again by name field = getattr(obj, field_name) @@ -281,29 +281,33 @@ def __setattr__(self, key, value): # Overriding fields but saving the hooks elif key in self._field_names: # Save the hooks from the field - hooks = getattr(getattr(self, key), '_from_bytes_hooks', []) + hooks = getattr(getattr(self, key), '_pre_deserialization_hooks', []) # Set the field to the new value super().__setattr__(key, value) # Inject the old hooks to the new field - setattr(getattr(self, key), '_from_bytes_hooks', hooks) + setattr(getattr(self, key), '_pre_deserialization_hooks', hooks) elif hasattr(self, key) or not self.__frozen: super().__setattr__(key, value) else: raise AttributeError("Struct doesn't allow defining new attributes") - def invoke_from_bytes_hooks(self, field: Field): - for f in getattr(field, '_from_bytes_hooks', ()): + def invoke_pre_deserialization_hooks(self, field: Field): + for f in getattr(field, '_pre_deserialization_hooks', ()): f(self) - + @classmethod def from_bytes_hook(cls, field): + return cls.pre_deserialization_hook(field) + + @classmethod + def pre_deserialization_hook(cls, field): # noinspection PyProtectedMember def register_field_hook(func: callable): - if hasattr(field, '_from_bytes_hooks'): - field._from_bytes_hooks.append(func) + if hasattr(field, '_pre_deserialization_hooks'): + field._pre_deserialization_hooks.append(func) else: - field._from_bytes_hooks = [func] + field._pre_deserialization_hooks = [func] return func return register_field_hook From e0560eb2237513541628eb2ee069fe037a77e772 Mon Sep 17 00:00:00 2001 From: alonalexander Date: Fri, 14 May 2021 15:52:14 +0300 Subject: [PATCH 3/4] Rename reader to read_func and add type annotations --- hydration/fields.py | 6 +++--- hydration/message.py | 6 +++--- hydration/scalars.py | 8 ++++---- hydration/vectors.py | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/hydration/fields.py b/hydration/fields.py index 3782350..2a92865 100644 --- a/hydration/fields.py +++ b/hydration/fields.py @@ -1,6 +1,6 @@ import abc from abc import ABC -from typing import Union +from typing import Union, Callable from .validators import ValidatorABC @@ -52,7 +52,7 @@ def from_bytes(self, data: bytes): raise NotImplementedError @abc.abstractmethod - def from_stream(self, reader): + def from_stream(self, read_func: Callable[[int], bytes]): raise NotImplementedError def __eq__(self, other): @@ -123,5 +123,5 @@ def __bytes__(self) -> bytes: def from_bytes(self, data: bytes): raise AttributeError('Placeholders cannot be deserialized') - def from_stream(self, reader): + def from_stream(self, read_func: Callable[[int], bytes]): raise AttributeError('Placeholders cannot be deserialized') diff --git a/hydration/message.py b/hydration/message.py index 02229cc..be440cf 100644 --- a/hydration/message.py +++ b/hydration/message.py @@ -1,7 +1,7 @@ import inspect from abc import ABC, abstractmethod from contextlib import suppress -from typing import List, Union, Type, Mapping +from typing import List, Union, Type, Mapping, Callable from hydration.helpers import as_obj from .base import Struct @@ -183,8 +183,8 @@ def size(self): def __bytes__(self) -> bytes: return bytes(self.data_field) - def from_stream(self, reader): - return self.data_field.from_stream(reader) + def from_stream(self, read_func: Callable[[int], bytes]): + return self.data_field.from_stream(read_func) def from_bytes(self, data: bytes): return self.data_field.from_bytes(data) diff --git a/hydration/scalars.py b/hydration/scalars.py index 5c1300d..4cd7951 100644 --- a/hydration/scalars.py +++ b/hydration/scalars.py @@ -125,9 +125,9 @@ def __float__(self) -> float: def __get_format_string(self): return '{}{}'.format(self.endianness_format, self.scalar_format) - def from_stream(self, reader): + def from_stream(self, read_func: Callable[[int], bytes]): format_string = self.__get_format_string() - data = reader(struct.calcsize(format_string)) + data = read_func(struct.calcsize(format_string)) return self.from_bytes(data) def from_bytes(self, data: bytes): @@ -320,8 +320,8 @@ def __bytes__(self) -> bytes: except ValueError as e: raise ValueError(f'Error serializing {repr(self)}:\n{str(e)}') - def from_stream(self, reader): - self.type.from_stream(reader) + def from_stream(self, read_func: Callable[[int], bytes]): + self.type.from_stream(read_func) self.value = self.type.value return self diff --git a/hydration/vectors.py b/hydration/vectors.py index 3aba15a..2c21a79 100644 --- a/hydration/vectors.py +++ b/hydration/vectors.py @@ -1,7 +1,7 @@ import copy from abc import ABC from collections import UserList -from typing import Sequence, Optional, Any, Union, Iterable +from typing import Sequence, Optional, Any, Union, Iterable, Callable from itertools import islice from .base import Struct @@ -153,8 +153,8 @@ def __delitem__(self, key) -> None: def size(self): return len(self) * len(self.type) - def from_stream(self, reader): - return self.from_bytes(reader(self.size)) + def from_stream(self, read_func: Callable[[int], bytes]): + return self.from_bytes(read_func(self.size)) class Vector(_Sequence, VLA): @@ -175,13 +175,13 @@ def value(self, value): # This assumes that the Struct will update the length field's value self.length = len(value) - def from_stream(self, reader): + def from_stream(self, read_func: Callable[[int], bytes]): if isinstance(self.type, Field): - return super().from_bytes(reader(len(self) * len(self.type))) + return super().from_bytes(read_func(len(self) * len(self.type))) else: val = [] for _ in range(len(self)): - next_obj = self.type.from_stream(reader) + next_obj = self.type.from_stream(read_func) val.append(next_obj) self.value = val return self From 954aff586a0111a53ec14b4f0b504f837de57f22 Mon Sep 17 00:00:00 2001 From: alonalexander Date: Sun, 16 May 2021 16:46:35 +0300 Subject: [PATCH 4/4] Fix comments on PR #36 --- hydration/__init__.py | 14 +++++++++----- hydration/base.py | 28 ++++++++++++++-------------- hydration/fields.py | 7 ++----- hydration/message.py | 3 --- hydration/scalars.py | 14 +------------- hydration/vectors.py | 12 ------------ 6 files changed, 26 insertions(+), 52 deletions(-) diff --git a/hydration/__init__.py b/hydration/__init__.py index fd88f1b..67733b7 100644 --- a/hydration/__init__.py +++ b/hydration/__init__.py @@ -8,10 +8,13 @@ from .message import Message, InclusiveLengthField, ExclusiveLengthField, OpcodeField from .fields import FieldPlaceholder -pre_bytes_hook = Struct.pre_bytes_hook -post_bytes_hook = Struct.post_bytes_hook -from_bytes_hook = Struct.pre_deserialization_hook -pre_deserialization_hook = Struct.pre_deserialization_hook +pre_serialization_hook = Struct.pre_serialization_hook +post_serialization_hook = Struct.post_serialization_hook +deserialization_hook = Struct.deserialization_hook + +pre_bytes_hook = Struct.pre_serialization_hook +post_bytes_hook = Struct.post_serialization_hook +from_bytes_hook = Struct.deserialization_hook LittleEndian = Endianness.LittleEndian BigEndian = Endianness.BigEndian @@ -26,5 +29,6 @@ 'Array', 'Vector', 'IPv4', 'FieldPlaceholder', 'ExactValueValidator', 'RangeValidator', 'FunctionValidator', 'SetValidator', 'Message', 'InclusiveLengthField', 'ExclusiveLengthField', 'OpcodeField', - 'pre_bytes_hook', 'post_bytes_hook', 'pre_deserialization_hook', 'from_bytes_hook', + 'pre_serialization_hook', 'post_serialization_hook', 'deserialization_hook', + 'from_bytes_hook', 'pre_bytes_hook', 'post_bytes_hook', 'LittleEndian', 'BigEndian', 'NativeEndian', 'NetworkEndian'] diff --git a/hydration/base.py b/hydration/base.py index 697cf73..a78ea59 100644 --- a/hydration/base.py +++ b/hydration/base.py @@ -101,7 +101,7 @@ def __prepare__(mcs, name, bases, *args, **kwargs): class Struct(metaclass=StructMeta): __frozen = False _field_names: List[str] - _pre_deserialization_hooks = {} + _deserialization_hooks = {} @property def value(self): @@ -144,7 +144,7 @@ def __init__(self, *args, **kwargs): self.from_bytes = lambda data: self._from_bytes(data, *args) self.from_stream = lambda data: self._from_stream(data, *args) - self._pre_deserialization_hooks = {} + self._deserialization_hooks = {} # Deepcopy the fields so different instances of Struct have unique fields for name, field in self: @@ -207,8 +207,8 @@ def serialize(self) -> bytes: except struct.error as e: raise ValueError(str(e)) from e - pre_bytes_hook = precall_register('__bytes__') - post_bytes_hook = postcall_register('__bytes__') + pre_serialization_hook = precall_register('__bytes__') + post_serialization_hook = postcall_register('__bytes__') @classmethod def from_bytes(cls, data: bytes, *args): @@ -240,7 +240,7 @@ def from_stream(cls, read_func: Callable[[int], bytes], *args): # Get field for current field name field = getattr(obj, field_name) - obj.invoke_pre_deserialization_hooks(field) + obj.invoke_deserialization_hooks(field) # Bytes hooks can change the field object, so get it again by name field = getattr(obj, field_name) @@ -281,33 +281,33 @@ def __setattr__(self, key, value): # Overriding fields but saving the hooks elif key in self._field_names: # Save the hooks from the field - hooks = getattr(getattr(self, key), '_pre_deserialization_hooks', []) + hooks = getattr(getattr(self, key), '_deserialization_hooks', []) # Set the field to the new value super().__setattr__(key, value) # Inject the old hooks to the new field - setattr(getattr(self, key), '_pre_deserialization_hooks', hooks) + setattr(getattr(self, key), '_deserialization_hooks', hooks) elif hasattr(self, key) or not self.__frozen: super().__setattr__(key, value) else: raise AttributeError("Struct doesn't allow defining new attributes") - def invoke_pre_deserialization_hooks(self, field: Field): - for f in getattr(field, '_pre_deserialization_hooks', ()): + def invoke_deserialization_hooks(self, field: Field): + for f in getattr(field, '_deserialization_hooks', ()): f(self) @classmethod def from_bytes_hook(cls, field): - return cls.pre_deserialization_hook(field) + return cls.deserialization_hook(field) @classmethod - def pre_deserialization_hook(cls, field): + def deserialization_hook(cls, field): # noinspection PyProtectedMember def register_field_hook(func: callable): - if hasattr(field, '_pre_deserialization_hooks'): - field._pre_deserialization_hooks.append(func) + if hasattr(field, '_deserialization_hooks'): + field._deserialization_hooks.append(func) else: - field._pre_deserialization_hooks = [func] + field._deserialization_hooks = [func] return func return register_field_hook diff --git a/hydration/fields.py b/hydration/fields.py index 2a92865..408d2e4 100644 --- a/hydration/fields.py +++ b/hydration/fields.py @@ -1,5 +1,6 @@ import abc from abc import ABC +from hydration.helpers import as_stream from typing import Union, Callable from .validators import ValidatorABC @@ -47,9 +48,8 @@ def size(self): def __bytes__(self) -> bytes: raise NotImplementedError - @abc.abstractmethod def from_bytes(self, data: bytes): - raise NotImplementedError + return self.from_stream(as_stream(data)) @abc.abstractmethod def from_stream(self, read_func: Callable[[int], bytes]): @@ -120,8 +120,5 @@ def __len__(self) -> int: def __bytes__(self) -> bytes: raise AttributeError('Placeholders cannot be serialized') - def from_bytes(self, data: bytes): - raise AttributeError('Placeholders cannot be deserialized') - def from_stream(self, read_func: Callable[[int], bytes]): raise AttributeError('Placeholders cannot be deserialized') diff --git a/hydration/message.py b/hydration/message.py index be440cf..76eb35f 100644 --- a/hydration/message.py +++ b/hydration/message.py @@ -186,9 +186,6 @@ def __bytes__(self) -> bytes: def from_stream(self, read_func: Callable[[int], bytes]): return self.data_field.from_stream(read_func) - def from_bytes(self, data: bytes): - return self.data_field.from_bytes(data) - @abstractmethod def update(self, message: Message, struct: Struct, struct_index: int): raise NotImplementedError diff --git a/hydration/scalars.py b/hydration/scalars.py index 4cd7951..2de934d 100644 --- a/hydration/scalars.py +++ b/hydration/scalars.py @@ -122,16 +122,9 @@ def __int__(self) -> int: def __float__(self) -> float: return float(self.value) - def __get_format_string(self): - return '{}{}'.format(self.endianness_format, self.scalar_format) - def from_stream(self, read_func: Callable[[int], bytes]): - format_string = self.__get_format_string() + format_string = '{}{}'.format(self.endianness_format, self.scalar_format) data = read_func(struct.calcsize(format_string)) - return self.from_bytes(data) - - def from_bytes(self, data: bytes): - format_string = self.__get_format_string() # noinspection PyAttributeOutsideInit self.value = struct.unpack(format_string, data)[0] return self @@ -325,11 +318,6 @@ def from_stream(self, read_func: Callable[[int], bytes]): self.value = self.type.value return self - def from_bytes(self, data: bytes): - self.type.from_bytes(data) - self.value = self.type.value - return self - @property def name(self): return self.enum_class(self.value).name diff --git a/hydration/vectors.py b/hydration/vectors.py index 2c21a79..3bd0f9c 100644 --- a/hydration/vectors.py +++ b/hydration/vectors.py @@ -186,18 +186,6 @@ def from_stream(self, read_func: Callable[[int], bytes]): self.value = val return self - def from_bytes(self, data: bytes): - if isinstance(self.type, Field): - return super().from_bytes(data[:len(self) * len(self.type)]) - else: - val = [] - for _ in range(len(self)): - next_obj = self.type.from_bytes(data) - val.append(next_obj) - data = data[len(bytes(next_obj)):] - self.value = val - return self - def __len__(self) -> int: return VLA.__len__(self)