Skip to content

Bug/update to proto v5 #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pw42020 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Bug/update to proto v5 #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8a45bc1
update _proto.py to have both including_default_value_fields and alwa…
pw42020 Aug 12, 2025
fcd047d
fix comments
pw42020 Aug 12, 2025
d070f1f
update unit tests to pass with update to always print fields with no …
pw42020 Aug 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 52 additions & 30 deletions pbspark/_proto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ def message_to_dict(
self,
message: Message,
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
descriptor_pool: t.Optional[DescriptorPool] = None,
Expand All @@ -212,10 +213,10 @@ def message_to_dict(

Args:
message: The protocol buffers message instance to serialize.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -224,14 +225,23 @@ def message_to_dict(
default.
float_precision: If set, use this to specify float field valid digits.
"""
printer = _Printer(
printer_kwargs = dict(
custom_serializers=self._custom_serializers,
including_default_value_fields=including_default_value_fields,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
descriptor_pool=descriptor_pool,
float_precision=float_precision,
)
# protobuf versions >=3.20.0,<5.26.1
if including_default_value_fields:
printer_kwargs.update({"including_default_value_fields": including_default_value_fields})
# protobuf version >=5.26.1
elif always_print_fields_with_no_presence:
printer_kwargs.update({"always_print_fields_with_no_presence": always_print_fields_with_no_presence})

printer = _Printer(
**printer_kwargs
)
return printer._MessageToJsonObject(message=message)

def parse_dict(
Expand Down Expand Up @@ -315,6 +325,7 @@ def get_decoder(
self,
message_type: t.Type[Message],
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
float_precision: t.Optional[int] = None,
Expand All @@ -326,10 +337,10 @@ def get_decoder(

Args:
message_type: The message type for decoding.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -343,6 +354,7 @@ def decoder(s: bytes) -> dict:
return self.message_to_dict(
message_type.FromString(s),
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
float_precision=float_precision,
Expand All @@ -354,6 +366,7 @@ def get_decoder_udf(
self,
message_type: t.Type[Message],
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
float_precision: t.Optional[int] = None,
Expand All @@ -365,10 +378,10 @@ def get_decoder_udf(

Args:
message_type: The message type for decoding.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -379,6 +392,7 @@ def get_decoder_udf(
self.get_decoder(
message_type=message_type,
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
float_precision=float_precision,
Expand All @@ -395,6 +409,7 @@ def from_protobuf(
data: t.Union[Column, str],
message_type: t.Type[Message],
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
float_precision: t.Optional[int] = None,
Expand All @@ -406,10 +421,10 @@ def from_protobuf(

Args:
message_type: The message type for decoding.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -420,6 +435,7 @@ def from_protobuf(
protobuf_decoder_udf = self.get_decoder_udf(
message_type=message_type,
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
float_precision=float_precision,
Expand Down Expand Up @@ -517,6 +533,7 @@ def df_from_protobuf(
df: DataFrame,
message_type: t.Type[Message],
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
float_precision: t.Optional[int] = None,
Expand All @@ -527,10 +544,10 @@ def df_from_protobuf(
Args:
df: A pyspark dataframe with encoded protobuf in the column at index 0.
message_type: The message type for decoding.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -545,6 +562,7 @@ def df_from_protobuf(
data=df.columns[0],
message_type=message_type,
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
float_precision=float_precision,
Expand Down Expand Up @@ -599,6 +617,7 @@ def from_protobuf(
data: t.Union[Column, str],
message_type: t.Type[Message],
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
float_precision: t.Optional[int] = None,
Expand All @@ -609,10 +628,10 @@ def from_protobuf(
Args:
data: A pyspark column.
message_type: The message type for decoding.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -625,6 +644,7 @@ def from_protobuf(
data=data,
message_type=message_type,
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
float_precision=float_precision,
Expand Down Expand Up @@ -665,6 +685,7 @@ def df_from_protobuf(
df: DataFrame,
message_type: t.Type[Message],
including_default_value_fields: bool = False,
always_print_fields_with_no_presence: bool = False,
preserving_proto_field_name: bool = False,
use_integers_for_enums: bool = False,
float_precision: t.Optional[int] = None,
Expand All @@ -676,10 +697,10 @@ def df_from_protobuf(
Args:
df: A pyspark dataframe with encoded protobuf in the column at index 0.
message_type: The message type for decoding.
including_default_value_fields: If True, singular primitive fields,
repeated fields, and map fields will always be serialized. If
False, only serialize non-empty fields. Singular message fields
and oneof fields are not affected by this option.
including_default_value_fields and always_print_fields_with_no_presence: If True,
singular primitive fields, repeated fields, and map fields will always
be serialized. If False, only serialize non-empty fields. Singular
message fields and one of fields are not affected by this option.
preserving_proto_field_name: If True, use the original proto field
names as defined in the .proto file. If False, convert the field
names to lowerCamelCase.
Expand All @@ -695,6 +716,7 @@ def df_from_protobuf(
df=df,
message_type=message_type,
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
preserving_proto_field_name=preserving_proto_field_name,
use_integers_for_enums=use_integers_for_enums,
float_precision=float_precision,
Expand Down
9 changes: 4 additions & 5 deletions tests/test_proto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def expanded(request):


@pytest.fixture(params=[True, False])
def including_default_value_fields(request):
def always_print_fields_with_no_presence(request):
return request.param


Expand Down Expand Up @@ -386,7 +386,7 @@ def test_df_to_from_protobuf(example, spark, expanded):
assert df.collect() == df_encoded.collect()


def test_including_default_value_fields(spark, including_default_value_fields):
def test_always_print_fields_with_no_presence(spark, always_print_fields_with_no_presence):
example = ExampleMessage(string="asdf")
data = [{"value": example.SerializeToString()}]

Expand All @@ -396,15 +396,14 @@ def test_including_default_value_fields(spark, including_default_value_fields):
df=df,
message_type=ExampleMessage,
expanded=True,
including_default_value_fields=including_default_value_fields,
always_print_fields_with_no_presence=always_print_fields_with_no_presence,
)
data = df_decoded.collect()
if including_default_value_fields:
if always_print_fields_with_no_presence:
assert data[0].asDict(True)["int32"] == 0
else:
assert data[0].asDict(True)["int32"] is None


def test_use_integers_for_enums(spark, use_integers_for_enums):
example = ExampleMessage(enum=ExampleMessage.SomeEnum.first)
data = [{"value": example.SerializeToString()}]
Expand Down