From cd39656d12f95d2d2714db300a53a36cea2c3612 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 27 Dec 2023 16:55:19 -0500 Subject: [PATCH 1/9] betterproto: support `Struct` and `Value` Signed-off-by: William Woodruff --- src/betterproto/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index da6893db7..ef5f4eb11 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -55,7 +55,6 @@ hybridmethod, ) - if TYPE_CHECKING: from _typeshed import ( SupportsRead, @@ -1522,6 +1521,12 @@ def to_dict( @classmethod def _from_dict_init(cls, mapping: Mapping[str, Any]) -> Mapping[str, Any]: + # Special case: google.protobuf.Struct has a single fields member but + # behaves like a transparent JSON object, so it needs to first be munged + # into `{fields: mapping}`. + if cls == Struct: + mapping = {"fields": mapping} + init_kwargs: Dict[str, Any] = {} for key, value in mapping.items(): field_name = safe_snake_case(key) @@ -1552,7 +1557,7 @@ def _from_dict_init(cls, mapping: Mapping[str, Any]) -> Mapping[str, Any]: if isinstance(value, list) else sub_cls.from_dict(value) ) - elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE: + elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE and cls != Struct: sub_cls = cls._betterproto.cls_by_field[f"{field_name}.value"] value = {k: sub_cls.from_dict(v) for k, v in value.items()} else: @@ -1582,6 +1587,7 @@ def _from_dict_init(cls, mapping: Mapping[str, Any]) -> Mapping[str, Any]: ) init_kwargs[field_name] = value + return init_kwargs @hybridmethod @@ -1926,6 +1932,7 @@ def which_one_of(message: Message, group_name: str) -> Tuple[str, Optional[Any]] Int32Value, Int64Value, StringValue, + Struct, Timestamp, UInt32Value, UInt64Value, From f5a3eee737eb7cb1c51e37315a6c13294dbe608d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 27 Dec 2023 17:15:38 -0500 Subject: [PATCH 2/9] betterproto: handle struct in to_dict as well Signed-off-by: William Woodruff --- src/betterproto/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index ef5f4eb11..5436d90d0 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1403,6 +1403,15 @@ def to_dict( Dict[:class:`str`, Any] The JSON serializable dict representation of this object. """ + # Mirror of from_dict: Struct's `fields` member is transparently + # dispatched through instead. + if isinstance(self, Struct): + output = {**self.fields} + for k in self.fields: + if hasattr(self.fields[k], "to_dict"): + output[k] = self.fields[k].to_dict(casing, include_default_values) + return output + output: Dict[str, Any] = {} field_types = self._type_hints() defaults = self._betterproto.default_gen From c8e6244098a1e7b45d4cb81acbf0c51941d60cf5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 27 Dec 2023 17:16:07 -0500 Subject: [PATCH 3/9] tests: add Struct roundtrip tests Signed-off-by: William Woodruff --- tests/test_struct.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/test_struct.py diff --git a/tests/test_struct.py b/tests/test_struct.py new file mode 100644 index 000000000..c28dd67d6 --- /dev/null +++ b/tests/test_struct.py @@ -0,0 +1,22 @@ +import json + +from betterproto.lib.google.protobuf import Struct + + +def test_struct_roundtrip(): + data = { + "foo": "bar", + "baz": None, + "quux": 123, + "zap": [1, {"two": 3}, "four"], + } + data_json = json.dumps(data) + + struct_from_dict = Struct().from_dict(data) + assert struct_from_dict.fields == data + assert struct_from_dict.to_json() == data_json + + struct_from_json = Struct().from_json(data_json) + assert struct_from_json.fields == data + assert struct_from_json == struct_from_dict + assert struct_from_json.to_json() == data_json From 8ed15a11ae0ac801c7430772474822f2fee08abc Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 27 Dec 2023 18:28:01 -0500 Subject: [PATCH 4/9] specialize from_dict and to_dict on Struct ...rather than special-casing in the Message ABC. Signed-off-by: William Woodruff --- src/betterproto/__init__.py | 18 +--------- .../lib/google/protobuf/__init__.py | 33 +++++++++++++++++++ tests/test_struct.py | 2 ++ 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index 5436d90d0..26e0e9545 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1403,15 +1403,6 @@ def to_dict( Dict[:class:`str`, Any] The JSON serializable dict representation of this object. """ - # Mirror of from_dict: Struct's `fields` member is transparently - # dispatched through instead. - if isinstance(self, Struct): - output = {**self.fields} - for k in self.fields: - if hasattr(self.fields[k], "to_dict"): - output[k] = self.fields[k].to_dict(casing, include_default_values) - return output - output: Dict[str, Any] = {} field_types = self._type_hints() defaults = self._betterproto.default_gen @@ -1530,12 +1521,6 @@ def to_dict( @classmethod def _from_dict_init(cls, mapping: Mapping[str, Any]) -> Mapping[str, Any]: - # Special case: google.protobuf.Struct has a single fields member but - # behaves like a transparent JSON object, so it needs to first be munged - # into `{fields: mapping}`. - if cls == Struct: - mapping = {"fields": mapping} - init_kwargs: Dict[str, Any] = {} for key, value in mapping.items(): field_name = safe_snake_case(key) @@ -1566,7 +1551,7 @@ def _from_dict_init(cls, mapping: Mapping[str, Any]) -> Mapping[str, Any]: if isinstance(value, list) else sub_cls.from_dict(value) ) - elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE and cls != Struct: + elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE: sub_cls = cls._betterproto.cls_by_field[f"{field_name}.value"] value = {k: sub_cls.from_dict(v) for k, v in value.items()} else: @@ -1941,7 +1926,6 @@ def which_one_of(message: Message, group_name: str) -> Tuple[str, Optional[Any]] Int32Value, Int64Value, StringValue, - Struct, Timestamp, UInt32Value, UInt64Value, diff --git a/src/betterproto/lib/google/protobuf/__init__.py b/src/betterproto/lib/google/protobuf/__init__.py index f59e4a17a..99741938e 100644 --- a/src/betterproto/lib/google/protobuf/__init__.py +++ b/src/betterproto/lib/google/protobuf/__init__.py @@ -2,14 +2,21 @@ # sources: google/protobuf/any.proto, google/protobuf/api.proto, google/protobuf/descriptor.proto, google/protobuf/duration.proto, google/protobuf/empty.proto, google/protobuf/field_mask.proto, google/protobuf/source_context.proto, google/protobuf/struct.proto, google/protobuf/timestamp.proto, google/protobuf/type.proto, google/protobuf/wrappers.proto # plugin: python-betterproto # This file has been @generated + +from __future__ import annotations + import warnings from dataclasses import dataclass from typing import ( Dict, List, + Mapping, ) +from typing_extensions import Self + import betterproto +from betterproto.utils import hybridmethod class Syntax(betterproto.Enum): @@ -1458,6 +1465,32 @@ class Struct(betterproto.Message): ) """Unordered map of dynamically typed values.""" + @hybridmethod + def from_dict(cls: type[Self], value: Mapping[str, Any]) -> Self: # type: ignore + self = cls() + return self.from_dict(value) + + @from_dict.instancemethod + def from_dict(self, value: Mapping[str, Any]) -> Self: + fields = {**value} + for k in fields: + if hasattr(fields[k], "from_dict"): + fields[k] = fields[k].from_dict() + + self.fields = fields + return self + + def to_dict( + self, + casing: betterproto.Casing = betterproto.Casing.CAMEL, + include_default_values: bool = False, + ) -> Dict[str, Any]: + output = {**self.fields} + for k in self.fields: + if hasattr(self.fields[k], "to_dict"): + output[k] = self.fields[k].to_dict(casing, include_default_values) + return output + @dataclass(eq=False, repr=False) class Value(betterproto.Message): diff --git a/tests/test_struct.py b/tests/test_struct.py index c28dd67d6..f266bc892 100644 --- a/tests/test_struct.py +++ b/tests/test_struct.py @@ -14,9 +14,11 @@ def test_struct_roundtrip(): struct_from_dict = Struct().from_dict(data) assert struct_from_dict.fields == data + assert struct_from_dict.to_dict() == data assert struct_from_dict.to_json() == data_json struct_from_json = Struct().from_json(data_json) assert struct_from_json.fields == data + assert struct_from_json.to_dict() == data assert struct_from_json == struct_from_dict assert struct_from_json.to_json() == data_json From d7ef248edb381c52505d7b5568973c4cd219b2d2 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 28 Dec 2023 10:40:20 -0500 Subject: [PATCH 5/9] betterproto: `poe format` Signed-off-by: William Woodruff --- src/betterproto/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index 26e0e9545..785fb90a2 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -55,6 +55,7 @@ hybridmethod, ) + if TYPE_CHECKING: from _typeshed import ( SupportsRead, From 1b887ba4e71812756d6f7b964a0dae822b94bf82 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 2 Jan 2024 09:40:41 -0500 Subject: [PATCH 6/9] Update src/betterproto/__init__.py Co-authored-by: James Hilton-Balfe --- src/betterproto/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index 785fb90a2..da6893db7 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1582,7 +1582,6 @@ def _from_dict_init(cls, mapping: Mapping[str, Any]) -> Mapping[str, Any]: ) init_kwargs[field_name] = value - return init_kwargs @hybridmethod From 49df234026ddec029f16785ae5a8ce370f1e1dcb Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 2 Jan 2024 10:57:55 -0500 Subject: [PATCH 7/9] remove future annotations Signed-off-by: William Woodruff --- src/betterproto/lib/google/protobuf/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/betterproto/lib/google/protobuf/__init__.py b/src/betterproto/lib/google/protobuf/__init__.py index 99741938e..e29ee5862 100644 --- a/src/betterproto/lib/google/protobuf/__init__.py +++ b/src/betterproto/lib/google/protobuf/__init__.py @@ -3,8 +3,6 @@ # plugin: python-betterproto # This file has been @generated -from __future__ import annotations - import warnings from dataclasses import dataclass from typing import ( From dde445d81b3c509151ec5256ae79f5cf7a1f104a Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 2 Jan 2024 11:06:20 -0500 Subject: [PATCH 8/9] replace type[...] with typing.T Signed-off-by: William Woodruff --- src/betterproto/lib/google/protobuf/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/betterproto/lib/google/protobuf/__init__.py b/src/betterproto/lib/google/protobuf/__init__.py index e29ee5862..ec83e4641 100644 --- a/src/betterproto/lib/google/protobuf/__init__.py +++ b/src/betterproto/lib/google/protobuf/__init__.py @@ -10,6 +10,9 @@ List, Mapping, ) +from typing import ( + Type as T, +) from typing_extensions import Self @@ -1464,7 +1467,7 @@ class Struct(betterproto.Message): """Unordered map of dynamically typed values.""" @hybridmethod - def from_dict(cls: type[Self], value: Mapping[str, Any]) -> Self: # type: ignore + def from_dict(cls: T[Self], value: Mapping[str, Any]) -> Self: # type: ignore self = cls() return self.from_dict(value) From bc527f5f39cacf0dc71a8c60c63b252dc6808753 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 2 Jan 2024 11:13:22 -0500 Subject: [PATCH 9/9] quote instead Signed-off-by: William Woodruff --- src/betterproto/lib/google/protobuf/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/betterproto/lib/google/protobuf/__init__.py b/src/betterproto/lib/google/protobuf/__init__.py index ec83e4641..9976bda86 100644 --- a/src/betterproto/lib/google/protobuf/__init__.py +++ b/src/betterproto/lib/google/protobuf/__init__.py @@ -10,9 +10,6 @@ List, Mapping, ) -from typing import ( - Type as T, -) from typing_extensions import Self @@ -1467,7 +1464,7 @@ class Struct(betterproto.Message): """Unordered map of dynamically typed values.""" @hybridmethod - def from_dict(cls: T[Self], value: Mapping[str, Any]) -> Self: # type: ignore + def from_dict(cls: "type[Self]", value: Mapping[str, Any]) -> Self: # type: ignore self = cls() return self.from_dict(value)