feat: add support for lazy annotations PEP563 (yukinarit#112)

* also move SerdeError to compat.py

Author: ydylla

serde/compat.py

@@ -3,8 +3,9 @@
 """
 import dataclasses
 import enum
+import sys
 import typing
-from dataclasses import fields, is_dataclass
+from dataclasses import is_dataclass
 from itertools import zip_longest
 from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union
 
@@ -15,6 +16,13 @@
 T = TypeVar('T')
 
 
+# moved SerdeError from core.py to compat.py to prevent circular dependency issues
+class SerdeError(TypeError):
+    """
+    Serde error class.
+    """
+
+
 def get_origin(typ):
     """
     Provide `get_origin` that works in all python versions.
@@ -128,13 +136,41 @@ def union_args(typ: Union) -> Tuple:
     return tuple(types)
 
 
+def dataclass_fields(cls: Type) -> Iterator:
+    raw_fields = dataclasses.fields(cls)
+
+    try:
+        # this resolves types when string forward reference
+        # or PEP 563: "from __future__ import annotations" are used
+        resolved_hints = typing.get_type_hints(cls)
+    except Exception as e:
+        raise SerdeError(
+            f"Failed to resolve type hints for {typename(cls)}:\n"
+            f"{e.__class__.__name__}: {e}\n\n"
+            f"If you are using forward references make sure you are calling deserialize & serialize after all classes are globally visible."
+        )
+
+    for f in raw_fields:
+        real_type = resolved_hints.get(f.name)
+        # python <= 3.6 has no typing.ForwardRef so we need to skip the check
+        if sys.version_info[:2] != (3, 6) and isinstance(real_type, typing.ForwardRef):
+            raise SerdeError(
+                f"Failed to resolve {real_type} for {typename(cls)}.\n\n"
+                f"Make sure you are calling deserialize & serialize after all classes are globally visible."
+            )
+        if real_type is not None:
+            f.type = real_type
+
+    return iter(raw_fields)
+
+
 def iter_types(cls: Type) -> Iterator[Type]:
     """
     Iterate field types recursively.
     """
     if is_dataclass(cls):
         yield cls
-        for f in fields(cls):
+        for f in dataclass_fields(cls):
             yield from iter_types(f.type)
     elif isinstance(cls, str):
         yield cls
@@ -170,7 +206,7 @@ def iter_unions(cls: Type) -> Iterator[Type]:
         for arg in type_args(cls):
             yield from iter_unions(arg)
     if is_dataclass(cls):
-        for f in fields(cls):
+        for f in dataclass_fields(cls):
             yield from iter_unions(f.type)
     elif is_opt(cls):
         arg = type_args(cls)
@@ -369,9 +405,9 @@ def has_default(field) -> bool:
     ... class C:
     ...     a: int
     ...     d: int = 10
-    >>> has_default(fields(C)[0])
+    >>> has_default(dataclasses.fields(C)[0])
     False
-    >>> has_default(fields(C)[1])
+    >>> has_default(dataclasses.fields(C)[1])
     True
     """
     return not isinstance(field.default, dataclasses._MISSING_TYPE)
@@ -386,9 +422,9 @@ def has_default_factory(field) -> bool:
     ... class C:
     ...     a: int
     ...     d: Dict = dataclasses.field(default_factory=dict)
-    >>> has_default_factory(fields(C)[0])
+    >>> has_default_factory(dataclasses.fields(C)[0])
     False
-    >>> has_default_factory(fields(C)[1])
+    >>> has_default_factory(dataclasses.fields(C)[1])
     True
     """
     return not isinstance(field.default_factory, dataclasses._MISSING_TYPE)

serde/core.py

@@ -10,6 +10,8 @@
 import stringcase
 
 from .compat import (
+    SerdeError,
+    dataclass_fields,
     is_bare_dict,
     is_bare_list,
     is_bare_set,
@@ -124,12 +126,6 @@ def _justify(self, s: str, length=50) -> str:
         return ' ' * (white_spaces if white_spaces > 0 else 0) + s
 
 
-class SerdeError(TypeError):
-    """
-    Serde error class.
-    """
-
-
 def raise_unsupported_type(obj):
     # needed because we can not render a raise statement everywhere, e.g. as argument
     raise SerdeError(f"Unsupported type: {typename(type(obj))}")
@@ -297,7 +293,7 @@ def conv_name(self) -> str:
 
 
 def fields(FieldCls: Type, cls: Type) -> Iterator[Field]:
-    return iter(FieldCls.from_dataclass(f) for f in dataclasses.fields(cls))
+    return iter(FieldCls.from_dataclass(f) for f in dataclass_fields(cls))
 
 
 def conv(f: Field, case: Optional[str] = None) -> str:

tests/test_basics.py

@@ -15,8 +15,9 @@
 import more_itertools
 import pytest
 
-import serde.compat
+import serde
 from serde import SerdeError, deserialize, from_dict, from_tuple, serialize, to_dict, to_tuple
+from serde.compat import dataclass_fields
 from serde.core import SERDE_SCOPE
 from serde.json import from_json, to_json
 from serde.msgpack import from_msgpack, to_msgpack
@@ -211,22 +212,57 @@ class Foo:
             i: int
 
 
-def test_forward_declaration():
-    @serialize
-    @deserialize
-    @dataclass
-    class Foo:
-        bar: 'Bar'
+# test_string_forward_reference_works currently only works with global visible classes
+# and can not be mixed with PEP 563 "from __future__ import annotations"
+@dataclass
+class ForwardReferenceFoo:
+    bar: 'ForwardReferenceBar'
 
-    @serialize
-    @deserialize
-    @dataclass
-    class Bar:
-        i: int
 
-    h = Foo(bar=Bar(i=10))
-    assert h.bar.i == 10
-    assert 'Bar' == dataclasses.fields(Foo)[0].type
+@serialize
+@deserialize
+@dataclass
+class ForwardReferenceBar:
+    i: int
+
+
+# assert type is str
+assert 'ForwardReferenceBar' == dataclasses.fields(ForwardReferenceFoo)[0].type
+
+# setup pyserde for Foo after Bar becomes visible to global scope
+deserialize(ForwardReferenceFoo)
+serialize(ForwardReferenceFoo)
+
+# now the type really is of type Bar
+assert ForwardReferenceBar == dataclasses.fields(ForwardReferenceFoo)[0].type
+assert ForwardReferenceBar == next(dataclass_fields(ForwardReferenceFoo)).type
+
+# verify usage works
+def test_string_forward_reference_works():
+    h = ForwardReferenceFoo(bar=ForwardReferenceBar(i=10))
+    h_dict = {"bar": {"i": 10}}
+
+    assert to_dict(h) == h_dict
+    assert from_dict(ForwardReferenceFoo, h_dict) == h
+
+
+# trying to use string forward reference normally will throw
+def test_unresolved_forward_reference_throws():
+    with pytest.raises(SerdeError) as e:
+
+        @serialize
+        @deserialize
+        @dataclass
+        class UnresolvedForwardFoo:
+            bar: 'UnresolvedForwardBar'
+
+        @serialize
+        @deserialize
+        @dataclass
+        class UnresolvedForwardBar:
+            i: int
+
+    assert "Failed to resolve type hints for UnresolvedForwardFoo" in str(e)
 
 
 @pytest.mark.parametrize('opt', opt_case, ids=opt_case_ids())

tests/test_lazy_type_evaluation.py

@@ -0,0 +1,126 @@
+from __future__ import annotations  # this is the line this test file is all about
+
+import dataclasses
+from dataclasses import dataclass
+from enum import Enum
+from typing import List, Tuple
+
+import pytest
+
+import serde
+from serde import SerdeError, deserialize, from_dict, serialize, to_dict
+from serde.compat import dataclass_fields
+
+serde.init(True)
+
+
+class Status(Enum):
+    OK = "ok"
+    ERR = "err"
+
+
+@deserialize
+@serialize
+@dataclass
+class A:
+    a: int
+    b: Status
+    c: List[str]
+
+
+@deserialize
+@serialize
+@dataclass
+class B:
+    a: A
+    b: Tuple[str, A]
+    c: Status
+
+
+# only works with global classes
+def test_serde_with_lazy_type_annotations():
+    a = A(1, Status.ERR, ["foo"])
+    a_dict = {"a": 1, "b": "err", "c": ["foo"]}
+
+    assert a == from_dict(A, a_dict)
+    assert a_dict == to_dict(a)
+
+    b = B(a, ("foo", a), Status.OK)
+    b_dict = {"a": a_dict, "b": ("foo", a_dict), "c": "ok"}
+
+    assert b == from_dict(B, b_dict)
+    assert b_dict == to_dict(b)
+
+
+# test_forward_reference_works currently only works with global visible classes
+@dataclass
+class ForwardReferenceFoo:
+    # this is not a string forward reference because we use PEP 563 (see 1st line of this file)
+    bar: ForwardReferenceBar
+
+
+@serialize
+@deserialize
+@dataclass
+class ForwardReferenceBar:
+    i: int
+
+
+# assert type is str
+assert 'ForwardReferenceBar' == dataclasses.fields(ForwardReferenceFoo)[0].type
+
+# setup pyserde for Foo after Bar becomes visible to global scope
+deserialize(ForwardReferenceFoo)
+serialize(ForwardReferenceFoo)
+
+# now the type really is of type Bar
+assert ForwardReferenceBar == dataclasses.fields(ForwardReferenceFoo)[0].type
+assert ForwardReferenceBar == next(dataclass_fields(ForwardReferenceFoo)).type
+
+# verify usage works
+def test_forward_reference_works():
+    h = ForwardReferenceFoo(bar=ForwardReferenceBar(i=10))
+    h_dict = {"bar": {"i": 10}}
+
+    assert to_dict(h) == h_dict
+    assert from_dict(ForwardReferenceFoo, h_dict) == h
+
+
+# trying to use forward reference normally will throw
+def test_unresolved_forward_reference_throws():
+    with pytest.raises(SerdeError) as e:
+
+        @serialize
+        @deserialize
+        @dataclass
+        class UnresolvedForwardFoo:
+            bar: UnresolvedForwardBar
+
+        @serialize
+        @deserialize
+        @dataclass
+        class UnresolvedForwardBar:
+            i: int
+
+    assert "Failed to resolve type hints for UnresolvedForwardFoo" in str(e)
+
+
+# trying to use string forward reference will throw
+def test_string_forward_reference_throws():
+    with pytest.raises(SerdeError) as e:
+
+        @serialize
+        @deserialize
+        @dataclass
+        class UnresolvedStringForwardFoo:
+            # string forward references are not compatible with PEP 563 and will throw
+            bar: 'UnresolvedStringForwardBar'
+
+        @serialize
+        @deserialize
+        @dataclass
+        class UnresolvedStringForwardBar:
+            i: int
+
+    # message is different between <= 3.8 & >= 3.9
+    assert "Failed to resolve " in str(e.value)