Merge pull request yukinarit#209 from kmsquire/feature/numpy

feat: Support numpy types

Author: yukinarit

.github/workflows/test.yml

@@ -9,7 +9,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "pypy-3.8"]
+        # Remove pypy-3.8 until it supports numpy
+        python-version: ["3.7", "3.8", "3.9", "3.10"]  # "pypy-3.8"
         os: [ubuntu-20.04, macos-10.15, windows-2019]
     steps:
     - name: Checkout

.gitignore

@@ -17,3 +17,4 @@ venv/
 .idea/
 out/
 docs/CHANGELOG.md
+poetry.lock

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
 
   - repo: https://github.com/psf/black
-    rev: 19.3b0
+    rev: 22.3.0
     hooks:
       - id: black
         args: [

docs/supported-data-formats.md

@@ -44,7 +44,7 @@ Call `to_json` to serialize `Foo` object into JSON string and `from_json` to des
 >>> from serde.json import to_json, from_json
 
 >>> to_json(Foo(i=10, s='foo', f=100.0, b=True))
-{"i": 10, "s": "foo", "f": 100.0, "b": true}
+'{"i": 10, "s": "foo", "f": 100.0, "b": true}'
 
 >>> from_json(Foo, '{"i": 10, "s": "foo", "f": 100.0, "b": true}')
 Foo(i=10, s='foo', f=100.0, b=True)
@@ -98,3 +98,75 @@ b'\x84\xa1i\n\xa1s\xa3foo\xa1f\xcb@Y\x00\x00\x00\x00\x00\x00\xa1b\xc3'
 >>> from_msgpack(Foo, b'\x84\xa1i\n\xa1s\xa3foo\xa1f\xcb@Y\x00\x00\x00\x00\x00\x00\xa1b\xc3')
 Foo(i=10, s='foo', f=100.0, b=True)
 ```
+
+## Numpy
+
+All of the above (de)serialization methods can transparently handle most numpy
+types with the "numpy" extras package.
+
+```python
+import numpy as np
+import numpy.typing as npt
+
+@serde
+class NPFoo:
+    i: np.int32
+    j: np.int64
+    f: np.float32
+    g: np.float64
+    h: np.bool_
+    u: np.ndarray
+    v: npt.NDArray
+    w: npt.NDArray[np.int32]
+    x: npt.NDArray[np.int64]
+    y: npt.NDArray[np.float32]
+    z: npt.NDArray[np.float64]
+
+npfoo = NPFoo(
+    np.int32(1),
+    np.int64(2),
+    np.float32(3.0),
+    np.float64(4.0),
+    np.bool_(False),
+    np.array([1, 2]),
+    np.array([3, 4]),
+    np.array([np.int32(i) for i in [1, 2]]),
+    np.array([np.int64(i) for i in [3, 4]]),
+    np.array([np.float32(i) for i in [5.0, 6.0]]),
+    np.array([np.float64(i) for i in [7.0, 8.0]]),
+)
+```
+
+```python
+>>> from serde.json import to_json, from_json
+
+>>> to_json(npfoo)
+'{"i": 1, "j": 2, "f": 3.0, "g": 4.0, "h": false, "u": [1, 2], "v": [3, 4], "w": [1, 2], "x": [3, 4], "y": [5.0, 6.0], "z": [7.0, 8.0]}'
+
+>>> from_json(NPFoo, to_json(npfoo))
+NPFoo(i=1, j=2, f=3.0, g=4.0, h=False, u=array([1, 2]), v=array([3, 4]), w=array([1, 2], dtype=int32), x=array([3, 4]), y=array([5., 6.], dtype=float32), z=array([7., 8.]))
+
+>>> from serde.yaml import from_yaml, to_yaml
+
+>>> to_yaml(npfoo)
+'f: 3.0\ng: 4.0\nh: false\ni: 1\nj: 2\nu:\n- 1\n- 2\nv:\n- 3\n- 4\nw:\n- 1\n- 2\nx:\n- 3\n- 4\ny:\n- 5.0\n- 6.0\nz:\n- 7.0\n- 8.0\n'
+
+>>> from_yaml(NPFoo, to_yaml(npfoo))
+NPFoo(i=1, j=2, f=3.0, g=4.0, h=False, u=array([1, 2]), v=array([3, 4]), w=array([1, 2], dtype=int32), x=array([3, 4]), y=array([5., 6.], dtype=float32), z=array([7., 8.]))
+
+>>> from serde.toml import from_toml, to_toml
+
+>>> to_toml(npfoo)
+'i = 1\nj = 2\nf = 3.0\ng = 4.0\nh = false\nu = [ 1, 2,]\nv = [ 3, 4,]\nw = [ 1, 2,]\nx = [ 3, 4,]\ny = [ 5.0, 6.0,]\nz = [ 7.0, 8.0,]\n'
+
+>>> from_toml(NPFoo, to_toml(npfoo))
+NPFoo(i=1, j=2, f=3.0, g=4.0, h=False, u=array([1, 2]), v=array([3, 4]), w=array([1, 2], dtype=int32), x=array([3, 4]), y=array([5., 6.], dtype=float32), z=array([7., 8.]))
+
+>>> from serde.msgpack import from_msgpack, to_msgpack
+
+>>> to_msgpack(npfoo)
+b'\x8b\xa1i\x01\xa1j\x02\xa1f\xcb@\x08\x00\x00\x00\x00\x00\x00\xa1g\xcb@\x10\x00\x00\x00\x00\x00\x00\xa1h\xc2\xa1u\x92\x01\x02\xa1v\x92\x03\x04\xa1w\x92\x01\x02\xa1x\x92\x03\x04\xa1y\x92\xcb@\x14\x00\x00\x00\x00\x00\x00\xcb@\x18\x00\x00\x00\x00\x00\x00\xa1z\x92\xcb@\x1c\x00\x00\x00\x00\x00\x00\xcb@ \x00\x00\x00\x00\x00\x00'
+
+>>> from_msgpack(NPFoo, to_msgpack(npfoo))
+NPFoo(i=1, j=2, f=3.0, g=4.0, h=False, u=array([1, 2]), v=array([3, 4]), w=array([1, 2], dtype=int32), x=array([3, 4]), y=array([5., 6.], dtype=float32), z=array([7., 8.]))
+```

pyproject.toml

@@ -30,11 +30,23 @@ jinja2 = "*"
 msgpack = { version = "*", markers = "extra == 'msgpack' or extra == 'all'", optional = true }
 toml = { version = "*", markers = "extra == 'toml' or extra == 'all'", optional = true }
 pyyaml = { version = "*", markers = "extra == 'yaml' or extra == 'all'", optional = true }
+numpy = [
+    { version = "~1.21.0", markers = "python_version ~= '3.7.0' and (extra == 'numpy' or extra == 'all')", optional = true },
+    { version = ">1.21.0", markers = "python_version ~= '3.8.0' and (extra == 'numpy' or extra == 'all')", optional = true },
+    { version = ">1.21.0", markers = "python_version ~= '3.9.0' and (extra == 'numpy' or extra == 'all')", optional = true },
+    { version = ">1.22.0", markers = "python_version ~= '3.10' and (extra == 'numpy' or extra == 'all')", optional = true },
+]
 
 [tool.poetry.dev-dependencies]
 pyyaml = "*"
 toml = "*"
 msgpack = "*"
+numpy = [
+    { version = "~1.21.0", markers = "python_version ~= '3.7.0'" },
+    { version = ">1.21.0", markers = "python_version ~= '3.8.0'" },
+    { version = ">1.21.0", markers = "python_version ~= '3.9.0'" },
+    { version = ">1.22.0", markers = "python_version ~= '3.10'" },
+]
 flake8 = "*"
 pytest = "*"
 pytest-cov = "*"
@@ -49,10 +61,12 @@ pytest-xdist = "^2.3.0"
 
 [tool.poetry.extras]
 msgpack = ["msgpack"]
+numpy = ["numpy"]
 toml = ["toml"]
 yaml = ["pyyaml"]
-all = ["msgpack", "toml", "pyyaml"]
+all = ["msgpack", "toml", "pyyaml", "numpy"]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+

serde/compat.py

@@ -8,10 +8,24 @@
 import types
 import typing
 from dataclasses import is_dataclass
-from typing import Any, Dict, Generic, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union
+from typing import Any, ClassVar, Dict, Generic, Iterator, List, Optional, Set, Tuple, Type, TypeVar, Union
 
 import typing_inspect
 
+try:
+    import numpy.typing as npt
+
+    def get_np_origin(tp):
+        if isinstance(tp, npt._generic_alias._GenericAlias) and tp.__origin__ is not ClassVar:
+            return tp.__origin__
+        return None
+
+except ImportError:
+
+    def get_np_origin(tp):
+        return None
+
+
 __all__: List = []
 
 T = TypeVar('T')
@@ -34,9 +48,9 @@ def get_origin(typ):
     Provide `get_origin` that works in all python versions.
     """
     try:
-        return typing.get_origin(typ)  # python>=3.8 typing module has get_origin.
+        return typing.get_origin(typ) or get_np_origin(typ)  # python>=3.8 typing module has get_origin.
     except AttributeError:
-        return typing_inspect.get_origin(typ)
+        return typing_inspect.get_origin(typ) or get_np_origin(typ)
 
 
 def get_args(typ):

serde/de.py

@@ -57,6 +57,7 @@
     raise_unsupported_type,
     union_func_name,
 )
+from .numpy import deserialize_numpy_array, deserialize_numpy_scalar, is_numpy_array, is_numpy_scalar
 
 __all__: List = ['deserialize', 'is_deserializable', 'from_dict', 'from_tuple']
 
@@ -488,6 +489,7 @@ class Renderer:
     func: str
     cls: Optional[Type] = None
     custom: Optional[DeserializeFunc] = None  # Custom class level deserializer.
+    import_numpy: bool = False
 
     def render(self, arg: DeField) -> str:
         """
@@ -509,7 +511,7 @@ def render(self, arg: DeField) -> str:
             res = self.tuple(arg)
         elif is_enum(arg.type):
             res = self.enum(arg)
-        elif is_primitive(arg.type):
+        elif is_primitive(arg.type) and not is_numpy_scalar(arg.type):
             res = self.primitive(arg)
         elif is_union(arg.type):
             res = self.union_func(arg)
@@ -529,6 +531,12 @@ def render(self, arg: DeField) -> str:
         elif is_generic(arg.type):
             arg.type = get_origin(arg.type)
             res = self.render(arg)
+        elif is_numpy_scalar(arg.type):
+            self.import_numpy = True
+            res = deserialize_numpy_scalar(arg)
+        elif is_numpy_array(arg.type):
+            self.import_numpy = True
+            res = deserialize_numpy_array(arg)
         else:
             return f"raise_unsupported_type({arg.data})"
 
@@ -755,7 +763,12 @@ def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_s
     env = jinja2.Environment(loader=jinja2.DictLoader({'iter': template}))
     env.filters.update({'rvalue': renderer.render})
     env.filters.update({'arg': to_iter_arg})
-    return env.get_template('iter').render(func=FROM_ITER, serde_scope=getattr(cls, SERDE_SCOPE), fields=defields(cls))
+    res = env.get_template('iter').render(func=FROM_ITER, serde_scope=getattr(cls, SERDE_SCOPE), fields=defields(cls))
+
+    if renderer.import_numpy:
+        res = "import numpy\n" + res
+
+    return res
 
 
 def render_from_dict(cls: Type, rename_all: Optional[str] = None, custom: Optional[DeserializeFunc] = None) -> str:
@@ -779,7 +792,12 @@ def {{func}}(cls=cls, maybe_generic=None, data=None, reuse_instances = {{serde_s
     env = jinja2.Environment(loader=jinja2.DictLoader({'dict': template}))
     env.filters.update({'rvalue': renderer.render})
     env.filters.update({'arg': functools.partial(to_arg, rename_all=rename_all)})
-    return env.get_template('dict').render(func=FROM_DICT, serde_scope=getattr(cls, SERDE_SCOPE), fields=defields(cls))
+    res = env.get_template('dict').render(func=FROM_DICT, serde_scope=getattr(cls, SERDE_SCOPE), fields=defields(cls))
+
+    if renderer.import_numpy:
+        res = "import numpy\n" + res
+
+    return res
 
 
 def render_union_func(cls: Type, union_args: List[Type], tagging: Tagging = DefaultTagging) -> str:

serde/json.py

@@ -6,6 +6,7 @@
 
 from .compat import T
 from .de import Deserializer, from_dict
+from .numpy import encode_numpy
 from .se import Serializer, to_dict
 
 __all__ = ["from_json", "to_json"]
@@ -14,6 +15,8 @@
 class JsonSerializer(Serializer):
     @classmethod
     def serialize(cls, obj: Any, **opts) -> str:
+        if "default" not in opts:
+            opts["default"] = encode_numpy
         return json.dumps(obj, **opts)
 
 

serde/msgpack.py

@@ -9,6 +9,7 @@
 from .compat import T
 from .core import SerdeError
 from .de import Deserializer, from_dict, from_tuple
+from .numpy import encode_numpy
 from .se import Serializer, to_dict, to_tuple
 
 __all__ = ["from_msgpack", "to_msgpack"]
@@ -17,6 +18,8 @@
 class MsgPackSerializer(Serializer):
     @classmethod
     def serialize(cls, obj, use_bin_type: bool = True, ext_type_code: int = None, **opts) -> bytes:
+        if "default" not in opts:
+            opts["default"] = encode_numpy
         if ext_type_code is not None:
             obj_bytes = msgpack.packb(obj, use_bin_type=use_bin_type, **opts)
             obj_or_ext = msgpack.ExtType(ext_type_code, obj_bytes)

serde/numpy.py

@@ -0,0 +1,88 @@
+from typing import Any, Callable, Optional
+
+from serde.compat import get_origin
+
+
+def fullname(klass):
+    module = klass.__module__
+    if module == 'builtins':
+        return klass.__qualname__  # avoid outputs like 'builtins.str'
+    return module + '.' + klass.__qualname__
+
+
+try:
+    import numpy as np
+    import numpy.typing as npt
+
+    encode_numpy: Optional[Callable[[Any], Any]]
+
+    def encode_numpy(obj: Any):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if isinstance(obj, np.generic):
+            return obj.item()
+        raise TypeError(f"Object of type {fullname(type(obj))} is not serializable")
+
+    def is_bare_numpy_array(typ) -> bool:
+        """
+        Test if the type is `np.ndarray` or `npt.NDArray` without type args.
+
+        >>> import numpy as np
+        >>> import numpy.typing as npt
+        >>> is_bare_numpy_array(npt.NDArray[np.int64])
+        False
+        >>> is_bare_numpy_array(npt.NDArray)
+        True
+        >>> is_bare_numpy_array(np.ndarray)
+        True
+        """
+        return typ in (np.ndarray, npt.NDArray)
+
+    def is_numpy_scalar(typ) -> bool:
+        try:
+            return issubclass(typ, np.generic)
+        except TypeError:
+            return False
+
+    def serialize_numpy_scalar(arg) -> str:
+        return f"{arg.varname}.item()"
+
+    def deserialize_numpy_scalar(arg):
+        return f"{fullname(arg.type)}({arg.data})"
+
+    def is_numpy_array(typ) -> bool:
+        origin = get_origin(typ)
+        if origin is not None:
+            typ = origin
+        return typ is np.ndarray
+
+    def serialize_numpy_array(arg) -> str:
+        return f"{arg.varname}.tolist()"
+
+    def deserialize_numpy_array(arg) -> str:
+        if is_bare_numpy_array(arg.type):
+            return f"numpy.array({arg.data})"
+
+        dtype = fullname(arg[1][0].type)
+        return f"numpy.array({arg.data}, dtype={dtype})"
+
+except ImportError:
+    encode_numpy = None
+
+    def is_numpy_scalar(typ) -> bool:
+        return False
+
+    def serialize_numpy_scalar(arg) -> str:
+        return ""
+
+    def deserialize_numpy_scalar(arg):
+        return ""
+
+    def is_numpy_array(typ) -> bool:
+        return False
+
+    def serialize_numpy_array(arg) -> str:
+        return ""
+
+    def deserialize_numpy_array(arg) -> str:
+        return ""