yaml loading: avoid datetime objects

Author: mr-c

Makefile

@@ -33,7 +33,7 @@ COVBASE=coverage run --append
 
 # Updating the Major & Minor version below?
 # Don't forget to update setup.py as well
-VERSION=8.1.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
+VERSION=8.2.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 
 ## all         : default task
@@ -70,7 +70,8 @@ docs: FORCE
 
 ## clean       : clean up all temporary / machine-generated files
 clean: FORCE
-	rm -f ${MODILE}/*.pyc tests/*.pyc
+	rm -rf ${MODULE}/__pycache__ ${MODULE}/tests/__pycache__
+	rm -f *.so ${MODULE}/*.so ${MODULE}/tests/*.so ${MODULE}/avro/*.so
 	python setup.py clean --all || true
 	rm -Rf .coverage
 	rm -f diff-cover.html

schema_salad/metaschema.py

@@ -25,11 +25,11 @@
 from urllib.request import pathname2url
 
 from ruamel.yaml.comments import CommentedMap
-from ruamel.yaml.main import YAML
 
 from schema_salad.exceptions import SchemaSaladException, ValidationException
 from schema_salad.fetcher import DefaultFetcher, Fetcher
 from schema_salad.sourceline import SourceLine, add_lc_filename
+from schema_salad.utils import yaml_no_ts  # requires schema-salad v8.2+
 
 _vocab = {}  # type: Dict[str, str]
 _rvocab = {}  # type: Dict[str, str]
@@ -561,8 +561,7 @@ def _document_load_by_url(loader, url, loadingOptions):
     else:
         textIO = StringIO(text)
     textIO.name = str(url)
-    yaml = YAML()
-    yaml.preserve_quotes = True  # type: ignore
+    yaml = yaml_no_ts()
     result = yaml.load(textIO)
     add_lc_filename(result, url)
 
@@ -2866,8 +2865,7 @@ def load_document(doc, baseuri=None, loadingOptions=None):
 
 def load_document_by_string(string, uri, loadingOptions=None):
     # type: (Any, str, Optional[LoadingOptions]) -> Any
-    yaml = YAML()
-    yaml.preserve_quotes = True  # type: ignore
+    yaml = yaml_no_ts()
     result = yaml.load(string)
     add_lc_filename(result, uri)
 

schema_salad/python_codegen.py

@@ -541,8 +541,7 @@ def load_document(doc, baseuri=None, loadingOptions=None):
 
 def load_document_by_string(string, uri, loadingOptions=None):
     # type: (Any, str, Optional[LoadingOptions]) -> Any
-    yaml = YAML()
-    yaml.preserve_quotes = True  # type: ignore
+    yaml = yaml_no_ts()
     result = yaml.load(string)
     add_lc_filename(result, uri)
 

schema_salad/python_codegen_support.py

@@ -21,11 +21,11 @@
 from urllib.request import pathname2url
 
 from ruamel.yaml.comments import CommentedMap
-from ruamel.yaml.main import YAML
 
 from schema_salad.exceptions import SchemaSaladException, ValidationException
 from schema_salad.fetcher import DefaultFetcher, Fetcher
 from schema_salad.sourceline import SourceLine, add_lc_filename
+from schema_salad.utils import yaml_no_ts  # requires schema-salad v8.2+
 
 _vocab = {}  # type: Dict[str, str]
 _rvocab = {}  # type: Dict[str, str]
@@ -557,8 +557,7 @@ def _document_load_by_url(loader, url, loadingOptions):
     else:
         textIO = StringIO(text)
     textIO.name = str(url)
-    yaml = YAML()
-    yaml.preserve_quotes = True  # type: ignore
+    yaml = yaml_no_ts()
     result = yaml.load(textIO)
     add_lc_filename(result, url)
 

schema_salad/ref_resolver.py

@@ -28,7 +28,6 @@
 from rdflib.plugins.parsers.notation3 import BadSyntax
 from ruamel.yaml.comments import CommentedMap, CommentedSeq, LineCol
 from ruamel.yaml.error import MarkedYAMLError
-from ruamel.yaml.main import YAML
 
 from .exceptions import SchemaSaladException, ValidationException
 from .fetcher import DefaultFetcher
@@ -44,6 +43,7 @@
     ResolveType,
     aslist,
     onWindows,
+    yaml_no_ts,
 )
 
 _logger = logging.getLogger("salad")
@@ -984,8 +984,7 @@ def fetch(
             else:
                 textIO = StringIO(text)
             textIO.name = str(url)
-            yaml = YAML()
-            yaml.preserve_quotes = True  # type: ignore
+            yaml = yaml_no_ts()
             attachments = yaml.load_all(textIO)
             result = cast(Union[CommentedSeq, CommentedMap], next(attachments))
 

schema_salad/schema.py

@@ -21,7 +21,6 @@
 
 from pkg_resources import resource_stream
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from ruamel.yaml.main import YAML
 
 from schema_salad.utils import (
     CacheType,
@@ -31,6 +30,7 @@
     convert_to_dict,
     flatten,
     json_dumps,
+    yaml_no_ts,
 )
 
 from . import _logger, jsonld_context, ref_resolver, validate
@@ -183,7 +183,7 @@ def get_metaschema() -> Tuple[Names, List[Dict[str, str]], Loader]:
     with resource_stream("schema_salad", "metaschema/metaschema.yml") as stream:
         loader.cache["https://w3id.org/cwl/salad"] = stream.read().decode("UTF-8")
 
-    yaml = YAML()
+    yaml = yaml_no_ts()
     j = yaml.load(loader.cache["https://w3id.org/cwl/salad"])
     add_lc_filename(j, "metaschema.yml")
     j2 = loader.resolve_all(j, saladp)[0]

schema_salad/tests/test_cg.py

@@ -3,11 +3,11 @@
 from typing import Any
 
 import pytest
-from ruamel.yaml.main import YAML
 
 import schema_salad.metaschema as cg_metaschema
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import file_uri
+from schema_salad.utils import yaml_no_ts
 
 from .matcher import JsonDiffMatcher
 from .util import get_data
@@ -203,8 +203,7 @@ def test_load_by_yaml_metaschema(metaschema_pre: Any) -> None:
     path = get_data("metaschema/metaschema.yml")
     assert path
     with open(path) as path_handle:
-        yaml = YAML()
-        yaml.preserve_quotes = True  # type: ignore
+        yaml = yaml_no_ts()
         yaml_doc = yaml.load(path_handle)
     doc = cg_metaschema.load_document_by_yaml(
         yaml_doc,

schema_salad/tests/test_examples.py

@@ -1,18 +1,18 @@
 """Test examples."""
 import datetime
 import os
-from typing import cast
+from io import StringIO
+from typing import cast, Any, Dict
 
 from pytest import CaptureFixture
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from ruamel.yaml.main import YAML
 
 import schema_salad.main
 import schema_salad.schema
 from schema_salad.jsonld_context import makerdf
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine, cmap
-from schema_salad.utils import ContextType, stdout
+from schema_salad.utils import ContextType, stdout, yaml_no_ts
 
 from .util import get_data
 
@@ -278,7 +278,7 @@ def test_examples() -> None:
         ldr, _, _, _ = schema_salad.schema.load_schema(path)
         path2 = get_data(f"metaschema/{a}_src.yml")
         assert path2
-        yaml = YAML()
+        yaml = yaml_no_ts()
         with open(path2) as src_fp:
             src = ldr.resolve_all(yaml.load(src_fp), "", checklinks=False)[0]
         path3 = get_data(f"metaschema/{a}_proc.yml")
@@ -289,7 +289,7 @@ def test_examples() -> None:
 
 
 def test_yaml_float_test() -> None:
-    assert YAML().load("float-test: 2e-10")["float-test"] == 2e-10
+    assert yaml_no_ts().load("float-test: 2e-10")["float-test"] == 2e-10
 
 
 def test_typedsl_ref() -> None:
@@ -425,6 +425,21 @@ def test_rdf_datetime() -> None:
     g2.serialize(destination=stdout(), format="n3")
 
 
+def test_yaml_datetime() -> None:
+    """Affirm that yaml_no_ts prevents the creation of datetime objects."""
+    example: Dict[str, Any] = {
+        "id": "foo",
+        "bar": {"id": "baz"},
+    }
+    example["s:dateCreated"] = datetime.datetime(2020, 10, 8)
+    yaml = yaml_no_ts()
+    stream = StringIO()
+    yaml.dump(example, stream)
+    stream2 = StringIO(stream.getvalue())
+    example2 = yaml.load(stream2)
+    assert isinstance(example2["s:dateCreated"], str)
+
+
 def test_subscoped_id() -> None:
     ldr = Loader({})
     ctx = {

schema_salad/utils.py

@@ -19,6 +19,8 @@
 import requests
 from rdflib.graph import Graph
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
+from ruamel.yaml.constructor import RoundTripConstructor
+from ruamel.yaml.main import YAML
 
 if TYPE_CHECKING:
     from .fetcher import Fetcher
@@ -115,3 +117,26 @@ def json_dumps(
 def stdout() -> BufferedWriter:
     """Build a replacement for sys.stdout that allow for writing binary data."""
     return os.fdopen(sys.stdout.fileno(), "wb", closefd=False)
+
+
+class _RoundTripNoTimeStampConstructor(RoundTripConstructor):
+    def construct_yaml_timestamp(self: Any, node: Any, values: Any = None) -> Any:
+        return node.value
+
+
+_RoundTripNoTimeStampConstructor.add_constructor(
+    u"tag:yaml.org,2002:timestamp",
+    _RoundTripNoTimeStampConstructor.construct_yaml_timestamp,
+)
+
+
+def yaml_no_ts() -> YAML:
+    """
+    Get a YAML loader that won't parse timestamps into datetime objects.
+
+    Such datetime objects can't be easily dumped into JSON.
+    """
+    yaml = YAML(typ="rt")
+    yaml.preserve_quotes = True  # type: ignore
+    yaml.Constructor = _RoundTripNoTimeStampConstructor
+    return yaml

setup.py

@@ -89,7 +89,7 @@
 
 setup(
     name="schema-salad",
-    version="8.1",  # update the VERSION prefix in the Makefile as well 🙂
+    version="8.2",  # update the VERSION prefix in the Makefile as well 🙂
     description="Schema Annotations for Linked Avro Data (SALAD)",
     long_description=open(README).read(),
     long_description_content_type="text/x-rst",