Merge pull request #412 from common-workflow-language/class-vocab

Class vocab

Author: mergify[bot]

schema_salad/avro/schema.py

@@ -579,10 +579,13 @@ def make_avsc_object(json_data: JsonDataType, names: Optional[Names] = None) ->
     if names is None:
         names = Names()
 
-    if isinstance(json_data, Dict) and json_data.get("name") == "Any":
-        del names.names["Any"]
-    elif not names.has_name("Any", None):
-        EnumSchema("Any", None, ["Any"], names=names)
+    if (
+        isinstance(json_data, Dict)
+        and json_data.get("name") == "org.w3id.cwl.salad.Any"
+    ):
+        del names.names["org.w3id.cwl.salad.Any"]
+    elif not names.has_name("org.w3id.cwl.salad.Any", None):
+        EnumSchema("org.w3id.cwl.salad.Any", None, ["Any"], names=names)
 
     # JSON object (non-union)
     if isinstance(json_data, dict):

schema_salad/makedoc.py

@@ -24,6 +24,7 @@
 import mistune
 
 from . import schema
+from .validate import avro_type_name
 from .exceptions import SchemaSaladException, ValidationException
 from .utils import add_dictlist, aslist
 
@@ -315,7 +316,7 @@ def typefmt(
                 "https://w3id.org/cwl/salad#record",
                 "https://w3id.org/cwl/salad#enum",
             ):
-                frg = schema.avro_type_name(tp["name"])
+                frg = avro_type_name(tp["name"])
                 if tp["name"] in redirects:
                     return """<a href="{}">{}</a>""".format(redirects[tp["name"]], frg)
                 if tp["name"] in self.typemap:
@@ -335,7 +336,7 @@ def typefmt(
                 return """<a href="{}">{}</a>""".format(redirects[tp], redirects[tp])
             if str(tp) in basicTypes:
                 return """<a href="{}">{}</a>""".format(
-                    self.primitiveType, schema.avro_type_name(str(tp))
+                    self.primitiveType, avro_type_name(str(tp))
                 )
             frg2 = urldefrag(tp)[1]
             if frg2 != "":

schema_salad/schema.py

@@ -192,7 +192,7 @@ def get_metaschema() -> Tuple[Names, List[Dict[str, str]], Loader]:
         _logger.error("%s", j2)
         raise SchemaParseException(f"Not a list: {j2}")
     else:
-        sch_obj = make_avro(j2, loader)
+        sch_obj = make_avro(j2, loader, loader.vocab)
     try:
         sch_names = make_avro_schema_from_avro(sch_obj)
     except SchemaParseException:
@@ -258,7 +258,7 @@ def load_schema(
 
     # Make the Avro validation that will be used to validate the target
     # document
-    avsc_names = make_avro_schema(schema_doc, document_loader)
+    avsc_names = make_avro_schema(schema_doc, document_loader, metaschema_loader.vocab)
 
     return document_loader, avsc_names, schema_metadata, metaschema_loader
 
@@ -350,6 +350,7 @@ def validate_doc(
                 raise_ex=False,
                 skip_foreign_properties=loader.skip_schemas,
                 strict_foreign_properties=strict_foreign_properties,
+                vocab=loader.vocab,
             )
             if success:
                 break
@@ -372,6 +373,7 @@ def validate_doc(
                         raise_ex=True,
                         skip_foreign_properties=loader.skip_schemas,
                         strict_foreign_properties=strict_foreign_properties,
+                        vocab=loader.vocab,
                     )
                 except ClassValidationException as exc1:
                     errors = [
@@ -493,42 +495,6 @@ def replace_type(
     return items
 
 
-primitives = {
-    "http://www.w3.org/2001/XMLSchema#string": "string",
-    "http://www.w3.org/2001/XMLSchema#boolean": "boolean",
-    "http://www.w3.org/2001/XMLSchema#int": "int",
-    "http://www.w3.org/2001/XMLSchema#long": "long",
-    "http://www.w3.org/2001/XMLSchema#float": "float",
-    "http://www.w3.org/2001/XMLSchema#double": "double",
-    saladp + "null": "null",
-    saladp + "enum": "enum",
-    saladp + "array": "array",
-    saladp + "record": "record",
-}
-
-
-def avro_type_name(url: str) -> str:
-    """
-    Turn a URL into an Avro-safe name.
-
-    If the URL has no fragment, return this plain URL.
-
-    Extract either the last part of the URL fragment past the slash, otherwise
-    the whole fragment.
-    """
-    global primitives
-
-    if url in primitives:
-        return primitives[url]
-
-    if url.startswith("http://"):
-        url = url[7:]
-    elif url.startswith("https://"):
-        url = url[8:]
-    url = url.replace("/", ".").replace("#", ".")
-    return url
-
-
 def avro_field_name(url: str) -> str:
     """
     Turn a URL into an Avro-safe name.
@@ -553,18 +519,24 @@ def make_valid_avro(
     found: Set[str],
     union: bool = False,
     fielddef: bool = False,
+    vocab: Optional[Dict[str, str]] = None,
 ) -> Union[
     Avro, MutableMapping[str, str], str, List[Union[Any, MutableMapping[str, str], str]]
 ]:
     """Convert our schema to be more avro like."""
+
+    if vocab is None:
+        _, _, metaschema_loader = get_metaschema()
+        vocab = metaschema_loader.vocab
+
     # Possibly could be integrated into our fork of avro/schema.py?
     if isinstance(items, MutableMapping):
         avro = copy.copy(items)
         if avro.get("name") and avro.get("inVocab", True):
             if fielddef:
                 avro["name"] = avro_field_name(avro["name"])
             else:
-                avro["name"] = avro_type_name(avro["name"])
+                avro["name"] = validate.avro_type_name(avro["name"])
 
         if "type" in avro and avro["type"] in (
             saladp + "record",
@@ -585,6 +557,7 @@ def make_valid_avro(
                     found,
                     union=True,
                     fielddef=(field == "fields"),
+                    vocab=vocab,
                 )
         if "symbols" in avro:
             avro["symbols"] = [avro_field_name(sym) for sym in avro["symbols"]]
@@ -593,13 +566,20 @@ def make_valid_avro(
         ret = []
         for i in items:
             ret.append(
-                make_valid_avro(i, alltypes, found, union=union, fielddef=fielddef)
+                make_valid_avro(
+                    i, alltypes, found, union=union, fielddef=fielddef, vocab=vocab
+                )
             )
         return ret
     if union and isinstance(items, str):
-        if items in alltypes and avro_type_name(items) not in found:
-            return make_valid_avro(alltypes[items], alltypes, found, union=union)
-        return avro_type_name(items)
+        if items in alltypes and validate.avro_type_name(items) not in found:
+            return make_valid_avro(
+                alltypes[items], alltypes, found, union=union, vocab=vocab
+            )
+        if items in vocab:
+            return validate.avro_type_name(vocab[items])
+        else:
+            return validate.avro_type_name(items)
     else:
         return items
 
@@ -697,7 +677,7 @@ def extend_and_specialize(
             for ex in aslist(result["extends"]):
                 if ex_types[ex].get("abstract"):
                     add_dictlist(extended_by, ex, ex_types[result["name"]])
-                    add_dictlist(extended_by, avro_type_name(ex), ex_types[ex])
+                    add_dictlist(extended_by, validate.avro_type_name(ex), ex_types[ex])
 
     for result in results:
         if result.get("abstract") and result["name"] not in extended_by:
@@ -717,27 +697,28 @@ def extend_and_specialize(
 def make_avro(
     i: List[Dict[str, Any]],
     loader: Loader,
+    metaschema_vocab: Optional[Dict[str, str]] = None,
 ) -> List[Any]:
 
     j = extend_and_specialize(i, loader)
 
     name_dict = {}  # type: Dict[str, Dict[str, Any]]
     for entry in j:
         name_dict[entry["name"]] = entry
-    avro = make_valid_avro(j, name_dict, set())
+
+    avro = make_valid_avro(j, name_dict, set(), vocab=metaschema_vocab)
 
     return [
         t
         for t in avro
         if isinstance(t, MutableMapping)
         and not t.get("abstract")
-        and t.get("type") != "documentation"
+        and t.get("type") != "org.w3id.cwl.salad.documentation"
     ]
 
 
 def make_avro_schema(
-    i: List[Any],
-    loader: Loader,
+    i: List[Any], loader: Loader, metaschema_vocab: Optional[Dict[str, str]] = None
 ) -> Names:
     """
     All in one convenience function.
@@ -746,7 +727,7 @@ def make_avro_schema(
     the intermediate result for diagnostic output.
     """
     names = Names()
-    avro = make_avro(i, loader)
+    avro = make_avro(i, loader, metaschema_vocab)
     make_avsc_object(convert_to_dict(avro), names)
     return names
 

schema_salad/validate.py

@@ -1,6 +1,6 @@
 import logging
 import pprint
-from typing import Any, List, MutableMapping, MutableSequence, Optional, Set
+from typing import Any, List, Mapping, MutableMapping, MutableSequence, Optional, Set
 from urllib.parse import urlsplit
 
 from . import avro
@@ -21,6 +21,7 @@ def validate(
     identifiers: Optional[List[str]] = None,
     strict: bool = False,
     foreign_properties: Optional[Set[str]] = None,
+    vocab: Optional[Mapping[str, str]] = None,
 ) -> bool:
     if not identifiers:
         identifiers = []
@@ -33,6 +34,7 @@ def validate(
         strict=strict,
         foreign_properties=foreign_properties,
         raise_ex=False,
+        vocab=vocab,
     )
 
 
@@ -47,6 +49,43 @@ def avro_shortname(name: str) -> str:
     return name.split(".")[-1]
 
 
+saladp = "https://w3id.org/cwl/salad#"
+primitives = {
+    "http://www.w3.org/2001/XMLSchema#string": "string",
+    "http://www.w3.org/2001/XMLSchema#boolean": "boolean",
+    "http://www.w3.org/2001/XMLSchema#int": "int",
+    "http://www.w3.org/2001/XMLSchema#long": "long",
+    "http://www.w3.org/2001/XMLSchema#float": "float",
+    "http://www.w3.org/2001/XMLSchema#double": "double",
+    saladp + "null": "null",
+    saladp + "enum": "enum",
+    saladp + "array": "array",
+    saladp + "record": "record",
+}
+
+
+def avro_type_name(url: str) -> str:
+    """
+    Turn a URL into an Avro-safe name.
+
+    If the URL has no fragment, return this plain URL.
+
+    Extract either the last part of the URL fragment past the slash, otherwise
+    the whole fragment.
+    """
+    global primitives
+
+    if url in primitives:
+        return primitives[url]
+
+    u = urlsplit(url)
+    joined = filter(
+        lambda x: x,
+        list(reversed(u.netloc.split("."))) + u.path.split("/") + u.fragment.split("/"),
+    )
+    return ".".join(joined)
+
+
 def friendly(v):  # type: (Any) -> Any
     if isinstance(v, avro.schema.NamedSchema):
         return avro_shortname(v.name)
@@ -77,6 +116,7 @@ def validate_ex(
     strict_foreign_properties=False,  # type: bool
     logger=_logger,  # type: logging.Logger
     skip_foreign_properties=False,  # type: bool
+    vocab=None,  # type: Optional[Mapping[str, str]]
 ):
     # type: (...) -> bool
     """Determine if a python datum is an instance of a schema."""
@@ -87,6 +127,9 @@ def validate_ex(
     if not foreign_properties:
         foreign_properties = set()
 
+    if vocab is None:
+        raise Exception("vocab must be provided")
+
     schema_type = expected_schema.type
 
     if schema_type == "null":
@@ -132,7 +175,7 @@ def validate_ex(
             )
         return False
     elif isinstance(expected_schema, avro.schema.EnumSchema):
-        if expected_schema.name == "w3id.org.cwl.salad.Any":
+        if expected_schema.name in ("org.w3id.cwl.salad.Any", "Any"):
             if datum is not None:
                 return True
             if raise_ex:
@@ -144,7 +187,7 @@ def validate_ex(
                     "value is a {} but expected a string".format(type(datum).__name__)
                 )
             return False
-        if expected_schema.name == "w3id.org.cwl.cwl.Expression":
+        if expected_schema.name == "org.w3id.cwl.cwl.Expression":
             if "$(" in datum or "${" in datum:
                 return True
             if raise_ex:
@@ -182,6 +225,7 @@ def validate_ex(
                         strict_foreign_properties=strict_foreign_properties,
                         logger=logger,
                         skip_foreign_properties=skip_foreign_properties,
+                        vocab=vocab,
                     ):
                         return False
                 except ValidationException as v:
@@ -208,6 +252,7 @@ def validate_ex(
                 strict_foreign_properties=strict_foreign_properties,
                 logger=logger,
                 skip_foreign_properties=skip_foreign_properties,
+                vocab=vocab,
             ):
                 return True
 
@@ -244,6 +289,7 @@ def validate_ex(
                     strict_foreign_properties=strict_foreign_properties,
                     logger=logger,
                     skip_foreign_properties=skip_foreign_properties,
+                    vocab=vocab,
                 )
             except ClassValidationException:
                 raise
@@ -284,7 +330,10 @@ def validate_ex(
                         raise ValidationException(f"Missing '{f.name}' field")
                     else:
                         return False
-                if avro_shortname(expected_schema.name) != d:
+                avroname = None
+                if d in vocab:
+                    avroname = avro_type_name(vocab[d])
+                if expected_schema.name != d and expected_schema.name != avroname:
                     if raise_ex:
                         raise ValidationException(
                             "Expected class '{}' but this is '{}'".format(
@@ -321,6 +370,7 @@ def validate_ex(
                     strict_foreign_properties=strict_foreign_properties,
                     logger=logger,
                     skip_foreign_properties=skip_foreign_properties,
+                    vocab=vocab,
                 ):
                     return False
             except ValidationException as v:

setup.py

@@ -98,7 +98,7 @@
 
 setup(
     name="schema-salad",
-    version="8.0",  # update the VERSION prefix in the Makefile as well 🙂
+    version="8.1",  # update the VERSION prefix in the Makefile as well 🙂
     description="Schema Annotations for Linked Avro Data (SALAD)",
     long_description=open(README).read(),
     long_description_content_type="text/x-rst",