From 54874a14e385c33c3d747d8965b630dbd37626a8 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 17:08:11 +0200 Subject: [PATCH 01/17] added script to make conversion from multilingual json schema to one that can be used in the IDE a lot easier. --- tested/transform_json.py | 76 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tested/transform_json.py diff --git a/tested/transform_json.py b/tested/transform_json.py new file mode 100644 index 00000000..0432befa --- /dev/null +++ b/tested/transform_json.py @@ -0,0 +1,76 @@ +import json +import os +import sys +from pathlib import Path +from typing import Any + + +def transform(data: Any) -> Any: + if isinstance(data, list): + return [transformed for ele in data if (transformed := transform(ele)) != {}] + elif isinstance(data, dict): + if "return" in data: + # This is necessary since tags aren't recognized in the Json schema + # and such natural_language maps wil always be seen as yamlValue. + assert isinstance(data["return"], dict) and "oneOf" in data["return"] + data["return"]["anyOf"] = data["return"].pop("oneOf") + + if "yamlValue" in data: + data["yamlValue"] = { + "description": "A value represented as YAML.", + } + + if "required" in data and "properties" in data: + required = data["required"] + properties = data["properties"] + assert isinstance(required, list) + assert isinstance(properties, dict) + if "__tag__" in required and "value" in required: + assert "__tag__" in properties and "value" in properties + value = properties["value"] + tag = properties["__tag__"] + assert isinstance(value, dict) + assert isinstance(tag, dict) and "const" in tag + + if tag["const"] == "!programming_language" or tag["const"] == "!expression": + return {} + + if tag["const"] == "!natural_language": + value["not"] = { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + } + data = value + + return {k: transform(v) for k, v in data.items()} + return data + +def transform_json(json_file): + _, ext = os.path.splitext(json_file) + assert ext.lower() == ".json", f"expected a json file, got {ext}." + try: + with open(json_file, "r") as stream: + json_stream = json.load(stream) + except FileNotFoundError as e: + print("The json file was not found.") + raise e + + result = transform(json_stream) + + print(json.dumps(result, indent=2)) + with open("output.json", "w", encoding='utf-8') as f: + json.dump(result, f, indent=2) + #print(result) + + + + + +if __name__ == "__main__": + n = len(sys.argv) + assert n > 1, "Expected path to multilingual json schema." + + transform_json(Path(sys.argv[1])) From 2bb7fd7ae43f125a3bbd532d04440a256a4c1253 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 17:13:19 +0200 Subject: [PATCH 02/17] fixed linting --- tested/transform_json.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 0432befa..acb8dad7 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -32,7 +32,10 @@ def transform(data: Any) -> Any: assert isinstance(value, dict) assert isinstance(tag, dict) and "const" in tag - if tag["const"] == "!programming_language" or tag["const"] == "!expression": + if ( + tag["const"] == "!programming_language" + or tag["const"] == "!expression" + ): return {} if tag["const"] == "!natural_language": @@ -48,6 +51,7 @@ def transform(data: Any) -> Any: return {k: transform(v) for k, v in data.items()} return data + def transform_json(json_file): _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." @@ -61,12 +65,9 @@ def transform_json(json_file): result = transform(json_stream) print(json.dumps(result, indent=2)) - with open("output.json", "w", encoding='utf-8') as f: + with open("output.json", "w", encoding="utf-8") as f: json.dump(result, f, indent=2) - #print(result) - - - + # print(result) if __name__ == "__main__": From 525373eb04a483bddf4f58624ee9472ac806d544 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 18:49:46 +0200 Subject: [PATCH 03/17] added some extra tests --- tested/transform_json.py | 10 +-- tests/test_preprocess_dsl.py | 145 +++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 5 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index acb8dad7..36d1f330 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -10,8 +10,8 @@ def transform(data: Any) -> Any: return [transformed for ele in data if (transformed := transform(ele)) != {}] elif isinstance(data, dict): if "return" in data: - # This is necessary since tags aren't recognized in the Json schema - # and such natural_language maps wil always be seen as yamlValue. + # This is necessary since tags aren't recognized in the Json schema. + # So a natural_language maps wil always be seen as yamlValue. assert isinstance(data["return"], dict) and "oneOf" in data["return"] data["return"]["anyOf"] = data["return"].pop("oneOf") @@ -64,10 +64,10 @@ def transform_json(json_file): result = transform(json_stream) - print(json.dumps(result, indent=2)) - with open("output.json", "w", encoding="utf-8") as f: + with open( + json_file.parent / "multilingual-schema.json", "w", encoding="utf-8" + ) as f: json.dump(result, f, indent=2) - # print(result) if __name__ == "__main__": diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index c00d5fa6..78a8f2a0 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,6 +18,7 @@ translate_yaml, validate_pre_dsl, ) +from tested.transform_json import transform def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -666,3 +667,147 @@ def test_template_syntax_error(): return: 11{%{{works}} """.strip() validate_natural_translate(yaml_str, translated_yml) + + +def test_return_json_schema(): + json_schema = { + "return": { + "description": "Expected return value", + "oneOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + }, + }, + }, + ], + } + } + + json_schema_expected = { + "return": { + "description": "Expected return value", + "anyOf": [ + {"$ref": "#/definitions/returnOutputChannel"}, + { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/returnOutputChannel" + }, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + }, + ], + } + } + + result = transform(json_schema) + + assert result == json_schema_expected + + +def test_yaml_value_json_schema(): + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + result = transform(json_schema) + + assert result == json_schema_expected + + +def test_nat_lang_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": {"type": "object", "additionalProperties": {"type": "string"}}, + }, + } + + json_schema_expected = { + "type": "object", + "additionalProperties": {"type": "string"}, + "not": { + "anyOf": [ + {"required": ["description"]}, + {"required": ["value"]}, + {"required": ["types"]}, + ] + }, + } + + result = transform(json_schema) + + assert result == json_schema_expected + + +def test_expr_json_schema(): + json_schema = { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + } + + json_schema_expected = {} + + result = transform(json_schema) + + assert result == json_schema_expected + +def test_list_json_schema(): + json_schema = [{}, { + "type" : "string", + "description" : "A statement of expression in Python-like syntax as YAML string." + }] + + json_schema_expected = [{ + "type" : "string", + "description" : "A statement of expression in Python-like syntax as YAML string." + }] + + result = transform(json_schema) + + assert result == json_schema_expected From f5c1a5cb276d087bf414683f683e1e4389c9559a Mon Sep 17 00:00:00 2001 From: breblanc Date: Sat, 12 Apr 2025 19:58:27 +0200 Subject: [PATCH 04/17] added more text and documentation --- tested/transform_json.py | 12 ++++- tests/test_preprocess_dsl.py | 85 ++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 36d1f330..1cb39b6f 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -32,12 +32,16 @@ def transform(data: Any) -> Any: assert isinstance(value, dict) assert isinstance(tag, dict) and "const" in tag + # For !programming_language the same was already defined without the tag. + # The only usage for the expression tag is also redundant. if ( tag["const"] == "!programming_language" or tag["const"] == "!expression" ): return {} + # This to help the validator to somewhat distinguish between + # a natural_language map and a dictionary. if tag["const"] == "!natural_language": value["not"] = { "anyOf": [ @@ -52,7 +56,13 @@ def transform(data: Any) -> Any: return data -def transform_json(json_file): +def transform_json(json_file: Path): + """ + This function transforms the JSON schema used in the DSL translator into + a new JSON schema that can be used to validate the multilingual YAML in your IDE. + + :param json_file: The path to the JSON file. + """ _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." try: diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 78a8f2a0..2b5cfbbf 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,7 +18,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform +from tested.transform_json import transform, transform_json def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -797,17 +797,84 @@ def test_expr_json_schema(): assert result == json_schema_expected + def test_list_json_schema(): - json_schema = [{}, { - "type" : "string", - "description" : "A statement of expression in Python-like syntax as YAML string." - }] + json_schema = [ + {}, + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + ] - json_schema_expected = [{ - "type" : "string", - "description" : "A statement of expression in Python-like syntax as YAML string." - }] + json_schema_expected = [ + { + "type": "string", + "description": "A statement of expression in Python-like syntax as YAML string.", + } + ] result = transform(json_schema) assert result == json_schema_expected + + +def test_transform_executed_correct(mocker: MockerFixture): + s = mocker.spy( + tested.transform_json, name="transform" # type: ignore[reportAttributeAccessIssue] + ) + + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in [ + """ +{ + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + } +}""" + ] + ] + mock_files.append(mocker.mock_open(read_data="{}").return_value) + mock_files.append(mocker.mock_open().return_value) + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + + transform_json(Path("schema.json")) + + assert s.call_count == 25 + + # Check if the file was opened for writing + mock_opener.assert_any_call(Path("multilingual-schema.json"), "w", encoding="utf-8") From afb53c809937a129b8943d47a760f81104681f29 Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 13 Apr 2025 19:22:38 +0200 Subject: [PATCH 05/17] Also wrote a script to generate monolingual JSON schemas --- tested/dsl/schema-strict-nat-translation.json | 21 +-- tested/dsl/schema-strict.json | 5 +- tested/transform_json.py | 132 +++++++++++++++++- tests/test_preprocess_dsl.py | 12 +- 4 files changed, 148 insertions(+), 22 deletions(-) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index e598cc2b..adfeaa57 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -250,7 +250,7 @@ ], "description" : "The name of this tab." }, - "translation" : { + "translations" : { "type" : "object", "description": "Define translations in the tab scope." }, @@ -331,12 +331,14 @@ "description" : "Defines if the unit/tab is hidden for the student or not" }, "unit" : { - "anyOf" : [ + "oneOf" : [ { - "type" : "string" + "type" : "string", + "description" : "The name of this tab." }, { "type" : "object", + "description" : "The name of this tab.", "required": [ "__tag__", "value" @@ -394,7 +396,7 @@ ] }, "_contextList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -429,7 +431,7 @@ ] }, "_caseList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -464,7 +466,7 @@ ] }, "_testcaseList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -499,7 +501,7 @@ ] }, "_scriptList" : { - "anyOf" : [ + "oneOf" : [ { "type" : "array", "minItems" : 1, @@ -1250,6 +1252,7 @@ "oneOf" : [ { "type" : "string", + "format" : "tested-dsl-expression", "description" : "A statement of expression in Python-like syntax as YAML string." }, { @@ -1296,6 +1299,7 @@ "oneOf" : [ { "type" : "string", + "format" : "tested-dsl-expression", "description" : "A statement of expression in Python-like syntax as YAML string." }, { @@ -1955,7 +1959,8 @@ "kotlin", "python", "runhaskell", - "csharp" + "csharp", + "cpp" ] }, "message" : { diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 0471d175..0659aa42 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -449,6 +449,10 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -472,7 +476,6 @@ "type": "programming_language" } ], - "type" : "object", "minProperties" : 1, "propertyNames" : { "$ref" : "#/definitions/programmingLanguage" diff --git a/tested/transform_json.py b/tested/transform_json.py index 1cb39b6f..048295fc 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -1,13 +1,119 @@ import json import os import sys +from enum import Enum from pathlib import Path from typing import Any +class SpecialMap(Enum): + NATURAL_LANGUAGE = "natural_language" + PROGRAMMING_LANGUAGE = "programming_language" + ORACLE = "oracle" + EXPRESSION = "expression" + NONE = "none" -def transform(data: Any) -> Any: +def map_kind(element: dict) -> SpecialMap: + if "required" in element and "properties" in element: + required = element["required"] + properties = element["properties"] + assert isinstance(required, list) + assert isinstance(properties, dict) + if "__tag__" in required and "value" in required: + if "__tag__" in properties and "value" in properties: + tag = properties["__tag__"] + if isinstance(tag, dict) and "const" in tag and isinstance(tag["const"], str): + if tag["const"] == "!natural_language": + return SpecialMap.NATURAL_LANGUAGE + elif tag["const"] == "!programming_language": + return SpecialMap.PROGRAMMING_LANGUAGE + elif tag["const"] == "!oracle": + return SpecialMap.ORACLE + elif tag["const"] == "!expression": + return SpecialMap.EXPRESSION + return SpecialMap.NONE + +def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: + if element == prog_lang: + ele_type = element.pop("type") + element["anyOf"] = [ + {"type": ele_type}, + {"type": "programming_language"}, + ] + return element + +def transform_monolingual(data: Any, strict: bool) -> Any: + if isinstance(data, dict): + if "oneOf" in data: + new_one_of = [] + prog_lang = None + for element in data["oneOf"]: + assert isinstance(element, dict) + # Removes all natural translations + kind = map_kind(element) + if kind == SpecialMap.PROGRAMMING_LANGUAGE: + prog_lang = element["properties"]["value"] + elif kind != SpecialMap.NATURAL_LANGUAGE: + if kind == SpecialMap.EXPRESSION or kind == SpecialMap.ORACLE: + element = element["properties"]["value"] + if strict: + if kind == SpecialMap.EXPRESSION: + element["type"] = "expression" + else: + element["type"] = "oracle" + new_one_of.append(element) + + # A programming_langauge map was found. If not strict, just remove. + # If strict, still provide the type option for the corresponding object. + if prog_lang is not None and strict: + new_one_of = [change_prog_lang_type(ele, prog_lang) for ele in new_one_of] + + if len(new_one_of) <= 1: + data.pop("oneOf") + if len(new_one_of) == 1: + for key, value in new_one_of[0].items(): + data[key] = value + else: + data["oneOf"] = new_one_of + + if "expressionOrStatementWithNatTranslation" in data: + data.pop("expressionOrStatementWithNatTranslation") + + if "translations" in data: + data.pop("translations") + + if "$ref" in data: + if isinstance(data["$ref"], str): + if data["$ref"] == "#/definitions/expressionOrStatementWithNatTranslation": + data["$ref"] = "#/definitions/expressionOrStatement" + + if "yamlValue" in data: + if strict: + data["yamlValue"] = { + "description" : "A value represented as YAML.", + "not" : { + "type" : [ + "oracle", + "expression", + "programming_language" + ] + } + } + else: + data["yamlValue"] = { + "description": "A value represented as YAML.", + } + + + return {k: transform_monolingual(v, strict) for k, v in data.items()} + elif isinstance(data, list): + return [transformed for ele in data if + (transformed := transform_monolingual(ele, strict)) != {}] + return data + + +def transform_IDE(data: Any) -> Any: if isinstance(data, list): - return [transformed for ele in data if (transformed := transform(ele)) != {}] + return [transformed for ele in data if (transformed := transform_IDE(ele)) != {}] elif isinstance(data, dict): if "return" in data: # This is necessary since tags aren't recognized in the Json schema. @@ -52,11 +158,11 @@ def transform(data: Any) -> Any: } data = value - return {k: transform(v) for k, v in data.items()} + return {k: transform_IDE(v) for k, v in data.items()} return data -def transform_json(json_file: Path): +def transform_json(json_file: Path, monolingual: bool, strict: bool): """ This function transforms the JSON schema used in the DSL translator into a new JSON schema that can be used to validate the multilingual YAML in your IDE. @@ -72,10 +178,15 @@ def transform_json(json_file: Path): print("The json file was not found.") raise e - result = transform(json_stream) + if not monolingual: + result = transform_IDE(json_stream) + file_name = "multilingual-schema.json" + else: + result = transform_monolingual(json_stream, strict) + file_name = "multilingual-schema.json" with open( - json_file.parent / "multilingual-schema.json", "w", encoding="utf-8" + json_file.parent / file_name, "w", encoding="utf-8" ) as f: json.dump(result, f, indent=2) @@ -83,5 +194,12 @@ def transform_json(json_file: Path): if __name__ == "__main__": n = len(sys.argv) assert n > 1, "Expected path to multilingual json schema." + monolingual = False + strict = False + if n > 2: + assert sys.argv[2] in ("strict", "not-strict") + strict = sys.argv[2] == "strict" + + monolingual = True - transform_json(Path(sys.argv[1])) + transform_json(Path(sys.argv[1]), monolingual=monolingual, strict=strict) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 2b5cfbbf..035f83c0 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,7 +18,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform, transform_json +from tested.transform_json import transform_IDE, transform_json def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -718,7 +718,7 @@ def test_return_json_schema(): } } - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -737,7 +737,7 @@ def test_yaml_value_json_schema(): } } - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -768,7 +768,7 @@ def test_nat_lang_json_schema(): }, } - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -793,7 +793,7 @@ def test_expr_json_schema(): json_schema_expected = {} - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected @@ -814,7 +814,7 @@ def test_list_json_schema(): } ] - result = transform(json_schema) + result = transform_IDE(json_schema) assert result == json_schema_expected From 4b00b0f8febfc1178ed0830b13d68a6c78572c1b Mon Sep 17 00:00:00 2001 From: breblanc Date: Sun, 13 Apr 2025 19:51:33 +0200 Subject: [PATCH 06/17] wrote some more tests --- tested/transform_json.py | 49 ++++++----- tests/test_preprocess_dsl.py | 155 ++++++++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 22 deletions(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 048295fc..30e180a9 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any + class SpecialMap(Enum): NATURAL_LANGUAGE = "natural_language" PROGRAMMING_LANGUAGE = "programming_language" @@ -12,6 +13,7 @@ class SpecialMap(Enum): EXPRESSION = "expression" NONE = "none" + def map_kind(element: dict) -> SpecialMap: if "required" in element and "properties" in element: required = element["required"] @@ -19,9 +21,13 @@ def map_kind(element: dict) -> SpecialMap: assert isinstance(required, list) assert isinstance(properties, dict) if "__tag__" in required and "value" in required: - if "__tag__" in properties and "value" in properties: + if "__tag__" in properties and "value" in properties: tag = properties["__tag__"] - if isinstance(tag, dict) and "const" in tag and isinstance(tag["const"], str): + if ( + isinstance(tag, dict) + and "const" in tag + and isinstance(tag["const"], str) + ): if tag["const"] == "!natural_language": return SpecialMap.NATURAL_LANGUAGE elif tag["const"] == "!programming_language": @@ -32,6 +38,7 @@ def map_kind(element: dict) -> SpecialMap: return SpecialMap.EXPRESSION return SpecialMap.NONE + def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: if element == prog_lang: ele_type = element.pop("type") @@ -41,6 +48,7 @@ def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: ] return element + def transform_monolingual(data: Any, strict: bool) -> Any: if isinstance(data, dict): if "oneOf" in data: @@ -65,7 +73,9 @@ def transform_monolingual(data: Any, strict: bool) -> Any: # A programming_langauge map was found. If not strict, just remove. # If strict, still provide the type option for the corresponding object. if prog_lang is not None and strict: - new_one_of = [change_prog_lang_type(ele, prog_lang) for ele in new_one_of] + new_one_of = [ + change_prog_lang_type(ele, prog_lang) for ele in new_one_of + ] if len(new_one_of) <= 1: data.pop("oneOf") @@ -83,37 +93,38 @@ def transform_monolingual(data: Any, strict: bool) -> Any: if "$ref" in data: if isinstance(data["$ref"], str): - if data["$ref"] == "#/definitions/expressionOrStatementWithNatTranslation": + if ( + data["$ref"] + == "#/definitions/expressionOrStatementWithNatTranslation" + ): data["$ref"] = "#/definitions/expressionOrStatement" if "yamlValue" in data: if strict: data["yamlValue"] = { - "description" : "A value represented as YAML.", - "not" : { - "type" : [ - "oracle", - "expression", - "programming_language" - ] - } + "description": "A value represented as YAML.", + "not": {"type": ["oracle", "expression", "programming_language"]}, } else: data["yamlValue"] = { "description": "A value represented as YAML.", } - return {k: transform_monolingual(v, strict) for k, v in data.items()} elif isinstance(data, list): - return [transformed for ele in data if - (transformed := transform_monolingual(ele, strict)) != {}] + return [ + transformed + for ele in data + if (transformed := transform_monolingual(ele, strict)) != {} + ] return data def transform_IDE(data: Any) -> Any: if isinstance(data, list): - return [transformed for ele in data if (transformed := transform_IDE(ele)) != {}] + return [ + transformed for ele in data if (transformed := transform_IDE(ele)) != {} + ] elif isinstance(data, dict): if "return" in data: # This is necessary since tags aren't recognized in the Json schema. @@ -162,7 +173,7 @@ def transform_IDE(data: Any) -> Any: return data -def transform_json(json_file: Path, monolingual: bool, strict: bool): +def transform_json(json_file: Path, monolingual: bool, strict: bool): """ This function transforms the JSON schema used in the DSL translator into a new JSON schema that can be used to validate the multilingual YAML in your IDE. @@ -185,9 +196,7 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): result = transform_monolingual(json_stream, strict) file_name = "multilingual-schema.json" - with open( - json_file.parent / file_name, "w", encoding="utf-8" - ) as f: + with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 035f83c0..ed5cf605 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -18,7 +18,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform_IDE, transform_json +from tested.transform_json import transform_IDE, transform_json, transform_monolingual def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -872,9 +872,160 @@ def test_transform_executed_correct(mocker: MockerFixture): mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - transform_json(Path("schema.json")) + transform_json(Path("schema.json"), False, False) assert s.call_count == 25 # Check if the file was opened for writing mock_opener.assert_any_call(Path("multilingual-schema.json"), "w", encoding="utf-8") + + +def test_json_rm_nat_lang_json_schema(): + json_schema = { + "oneOf": [ + {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/tab"}}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + }, + }, + }, + }, + ] + } + + json_schema_expected = { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/definitions/tab"}, + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + +def test_json_rm_prog_lang_json_schema(): + json_schema = { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language", + }, + "value": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + }, + }, + }, + ] + }, + } + + json_schema_expected = { + "expressionOrStatement": { + "oneOf": [ + { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string.", + }, + { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": {"$ref": "#/definitions/programmingLanguage"}, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim.", + }, + "anyOf": [{"type": "object"}, {"type": "programming_language"}], + }, + ] + }, + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + +def test_json_schema_edge_cases(): + json_schema = { + "expressionOrStatementWithNatTranslation": {}, + "translations": {}, + "$ref": "#/definitions/expressionOrStatementWithNatTranslation", + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "$ref": "#/definitions/expressionOrStatement", + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"type": ["oracle", "expression", "programming_language"]}, + }, + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + json_schema = { + "yamlValue": { + "description": "A value represented as YAML.", + "not": {"properties": {"__tag__": {"type": "string"}}, "type": "object"}, + }, + } + + json_schema_expected = { + "yamlValue": { + "description": "A value represented as YAML.", + } + } + + result = transform_monolingual(json_schema, False) + + assert result == json_schema_expected From 943a0affbfa362913687d1115f77f49e32e3ed9c Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 14 Apr 2025 16:13:31 +0200 Subject: [PATCH 07/17] added extra bit to the schema --- tested/dsl/schema-strict-nat-translation.json | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index adfeaa57..51f0647c 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -1310,8 +1310,33 @@ "$ref" : "#/definitions/programmingLanguage" }, "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] } }, { @@ -1334,8 +1359,33 @@ "$ref" : "#/definitions/programmingLanguage" }, "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] } } } From 431b27df12dd9ad8786e353c8434ec77c45e4f61 Mon Sep 17 00:00:00 2001 From: breblanc Date: Mon, 14 Apr 2025 16:24:53 +0200 Subject: [PATCH 08/17] fixed tests --- tests/test_preprocess_dsl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index ed5cf605..bbedfcfa 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -821,7 +821,7 @@ def test_list_json_schema(): def test_transform_executed_correct(mocker: MockerFixture): s = mocker.spy( - tested.transform_json, name="transform" # type: ignore[reportAttributeAccessIssue] + tested.transform_json, name="transform_IDE" # type: ignore[reportAttributeAccessIssue] ) mock_files = [ From 23b3dbf57e8dd516d09cb18131bb6b56a50a74b4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 15 Apr 2025 15:46:31 +0200 Subject: [PATCH 09/17] fixed linting --- tested/nat_translation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tested/nat_translation.py b/tested/nat_translation.py index 7b4ebe34..5b656ebe 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -49,7 +49,6 @@ def custom_representer(dumper, data): if "__tag__" in data: return dumper.represent_with_tag(data["__tag__"], data["value"]) - return dumper.represent_mapping( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data ) From edce398720e25f07b909f8026d0610a1a242d966 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 16:34:43 +0200 Subject: [PATCH 10/17] updated schema-strict.json by generated version --- tested/dsl/schema-strict.json | 1076 ++++++++++++++++----------------- tested/transform_json.py | 30 +- tests/test_preprocess_dsl.py | 12 +- 3 files changed, 555 insertions(+), 563 deletions(-) diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 0659aa42..66d8cc0e 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -1,293 +1,297 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/definitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } } }, - "_tabList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "_tabList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, - "_unitList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "_unitList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "type" : "string", - "description" : "The name of this tab." + "tab": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "type" : "string", - "description" : "The name of this tab." + "unit": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "_contextList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, - "_caseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "_caseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, - "_testcaseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "_testcaseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, - "_scriptList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "_scriptList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "testcases": { + "$ref": "#/definitions/_testcaseList" } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "script" : { - "$ref" : "#/definitions/_scriptList" + "script": { + "$ref": "#/definitions/_scriptList" } } }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -295,95 +299,68 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -391,83 +368,64 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "expressionOrStatement" : { - "oneOf" : [ + "expressionOrStatement": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description" : "Programming-language-specific statement or expression.", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, "anyOf": [ { "type": "object" @@ -475,108 +433,131 @@ { "type": "programming_language" } - ], - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." - } + ] } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "$ref": "#/definitions/yamlValue" }, { - "type" : "expression", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } ] }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ + "file": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ "name", "url" ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } }, - "textOutputChannel" : { - "anyOf" : [ + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/definitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "config": { + "$ref": "#/definitions/textConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ + "type": "object", + "description": "Custom oracle for text values.", + "required": [ "oracle", "file", "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "custom_check" + "oracle": { + "const": "custom_check" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } @@ -584,202 +565,202 @@ ] }, "fileOutputChannel": { - "anyOf" : [ + "anyOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ + "type": "object", + "description": "Built-in oracle for files.", + "required": [ "content", "location" ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" + "config": { + "$ref": "#/definitions/fileConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ + "type": "object", + "description": "Custom oracle for file values.", + "required": [ "oracle", "content", "location", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } } ] }, - "returnOutputChannel" : { - "oneOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/definitions/yamlValueOrPythonExpression" }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } }, { - "type" : "oracle", - "additionalProperties" : false, - "required" : [ + "type": "oracle", + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "object", - "required" : [ + "items": { + "type": "object", + "required": [ "file" ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" } } } }, - "arguments" : { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ + "programmingLanguage": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ "bash", "c", "haskell", @@ -793,96 +774,99 @@ "cpp" ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ { - "type" : "string", - "description" : "A simple message to display." + "type": "string", + "description": "A simple message to display." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { - "type" : "string", - "description" : "The message to display." + "properties": { + "description": { + "description": "The message to display.", + "type": "string" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } } }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" } } } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ + "textualType": { + "description": "Simple textual value, converted to string.", + "type": [ "string", "number", "integer", "boolean" ] }, - "yamlValue" : { - "description" : "A value represented as YAML.", - "not" : { - "type" : [ + "yamlValue": { + "description": "A value represented as YAML.", + "not": { + "type": [ "oracle", "expression", "programming_language" @@ -891,17 +875,17 @@ }, "inheritableConfigObject": { "type": "object", - "properties" : { + "properties": { "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + "$ref": "#/definitions/fileConfigurationOptions" } } } } -} +} \ No newline at end of file diff --git a/tested/transform_json.py b/tested/transform_json.py index 30e180a9..0900a39f 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -49,7 +49,7 @@ def change_prog_lang_type(element: dict, prog_lang: dict) -> dict: return element -def transform_monolingual(data: Any, strict: bool) -> Any: +def transform_monolingual(data: Any, strict_schema: bool) -> Any: if isinstance(data, dict): if "oneOf" in data: new_one_of = [] @@ -63,7 +63,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: elif kind != SpecialMap.NATURAL_LANGUAGE: if kind == SpecialMap.EXPRESSION or kind == SpecialMap.ORACLE: element = element["properties"]["value"] - if strict: + if strict_schema: if kind == SpecialMap.EXPRESSION: element["type"] = "expression" else: @@ -72,7 +72,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: # A programming_langauge map was found. If not strict, just remove. # If strict, still provide the type option for the corresponding object. - if prog_lang is not None and strict: + if prog_lang is not None and strict_schema: new_one_of = [ change_prog_lang_type(ele, prog_lang) for ele in new_one_of ] @@ -85,6 +85,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: else: data["oneOf"] = new_one_of + # THe next changes are a few edge cases. if "expressionOrStatementWithNatTranslation" in data: data.pop("expressionOrStatementWithNatTranslation") @@ -100,7 +101,7 @@ def transform_monolingual(data: Any, strict: bool) -> Any: data["$ref"] = "#/definitions/expressionOrStatement" if "yamlValue" in data: - if strict: + if strict_schema: data["yamlValue"] = { "description": "A value represented as YAML.", "not": {"type": ["oracle", "expression", "programming_language"]}, @@ -110,20 +111,20 @@ def transform_monolingual(data: Any, strict: bool) -> Any: "description": "A value represented as YAML.", } - return {k: transform_monolingual(v, strict) for k, v in data.items()} + return {k: transform_monolingual(v, strict_schema) for k, v in data.items()} elif isinstance(data, list): return [ transformed for ele in data - if (transformed := transform_monolingual(ele, strict)) != {} + if (transformed := transform_monolingual(ele, strict_schema)) != {} ] return data -def transform_IDE(data: Any) -> Any: +def transform_ide(data: Any) -> Any: if isinstance(data, list): return [ - transformed for ele in data if (transformed := transform_IDE(ele)) != {} + transformed for ele in data if (transformed := transform_ide(ele)) != {} ] elif isinstance(data, dict): if "return" in data: @@ -137,6 +138,7 @@ def transform_IDE(data: Any) -> Any: "description": "A value represented as YAML.", } + # This is a speciale structure for tags. if "required" in data and "properties" in data: required = data["required"] properties = data["properties"] @@ -169,7 +171,7 @@ def transform_IDE(data: Any) -> Any: } data = value - return {k: transform_IDE(v) for k, v in data.items()} + return {k: transform_ide(v) for k, v in data.items()} return data @@ -179,6 +181,9 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): a new JSON schema that can be used to validate the multilingual YAML in your IDE. :param json_file: The path to the JSON file. + :param monolingual: indicator if it will generate a monolingual schema. + :param strict: indicator if it will generate a strict schema. This only applies + to monolingual transforms """ _, ext = os.path.splitext(json_file) assert ext.lower() == ".json", f"expected a json file, got {ext}." @@ -190,11 +195,14 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): raise e if not monolingual: - result = transform_IDE(json_stream) + result = transform_ide(json_stream) file_name = "multilingual-schema.json" else: result = transform_monolingual(json_stream, strict) - file_name = "multilingual-schema.json" + if strict: + file_name = "schema-strict.json" + else: + file_name = "multilingual-schema.json" with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 2a4a45d6..e72cd0e1 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -19,7 +19,7 @@ translate_yaml, validate_pre_dsl, ) -from tested.transform_json import transform_IDE, transform_json, transform_monolingual +from tested.transform_json import transform_ide, transform_json, transform_monolingual def validate_natural_translate(yaml_str: str, translated_yaml_str: str): @@ -692,7 +692,7 @@ def test_return_json_schema(): } } - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -711,7 +711,7 @@ def test_yaml_value_json_schema(): } } - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -742,7 +742,7 @@ def test_nat_lang_json_schema(): }, } - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -767,7 +767,7 @@ def test_expr_json_schema(): json_schema_expected = {} - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected @@ -788,7 +788,7 @@ def test_list_json_schema(): } ] - result = transform_IDE(json_schema) + result = transform_ide(json_schema) assert result == json_schema_expected From 66acff4c625f30ccf06671732ec06bcd8773e660 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 17:07:49 +0200 Subject: [PATCH 11/17] updated schema.json by generated version --- tested/dsl/schema.json | 1068 +++++++++++++++++----------------- tested/transform_json.py | 6 +- tests/test_preprocess_dsl.py | 2 +- 3 files changed, 532 insertions(+), 544 deletions(-) diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f8d066bc..6d8943c4 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -1,293 +1,297 @@ { - "$id" : "tested:dsl:schema7", - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "TESTed-DSL", - "oneOf" : [ + "$id": "tested:dsl:schema7", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TESTed-DSL", + "oneOf": [ { - "$ref" : "#/definitions/_rootObject" + "$ref": "#/definitions/_rootObject" }, { - "$ref" : "#/definitions/_tabList" + "$ref": "#/definitions/_tabList" }, { - "$ref" : "#/definitions/_unitList" + "$ref": "#/definitions/_unitList" } ], - "definitions" : { - "_rootObject" : { - "type" : "object", - "oneOf" : [ + "definitions": { + "_rootObject": { + "type": "object", + "oneOf": [ { - "required" : [ + "required": [ "tabs" ], - "not" : { - "required" : [ + "not": { + "required": [ "units" ] } }, { - "required" : [ + "required": [ "units" ], - "not" : { - "required" : [ + "not": { + "required": [ "tabs" ] } } ], - "properties" : { - "files" : { - "description" : "A list of files used in the test suite.", - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "namespace" : { - "type" : "string", - "description" : "Namespace of the submitted solution, in `snake_case`" + "namespace": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - "tabs" : { - "$ref" : "#/definitions/_tabList" + "tabs": { + "$ref": "#/definitions/_tabList" }, - "units" : { - "$ref" : "#/definitions/_unitList" + "units": { + "$ref": "#/definitions/_unitList" }, - "language" : { - "description" : "Indicate that all code is in a specific language.", - "oneOf" : [ + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "$ref" : "#/definitions/programmingLanguage" + "$ref": "#/definitions/programmingLanguage" }, { - "const" : "tested" + "const": "tested" } ] }, - "definitions" : { - "description" : "Define hashes to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } } }, - "_tabList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/tab" + "_tabList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/tab" } }, - "_unitList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/unit" + "_unitList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/unit" } }, - "tab" : { - "type" : "object", - "description" : "A tab in the test suite.", - "required" : [ + "tab": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ "tab" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "tab" : { - "type" : "string", - "description" : "The name of this tab." + "tab": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "contexts" ], - "properties" : { - "contexts" : { - "$ref" : "#/definitions/_contextList" + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" } } }, { - "required" : [ + "required": [ "testcases" ], - "properties" : { - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "unit" : { - "type" : "object", - "description" : "A unit in the test suite.", - "required" : [ + "unit": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ "unit" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "hidden" : { - "type" : "boolean", - "description" : "Defines if the unit/tab is hidden for the student or not" + "hidden": { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" }, - "unit" : { - "type" : "string", - "description" : "The name of this tab." + "unit": { + "description": "The name of this tab.", + "type": "string" }, - "definitions" : { - "description" : "Define objects to use elsewhere.", - "type" : "object" + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, "config": { "$ref": "#/definitions/inheritableConfigObject" } }, - "oneOf" : [ + "oneOf": [ { - "required" : [ + "required": [ "cases" ], - "properties" : { - "cases" : { - "$ref" : "#/definitions/_caseList" + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" } } }, { - "required" : [ + "required": [ "scripts" ], - "properties" : { - "scripts" : { - "$ref" : "#/definitions/_scriptList" + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } ] }, - "_contextList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/context" + "_contextList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } }, - "_caseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/case" + "_caseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } }, - "_testcaseList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/testcase" + "_testcaseList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } }, - "_scriptList" : { - "type" : "array", - "minItems" : 1, - "items" : { - "$ref" : "#/definitions/script" + "_scriptList": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/script" } }, - "context" : { - "type" : "object", - "description" : "A set of testcase in the same context.", - "required" : [ + "context": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ "testcases" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "testcases" : { - "$ref" : "#/definitions/_testcaseList" + "testcases": { + "$ref": "#/definitions/_testcaseList" } } }, - "case" : { - "type" : "object", - "description" : "A test case.", - "required" : [ + "case": { + "type": "object", + "description": "A test case.", + "required": [ "script" ], - "properties" : { - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "context" : { - "type" : "string", - "description" : "Description of this context." + "context": { + "type": "string", + "description": "Description of this context." }, - "script" : { - "$ref" : "#/definitions/_scriptList" + "script": { + "$ref": "#/definitions/_scriptList" } } }, - "testcase" : { - "type" : "object", - "description" : "An individual test for a statement or expression", - "additionalProperties" : false, - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "testcase": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -295,91 +299,68 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "script" : { - "type" : "object", - "description" : "An individual test (script) for a statement or expression", - "properties" : { - "description" : { - "$ref" : "#/definitions/message" - }, - "stdin" : { - "description" : "Stdin for this context", - "type" : [ + "script": { + "type": "object", + "description": "An individual test (script) for a statement or expression", + "properties": { + "description": { + "$ref": "#/definitions/message" + }, + "stdin": { + "description": "Stdin for this context", + "type": [ "string", "number", "integer", "boolean" ] }, - "arguments" : { - "type" : "array", - "description" : "Array of program call arguments", - "items" : { - "type" : [ + "arguments": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "type": [ "string", "number", "integer", @@ -387,185 +368,189 @@ ] } }, - "statement" : { - "description" : "The statement to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "expression" : { - "description" : "The expression to evaluate.", - "$ref" : "#/definitions/expressionOrStatement" + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/definitions/expressionOrStatement" }, - "exception" : { - "description" : "Expected exception message", - "oneOf" : [ - { - "type" : "string", - "description" : "Message of the expected exception." - }, - { - "type" : "object", - "required" : [ - "types" - ], - "properties" : { - "message" : { - "type" : "string", - "description" : "Message of the expected exception." - }, - "types" : { - "minProperties" : 1, - "description" : "Language mapping of expected exception types.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" - }, - "items" : { - "type" : "string" - } - } - } - } - ] + "exception": { + "description": "Expected exception message", + "$ref": "#/definitions/exceptionChannel" }, - "files" : { - "type" : "array", - "items" : { - "$ref" : "#/definitions/file" + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/file" } }, - "return" : { - "description" : "Expected return value", - "$ref" : "#/definitions/returnOutputChannel" + "return": { + "description": "Expected return value", + "$ref": "#/definitions/returnOutputChannel" }, - "stderr" : { - "description" : "Expected output at stderr", - "$ref" : "#/definitions/textOutputChannel" + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/definitions/textOutputChannel" }, - "stdout" : { - "description" : "Expected output at stdout", - "$ref" : "#/definitions/textOutputChannel" + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/definitions/textOutputChannel" }, "file": { - "description" : "Expected files generated by the submission.", - "$ref" : "#/definitions/fileOutputChannel" + "description": "Expected files generated by the submission.", + "$ref": "#/definitions/fileOutputChannel" }, - "exit_code" : { - "type" : "integer", - "description" : "Expected exit code for the run" + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run" } } }, - "expressionOrStatement" : { - "oneOf" : [ + "expressionOrStatement": { + "oneOf": [ { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "A statement of expression in Python-like syntax as YAML string." + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." }, { - "description" : "Programming-language-specific statement or expression.", - "type" : "object", - "minProperties" : 1, - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } ] }, - "yamlValueOrPythonExpression" : { - "oneOf" : [ + "yamlValueOrPythonExpression": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValue" + "$ref": "#/definitions/yamlValue" }, { - "type" : "string", - "format" : "tested-dsl-expression", - "description" : "An expression in Python-syntax." + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." } ] }, - "file" : { - "type" : "object", - "description" : "A file used in the test suite.", - "required" : [ + "file": { + "type": "object", + "description": "A file used in the test suite.", + "required": [ "name", "url" ], - "properties" : { - "name" : { - "type" : "string", - "description" : "The filename, including the file extension." - }, - "url" : { - "type" : "string", - "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "properties": { + "name": { + "type": "string", + "description": "The filename, including the file extension." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } }, - "textOutputChannel" : { - "anyOf" : [ + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ { - "$ref" : "#/definitions/textualType" + "$ref": "#/definitions/textualType" }, { - "type" : "object", - "description" : "Built-in oracle for text values.", - "required" : [ + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/textConfigurationOptions" + "config": { + "$ref": "#/definitions/textConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for text values.", - "required" : [ + "type": "object", + "description": "Custom oracle for text values.", + "required": [ "oracle", "file", "data" ], - "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "properties": { + "data": { + "$ref": "#/definitions/textualType" }, - "oracle" : { - "const" : "custom_check" + "oracle": { + "const": "custom_check" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } @@ -573,202 +558,202 @@ ] }, "fileOutputChannel": { - "anyOf" : [ + "anyOf": [ { - "type" : "object", - "description" : "Built-in oracle for files.", - "required" : [ + "type": "object", + "description": "Built-in oracle for files.", + "required": [ "content", "location" ], - "properties" : { - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "oracle" : { - "const" : "builtin" + "oracle": { + "const": "builtin" }, - "config" : { - "$ref" : "#/definitions/fileConfigurationOptions" + "config": { + "$ref": "#/definitions/fileConfigurationOptions" } } }, { - "type" : "object", - "description" : "Custom oracle for file values.", - "required" : [ + "type": "object", + "description": "Custom oracle for file values.", + "required": [ "oracle", "content", "location", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "content" : { - "type" : "string", - "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "location" : { - "type" : "string", - "description" : "Path to where the file generated by the submission should go." + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } } ] }, - "returnOutputChannel" : { - "oneOf" : [ + "returnOutputChannel": { + "oneOf": [ { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "$ref": "#/definitions/yamlValueOrPythonExpression" }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "value" ], - "properties" : { - "oracle" : { - "const" : "builtin" + "properties": { + "oracle": { + "const": "builtin" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "value", "oracle", "file" ], - "properties" : { - "oracle" : { - "const" : "custom_check" + "properties": { + "oracle": { + "const": "custom_check" }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" }, - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, - "arguments" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "arguments": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } }, "languages": { - "type" : "array", - "description" : "Which programming languages are supported by this oracle.", - "items" : { - "$ref" : "#/definitions/programmingLanguage" + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/definitions/programmingLanguage" } } } }, { - "type" : "object", - "additionalProperties" : false, - "required" : [ + "type": "object", + "additionalProperties": false, + "required": [ "oracle", "functions" ], - "properties" : { - "oracle" : { - "const" : "specific_check" + "properties": { + "oracle": { + "const": "specific_check" }, - "functions" : { - "minProperties" : 1, - "description" : "Language mapping of oracle functions.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "object", - "required" : [ + "items": { + "type": "object", + "required": [ "file" ], - "properties" : { - "file" : { - "type" : "string", - "description" : "The path to the file containing the custom check function." + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." }, - "name" : { - "type" : "string", - "description" : "The name of the custom check function.", - "default" : "evaluate" + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" } } } }, - "arguments" : { - "minProperties" : 1, - "description" : "Language mapping of oracle arguments.", - "type" : "object", - "propertyNames" : { - "$ref" : "#/definitions/programmingLanguage" + "arguments": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/definitions/programmingLanguage" }, - "items" : { - "type" : "array", - "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", - "items" : { - "type" : "string", - "description" : "A language-specific literal, which will be used verbatim." + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } }, - "value" : { - "$ref" : "#/definitions/yamlValueOrPythonExpression" + "value": { + "$ref": "#/definitions/yamlValueOrPythonExpression" } } } ] }, - "programmingLanguage" : { - "type" : "string", - "description" : "One of the programming languages supported by TESTed.", - "enum" : [ + "programmingLanguage": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ "bash", "c", "haskell", @@ -782,108 +767,111 @@ "cpp" ] }, - "message" : { - "oneOf" : [ + "message": { + "oneOf": [ { - "type" : "string", - "description" : "A simple message to display." + "type": "string", + "description": "A simple message to display." }, { - "type" : "object", - "required" : [ + "type": "object", + "required": [ "description" ], - "properties" : { - "description" : { - "type" : "string", - "description" : "The message to display." + "properties": { + "description": { + "description": "The message to display.", + "type": "string" }, - "format" : { - "type" : "string", - "default" : "text", - "description" : "The format of the message, either a programming language, 'text' or 'html'." + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." } } } ] }, - "textConfigurationOptions" : { - "type" : "object", - "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties" : 1, - "properties" : { - "applyRounding" : { - "description" : "Apply rounding when comparing as float", - "type" : "boolean" - }, - "caseInsensitive" : { - "description" : "Ignore case when comparing strings", - "type" : "boolean" - }, - "ignoreWhitespace" : { - "description" : "Ignore trailing whitespace", - "type" : "boolean" - }, - "normalizeTrailingNewlines" : { - "description" : "Normalize trailing newlines", - "type" : "boolean" - }, - "roundTo" : { - "description" : "The number of decimals to round at, when applying the rounding on floats", - "type" : "integer" - }, - "tryFloatingPoint" : { - "description" : "Try comparing text as floating point numbers", - "type" : "boolean" - }, - "hideExpected" : { - "description" : "Hide the expected value in feedback (default: false), not recommended to use!", - "type" : "boolean" + "textConfigurationOptions": { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "description": "Apply rounding when comparing as float", + "type": "boolean" + }, + "caseInsensitive": { + "description": "Ignore case when comparing strings", + "type": "boolean" + }, + "ignoreWhitespace": { + "description": "Ignore trailing whitespace", + "type": "boolean" + }, + "normalizeTrailingNewlines": { + "description": "Normalize trailing newlines", + "type": "boolean" + }, + "roundTo": { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + }, + "tryFloatingPoint": { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + }, + "hideExpected": { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } } }, "fileConfigurationOptions": { - "anyOf" : [ + "anyOf": [ { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "mode": { - "type" : "string", - "enum" : ["full", "line"], - "default" : "full" + "type": "string", + "enum": [ + "full", + "line" + ], + "default": "full" } } } ] }, - "textualType" : { - "description" : "Simple textual value, converted to string.", - "type" : [ + "textualType": { + "description": "Simple textual value, converted to string.", + "type": [ "string", "number", "integer", "boolean" ] }, - "yamlValue" : { - "description" : "A value represented as YAML." + "yamlValue": { + "description": "A value represented as YAML." }, "inheritableConfigObject": { "type": "object", - "properties" : { + "properties": { "stdout": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "stderr": { - "$ref" : "#/definitions/textConfigurationOptions" + "$ref": "#/definitions/textConfigurationOptions" }, "file": { - "$ref" : "#/definitions/fileConfigurationOptions" + "$ref": "#/definitions/fileConfigurationOptions" } } } } -} +} \ No newline at end of file diff --git a/tested/transform_json.py b/tested/transform_json.py index 0900a39f..c47d9cd9 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -1,12 +1,12 @@ import json import os import sys -from enum import Enum +from enum import StrEnum from pathlib import Path from typing import Any -class SpecialMap(Enum): +class SpecialMap(StrEnum): NATURAL_LANGUAGE = "natural_language" PROGRAMMING_LANGUAGE = "programming_language" ORACLE = "oracle" @@ -202,7 +202,7 @@ def transform_json(json_file: Path, monolingual: bool, strict: bool): if strict: file_name = "schema-strict.json" else: - file_name = "multilingual-schema.json" + file_name = "schema.json" with open(json_file.parent / file_name, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index e72cd0e1..136748ca 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -795,7 +795,7 @@ def test_list_json_schema(): def test_transform_executed_correct(mocker: MockerFixture): s = mocker.spy( - tested.transform_json, name="transform_IDE" # type: ignore[reportAttributeAccessIssue] + tested.transform_json, name="transform_ide" # type: ignore[reportAttributeAccessIssue] ) mock_files = [ From 88b0196d45718e639a64a4636214810a0951f0a8 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 17:17:16 +0200 Subject: [PATCH 12/17] small change --- tested/transform_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tested/transform_json.py b/tested/transform_json.py index c47d9cd9..ab1e059d 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -137,6 +137,7 @@ def transform_ide(data: Any) -> Any: data["yamlValue"] = { "description": "A value represented as YAML.", } + # This is a speciale structure for tags. if "required" in data and "properties" in data: From 2b65c0e42e9fb361471c58fb60b49d7f1f8b15b4 Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 17:20:59 +0200 Subject: [PATCH 13/17] fixed linting --- tested/transform_json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index ab1e059d..c47d9cd9 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -137,7 +137,6 @@ def transform_ide(data: Any) -> Any: data["yamlValue"] = { "description": "A value represented as YAML.", } - # This is a speciale structure for tags. if "required" in data and "properties" in data: From 6149de9178fac39c8757778dfee108fbcf470e0b Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 18:17:04 +0200 Subject: [PATCH 14/17] Wrote a few more tests --- tests/test_preprocess_dsl.py | 110 +++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 136748ca..588ae864 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -964,6 +964,116 @@ def test_json_rm_prog_lang_json_schema(): assert result == json_schema_expected +def test_strict_json_schema_oracle(): + json_schema = { + "oneOf": [ + {"$ref": "#/definitions/yamlValueOrPythonExpression"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle", + }, + "value": { + "type": "object", + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": { + "oneOf": [ + { + "$ref": "#/definitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language", + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/yamlValueOrPythonExpression" + }, + }, + }, + }, + ] + }, + }, + }, + }, + }, + ] + } + + json_schema_expected = { + "oneOf": [ + {"$ref": "#/definitions/yamlValueOrPythonExpression"}, + { + "type": "oracle", + "additionalProperties": False, + "required": ["value"], + "properties": { + "oracle": {"const": "builtin"}, + "value": {"$ref": "#/definitions/yamlValueOrPythonExpression"}, + }, + }, + ] + } + + result = transform_monolingual(json_schema, True) + + assert result == json_schema_expected + + +def test_strict_json_schema_expression(): + json_schema = { + "oneOf": [ + {"$ref": "#/definitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!expression", + }, + "value": { + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + }, + }, + ] + } + + json_schema_expected = { + "oneOf": [ + {"$ref": "#/definitions/yamlValue"}, + { + "type": "expression", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax.", + }, + ] + } + + result = transform_monolingual(json_schema, True) + print(result) + + assert result == json_schema_expected + + def test_json_schema_edge_cases(): json_schema = { "expressionOrStatementWithNatTranslation": {}, From 1364569eb7a66122cdf0ee392991b84b5ed3fb31 Mon Sep 17 00:00:00 2001 From: Brent Blanckaert <125136419+BrentBlanckaert@users.noreply.github.com> Date: Tue, 29 Apr 2025 18:21:21 +0200 Subject: [PATCH 15/17] Update tested/transform_json.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tested/transform_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index c47d9cd9..29945bf2 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -70,7 +70,7 @@ def transform_monolingual(data: Any, strict_schema: bool) -> Any: element["type"] = "oracle" new_one_of.append(element) - # A programming_langauge map was found. If not strict, just remove. + # A programming_language map was found. If not strict, just remove. # If strict, still provide the type option for the corresponding object. if prog_lang is not None and strict_schema: new_one_of = [ From 60c215c2cf1b187d5ccbf7d55f165b4f386df297 Mon Sep 17 00:00:00 2001 From: Brent Blanckaert <125136419+BrentBlanckaert@users.noreply.github.com> Date: Tue, 29 Apr 2025 18:21:57 +0200 Subject: [PATCH 16/17] Update tested/transform_json.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tested/transform_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tested/transform_json.py b/tested/transform_json.py index 29945bf2..6d35c2af 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -85,7 +85,7 @@ def transform_monolingual(data: Any, strict_schema: bool) -> Any: else: data["oneOf"] = new_one_of - # THe next changes are a few edge cases. + # The next changes are a few edge cases. if "expressionOrStatementWithNatTranslation" in data: data.pop("expressionOrStatementWithNatTranslation") From 6f047c66ac23f9371b0b91ca995a8b45706d1a6d Mon Sep 17 00:00:00 2001 From: breblanc Date: Tue, 29 Apr 2025 19:00:59 +0200 Subject: [PATCH 17/17] added an extra test --- tests/test_preprocess_dsl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 588ae864..209b66ae 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -1113,3 +1113,13 @@ def test_json_schema_edge_cases(): result = transform_monolingual(json_schema, False) assert result == json_schema_expected + + +def test_exception_when_file_not_found(): + + try: + transform_json(Path("test.json"), False, False) + except FileNotFoundError: + print("As expected") + else: + assert False, "Expected FileNotFoundError error"