Report a non-zero exit code if a test file has no tests

src/command_test.cc

@@ -107,6 +107,7 @@ auto parse_test_suite(const sourcemeta::jsonschema::InputJSON &entry,
 
 auto report_as_text(const sourcemeta::core::Options &options) -> void {
   bool result{true};
+  bool empty_test_suite{false};
   const auto verbose{options.contains("verbose") || options.contains("debug")};
 
   for (const auto &entry : sourcemeta::jsonschema::for_each_json(options)) {
@@ -202,6 +203,7 @@ auto report_as_text(const sourcemeta::core::Options &options) -> void {
     }
 
     if (suite_result.total == 0) {
+      empty_test_suite = true;
       std::cout << " NO TESTS\n";
     } else if (!verbose && suite_result.passed == suite_result.total) {
       std::cout << " PASS " << suite_result.passed << "/" << suite_result.total
@@ -213,6 +215,13 @@ auto report_as_text(const sourcemeta::core::Options &options) -> void {
     throw sourcemeta::jsonschema::Fail{
         sourcemeta::jsonschema::EXIT_EXPECTED_FAILURE};
   }
+
+  // An empty test suite likely means the author forgot to write the tests,
+  // so don't let it silently succeed
+  if (empty_test_suite) {
+    throw sourcemeta::jsonschema::Fail{
+        sourcemeta::jsonschema::EXIT_OTHER_INPUT_ERROR};
+  }
 }
 
 auto timestamp_to_unix_ms(
@@ -234,6 +243,7 @@ auto duration_ms(const sourcemeta::blaze::TestTimestamp &start,
 
 auto report_as_ctrf(const sourcemeta::core::Options &options) -> void {
   bool result{true};
+  bool empty_test_suite{false};
 
   const auto system_ref{std::chrono::system_clock::now()};
   const auto steady_ref{std::chrono::steady_clock::now()};
@@ -333,6 +343,10 @@ auto report_as_ctrf(const sourcemeta::core::Options &options) -> void {
     total_passed += suite_result.passed;
     total_failed += suite_result.total - suite_result.passed;
 
+    if (suite_result.total == 0) {
+      empty_test_suite = true;
+    }
+
     if (suite_result.passed != suite_result.total) {
       result = false;
     }
@@ -378,6 +392,13 @@ auto report_as_ctrf(const sourcemeta::core::Options &options) -> void {
     throw sourcemeta::jsonschema::Fail{
         sourcemeta::jsonschema::EXIT_EXPECTED_FAILURE};
   }
+
+  // An empty test suite likely means the author forgot to write the tests,
+  // so don't let it silently succeed
+  if (empty_test_suite) {
+    throw sourcemeta::jsonschema::Fail{
+        sourcemeta::jsonschema::EXIT_OTHER_INPUT_ERROR};
+  }
 }
 
 } // namespace

test/CMakeLists.txt

@@ -422,8 +422,8 @@ add_jsonschema_test_unix(test/fail_test_case_non_string_data_path)
 add_jsonschema_test_unix(test/fail_test_case_no_valid)
 add_jsonschema_test_unix(test/fail_test_case_non_boolean_valid)
 add_jsonschema_test_unix(test/fail_true_resolve_fragment)
-add_jsonschema_test_unix(test/pass_empty)
-add_jsonschema_test_unix(test/pass_empty_verbose)
+add_jsonschema_test_unix(test/fail_tests_empty)
+add_jsonschema_test_unix(test/fail_tests_empty_verbose)
 add_jsonschema_test_unix(test/pass_single_yaml)
 add_jsonschema_test_unix(test/pass_single_resolve)
 add_jsonschema_test_unix(test/pass_single_resolve_draft3_https)

test/test/fail_tests_empty.sh

@@ -0,0 +1,79 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "id": "https://example.com",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "string"
+}
+EOF
+
+cat << 'EOF' > "$TMP/test.json"
+{
+  "target": "https://example.com",
+  "tests": []
+}
+EOF
+
+# An empty `tests` array almost always means the author forgot to write the
+# tests. We still print NO TESTS, but exit with an error so that such test
+# suites cannot silently pass, i.e. on CI
+"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" 1> "$TMP/output.txt" 2>&1 \
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+# Other input error
+test "$EXIT_CODE" = "6"
+
+cat << EOF > "$TMP/expected.txt"
+$(realpath "$TMP")/test.json: NO TESTS
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"
+
+"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --json 1> "$TMP/output.json" 2>&1 \
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+# Other input error
+test "$EXIT_CODE" = "6"
+
+# Validate against CTRF schema
+CTRF_SCHEMA="$(dirname "$0")/../../vendor/ctrf/specification/schema-0.0.0.json"
+"$1" validate "$CTRF_SCHEMA" "$TMP/output.json"
+
+# Remove dynamic fields for comparison
+sed -e '/"start":/d' \
+    -e '/"stop":/d' \
+    "$TMP/output.json" > "$TMP/output_filtered.json"
+
+VERSION=$("$1" --version)
+
+cat << EOF > "$TMP/expected.json"
+{
+  "reportFormat": "CTRF",
+  "specVersion": "0.0.0",
+  "results": {
+    "tool": {
+      "name": "jsonschema",
+      "version": "$VERSION"
+    },
+    "summary": {
+      "tests": 0,
+      "passed": 0,
+      "failed": 0,
+      "pending": 0,
+      "skipped": 0,
+      "other": 0,
+    },
+    "tests": []
+  }
+}
+EOF
+
+diff "$TMP/output_filtered.json" "$TMP/expected.json"

test/test/pass_empty_verbose.sh → test/test/fail_tests_empty_verbose.sh

@@ -24,7 +24,13 @@ cat << 'EOF' > "$TMP/test.json"
 }
 EOF
 
-"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --verbose 1> "$TMP/output.txt" 2>&1
+# An empty `tests` array almost always means the author forgot to write the
+# tests. We still print NO TESTS, but exit with an error so that such test
+# suites cannot silently pass, i.e. on CI
+"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --verbose 1> "$TMP/output.txt" 2>&1 \
+  && EXIT_CODE="$?" || EXIT_CODE="$?"
+# Other input error
+test "$EXIT_CODE" = "6"
 
 cat << EOF > "$TMP/expected.txt"
 $(realpath "$TMP")/test.json: NO TESTS

test/test/pass_config_path.sh

@@ -23,7 +23,13 @@ EOF
 cat << 'EOF' > "$TMP/foo/test.json"
 {
   "target": "https://example.com",
-  "tests": []
+  "tests": [
+    {
+      "description": "First test",
+      "valid": true,
+      "data": "foo"
+    }
+  ]
 }
 EOF
 
@@ -40,7 +46,8 @@ cat << EOF > "$TMP/expected.txt"
 Using extension: .json
 Using extension: .yaml
 Using extension: .yml
-$(realpath "$TMP")/foo/test.json: NO TESTS
+$(realpath "$TMP")/foo/test.json:
+  1/1 PASS First test
 EOF
 
 diff "$TMP/output.txt" "$TMP/expected.txt"