Fix cstest_py and add negative tests.

.github/workflows/CITest.yml

@@ -147,6 +147,16 @@ jobs:
       run: |
         sh suite/run_invalid_cstool.sh
 
+    - name: cstest negatives
+      if: startsWith(matrix.config.build-system, 'cmake')
+      run: |
+        ctest --test-dir build --output-on-failure -R NegativeTests1
+        ctest --test-dir build --output-on-failure -R NegativeTests2
+        ctest --test-dir build --output-on-failure -R NegativeTests3
+        ctest --test-dir build --output-on-failure -R NegativeTests4
+        ctest --test-dir build --output-on-failure -R NegativeTests5
+        ctest --test-dir build --output-on-failure -R NegativeTests6
+
     - name: cstest MC
       if: startsWith(matrix.config.build-system, 'cmake')
       run: |
@@ -177,9 +187,12 @@ jobs:
       run: |
         sudo apt-get -y update
         sudo apt-get -y install valgrind
-        valgrind cstest tests
+        valgrind --leak-check=full cstest tests/details
+        valgrind --leak-check=full cstest tests/features
+        valgrind --leak-check=full cstest tests/issues
+        valgrind --leak-check=full cstest tests/MC
 
-    - name: Comaptibility header generation
+    - name: Compatibility header generation
       # clang-format-17 is only available in Ubuntu 24.04
       # Hence the check only happens for it.
       if: startsWith(matrix.config.build-system, 'cmake') && startsWith(matrix.config.os, 'ubuntu-24.04')

bindings/python/cstest_py/src/cstest_py/cstest.py

@@ -137,6 +137,7 @@ def print_evaluate(self):
             log.error(
                 "Inconsistent statistics: total != successful + failed + skipped\n"
             )
+            exit(-1)
 
         if self.errors != 0:
             log.error("Failed with errors\n")
@@ -236,28 +237,41 @@ def compare(self, actual_insns: list[CsInsn], bits: int) -> TestResult:
             return TestResult.FAILED
 
         for a_insn, e_insn in zip(actual_insns, self.insns):
-            def _check(res: bool):
-                if not res:
-                    log.error(f"Failed instruction: {a_insn}")
-                    return TestResult.FAILED
-
-            _check(compare_asm_text(a_insn, e_insn.get("asm_text"), bits))
+            if not compare_asm_text(a_insn, e_insn.get("asm_text"), bits):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_str(a_insn.mnemonic, e_insn.get("mnemonic"), "mnemonic"))
+            if not compare_str(a_insn.mnemonic, e_insn.get("mnemonic"), "mnemonic"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_str(a_insn.op_str, e_insn.get("op_str"), "op_str"))
+            if not compare_str(a_insn.op_str, e_insn.get("op_str"), "op_str"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_enum(a_insn.id, e_insn.get("id"), "id"))
+            if not compare_enum(a_insn.id, e_insn.get("id"), "id"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_tbool(a_insn.is_alias, e_insn.get("is_alias"), "is_alias"))
+            if not compare_tbool(a_insn.is_alias, e_insn.get("is_alias"), "is_alias"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_tbool(a_insn.illegal, e_insn.get("illegal"), "illegal"))
+            if not compare_tbool(a_insn.illegal, e_insn.get("illegal"), "illegal"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_uint32(a_insn.size, e_insn.get("size"), "size"))
+            if not compare_uint32(a_insn.size, e_insn.get("size"), "size"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_enum(a_insn.alias_id, e_insn.get("alias_id"), "alias_id"))
+            if not compare_enum(a_insn.alias_id, e_insn.get("alias_id"), "alias_id"):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
-            _check(compare_details(a_insn, e_insn.get("details")))
+            if not compare_details(a_insn, e_insn.get("details")):
+                log.error(f"Failed instruction: {a_insn}")
+                return TestResult.FAILED
 
         return TestResult.SUCCESS
 
@@ -401,7 +415,6 @@ def run_tests(self):
                     self.stats.add_failing_file(tf.path)
                 self.stats.add_test_case_data_point(result)
                 log.info(result)
-                print()
         self.stats.print_evaluate()
 
 

suite/cstest/CMakeLists.txt

@@ -88,6 +88,40 @@ add_test(FeaturesTests
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
 )
 
+# These tests must be run independently. Otherwise we won't detect when one succeeds.
+# Because cstest will always return not 0 from the first failing test onwards.
+# But we need to ensure all of them fail always.
+add_test(NegativeTests1
+    cstest ${PROJECT_SOURCE_DIR}/tests/negative/should_fail_I.yaml
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+set_property(TEST NegativeTests1 PROPERTY WILL_FAIL TRUE)
+add_test(NegativeTests2
+    cstest ${PROJECT_SOURCE_DIR}/tests/negative/should_fail_II.yaml
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+set_property(TEST NegativeTests2 PROPERTY WILL_FAIL TRUE)
+add_test(NegativeTests3
+    cstest ${PROJECT_SOURCE_DIR}/tests/negative/should_fail_III.yaml
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+set_property(TEST NegativeTests3 PROPERTY WILL_FAIL TRUE)
+add_test(NegativeTests4
+    cstest ${PROJECT_SOURCE_DIR}/tests/negative/should_fail_IV.yaml
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+set_property(TEST NegativeTests4 PROPERTY WILL_FAIL TRUE)
+add_test(NegativeTests5
+    cstest ${PROJECT_SOURCE_DIR}/tests/negative/should_fail_V.yaml
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+set_property(TEST NegativeTests5 PROPERTY WILL_FAIL TRUE)
+add_test(NegativeTests6
+    cstest ${PROJECT_SOURCE_DIR}/tests/negative/should_fail_VI.yaml
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+set_property(TEST NegativeTests6 PROPERTY WILL_FAIL TRUE)
+
 
 if(CAPSTONE_INSTALL)
     install(TARGETS cstest EXPORT capstone-targets DESTINATION ${CMAKE_INSTALL_BINDIR})

suite/cstest/src/cstest.c

@@ -74,6 +74,20 @@ void print_test_run_stats(const TestRunStats *stats)
 	printf("\n");
 }
 
+void cleanup_test_files()
+{
+	if (!test_files) {
+		return;
+	}
+	for (size_t k = 0; k < file_count; ++k) {
+		cs_mem_free(test_files[0][k]);
+	}
+	if (test_files[0]) {
+		cs_mem_free(test_files[0]);
+	}
+	cs_mem_free(test_files);
+}
+
 int main(int argc, const char **argv)
 {
 	if (argc < 2 || strcmp(argv[1], "-h") == 0 ||
@@ -87,6 +101,7 @@ int main(int argc, const char **argv)
 	get_tfiles(argc, argv);
 	if (!*test_files || file_count == 0) {
 		fprintf(stderr, "Arguments are invalid. No files found.\n");
+		cleanup_test_files();
 		exit(EXIT_FAILURE);
 	}
 
@@ -95,6 +110,7 @@ int main(int argc, const char **argv)
 	TestRunResult res = cstest_run_tests(*test_files, file_count, &stats);
 
 	print_test_run_stats(&stats);
+	cleanup_test_files();
 	if (res == TEST_RUN_ERROR) {
 		fprintf(stderr, "[!] An error occured.\n");
 		exit(EXIT_FAILURE);

suite/cstest/src/test_case.c

@@ -248,6 +248,7 @@ void test_expected_compare(csh *handle, TestExpected *expected, cs_insn *insns,
 
 #define CS_TEST_FAIL(msg) \
 	_print_insn(handle, &insns[i]); \
+	cs_free(insns, insns_count); \
 	fail_msg(msg);
 
 		if (!compare_asm_text(asm_text, expec_data->asm_text,

suite/run_tests.py

@@ -18,8 +18,22 @@
     f"cstest_py {root_dir}/tests/features/",
 ]
 
+expected_to_fail = [
+    f"cstest_py {root_dir}/tests/negative/",
+]
+
 for test in tests:
     logger.info(f'Running {test}')
     logger.info("#######################")
     subprocess.run(test.split(" "), check=True)
     logger.info("-----------------------")
+
+for test in expected_to_fail:
+    logger.info(f'Running {test}')
+    logger.info("#######################")
+    try:
+        subprocess.run(test.split(" "), check=True)
+        logger.error("Test was expected to fail.")
+        exit(-1)
+    except Exception as e:
+        logger.info("-----------------------")

tests/negative/should_fail_I.yaml

@@ -0,0 +1,21 @@
+test_cases:
+  -
+    input:
+      name: "Fail because of too few input bytes."
+      bytes: [ 0x00 ]
+      arch: "sparc"
+      options: [ CS_OPT_DETAIL, CS_MODE_BIG_ENDIAN ]
+      address: 0x1000
+    expected:
+      insns:
+      -
+        asm_text: "cmp %g1, %g2"
+        details:
+          sparc:
+            operands:
+              -
+                type: SPARC_OP_REG
+                reg: g1
+              -
+                type: SPARC_OP_REG
+                reg: g2

tests/negative/should_fail_II.yaml

@@ -0,0 +1,21 @@
+test_cases:
+  -
+    input:
+      name: "Fails due to mismatching asm text"
+      bytes: [ 0x80, 0xa0, 0x40, 0x02 ]
+      arch: "sparc"
+      options: [ CS_OPT_DETAIL, CS_MODE_BIG_ENDIAN ]
+      address: 0x1000
+    expected:
+      insns:
+      -
+        asm_text: "cmp %g1, %g3"
+        details:
+          sparc:
+            operands:
+              -
+                type: SPARC_OP_REG
+                reg: g1
+              -
+                type: SPARC_OP_REG
+                reg: g2

tests/negative/should_fail_III.yaml

@@ -0,0 +1,18 @@
+test_cases:
+  -
+    input:
+      name: "Fails due to missing detail operand"
+      bytes: [ 0x80, 0xa0, 0x40, 0x02 ]
+      arch: "sparc"
+      options: [ CS_OPT_DETAIL, CS_MODE_BIG_ENDIAN ]
+      address: 0x1000
+    expected:
+      insns:
+      -
+        asm_text: "cmp %g1, %g2"
+        details:
+          sparc:
+            operands:
+              -
+                type: SPARC_OP_REG
+                reg: g1

tests/negative/should_fail_IV.yaml

@@ -0,0 +1,21 @@
+test_cases:
+  -
+    input:
+      name: "Fails due to non existent architecture name."
+      bytes: [ 0x00, 0xa0, 0x40, 0x02 ]
+      arch: "not_existant"
+      options: [ CS_OPT_DETAIL, CS_MODE_BIG_ENDIAN ]
+      address: 0x1000
+    expected:
+      insns:
+      -
+        asm_text: "cmp %g1, %g2"
+        details:
+          sparc:
+            operands:
+              -
+                type: SPARC_OP_REG
+                reg: g1
+              -
+                type: SPARC_OP_REG
+                reg: g2

tests/negative/should_fail_V.yaml

@@ -0,0 +1,21 @@
+test_cases:
+  -
+    input:
+      name: "Fails due to not decoded details."
+      bytes: [ 0x80, 0xa0, 0x40, 0x02 ]
+      arch: "sparc"
+      options: [ CS_MODE_BIG_ENDIAN ]
+      address: 0x1000
+    expected:
+      insns:
+      -
+        asm_text: "cmp %g1, %g2"
+        details:
+          sparc:
+            operands:
+              -
+                type: SPARC_OP_REG
+                reg: g1
+              -
+                type: SPARC_OP_REG
+                reg: g2

tests/negative/should_fail_VI.yaml

@@ -0,0 +1,21 @@
+test_cases:
+  -
+    input
+      name: "Fails due to malformatted file"
+      bytes: [ 0x00 ]
+      arch: "sparc"
+      options: [ CS_OPT_DETAIL, CS_MODE_BIG_ENDIAN ]
+      address: 0x1000
+    expected:
+      insns:
+      -
+        asm_text: "cmp %g1, %g2"
+        details:
+          sparc:
+            operands:
+              -
+                type: SPARC_OP_REG
+                reg: g1
+              -
+                type: SPARC_OP_REG
+                reg: g2