add a dumb timeout logic

doc/adapters.md

@@ -32,6 +32,11 @@ The adapter must return exit code to the environment that was passed as an argum
 and check if the exit code is equal to `13`. There are also two test cases in Assembly Script test suite that verify the behavior:
 * [proc_exit-failure](../tests/assemblyscript/testsuite/proc_exit-failure.ts)
 * [proc_exit-success](../tests/assemblyscript/testsuite/proc_exit-success.ts)
+
+On a timeout, the test runner sends SIGTERM singal to the adapter process.
+When receiving the signal, the adapter process should clean up and exit
+as soon as possible.
+
 ### Examples:
 
 Print runtime version:
@@ -61,4 +66,4 @@ See the [`adapters`](../adapters) directory for example adapters.
 
 We prefer runtime maintainers to maintain adapters in their own repository We'll only maintain adapters for [Bytecode Alliance](https://bytecodealliance.org/) projects and we'll aim for compatibility with the most recent stable version.
 
-We'll accept pull requests for new adapters in this repository, but we can't guarantee we'll have the capacity to maintain them (so they might stop working with the new runtime release).
+We'll accept pull requests for new adapters in this repository, but we can't guarantee we'll have the capacity to maintain them (so they might stop working with the new runtime release).

test-runner/tests/test_test_case.py

@@ -42,12 +42,12 @@ def test_test_config_should_warn_when_unknown_field(_mock_file: Mock) -> None:
 
 
 def test_test_results_should_mark_failed_if_multiple_failures() -> None:
-    results = Result(Output(0, "", ""), True, [Failure("type", "message")])
+    results = Result(Output(0, "", ""), [Failure("type", "message")])
 
     assert results.failed is True
 
 
 def test_test_results_should_not_mark_failed_if_no_failure() -> None:
-    results = Result(Output(0, "", ""), True, [])
+    results = Result(Output(0, "", ""), [])
 
     assert results.failed is False

test-runner/tests/test_test_suite.py

@@ -9,7 +9,7 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa
     return tc.TestCase(
         name,
         tc.Config(),
-        tc.Result(tc.Output(0, "", ""), is_executed, failures),
+        tc.Result(tc.Output(0, "", ""), failures) if is_executed else tc.SkippedResult(),
         1.0,
     )
 

test-runner/tests/test_test_suite_runner.py

@@ -35,9 +35,9 @@ def test_runner_end_to_end() -> None:
         tc.Output(2, "test3", ""),
     ]
     expected_results = [
-        tc.Result(outputs[0], True, []),
-        tc.Result(outputs[1], True, [failures[1]]),
-        tc.Result(outputs[2], True, [failures[0], failures[2]]),
+        tc.Result(outputs[0], []),
+        tc.Result(outputs[1], [failures[1]]),
+        tc.Result(outputs[2], [failures[0], failures[2]]),
     ]
     expected_config = [
         tc.Config(dirs=[".", "deep/dir"]),

test-runner/wasi_test_runner/reporters/console.py

@@ -2,7 +2,7 @@
 from colorama import Fore, init
 
 from . import TestReporter
-from ..test_case import TestCase
+from ..test_case import TestCase, SkippedResult, TimedoutResult
 from ..test_suite import TestSuite
 from ..runtime_adapter import RuntimeVersion
 
@@ -20,18 +20,21 @@ def __init__(self, colored: bool = True) -> None:
         self._colored = colored
 
     def report_test(self, test: TestCase) -> None:
-        if test.result.failed:
-            self._print_fail(f"Test {test.name} failed")
-            for reason in test.result.failures:
-                self._print_fail(f"  [{reason.type}] {reason.message}")
-            print("STDOUT:")
-            print(test.result.output.stdout)
-            print("STDERR:")
-            print(test.result.output.stderr)
-        elif test.result.is_executed:
-            self._print_pass(f"Test {test.name} passed")
-        else:
+        if isinstance(test.result, TimedoutResult):
+            self._print_fail(f"Test {test.name} timed out")
+        elif isinstance(test.result, SkippedResult):
             self._print_skip(f"Test {test.name} skipped")
+        else:
+            if test.result.failed:
+                self._print_fail(f"Test {test.name} failed")
+                for reason in test.result.failures:
+                    self._print_fail(f"  [{reason.type}] {reason.message}")
+                print("STDOUT:")
+                print(test.result.output.stdout)
+                print("STDERR:")
+                print(test.result.output.stderr)
+            else:
+                self._print_pass(f"Test {test.name} passed")
 
     def report_test_suite(self, test_suite: TestSuite) -> None:
         self._test_suites.append(test_suite)
@@ -41,29 +44,31 @@ def finalize(self, version: RuntimeVersion) -> None:
         print("===== Test results =====")
         print(f"Runtime: {version.name} {version.version}")
 
-        total_skip = total_pass = total_fail = pass_suite = 0
+        total_skip = total_pass = total_fail = total_timedout = pass_suite = 0
 
         for suite in self._test_suites:
             total_pass += suite.pass_count
             total_fail += suite.fail_count
             total_skip += suite.skip_count
+            total_timedout += suite.timedout_count
 
-            if suite.fail_count == 0:
+            if suite.fail_count == 0 and suite.timedout_count == 0:
                 pass_suite += 1
 
             print(f"Suite: {suite.name}")
             print(f"  Total: {suite.test_count}")
-            self._print_pass(f"  Passed:  {suite.pass_count}")
-            self._print_fail(f"  Failed:  {suite.fail_count}")
-            self._print_skip(f"  Skipped: {suite.skip_count}")
+            self._print_pass(f"  Passed:    {suite.pass_count}")
+            self._print_fail(f"  Failed:    {suite.fail_count}")
+            self._print_skip(f"  Skipped:   {suite.skip_count}")
+            self._print_fail(f"  Timed out: {suite.timedout_count}")
             print("")
 
         print(
-            f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0)}"
+            f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0, 0)}"
         )
-        print(f"Tests:       {self._get_summary(total_fail, total_pass, total_skip)}")
+        print(f"Tests:       {self._get_summary(total_fail, total_pass, total_skip, total_timedout)}")
 
-    def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str:
+    def _get_summary(self, fail_count: int, pass_count: int, skip_count: int, timedout_count: int) -> str:
         items: List[str] = []
 
         if fail_count:
@@ -72,8 +77,10 @@ def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str
             items.append(f"{self._pass_color}{pass_count} passed")
         if skip_count:
             items.append(f"{self._skip_color}{skip_count} skipped")
+        if timedout_count:
+            items.append(f"{self._fail_color}{timedout_count} timed out")
 
-        total = fail_count + pass_count + skip_count
+        total = fail_count + pass_count + skip_count + timedout_count
         items.append(f"{self._reset_color}{total} total")
         return ", ".join(items)
 

test-runner/wasi_test_runner/reporters/json.py

@@ -5,6 +5,7 @@
 
 from . import TestReporter
 from ..test_suite import TestSuite
+from ..test_case import Result, SkippedResult, TimedoutResult
 from ..runtime_adapter import RuntimeVersion
 
 
@@ -28,16 +29,19 @@ def finalize(self, version: RuntimeVersion) -> None:
                     "duration_s": suite.duration_s,
                     "failed": suite.fail_count,
                     "skipped": suite.skip_count,
+                    "timedout": suite.timedout_count,
                     "passed": suite.pass_count,
                     "tests": [
                         {
                             "name": test.name,
-                            "executed": test.result.is_executed,
+                            "executed": isinstance(test.result, Result),
+                            "skipped": isinstance(test.result, SkippedResult),
+                            "timedout": isinstance(test.result, TimedoutResult),
                             "duration_s": test.duration_s,
                             "wasi_functions": test.config.wasi_functions,
                             "failures": [
                                 failure.message for failure in test.result.failures
-                            ],
+                            ] if isinstance(test.result, Result) else [],
                         }
                         for test in suite.test_cases
                     ],

test-runner/wasi_test_runner/runtime_adapter.py

@@ -17,7 +17,9 @@ def __init__(self, adapter_path: str) -> None:
 
     def get_version(self) -> RuntimeVersion:
         output = (
-            subprocess.check_output([sys.executable, self._adapter_path, "--version"], encoding="UTF-8")
+            subprocess.check_output(
+                [sys.executable, self._adapter_path, "--version"], encoding="UTF-8"
+            )
             .strip()
             .split(" ")
         )
@@ -42,14 +44,20 @@ def run_test(
             + [e for env in self._env_to_list(env_variables) for e in ("--env", env)]
         )
 
-        result = subprocess.run(
+        with subprocess.Popen(
             args,
-            capture_output=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
             text=True,
-            check=False,
             cwd=Path(test_path).parent,
-        )
-        return Output(result.returncode, result.stdout, result.stderr)
+        ) as proc:
+            try:
+                out, err = proc.communicate(timeout=3)
+            except subprocess.TimeoutExpired:
+                proc.terminate()
+                proc.wait()
+                raise
+            return Output(proc.returncode, out, err)
 
     @staticmethod
     def _abs(path: str) -> str:

test-runner/wasi_test_runner/test_case.py

@@ -1,6 +1,6 @@
 import logging
 import json
-from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional
+from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional, Union
 
 
 class Output(NamedTuple):
@@ -16,14 +16,21 @@ class Failure(NamedTuple):
 
 class Result(NamedTuple):
     output: Output
-    is_executed: bool
     failures: List[Failure]
 
     @property
     def failed(self) -> bool:
         return len(self.failures) > 0
 
 
+class TimedoutResult(NamedTuple):
+    pass
+
+
+class SkippedResult(NamedTuple):
+    pass
+
+
 T = TypeVar("T", bound="Config")
 
 
@@ -63,5 +70,5 @@ def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
 class TestCase(NamedTuple):
     name: str
     config: Config
-    result: Result
+    result: Union[Result | SkippedResult | TimedoutResult]
     duration_s: float

test-runner/wasi_test_runner/test_suite.py

@@ -1,6 +1,6 @@
 from typing import NamedTuple, List
 from datetime import datetime
-from .test_case import TestCase
+from .test_case import TestCase, Result, SkippedResult, TimedoutResult
 
 
 class TestSuite(NamedTuple):
@@ -19,7 +19,7 @@ def pass_count(self) -> int:
             [
                 1
                 for test in self.test_cases
-                if test.result.is_executed and test.result.failed is False
+                if isinstance(test.result, Result) and test.result.failed is False
             ]
         )
 
@@ -29,10 +29,14 @@ def fail_count(self) -> int:
             [
                 1
                 for test in self.test_cases
-                if test.result.is_executed and test.result.failed
+                if isinstance(test.result, Result) and test.result.failed
             ]
         )
 
     @property
     def skip_count(self) -> int:
-        return len([1 for test in self.test_cases if not test.result.is_executed])
+        return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)])
+
+    @property
+    def timedout_count(self) -> int:
+        return len([1 for test in self.test_cases if isinstance(test.result, TimedoutResult)])

test-runner/wasi_test_runner/test_suite_runner.py

@@ -3,15 +3,18 @@
 import os
 import re
 import shutil
+import subprocess
 import time
 
 from datetime import datetime
-from typing import List, cast
+from typing import List, cast, Union
 
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
     Result,
+    SkippedResult,
+    TimedoutResult,
     Config,
     Output,
     TestCase,
@@ -67,7 +70,7 @@ def _skip_single_test(
     return TestCase(
         name=os.path.splitext(os.path.basename(test_path))[0],
         config=config,
-        result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
+        result=SkippedResult(),
         duration_s=0,
     )
 
@@ -77,13 +80,17 @@ def _execute_single_test(
 ) -> TestCase:
     config = _read_test_config(test_path)
     test_start = time.time()
-    test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
+    try:
+        test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
+        result: Union[Result | TimedoutResult] = _validate(validators, config, test_output)
+    except subprocess.TimeoutExpired:
+        result = TimedoutResult()
     elapsed = time.time() - test_start
 
     return TestCase(
         name=os.path.splitext(os.path.basename(test_path))[0],
         config=config,
-        result=_validate(validators, config, test_output),
+        result=result,
         duration_s=elapsed,
     )
 
@@ -95,7 +102,7 @@ def _validate(validators: List[Validator], config: Config, output: Output) -> Re
         if result is not None
     ]
 
-    return Result(failures=failures, is_executed=True, output=output)
+    return Result(failures=failures, output=output)
 
 
 def _read_test_config(test_path: str) -> Config: