diff --git a/doc/adapters.md b/doc/adapters.md index 84010f29..a1cb25be 100644 --- a/doc/adapters.md +++ b/doc/adapters.md @@ -32,6 +32,11 @@ The adapter must return exit code to the environment that was passed as an argum and check if the exit code is equal to `13`. There are also two test cases in Assembly Script test suite that verify the behavior: * [proc_exit-failure](../tests/assemblyscript/testsuite/proc_exit-failure.ts) * [proc_exit-success](../tests/assemblyscript/testsuite/proc_exit-success.ts) + +On a timeout, the test runner sends SIGTERM singal to the adapter process. +When receiving the signal, the adapter process should clean up and exit +as soon as possible. + ### Examples: Print runtime version: @@ -61,4 +66,4 @@ See the [`adapters`](../adapters) directory for example adapters. We prefer runtime maintainers to maintain adapters in their own repository We'll only maintain adapters for [Bytecode Alliance](https://bytecodealliance.org/) projects and we'll aim for compatibility with the most recent stable version. -We'll accept pull requests for new adapters in this repository, but we can't guarantee we'll have the capacity to maintain them (so they might stop working with the new runtime release). \ No newline at end of file +We'll accept pull requests for new adapters in this repository, but we can't guarantee we'll have the capacity to maintain them (so they might stop working with the new runtime release). diff --git a/test-runner/tests/test_test_case.py b/test-runner/tests/test_test_case.py index ad87ab91..8fef8839 100644 --- a/test-runner/tests/test_test_case.py +++ b/test-runner/tests/test_test_case.py @@ -42,12 +42,12 @@ def test_test_config_should_warn_when_unknown_field(_mock_file: Mock) -> None: def test_test_results_should_mark_failed_if_multiple_failures() -> None: - results = Result(Output(0, "", ""), True, [Failure("type", "message")]) + results = Result(Output(0, "", ""), [Failure("type", "message")]) assert results.failed is True def test_test_results_should_not_mark_failed_if_no_failure() -> None: - results = Result(Output(0, "", ""), True, []) + results = Result(Output(0, "", ""), []) assert results.failed is False diff --git a/test-runner/tests/test_test_suite.py b/test-runner/tests/test_test_suite.py index 530ed6c2..06884b48 100644 --- a/test-runner/tests/test_test_suite.py +++ b/test-runner/tests/test_test_suite.py @@ -9,7 +9,7 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa return tc.TestCase( name, tc.Config(), - tc.Result(tc.Output(0, "", ""), is_executed, failures), + tc.Result(tc.Output(0, "", ""), failures) if is_executed else tc.SkippedResult(), 1.0, ) diff --git a/test-runner/tests/test_test_suite_runner.py b/test-runner/tests/test_test_suite_runner.py index d73438de..cc39a7c4 100644 --- a/test-runner/tests/test_test_suite_runner.py +++ b/test-runner/tests/test_test_suite_runner.py @@ -35,9 +35,9 @@ def test_runner_end_to_end() -> None: tc.Output(2, "test3", ""), ] expected_results = [ - tc.Result(outputs[0], True, []), - tc.Result(outputs[1], True, [failures[1]]), - tc.Result(outputs[2], True, [failures[0], failures[2]]), + tc.Result(outputs[0], []), + tc.Result(outputs[1], [failures[1]]), + tc.Result(outputs[2], [failures[0], failures[2]]), ] expected_config = [ tc.Config(dirs=[".", "deep/dir"]), diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index 8a849c2e..a5225b2e 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -2,7 +2,7 @@ from colorama import Fore, init from . import TestReporter -from ..test_case import TestCase +from ..test_case import TestCase, SkippedResult, TimedoutResult from ..test_suite import TestSuite from ..runtime_adapter import RuntimeVersion @@ -20,18 +20,21 @@ def __init__(self, colored: bool = True) -> None: self._colored = colored def report_test(self, test: TestCase) -> None: - if test.result.failed: - self._print_fail(f"Test {test.name} failed") - for reason in test.result.failures: - self._print_fail(f" [{reason.type}] {reason.message}") - print("STDOUT:") - print(test.result.output.stdout) - print("STDERR:") - print(test.result.output.stderr) - elif test.result.is_executed: - self._print_pass(f"Test {test.name} passed") - else: + if isinstance(test.result, TimedoutResult): + self._print_fail(f"Test {test.name} timed out") + elif isinstance(test.result, SkippedResult): self._print_skip(f"Test {test.name} skipped") + else: + if test.result.failed: + self._print_fail(f"Test {test.name} failed") + for reason in test.result.failures: + self._print_fail(f" [{reason.type}] {reason.message}") + print("STDOUT:") + print(test.result.output.stdout) + print("STDERR:") + print(test.result.output.stderr) + else: + self._print_pass(f"Test {test.name} passed") def report_test_suite(self, test_suite: TestSuite) -> None: self._test_suites.append(test_suite) @@ -41,29 +44,31 @@ def finalize(self, version: RuntimeVersion) -> None: print("===== Test results =====") print(f"Runtime: {version.name} {version.version}") - total_skip = total_pass = total_fail = pass_suite = 0 + total_skip = total_pass = total_fail = total_timedout = pass_suite = 0 for suite in self._test_suites: total_pass += suite.pass_count total_fail += suite.fail_count total_skip += suite.skip_count + total_timedout += suite.timedout_count - if suite.fail_count == 0: + if suite.fail_count == 0 and suite.timedout_count == 0: pass_suite += 1 print(f"Suite: {suite.name}") print(f" Total: {suite.test_count}") - self._print_pass(f" Passed: {suite.pass_count}") - self._print_fail(f" Failed: {suite.fail_count}") - self._print_skip(f" Skipped: {suite.skip_count}") + self._print_pass(f" Passed: {suite.pass_count}") + self._print_fail(f" Failed: {suite.fail_count}") + self._print_skip(f" Skipped: {suite.skip_count}") + self._print_fail(f" Timed out: {suite.timedout_count}") print("") print( - f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0)}" + f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0, 0)}" ) - print(f"Tests: {self._get_summary(total_fail, total_pass, total_skip)}") + print(f"Tests: {self._get_summary(total_fail, total_pass, total_skip, total_timedout)}") - def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str: + def _get_summary(self, fail_count: int, pass_count: int, skip_count: int, timedout_count: int) -> str: items: List[str] = [] if fail_count: @@ -72,8 +77,10 @@ def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str items.append(f"{self._pass_color}{pass_count} passed") if skip_count: items.append(f"{self._skip_color}{skip_count} skipped") + if timedout_count: + items.append(f"{self._fail_color}{timedout_count} timed out") - total = fail_count + pass_count + skip_count + total = fail_count + pass_count + skip_count + timedout_count items.append(f"{self._reset_color}{total} total") return ", ".join(items) diff --git a/test-runner/wasi_test_runner/reporters/json.py b/test-runner/wasi_test_runner/reporters/json.py index 023f5563..3a4e9606 100644 --- a/test-runner/wasi_test_runner/reporters/json.py +++ b/test-runner/wasi_test_runner/reporters/json.py @@ -5,6 +5,7 @@ from . import TestReporter from ..test_suite import TestSuite +from ..test_case import Result, SkippedResult, TimedoutResult from ..runtime_adapter import RuntimeVersion @@ -28,16 +29,19 @@ def finalize(self, version: RuntimeVersion) -> None: "duration_s": suite.duration_s, "failed": suite.fail_count, "skipped": suite.skip_count, + "timedout": suite.timedout_count, "passed": suite.pass_count, "tests": [ { "name": test.name, - "executed": test.result.is_executed, + "executed": isinstance(test.result, Result), + "skipped": isinstance(test.result, SkippedResult), + "timedout": isinstance(test.result, TimedoutResult), "duration_s": test.duration_s, "wasi_functions": test.config.wasi_functions, "failures": [ failure.message for failure in test.result.failures - ], + ] if isinstance(test.result, Result) else [], } for test in suite.test_cases ], diff --git a/test-runner/wasi_test_runner/runtime_adapter.py b/test-runner/wasi_test_runner/runtime_adapter.py index 940c2e2c..9a8c8885 100644 --- a/test-runner/wasi_test_runner/runtime_adapter.py +++ b/test-runner/wasi_test_runner/runtime_adapter.py @@ -17,7 +17,9 @@ def __init__(self, adapter_path: str) -> None: def get_version(self) -> RuntimeVersion: output = ( - subprocess.check_output([sys.executable, self._adapter_path, "--version"], encoding="UTF-8") + subprocess.check_output( + [sys.executable, self._adapter_path, "--version"], encoding="UTF-8" + ) .strip() .split(" ") ) @@ -42,14 +44,20 @@ def run_test( + [e for env in self._env_to_list(env_variables) for e in ("--env", env)] ) - result = subprocess.run( + with subprocess.Popen( args, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, - check=False, cwd=Path(test_path).parent, - ) - return Output(result.returncode, result.stdout, result.stderr) + ) as proc: + try: + out, err = proc.communicate(timeout=3) + except subprocess.TimeoutExpired: + proc.terminate() + proc.wait() + raise + return Output(proc.returncode, out, err) @staticmethod def _abs(path: str) -> str: diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eebb95d3..f091c213 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -1,6 +1,6 @@ import logging import json -from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional +from typing import List, NamedTuple, TypeVar, Type, Dict, Any, Optional, Union class Output(NamedTuple): @@ -16,7 +16,6 @@ class Failure(NamedTuple): class Result(NamedTuple): output: Output - is_executed: bool failures: List[Failure] @property @@ -24,6 +23,14 @@ def failed(self) -> bool: return len(self.failures) > 0 +class TimedoutResult(NamedTuple): + pass + + +class SkippedResult(NamedTuple): + pass + + T = TypeVar("T", bound="Config") @@ -63,5 +70,5 @@ def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None: class TestCase(NamedTuple): name: str config: Config - result: Result + result: Union[Result | SkippedResult | TimedoutResult] duration_s: float diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py index 4681cd9a..09b1b9d3 100644 --- a/test-runner/wasi_test_runner/test_suite.py +++ b/test-runner/wasi_test_runner/test_suite.py @@ -1,6 +1,6 @@ from typing import NamedTuple, List from datetime import datetime -from .test_case import TestCase +from .test_case import TestCase, Result, SkippedResult, TimedoutResult class TestSuite(NamedTuple): @@ -19,7 +19,7 @@ def pass_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed is False + if isinstance(test.result, Result) and test.result.failed is False ] ) @@ -29,10 +29,14 @@ def fail_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed + if isinstance(test.result, Result) and test.result.failed ] ) @property def skip_count(self) -> int: - return len([1 for test in self.test_cases if not test.result.is_executed]) + return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)]) + + @property + def timedout_count(self) -> int: + return len([1 for test in self.test_cases if isinstance(test.result, TimedoutResult)]) diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index c9a26b14..c3ee9c56 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -3,15 +3,18 @@ import os import re import shutil +import subprocess import time from datetime import datetime -from typing import List, cast +from typing import List, cast, Union from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( Result, + SkippedResult, + TimedoutResult, Config, Output, TestCase, @@ -67,7 +70,7 @@ def _skip_single_test( return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=Result(output=Output(0, "", ""), is_executed=False, failures=[]), + result=SkippedResult(), duration_s=0, ) @@ -77,13 +80,17 @@ def _execute_single_test( ) -> TestCase: config = _read_test_config(test_path) test_start = time.time() - test_output = runtime.run_test(test_path, config.args, config.env, config.dirs) + try: + test_output = runtime.run_test(test_path, config.args, config.env, config.dirs) + result: Union[Result | TimedoutResult] = _validate(validators, config, test_output) + except subprocess.TimeoutExpired: + result = TimedoutResult() elapsed = time.time() - test_start return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=_validate(validators, config, test_output), + result=result, duration_s=elapsed, ) @@ -95,7 +102,7 @@ def _validate(validators: List[Validator], config: Config, output: Output) -> Re if result is not None ] - return Result(failures=failures, is_executed=True, output=output) + return Result(failures=failures, output=output) def _read_test_config(test_path: str) -> Config: