Merge pull request #565 from launchableinc/aw/split-subset-zis

Add --output-exclusion-rules support to split-subset

Author: Ryosuke Yabuki

launchable/commands/split_subset.py

@@ -102,10 +102,24 @@ def split_subset(
     class SplitSubset(TestPathWriter):
         def __init__(self, dry_run: bool = False):
             super(SplitSubset, self).__init__(dry_run=dry_run)
+            self.rest = rest
+            self.output_handler = self._default_output_handler
+            self.exclusion_output_handler = self._default_exclusion_output_handler
             self.split_by_groups_output_handler = self._default_split_by_groups_output_handler
             self.split_by_groups_exclusion_output_handler = self._default_split_by_groups_exclusion_output_handler
             self.is_split_by_groups_with_rest = is_split_by_groups_with_rest
             self.split_by_groups_output_dir = split_by_groups_output_dir
+            self.is_output_exclusion_rules = is_output_exclusion_rules
+
+        def _default_output_handler(self, output: List[TestPath], rests: List[TestPath]):
+            if rest:
+                self.write_file(rest, rests)
+
+            if output:
+                self.print(output)
+
+        def _default_exclusion_output_handler(self, subset: List[TestPath], rest: List[TestPath]):
+            self.output_handler(rest, subset)
 
         def _default_split_by_groups_output_handler(self, group_name: str, subset: List[TestPath], rests: List[TestPath]):
             if is_split_by_groups_with_rest:
@@ -148,8 +162,8 @@ def split_by_bin(self):
                     )
                     return
 
-            output = []
-            rests = []
+            output_subset = []
+            output_rests = []
             is_observation = False
 
             try:
@@ -230,24 +244,24 @@ def split_by_bin(self):
                 res = client.request("POST", "{}/slice".format(subset_id), payload=payload)
                 res.raise_for_status()
 
-                output = res.json().get("testPaths", [])
-                rests = res.json().get("rest", [])
+                output_subset = res.json().get("testPaths", [])
+                output_rests = res.json().get("rest", [])
                 is_observation = res.json().get("isObservation", False)
 
-                if len(output) == 0:
+                if len(output_subset) == 0:
                     click.echo(click.style(
-                        "Error: no tests found in this subset id.", 'yellow'), err=True)
+                        "Error: no tests found for this subset id.", 'yellow'), err=True)
                     return
 
                 if is_observation:
-                    output = output + rests
-                if rest:
-                    if len(rests) == 0:
-                        rests.append(output[0])
+                    output_subset = output_subset + output_rests
+                    output_rests = []
 
-                    self.write_file(rest, rests)
+                if is_output_exclusion_rules:
+                    self.exclusion_output_handler(output_subset, output_rests)
+                else:
+                    self.output_handler(output_subset, output_rests)
 
-                self.print(output)
             except Exception as e:
                 if os.getenv(REPORT_ERROR_KEY):
                     raise e

launchable/test_runners/dotnet.py

@@ -53,6 +53,40 @@ def exclusion_output_handler(subset_tests: List[TestPath], rest_tests: List[Test
     client.run()
 
 
+@launchable.split_subset
+def split_subset(client):
+    # ref: https://github.com/Microsoft/vstest-docs/blob/main/docs/filter.md
+    separator = "|"
+    prefix = "FullyQualifiedName="
+
+    if client.is_output_exclusion_rules:
+        separator = "&"
+        prefix = "FullyQualifiedName!="
+
+    def formatter(test_path: TestPath):
+        paths = []
+
+        for path in test_path:
+            t = path.get("type", "")
+            if t == 'Assembly':
+                continue
+            paths.append(path.get("name", ""))
+
+        return prefix + ".".join(paths)
+
+    def exclusion_output_handler(subset_tests: List[TestPath], rest_tests: List[TestPath]):
+        if client.rest:
+            with open(client.rest, "w+", encoding="utf-8") as fp:
+                fp.write(client.separator.join(formatter(t) for t in subset_tests))
+
+        click.echo(client.separator.join(formatter(t) for t in rest_tests))
+
+    client.separator = separator
+    client.formatter = formatter
+    client.exclusion_output_handler = exclusion_output_handler
+    client.run()
+
+
 @click.argument('files', required=True, nargs=-1)
 @launchable.record.tests
 def record_tests(client, files):

tests/commands/test_split_subset.py

@@ -65,7 +65,7 @@ def test_split_subset_with_observation_mode(self):
 
         self.assertEqual(result.exit_code, 0)
         self.assertEqual(result.stdout, "test_1.py\ntest_3.py\ntest_5.py\n")
-        self.assertEqual(observation_mode_rest.read().decode(), os.linesep.join(["test_5.py"]))
+        self.assertEqual(observation_mode_rest.read().decode(), "")
         observation_mode_rest.close()
         os.unlink(observation_mode_rest.name)
 

tests/test_runners/test_dotnet.py

@@ -8,6 +8,7 @@
 
 from launchable.utils.http_client import get_base_url
 from tests.cli_test_case import CliTestCase
+from tests.helper import ignore_warnings
 
 
 class DotnetTest(CliTestCase):
@@ -93,6 +94,55 @@ def test_subset(self):
         output = "FullyQualifiedName!=rocket_car_dotnet.ExampleTest.TestAdd&FullyQualifiedName!=rocket_car_dotnet.ExampleTest.TestDiv\n"  # noqa: E501
         self.assertEqual(result.output, output)
 
+    @ignore_warnings
+    @responses.activate
+    @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token})
+    def test_split_subset(self):
+        responses.replace(
+            responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset/456/slice".format(
+                get_base_url(), self.organization, self.workspace), json={
+                "testPaths": [
+                    [
+                        {"type": "Assembly", "name": "rocket-car-dotnet.dll"},
+                        {"type": "TestSuite", "name": "rocket_car_dotnet"},
+                        {"type": "TestSuite", "name": "ExampleTest"},
+                        {"type": "TestCase", "name": "TestSub"},
+                    ],
+                ],
+                "rest": [
+                    [
+                        {"type": "Assembly", "name": "rocket-car-dotnet.dll"},
+                        {"type": "TestSuite", "name": "rocket_car_dotnet"},
+                        {"type": "TestSuite", "name": "ExampleTest"},
+                        {"type": "TestCase", "name": "TestAdd"},
+                    ],
+                ],
+                'subsettingId': 456,
+                'summary': {
+                    'subset': {
+                        'duration': 8, 'candidates': 1, 'rate': 50,
+                    },
+                    'rest': {
+                        'duration': 7, 'candidates': 1, 'rate': 50,
+                    },
+                },
+            },
+            status=200)
+
+        result = self.cli('split-subset', '--subset-id', 'subset/456',
+                          '--bin', '1/2', 'dotnet')
+
+        self.assertEqual(result.exit_code, 0)
+
+        output = "FullyQualifiedName=rocket_car_dotnet.ExampleTest.TestSub\n"  # noqa: E501
+        self.assertEqual(result.output, output)
+
+        result = self.cli('split-subset', '--subset-id', 'subset/456',
+                          '--bin', '1/2', '--output-exclusion-rules', 'dotnet')
+        self.assertEqual(result.exit_code, 0)
+        output = "FullyQualifiedName!=rocket_car_dotnet.ExampleTest.TestAdd\n"  # noqa: E501
+        self.assertEqual(result.output, output)
+
     @responses.activate
     @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token})
     def test_record_tests(self):
@@ -102,4 +152,4 @@ def test_record_tests(self):
 
         payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode())
         expected = self.load_json_from_file(self.test_files_dir.joinpath("record_test_result.json"))
-        self.assert_json_orderless_equal(payload, expected)
+        self.assert_json_orderless_equal(payload, expected)

tests/test_runners/test_minitest.py

@@ -134,6 +134,6 @@ def test_split_subset(self):
         self.assertEqual(result.exit_code, 0)
         output = Path(self.test_files_dir, "test", "example_test.rb")
         self.assertEqual(str(output), result.output.rstrip("\n"))
-        self.assertEqual(rest.read().decode().rstrip("\n"), str(output))
+        self.assertEqual(rest.read().decode().rstrip("\n"), "")
         rest.close()
         os.unlink(rest.name)