Adds CTS e2etests

e2etests/MODULE.bazel

@@ -1,5 +1,6 @@
 bazel_dep(name = "gazelle", version = "0.40.0")
 bazel_dep(name = "rules_go", version = "0.50.1")
+bazel_dep(name = "rules_python", version = "1.1.0")
 
 go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
 go_sdk.download(version = "1.23.1")

e2etests/cvd/BUILD.bazel

@@ -1,4 +1,25 @@
-load("boot_tests.bzl", "cvd_command_boot_test", "cvd_load_boot_test", "launch_cvd_boot_test")
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("boot_tests.bzl", "cvd_command_boot_test", "cvd_cts_test", "cvd_load_boot_test", "launch_cvd_boot_test")
+
+py_library(
+    name = "convert_xts_xml_to_junit_xml",
+    srcs = ["convert_xts_xml_to_junit_xml.py"],
+    srcs_version = "PY3",
+)
+
+py_test(
+    name = "convert_xts_xml_to_junit_xml_test",
+    srcs = ["convert_xts_xml_to_junit_xml_test.py"],
+    srcs_version = "PY3",
+    deps = [":convert_xts_xml_to_junit_xml"],
+)
+
+py_binary(
+    name = "convert_xts_xml_to_junit_xml_bin",
+    srcs = ["convert_xts_xml_to_junit_xml.py"],
+    srcs_version = "PY3",
+    main = "convert_xts_xml_to_junit_xml.py",
+)
 
 # cvd fetch + launch_cvd tests
 launch_cvd_boot_test(
@@ -106,13 +127,83 @@ cvd_command_boot_test(
     target = "aosp_cf_x86_64_only_phone-userdebug",
 )
 
-cvd_command_boot_test(
-    name = "this_test_will_fail",
-    branch = "aosp-android-latest-release",
-    cvd_command = [
-        "this_command_does_not_exist",
+cvd_cts_test(
+    name = "aosp_android_latest_swiftshader_deqp_vk_smoke_tests",
+    cuttlefish_branch = "aosp-android-latest-release",
+    cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug",
+    cuttlefish_create_args = [
+        "--gpu_mode=guest_swiftshader",
     ],
-    target = "aosp_cf_x86_64_only_phone-userdebug",
+    cts_branch = "aosp-android15-tests-release",
+    cts_target = "test_suites_x86_64",
+    cts_args = [
+        "--include-filter=CtsDeqpTestCases",
+        "--module-arg=CtsDeqpTestCases:include-filter:dEQP-VK.api.smoke*",
+    ],
+)
+
+cvd_cts_test(
+    name = "aosp_android_latest_auto_enables_gfxstream_test",
+    cuttlefish_branch = "aosp-android-latest-release",
+    cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug",
+    cuttlefish_create_args = [
+        "--gpu_mode=auto",
+    ],
+    cts_branch = "aosp-android15-tests-release",
+    cts_target = "test_suites_x86_64",
+    cts_args = [
+        "--include-filter=CuttlefishGraphicsConfigurationTest",
+        "--module-arg=CuttlefishGraphicsConfigurationTest:set-option:expected-egl:emulation",
+    ],
+    tags = ["requires_gpu"],
+)
+
+cvd_cts_test(
+    name = "aosp_android_latest_gfxstream_deqp_vk_smoke_tests",
+    cuttlefish_branch = "aosp-android-latest-release",
+    cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug",
+    cuttlefish_create_args = [
+        "--gpu_mode=gfxstream",
+    ],
+    cts_branch = "aosp-android15-tests-release",
+    cts_target = "test_suites_x86_64",
+    cts_args = [
+        "--include-filter=CtsDeqpTestCases",
+        "--module-arg=CtsDeqpTestCases:include-filter:dEQP-VK.api.smoke*",
+    ],
+    tags = ["requires_gpu"],
+)
+
+cvd_cts_test(
+    name = "aosp_android_latest_gfxstream_guest_angle_host_swiftshader_deqp_vk_smoke_tests",
+    cuttlefish_branch = "aosp-android-latest-release",
+    cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug",
+    cuttlefish_create_args = [
+        "--gpu_mode=gfxstream_guest_angle_host_swiftshader",
+    ],
+    cts_branch = "aosp-android15-tests-release",
+    cts_target = "test_suites_x86_64",
+    cts_args = [
+        "--include-filter=CtsDeqpTestCases",
+        "--module-arg=CtsDeqpTestCases:include-filter:dEQP-VK.api.smoke*",
+    ],
+    # TODO: still need vulkan loader available in kokoro images.
+    tags = ["requires_gpu"],
+)
+
+cvd_cts_test(
+    name = "aosp_android_latest_gfxstream_guest_angle_host_swiftshader_graphics_cts_tests",
+    cuttlefish_branch = "aosp-android-latest-release",
+    cuttlefish_target = "aosp_cf_x86_64_only_phone-userdebug",
+    cuttlefish_create_args = [
+        "--gpu_mode=gfxstream_guest_angle_host_swiftshader",
+    ],
+    cts_branch = "aosp-android15-tests-release",
+    cts_target = "test_suites_x86_64",
+    cts_args = [
+        "--include-filter=CtsGraphicsTestCases",
+    ],
+    # TODO: still need vulkan loader available in kokoro images.
     tags = ["requires_gpu"],
 )
 

e2etests/cvd/boot_tests.bzl

@@ -49,3 +49,49 @@ def cvd_command_boot_test(name, branch, target, cvd_command = [], credential_sou
             "no-sandbox",
         ] + tags,
     )
+
+def cvd_cts_test(
+        name,
+        cuttlefish_branch,
+        cuttlefish_target,
+        cts_branch,
+        cts_target,
+        cts_args,
+        cuttlefish_create_args = [],
+        credential_source = "",
+        substitutions = "",
+        tags = []):
+
+    args = [
+        "--cuttlefish-create-args=\"" + " ".join(cuttlefish_create_args) + "\"",
+        "--cuttlefish-fetch-branch=" + cuttlefish_branch,
+        "--cuttlefish-fetch-target=" + cuttlefish_target,
+        "--xml-test-result-converter-path=$(location //cvd:convert_xts_xml_to_junit_xml_bin)",
+        "--xts-args=\"" + " ".join(cts_args) + "\"",
+        "--xts-fetch-branch=" + cts_branch,
+        "--xts-fetch-target=" + cts_target,
+        "--xts-type=cts",
+    ]
+
+    if credential_source:
+        args += ["--credential-source=" + credential_source]
+
+    if substitutions:
+        args += ["--substitutions=\"" + ",".join(substitutions) + "\""]
+
+
+    native.sh_test(
+        name = name,
+        size = "enormous",
+        srcs = ["cvd_xts_test.sh"],
+        args = args,
+        data = [
+            "//cvd:convert_xts_xml_to_junit_xml_bin",
+        ],
+        tags = [
+            "exclusive",
+            "external",
+            "no-sandbox",
+        ] + tags,
+    )
+

e2etests/cvd/convert_xts_xml_to_junit_xml.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0(the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Converts Android CTS test_result.xml files to Bazel result xml files.
+
+JUnit XML structure: https://github.com/testmoapp/junitxml
+
+"""
+
+import argparse
+import os
+import re
+import sys
+
+from xml.etree import ElementTree
+import xml.dom.minidom
+
+def to_pretty_xml(root, indent="    "):
+    return xml.dom.minidom.parseString(ElementTree.tostring(root, encoding='UTF-8', xml_declaration=True)).toprettyxml(indent)
+
+def convert_xts_xml_to_junit_xml(input_xml_string):
+    input_xml_root = ElementTree.ElementTree(ElementTree.fromstring(input_xml_string)).getroot()
+
+    output_xml_testsuites = ElementTree.Element("testsuites")
+    output_xml_testsuites.set('name', 'AllTests')
+
+    output_xml_testsuites_fail = 0
+    output_xml_testsuites_total = 0
+
+    for input_xml_result in input_xml_root.iter('Result'):
+        for input_xml_module in input_xml_result.iter('Module'):
+            # e.g. CtsDeqpTestCases
+            input_module_name = input_xml_module.get('name')
+
+            output_xml_module_testsuite = ElementTree.SubElement(output_xml_testsuites, "testsuite")
+            output_xml_module_testsuite.set('name', input_module_name)
+
+            output_xml_module_testsuite_fail = 0
+            output_xml_module_testsuite_total = 0
+
+            for input_xml_testcase in input_xml_module.iter('TestCase'):
+                # e.g. dEQP-EGL.functional.image.create
+                input_testcase_name = input_xml_testcase.get('name')
+
+                output_xml_testcase_testsuite = ElementTree.SubElement(output_xml_module_testsuite, "testsuite")
+                output_xml_testcase_testsuite.set('name', input_testcase_name)
+
+                output_xml_testcase_testsuite_fail = 0
+                output_xml_testcase_testsuite_total = 0
+
+                for input_xml_test in input_xml_testcase.iter('Test'):
+                    # e.g. gles2_android_native_rgba4_texture
+                    input_test_name = input_xml_test.get('name')
+
+                    # e.g. dEQP-EGL.functional.image.create#gles2_android_native_rgba4_texture
+                    output_test_name = input_testcase_name + '#' + input_test_name
+
+                    output_xml_testcase = ElementTree.SubElement(output_xml_testcase_testsuite, "testcase")
+                    output_xml_testcase.set('name', output_test_name)
+
+                    input_test_result = input_xml_test.get('result')
+                    if input_test_result == 'failed':
+                        output_xml_testcase_failure = ElementTree.SubElement(output_xml_testcase, "failure")
+                        output_xml_testcase_failure.set('name', output_test_name)
+
+                        output_xml_testcase_testsuite_fail += 1
+
+                    output_xml_testcase_testsuite_total += 1
+
+                output_xml_testcase_testsuite.set('tests', str(output_xml_testcase_testsuite_total))
+                output_xml_testcase_testsuite.set('failures', str(output_xml_testcase_testsuite_fail))
+
+                output_xml_module_testsuite_fail += output_xml_testcase_testsuite_fail
+                output_xml_module_testsuite_total += output_xml_testcase_testsuite_total
+
+            output_xml_module_testsuite.set('tests', str(output_xml_module_testsuite_total))
+            output_xml_module_testsuite.set('failures', str(output_xml_module_testsuite_fail))
+
+            output_xml_testsuites_fail += output_xml_module_testsuite_fail
+            output_xml_testsuites_total += output_xml_testcase_testsuite_total
+
+    output_xml_testsuites.set('tests', str(output_xml_module_testsuite_total))
+    output_xml_testsuites.set('failures', str(output_xml_module_testsuite_fail))
+
+    output_xml_string = to_pretty_xml(output_xml_testsuites)
+
+    return output_xml_string
+
+def main():
+    parser = argparse.ArgumentParser(description="Converts Android CTS test_result.xml files to Bazel result xml files")
+    parser.add_argument("--input_xml_file", help="The path to input Android CTS test_result.xml file.")
+    parser.add_argument("--output_xml_file", help="The path for the output xml file that is digestable by bazel.")
+    args = parser.parse_args()
+
+    input_xml_string = None
+    with open(args.input_xml_file, 'r') as intput_xml_file:
+        input_xml_string = intput_xml_file.read()
+
+    output_xml_string = convert_xts_xml_to_junit_xml(input_xml_string)
+
+    with open(args.output_xml_file, 'w') as output_xml_file:
+        output_xml_file.write(output_xml_string)
+
+if __name__ == '__main__':
+    main()

e2etests/cvd/convert_xts_xml_to_junit_xml_test.py

@@ -0,0 +1,60 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2025 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0(the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Tests for convert_xts_xml_to_junit_xml.py"""
+
+import unittest
+
+from cvd.convert_xts_xml_to_junit_xml import *
+
+class TestConvertXtsXmlToJunitXml(unittest.TestCase):
+
+    def test_basic_conversion(self):
+        input = """<?xml version='1.0' encoding='UTF-8' standalone='no' ?><?xml-stylesheet type="text/xsl" href="compatibility_result.xsl"?>
+<Result start="1765488789733" end="1765488875726" start_display="Thu Dec 11 21:33:09 UTC 2025" end_display="Thu Dec 11 21:34:35 UTC 2025" devices="0.0.0.0:6520" host_name="jason" report_version="5.0" suite_build_number="14561884" suite_name="CTS" suite_plan="cts" suite_variant="CTS" suite_version="13_r15" command_line_args="cts --log-level-display=INFO --include-filter=CtsDeqpTestCases --module-arg=CtsDeqpTestCases:include-filter:dEQP-VK.api.smoke*">
+  <Summary pass="6" failed="0" modules_done="1" modules_total="1" />
+  <Module name="CtsDeqpTestCases" abi="x86_64" runtime="7355" done="true" pass="6" total_tests="6">
+    <TestCase name="dEQP-VK.api.smoke">
+      <Test result="pass" name="create_sampler" />
+      <Test result="pass" name="create_shader" />
+      <Test result="pass" name="triangle" />
+      <Test result="pass" name="asm_triangle" />
+      <Test result="pass" name="asm_triangle_no_opname" />
+      <Test result="pass" name="unused_resolve_attachment" />
+    </TestCase>
+  </Module>
+</Result>
+        """
+        actual = convert_xts_xml_to_junit_xml(input)
+        expected = """<?xml version="1.0" ?>
+<testsuites name="AllTests" tests="6" failures="0">
+    <testsuite name="CtsDeqpTestCases" tests="6" failures="0">
+        <testsuite name="dEQP-VK.api.smoke" tests="6" failures="0">
+            <testcase name="dEQP-VK.api.smoke#create_sampler"/>
+            <testcase name="dEQP-VK.api.smoke#create_shader"/>
+            <testcase name="dEQP-VK.api.smoke#triangle"/>
+            <testcase name="dEQP-VK.api.smoke#asm_triangle"/>
+            <testcase name="dEQP-VK.api.smoke#asm_triangle_no_opname"/>
+            <testcase name="dEQP-VK.api.smoke#unused_resolve_attachment"/>
+        </testsuite>
+    </testsuite>
+</testsuites>
+"""
+        self.assertEqual(actual, expected)
+
+
+if __name__ == '__main__':
+    unittest.main()