FIX: Test instabilities in CI (#2190)

Author: windxu88

.yamato/config.metadata

@@ -6,6 +6,8 @@ editors:
     run_tvos: false
   - version: 6000.1
     run_tvos: false
+  - version: 6000.2
+    run_tvos: false
   - version: trunk
     run_tvos: false    
 

.yamato/upm-ci.yml

@@ -13,7 +13,6 @@
   variables:
     EDITOR_VERSION: {{ editor.version }}
   commands:        
-    - {{ utr_install_win }}
     - {{ upm_ci_install }}
     # Get version 2.3.0-preview of doctools package (it currently fails for 3.0.0-preview).
     # Automatically makes the documentation tests in APIVerification go live.
@@ -28,15 +27,14 @@
     - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION%
       {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}}_pkg;flags:{{platform.name}}_{{editor.version}}_pkg\"" {% endif %}
     {% if platform.installscript %}
-    - {{ unity_downloader_install }}
     - {{ platform.installscript }} %EDITOR_VERSION%
     {% endif %}
     # ADBv2 on 2019.4 causes the test runner to not start on initial import when the
     # samples are in the package. Move the samples back into the project.
     - move /Y .\Packages\com.unity.inputsystem\Samples .\Assets
     - move /Y .\Packages\com.unity.inputsystem\Samples.meta .\Assets
     # Now run our full test suite that sits in Assets/Tests by running UTR on our project.
-    - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if platform.name == "win" and category.name == "functional" %}--enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %}
+    - UnifiedTestRunner --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if platform.name == "win" and category.name == "functional" %}--enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %}
     {% if platform.name == "win" and category.name == "functional" %}
     # Delete the Package and Project reports & raw coverage data, to keep Artifacts.zip smaller
     - rmdir /s/q "{{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package"
@@ -68,7 +66,6 @@
   variables:
     EDITOR_VERSION: {{ editor.version }}  
   commands:
-    - {{ utr_install_nix }}
     - {{ upm_ci_install }}
     # Get version 2.3.0-preview of doctools package (it currently fails for 3.0.0-preview).
     # Automatically makes the documentation tests in APIVerification go live.
@@ -81,15 +78,14 @@
     # Run upm-ci verification tests as well as tests contained in the package.
     - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u $EDITOR_VERSION {% if category.name == "functional" and platform.name == "mac" or platform.name == "linux" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_nix }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}}_pkg;flags:{{platform.name}}_{{editor.version}}_pkg\"" {% endif %}
     {% if platform.installscript %}
-    - {{ unity_downloader_install }}
     - {{ platform.installscript }} $EDITOR_VERSION
     {% endif %}
     # ADBv2 on 2019.4 causes the test runner to not start on initial import when the
     # samples are in the package. Move the samples back into the project.
     - mv ./Packages/com.unity.inputsystem/Samples ./Assets
     - mv ./Packages/com.unity.inputsystem/Samples.meta ./Assets
     # Now run our full test suite that sits in Assets/Tests by running UTR on our project.
-    - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "mac" or platform.name == "linux"%} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if category.name == "functional" and platform.name == "mac" or platform.name == "linux" %} --enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_nix }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %}
+    - UnifiedTestRunner --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "mac" or platform.name == "linux"%} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if category.name == "functional" and platform.name == "mac" or platform.name == "linux" %} --enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_nix }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %}
     {% if category.name == "functional" and platform.name == "mac" or platform.name == "linux"%}
     # Delete the Package and Project reports & raw coverage data, to keep Artifacts.zip smaller
     - rm -rf {{ yamato_source_dir_nix }}/upm-ci~/test-results/CodeCoverage/Package
@@ -123,10 +119,8 @@ build_ios_{{ editor.version }}_{{ category.name }}:
     EDITOR_VERSION: {{ editor.version }}
     UTR_VERSION: 1.35.1
   commands:
-    - {{ utr_install_nix }}
-    - {{ unity_downloader_install }}
     - unity-downloader-cli -c Editor -c iOS -u $EDITOR_VERSION --fast --wait
-    - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=iOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
+    - UnifiedTestRunner --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=iOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
   after:
     - {{ instabilities_install_nix }}
     - {{ instabilities_run_mac }}
@@ -153,8 +147,7 @@ run_ios_{{ editor.version }}_{{ category.name }}:
   dependencies:
     - .yamato/upm-ci.yml#build_ios_{{ editor.version }}_{{ category.name }}
   commands:
-    - {{ utr_install_nix }}
-    - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=iOS --player-load-path=build/players --artifacts_path=build/test-results {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
+    - UnifiedTestRunner --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=iOS --player-load-path=build/players --artifacts_path=build/test-results {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
   after:
     - {{ instabilities_install_nix }}
     - {{ instabilities_run_mac }}
@@ -173,10 +166,8 @@ build_tvos_{{ editor.version }}_{{ category.name }}:
     EDITOR_VERSION: {{ editor.version }}
     UTR_VERSION: 1.35.1    
   commands:
-    - {{ utr_install_nix }}
-    - {{ unity_downloader_install }}
     - unity-downloader-cli -c Editor -c AppleTV -u $EDITOR_VERSION --fast --wait
-    - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=tvOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
+    - UnifiedTestRunner --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=tvOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
   after:
     - {{ instabilities_install_nix }}
     - {{ instabilities_run_mac }}
@@ -202,8 +193,7 @@ run_tvos_{{ editor.version }}_{{ category.name }}:
   dependencies:
     - .yamato/upm-ci.yml#build_tvos_{{ editor.version }}_{{ category.name }}
   commands:
-    - {{ utr_install_nix }}
-    - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=tvOS --player-load-path=build/players --artifacts_path=build/test-results {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
+    - UnifiedTestRunner --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=tvOS --player-load-path=build/players --artifacts_path=build/test-results {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
   after:
     - {{ instabilities_install_nix }}
     - {{ instabilities_run_mac }}
@@ -224,10 +214,8 @@ build_android_{{ editor.version }}_{{ backend.name }}_{{ category.name }}:
   variables:
     EDITOR_VERSION: {{ editor.version }}
   commands:
-    - {{ utr_install_win }}
-    - {{ unity_downloader_install }}
     - unity-downloader-cli -c Editor -c Android -u %EDITOR_VERSION% --fast --wait
-    - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=Android --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --scripting-backend={{ backend.name }} --build-only --repository {% if category.name == "performance" %} --performance-project-id=InputSystem {% endif %}
+    - UnifiedTestRunner --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=Android --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --scripting-backend={{ backend.name }} --build-only --repository {% if category.name == "performance" %} --performance-project-id=InputSystem {% endif %}
   after:
     - {{ instabilities_install_win }}
     - {{ instabilities_run_win }}  
@@ -253,15 +241,14 @@ run_android_{{ editor.version }}_{{ backend.name }}_{{ category.name }}:
   dependencies:
     - .yamato/upm-ci.yml#build_android_{{ editor.version }}_{{ backend.name }}_{{ category.name }}
   commands:
-    - {{ utr_install_win }}
     - |
        # Set the IP of the device. In case device gets lost, UTR will try to recconect to ANDROID_DEVICE_CONNECTION
        set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP%
        # Establish an ADB connection with the device
        start %ANDROID_SDK_ROOT%\platform-tools\adb.exe connect %BOKKEN_DEVICE_IP%
        # List the connected devices
        start %ANDROID_SDK_ROOT%\platform-tools\adb.exe devices
-       ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=android --player-load-path=build/players --artifacts_path=build/test-results {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
+       UnifiedTestRunner --suite=playmode {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --platform=android --player-load-path=build/players --artifacts_path=build/test-results {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %}
   after:
     - start %ANDROID_SDK_ROOT%\platform-tools\adb.exe connect %BOKKEN_DEVICE_IP%
     - if not exist build\test-results mkdir build\test-results