ci: [1.X] CI failures fixes and disabled tests Jira tracking informations

.yamato/_run-all.yml

@@ -130,7 +130,13 @@ run_all_project_tests_desktop_standalone:
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.all -%}
 {% for backend in scripting_backends -%}
+# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since 
+# 1) The 2021 editor will soon be out of support
+# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks
+# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on
+{% if editor != "2021.3" or platform.name != "ubuntu" %}
     - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -158,7 +164,13 @@ run_all_project_tests_desktop_standalone_2021:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
 {% for backend in scripting_backends -%}
+# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since 
+# 1) The 2021 editor will soon be out of support
+# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks
+# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on
+{% if platform.name != "ubuntu" %}
     - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_2021.3
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -206,7 +218,10 @@ run_all_project_tests_console_standalone:
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_test -%}
 {% for editor in validation_editors.all -%}
+# When building consoles with 2023 editor there are errors related to SDK version or uninstalled components. I will disable those for now and will add to my investigation list 
+{% if editor != "2023.3" %}
     - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}

.yamato/code-coverage.yml

@@ -16,12 +16,14 @@ code_coverage_{{ platform.name }}_{{ editor }}:
   commands:
     - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
     - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait
-    - {% if platform.name == "ubuntu" %}DISPLAY=:0 {% endif %} upm-ci package test -u {{ editor }} --package-path com.unity.netcode.gameobjects --enable-code-coverage --code-coverage-options 'generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime' --extra-utr-arg="--extra-editor-arg=--burst-disable-compilation --extra-editor-arg=-testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun"
+
+    - upm-ci package pack --package-path com.unity.netcode.gameobjects
+
+    - DISPLAY=:0 upm-ci package test -u {{ editor }} --package-path upm-ci~/packages/com.unity.netcode.gameobjects-*.tgz --artifacts-path=test-results --enable-code-coverage --coverage-results-path=CoverageResults --code-coverage-options 'generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime' --extra-utr-arg="--extra-editor-arg=--burst-disable-compilation --extra-editor-arg=-testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun"
   artifacts:
     logs:
       paths:
-        - "upm-ci~/test-results/**/*"
-  dependencies:
-    - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}
+        - "test-results/**/*"
+        - "**/CoverageResults/**/*"
 {% endfor -%}
 {% endfor -%}

.yamato/console-standalone-test.yml

@@ -15,6 +15,8 @@
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_build -%}
 {% for editor in validation_editors.all -%}
+# When building consoles with 2023 editor there are errors related to SDK version or uninstalled components. I will disable those for now and will add to my investigation list 
+{% if editor != "2023.3" %}
 build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
   name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp{% if platform.name == "switch" %}, arm64{% endif %}]
   agent:
@@ -47,6 +49,7 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     logs:
       paths:
         - "artifacts/**/*"
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -57,6 +60,8 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_test -%}
 {% for editor in validation_editors.all -%}
+# When building consoles with 2023 editor there are errors related to SDK version or uninstalled components. I will disable those for now and will add to my investigation list 
+{% if editor != "2023.3" %}
 run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
   name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp]
   agent:
@@ -91,6 +96,7 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
         - "test-results/**/*"
   dependencies:
     - .yamato/console-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }}
+{% endif -%}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}

.yamato/desktop-standalone-tests.yml

@@ -13,6 +13,11 @@
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.all -%}
 {% for backend in scripting_backends -%}
+# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since 
+# 1) The 2021 editor will soon be out of support
+# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks
+# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on
+{% if editor != "2021.3" or platform.name != "ubuntu" %}
 desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}:
   name : Standalone Build - NGO {{ project.name }} - [{{ platform.name }}, {{ editor }}, {{ backend }}]
   agent:
@@ -57,6 +62,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 
   dependencies:
     - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }}
+{% endif %}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -70,6 +76,11 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.all -%}
 {% for backend in scripting_backends -%}
+# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since 
+# 1) The 2021 editor will soon be out of support
+# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks
+# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on
+{% if editor != "2021.3" or platform.name != "ubuntu" %}
 desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}:
   name : Standalone Test - NGO {{ project.name }} - [{{ platform.name }}, {{ editor }}, {{ backend }}]
   agent:
@@ -102,6 +113,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
         - "test-results/**/*"
   dependencies:
     - .yamato/desktop-standalone-tests.yml#desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}
+{% endif %}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}

com.unity.netcode.gameobjects/Tests/Editor/InterpolatorTests.cs

@@ -105,7 +105,7 @@ public void OutOfOrderShouldStillWork()
             Assert.That(interpolator.GetInterpolatedValue(), Is.EqualTo(2f).Within(k_Precision));
         }
 
-        [Ignore("TODO: Fix this test to still handle testing message loss without extrapolation")]
+        [Ignore("TODO: Fix this test to still handle testing message loss without extrapolation. This is tracked in MTT-11338")]
         [Test]
         public void MessageLoss()
         {
@@ -306,7 +306,7 @@ public void TestUpdatingInterpolatorWithNoData()
             Assert.Throws<InvalidOperationException>(() => interpolator.Update(1f, serverTime));
         }
 
-        [Ignore("TODO: Fix this test to still test duplicated values without extrapolation")]
+        [Ignore("TODO: Fix this test to still test duplicated values without extrapolation. This is tracked in MTT-11338")]
         [Test]
         public void TestDuplicatedValues()
         {

com.unity.netcode.gameobjects/Tests/Editor/Metrics/NetworkMetricsRegistrationTests.cs

@@ -16,7 +16,7 @@ public class NetworkMetricsRegistrationTests
             .ToArray();
 
         [TestCaseSource(nameof(s_MetricTypes))]
-        [Ignore("Disable test while we reevaluate the assumption that INetworkMetricEvent interfaces must be reported from MLAPI.")]
+        [Ignore("Disable test while we reevaluate the assumption that INetworkMetricEvent interfaces must be reported from MLAPI. This is tracked in MTT-11339")]
         public void ValidateThatAllMetricTypesAreRegistered(Type metricType)
         {
             var dispatcher = new NetworkMetrics().Dispatcher as MetricDispatcher;

com.unity.netcode.gameobjects/Tests/Runtime/DeferredMessagingTests.cs

@@ -492,7 +492,6 @@ public void WhenANetworkVariableDeltaMessageArrivesBeforeASpawnArrives_ItIsDefer
         }
 
         [Test]
-        //[Ignore("Disabling this temporarily until it is migrated into new integration test.")]
         public void WhenASpawnMessageArrivesBeforeThePrefabIsAvailable_ItIsDeferred()
         {
             m_SkipAddingPrefabsToClient = true;

com.unity.netcode.gameobjects/Tests/Runtime/NetworkObject/NetworkObjectSpawnManyObjectsTests.cs

@@ -48,7 +48,7 @@ protected override void OnServerAndClientsCreated()
 
         [UnityTest]
         // When this test fails it does so without an exception and will wait the default ~6 minutes
-        [Timeout(360000)]
+        [Timeout(360000)] // Tracked in MTT-11360
         public IEnumerator WhenManyObjectsAreSpawnedAtOnce_AllAreReceived()
         {
             for (int x = 0; x < k_SpawnedObjects; x++)

com.unity.netcode.gameobjects/Tests/Runtime/NetworkTransformAnticipationTests.cs

@@ -115,8 +115,7 @@ public AnticipatedNetworkTransform GetOtherClientComponent()
         }
 
         [Test]
-        // I will add this test to track in Jira as next step (after having working CI in place). The job is failing on Android
-        [UnityPlatform(exclude = new[] { RuntimePlatform.Android })]
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android })] // Tracked in MTT-11361
         public void WhenAnticipating_ValueChangesImmediately()
         {
             var testComponent = GetTestComponent();

com.unity.netcode.gameobjects/Tests/Runtime/NetworkVariableTests.cs

@@ -1844,6 +1844,7 @@ public void WhenSerializingAndDeserializingValueTypeNetworkVariables_ValuesAreSe
         }
 
         [Test]
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Tracked in MTT-11356. The job is failing on mobile devices on 2021 editor specifically
         public void WhenSerializingAndDeserializingValueTypeNativeArrayNetworkVariables_ValuesAreSerializedCorrectly(
 
             [Values(typeof(byte), typeof(sbyte), typeof(short), typeof(ushort), typeof(int), typeof(uint),
@@ -2179,7 +2180,7 @@ public string ArrayStr<T>(NativeArray<T> arr) where T : unmanaged
         }
 
         [Test]
-        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // I will add this test to track in Jira as next step (after having working CI in place). The job is failing on iOS
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Tracked in MTT-11356.
         [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeArrayNetworkVariables_ValuesAreSerializedCorrectly(
 
@@ -2645,7 +2646,7 @@ public string DictionaryStr<TKey, TVal>(Dictionary<TKey, TVal> list)
         }
 
         [Test]
-        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // I will add this test to track in Jira as next step (after having working CI in place). The job is failing on iOS
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Tracked in MTT-11356.
         [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeListNetworkVariables_ValuesAreSerializedCorrectly(
 
@@ -2832,7 +2833,7 @@ public void WhenSerializingAndDeserializingVeryLargeListNetworkVariables_ValuesA
         }
 
         [Test]
-        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // I will add this test to track in Jira as next step (after having working CI in place). The job is failing on iOS
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Tracked in MTT-11356.
         [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeHashSetNetworkVariables_ValuesAreSerializedCorrectly(
 
@@ -2988,7 +2989,7 @@ public void WhenSerializingAndDeserializingVeryLargeHashSetNetworkVariables_Valu
         }
 
         [Test]
-        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // I will add this test to track in Jira as next step (after having working CI in place). The job is unstable on mobile devices
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Tracked in MTT-11356.
         [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeDictionaryNetworkVariables_ValuesAreSerializedCorrectly(
 

com.unity.netcode.gameobjects/Tests/Runtime/RpcTypeSerializationTests.cs

@@ -4,6 +4,7 @@
 using Unity.Collections;
 using Unity.Netcode.TestHelpers.Runtime;
 using UnityEngine;
+using UnityEngine.TestTools;
 using Quaternion = UnityEngine.Quaternion;
 using Vector2 = UnityEngine.Vector2;
 using Vector3 = UnityEngine.Vector3;
@@ -1503,6 +1504,7 @@ public void WhenSendingAnArrayOfValueTypesOverAnRpc_ValuesAreSerializedCorrectly
         }
 
         [Test]
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Tracked in MTT-11356. The job is failing on mobile devices on 2021 editor specifically
         public void WhenSendingANativeArrayOfValueTypesOverAnRpc_ValuesAreSerializedCorrectly(
 
             [Values(typeof(byte), typeof(sbyte), typeof(short), typeof(ushort), typeof(int), typeof(uint),

com.unity.netcode.gameobjects/Tests/Runtime/Timing/TimeInitializationTest.cs

@@ -16,6 +16,7 @@ public class TimeInitializationTest
         private NetworkManager m_Client;
 
         [UnityTest]
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android })] // Tracked in MTT-11356.
         public IEnumerator TestClientTimeInitializationOnConnect([Values(0, 1f)] float serverStartDelay, [Values(0, 1f)] float clientStartDelay, [Values(true, false)] bool isHost)
         {
             // Create multiple NetworkManager instances