fix: Yamato jobs not testing built player (Backport of: #4194)

.pr_agent.toml

@@ -4,6 +4,5 @@
 # https://developer.portal.internal.unity.com/catalog/default/component/u-pr/docs/configuration/automatic_runs/
 
 [github_app]
-handle_pr_actions = ['opened', 'ready_for_review']  # PR events that auto-run pr_commands
 pr_commands       = ["/harness-review"]             # what runs on those events
 reviewer_commands = ["/harness-review"]             # what runs when @u-pr is added as reviewer

.yamato/cmb-service-standalone-tests.yml

@@ -56,8 +56,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}
     # run_cmb_service.sh builds and starts a release version of the full CMB service (along with the limited echo server)
     - ./Tools/CI/service.cmb/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT -l artifacts
 
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
-    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --scripting-backend={{ backend }} --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --extra-player-arg=-batchmode --extra-player-arg=-nographics --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
   artifacts:
     logs:
       paths:

.yamato/console-standalone-test.yml

@@ -88,8 +88,17 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile)
 {% endif %}
   commands:
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
-    - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+    # Console players connect back to UTR's player connection listener on this VM, so the firewall must allow it
+    - command: gsudo NetSh Advfirewall set allprofiles state off
+    # Register the Bokken devkit with the platform's target tools so UTR can deploy and run the prebuilt player
+{% if platform.name == "ps5" %}
+    - command: gsudo prospero-ctrl target add %BOKKEN_DEVICE_IP%
+{% elsif platform.name == "ps4" %}
+    - command: orbis-ctrl add %BOKKEN_DEVICE_IP%
+{% else %}
+    - command: '"%GameDK%/bin/xbconnect" /WS %BOKKEN_DEVICE_IP%'
+{% endif %}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --scripting-backend=il2cpp --testproject={{ project.path }} --artifacts-path=test-results --player-load-path=build/players --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'

.yamato/desktop-standalone-tests.yml

@@ -44,7 +44,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% endif %}
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %}
-    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*;TestProject.RuntimeTests.*" --playergraphicsapi=Null --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout={{ test_timeout }}
   artifacts:
     players:
       paths:
@@ -79,8 +79,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% endif %}
 
   commands:
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
-    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --scripting-backend={{ backend }} --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --extra-player-arg=-batchmode --extra-player-arg=-nographics --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
   artifacts:
     logs:
       paths:

.yamato/mobile-standalone-test.yml

@@ -82,9 +82,6 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile)
 {% endif %}
   commands:
-    # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required
-    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %}
-
 {% if platform.standalone == "Android" %}
     # Download standalone UnityTestRunner and ADB setup
     - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub
@@ -97,11 +94,11 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     # Run tests for Android devices
     - |
       set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP%
-      UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }}
+      UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }}
 {% else %}
 
     # Run tests for non-Android devices
-    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --timeout={{ test_timeout }} --device-id=%BOKKEN_DEVICE_ID%
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --fail-on-assert --rerun-strategy=Test --retry={{ num_test_retries }} --timeout={{ test_timeout }} --device-id=%BOKKEN_DEVICE_ID%
 {% endif %}
   artifacts:
     logs:

Tools/CI/service.cmb/run_cmb_service.sh

@@ -171,8 +171,8 @@ cargo build --release --locked
 # The infinite loop is required as the service will exit each time all connected clients disconnect.
 # This means the service will exit after each test. The infinite loop will immediately restart the service each time it exits.
 logMessage "Running service integration tests..."
-echo "comb-server -l error --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version"
+echo "comb-server -l warn -l ngo::session=info -l comb_server::session=info --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version"
 
 while :; do
-  ./target/release/comb-server -l error --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version;
+  ./target/release/comb-server -l warn -l ngo::session=info -l comb_server::session=info --metrics-port 5000 standalone --port $service_port -t 60m --unity-version $unity_version;
 done & # <- use & to run the entire loop in the background

com.unity.netcode.gameobjects/Tests/Runtime/NetworkTransform/NetworkTransformOrderOfOperations.cs

@@ -24,6 +24,7 @@ namespace Unity.Netcode.RuntimeTests
     [TestFixture(HostOrServer.DAHost)]
     [TestFixture(HostOrServer.Host)]
     [TestFixture(HostOrServer.Server)]
+    [UnityPlatform(exclude = new[] { RuntimePlatform.PS4 })] // Ignored test tracked in MTT-16494
     internal class NetworkTransformOrderOfOperations : IntegrationTestWithApproximation
     {
         #region Configuration, Properties, and Overrides

com.unity.netcode.gameobjects/Tests/Runtime/NetworkVariable/NetworkVariableTests.cs

@@ -1935,7 +1935,6 @@ public string ArrayStr<T>(NativeArray<T> arr) where T : unmanaged
 
         [Test]
         [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Ignored test tracked in MTT-11343
-        [Repeat(5)]
 #if ENABLE_CORECLR
         [Explicit("NGO NetworkVariable serialization codegen not generated for some types on CoreCLR (falls back to FallbackSerializer), see https://jira.unity3d.com/browse/UUM-149592")]
 #endif
@@ -1998,7 +1997,7 @@ public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeArrayNetworkV
             }
             else if (testType == typeof(bool))
             {
-                (var original, var original2, var changed, var changed2) = GetArarys(RandGenBytes<bool>);
+                (var original, var original2, var changed, var changed2) = GetArarys((rand) => rand.Next(2) == 1);
                 TestValueTypeNativeArray(original, changed);
                 TestValueTypeNativeArray(original2, changed2);
             }
@@ -2400,7 +2399,6 @@ public string DictionaryStr<TKey, TVal>(Dictionary<TKey, TVal> list)
 
         [Test]
         [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Ignored test tracked in MTT-11343
-        [Repeat(5)]
 #if ENABLE_CORECLR
         [Explicit("NGO NetworkVariable serialization codegen not generated for some types on CoreCLR (falls back to FallbackSerializer), see https://jira.unity3d.com/browse/UUM-149592")]
 #endif
@@ -2598,7 +2596,6 @@ public void WhenSerializingAndDeserializingVeryLargeListNetworkVariables_ValuesA
 
         [Test]
         [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Ignored test tracked in MTT-11343
-        [Repeat(5)]
 #if ENABLE_CORECLR
         [Explicit("NGO NetworkVariable serialization codegen not generated for some types on CoreCLR (falls back to FallbackSerializer), see https://jira.unity3d.com/browse/UUM-149592")]
 #endif
@@ -2764,8 +2761,7 @@ public void WhenSerializingAndDeserializingVeryLargeHashSetNetworkVariables_Valu
         }
 
         [Test]
-        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer })] // Ignored test tracked in MTT-11343
-        [Repeat(5)]
+        [UnityPlatform(exclude = new[] { RuntimePlatform.Android, RuntimePlatform.IPhonePlayer, RuntimePlatform.PS4, RuntimePlatform.PS5 })] // Ignored test tracked in MTT-11343, MTT-16494
 #if ENABLE_CORECLR
         [Explicit("NGO NetworkVariable serialization codegen not generated for some types on CoreCLR (falls back to FallbackSerializer), see https://jira.unity3d.com/browse/UUM-149592")]
 #endif
@@ -3910,7 +3906,6 @@ public string NativeHashMapStr<TKey, TVal>(NativeHashMap<TKey, TVal> list)
         }
 
         [Test]
-        [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeListNetworkVariables_ValuesAreSerializedCorrectly(
 
             [Values(typeof(byte), typeof(sbyte), typeof(short), typeof(ushort), typeof(int), typeof(uint),
@@ -4101,11 +4096,10 @@ public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeListNetworkVa
         }
 
         [Test]
-        [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeHashSetNetworkVariables_ValuesAreSerializedCorrectly(
 
             [Values(typeof(byte), typeof(sbyte), typeof(short), typeof(ushort), typeof(int), typeof(uint),
-                typeof(long), typeof(ulong), typeof(bool), typeof(char), typeof(float), typeof(double),
+                typeof(long), typeof(ulong), typeof(char), typeof(float), typeof(double),
                 typeof(Vector2), typeof(Vector3), typeof(Vector2Int), typeof(Vector3Int), typeof(Vector4),
                 typeof(Quaternion), typeof(Pose), typeof(HashableNetworkVariableTestStruct), typeof(FixedString32Bytes))]
             Type testType)
@@ -4261,7 +4255,6 @@ public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeHashSetNetwor
         }
 
         [Test]
-        [Repeat(5)]
         public void WhenSerializingAndDeserializingVeryLargeValueTypeNativeHashMapNetworkVariables_ValuesAreSerializedCorrectly(
 
             [Values(typeof(byte), typeof(ulong), typeof(Vector2), typeof(HashMapKeyStruct))] Type keyType,

com.unity.netcode.gameobjects/Tests/Runtime/NetworkVariable/NetworkVariableTraitsTests.cs

@@ -100,7 +100,7 @@ public IEnumerator WhenNewValueIsLessThanThresholdButMaxTimeHasPassed_VariableIs
             var authorityComponent = GetAuthorityComponent();
             authorityComponent.TheVariable.CheckExceedsDirtinessThreshold = (in float value, in float newValue) => Mathf.Abs(newValue - value) >= 0.1;
             authorityComponent.TheVariable.SetUpdateTraits(new NetworkVariableUpdateTraits { MaxSecondsBetweenUpdates = 1.0f });
-            authorityComponent.TheVariable.LastUpdateSent = authorityComponent.NetworkManager.NetworkTimeSystem.LocalTime;
+            authorityComponent.TheVariable.UpdateLastSentTime();
 
             var timeoutHelper = new TimeoutHelper(0.62f);
             var newValue = 0.05f;
@@ -120,7 +120,7 @@ public IEnumerator WhenNewValueIsGreaterThanThresholdButMinTimeHasNotPassed_Vari
             var authorityComponent = GetAuthorityComponent();
             authorityComponent.TheVariable.CheckExceedsDirtinessThreshold = (in float value, in float newValue) => Mathf.Abs(newValue - value) >= 0.1;
             authorityComponent.TheVariable.SetUpdateTraits(new NetworkVariableUpdateTraits { MinSecondsBetweenUpdates = 1 });
-            authorityComponent.TheVariable.LastUpdateSent = authorityComponent.NetworkManager.NetworkTimeSystem.LocalTime;
+            authorityComponent.TheVariable.UpdateLastSentTime();
 
             var timeoutHelper = new TimeoutHelper(0.62f);
             var newValue = 0.15f;
@@ -139,7 +139,7 @@ public IEnumerator WhenNoThresholdIsSetButMinTimeHasNotPassed_VariableIsNotSeria
         {
             var authorityComponent = GetAuthorityComponent();
             authorityComponent.TheVariable.SetUpdateTraits(new NetworkVariableUpdateTraits { MinSecondsBetweenUpdates = 1 });
-            authorityComponent.TheVariable.LastUpdateSent = authorityComponent.NetworkManager.NetworkTimeSystem.LocalTime;
+            authorityComponent.TheVariable.UpdateLastSentTime();
 
             var timeoutHelper = new TimeoutHelper(0.62f);
             var newValue = 0.15f;
@@ -163,7 +163,7 @@ public IEnumerator WhenNonTraitsIsDirtyButTraitsIsNotReadyToSend()
         {
             var authorityComponent = GetAuthorityComponent();
             authorityComponent.TheVariable.SetUpdateTraits(new NetworkVariableUpdateTraits { MinSecondsBetweenUpdates = 1 });
-            authorityComponent.TheVariable.LastUpdateSent = authorityComponent.NetworkManager.NetworkTimeSystem.LocalTime;
+            authorityComponent.TheVariable.UpdateLastSentTime();
 
             var timeoutHelper = new TimeoutHelper(0.62f);
             var firstValue = 0.15f;

com.unity.netcode.gameobjects/Tests/Runtime/TestHelpers/Components/ObjectNameIdentifier.cs

@@ -9,7 +9,7 @@ namespace Unity.Netcode.TestHelpers.Runtime
     /// </summary>
     public class ObjectNameIdentifier : NetworkBehaviour
     {
-        private ulong m_CurrentOwner;
+        private ulong m_RegisteredLocalClientId;
         private ulong m_CurrentNetworkObjectId;
         private bool m_IsRegistered;
 
@@ -45,18 +45,18 @@ protected void RegisterAndLabelNetworkObject()
                 // This is required otherwise it will try to continue to update the NetworkBehaviour even if
                 // it has been destroyed.
                 m_NetworkObject = NetworkObject;
-                m_CurrentOwner = OwnerClientId;
+                m_RegisteredLocalClientId = NetworkManager.LocalClientId;
                 m_CurrentNetworkObjectId = NetworkObjectId;
 
                 var serverOrClient = IsServer ? "Server" : "Client";
                 if (NetworkObject.IsPlayerObject)
                 {
-                    gameObject.name = NetworkManager.LocalClientId == OwnerClientId ? $"{m_OriginalName}-{k_TagInfoStart}{OwnerClientId}{k_TagInfoStop}-Local{m_OriginalName}" :
-                        $"{m_OriginalName}-{k_TagInfoStart}{OwnerClientId}{k_TagInfoStop}- On{serverOrClient}{k_TagInfoStart}{NetworkManager.LocalClientId}{k_TagInfoStop}";
+                    gameObject.name = m_RegisteredLocalClientId == OwnerClientId ? $"{m_OriginalName}-{k_TagInfoStart}{OwnerClientId}{k_TagInfoStop}-Local{m_OriginalName}" :
+                        $"{m_OriginalName}-{k_TagInfoStart}{OwnerClientId}{k_TagInfoStop}- On{serverOrClient}{k_TagInfoStart}{m_RegisteredLocalClientId}{k_TagInfoStop}";
                 }
                 else
                 {
-                    gameObject.name = $"{m_OriginalName}{k_TagInfoStart}{NetworkObjectId}{k_TagInfoStop}-On{serverOrClient}{k_TagInfoStart}{NetworkManager.LocalClientId}{k_TagInfoStop}";
+                    gameObject.name = $"{m_OriginalName}{k_TagInfoStart}{NetworkObjectId}{k_TagInfoStop}-On{serverOrClient}{k_TagInfoStart}{m_RegisteredLocalClientId}{k_TagInfoStop}";
                 }
 
                 // Don't add the player objects to the global list of NetworkObjects
@@ -75,7 +75,7 @@ protected void DeRegisterNetworkObject()
         {
             if (m_IsRegistered)
             {
-                NetcodeIntegrationTest.DeregisterNetworkObject(m_CurrentOwner, m_CurrentNetworkObjectId);
+                NetcodeIntegrationTest.DeregisterNetworkObject(m_RegisteredLocalClientId, m_CurrentNetworkObjectId);
                 m_IsRegistered = false;
             }
         }

com.unity.netcode.gameobjects/Tests/Runtime/TestHelpers/IntegrationTestSceneHandler.cs

@@ -372,7 +372,8 @@ private bool ExcludeSceneFromSynchronizationCheck(Scene scene)
 
         public void ClearSceneTracking(NetworkManager networkManager)
         {
-            SceneNameToSceneHandles.Clear();
+            // Only clear the calling NetworkManager's tracking, other NetworkManagers may still be tracking their own scenes
+            SceneNameToSceneHandles.Remove(networkManager);
         }
 
         public void StopTrackingScene(NetworkSceneHandle handle, string name, NetworkManager networkManager)