[ci] Add Windows 'DebuggingTest' job (#9951)

A new [Intel based Windows pool][0] has been configured to run Android
emulator test jobs on. This pool currently has capacity to scale up to
10 agents.

The `StartAndroidEmulator` task and the execution environment for the
`AcquireAndroidTarget` target have been updated to address issues with
emulator launch attempts hanging on these Windows VMs with this
seemingly related output:

    The STDIO streams did not close within 10 seconds of the exit event from process 'C:\Program Files\dotnet\dotnet.exe'. This may indicate a child process inherited the STDIO streams and has not yet exited.

A new "Windows > Tests > Debugging" job has been added to the MSBuild
Device tests stage that currently runs all tests in the `DebuggingTest`
category.

Starting with only one new job will allow us to start to scale up on
Windows while monitoring for potential performance/reliabilty issues.

[0]: https://ms.portal.azure.com/?feature.customportal=false#@microsoft.onmicrosoft.com/resource/subscriptions/cd4829e2-e38b-43d2-8316-2f2009f36f97/resourcegroups/1ESobjects/providers/Microsoft.CloudTest/hostedpools/Android-1ESPT/overview

Author: pjcollins

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs

@@ -37,13 +37,13 @@ public class StartAndroidEmulator : Task
 
 		public override bool Execute ()
 		{
-			Run (GetEmulatorPath ());
+			var ranSuccessfully = Run (GetEmulatorPath ());
 
 			if (!string.IsNullOrEmpty (Port)) {
 				AdbTarget   = $"-s emulator-{Port}";
 			}
 
-			return !Log.HasLoggedErrors;
+			return ranSuccessfully && !Log.HasLoggedErrors;
 		}
 
 		string GetEmulatorPath ()
@@ -63,10 +63,10 @@ string GetEmulatorPath ()
 			return path;
 		}
 
-		void Run (string emulator)
+		bool Run (string emulator)
 		{
 			if (emulator == null)
-				return;
+				return false;
 
 			var port = string.IsNullOrEmpty (Port) ? "" : $"-port {Port}";
 			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {port} -avd {ImageName}";
@@ -136,40 +136,26 @@ void Run (string emulator)
 			p.OutputDataReceived  += output;
 			p.ErrorDataReceived   += error;
 
-			p.Start ();
-			p.BeginOutputReadLine ();
-			p.BeginErrorReadLine ();
-
-			const int Timeout = 20*1000;
-			int i = WaitHandle.WaitAny (new[]{sawError}, millisecondsTimeout: Timeout);
-			if (i == 0 || Log.HasLoggedErrors) {
-				p.Kill ();
-				return;
+			try {
+				p.Start ();
+				EmulatorProcessId = p.Id;
+				p.BeginOutputReadLine ();
+				p.BeginErrorReadLine ();
+
+				const int Timeout = 20*1000;
+				int i = WaitHandle.WaitAny (new[]{sawError}, millisecondsTimeout: Timeout);
+				if (i == 0 || Log.HasLoggedErrors) {
+					p.Kill ();
+					return false;
+				}
+			} finally {
+				p.CancelOutputRead ();
+				p.CancelErrorRead ();
+				p.OutputDataReceived  -= output;
+				p.ErrorDataReceived   -= error;
 			}
-
-			p.CancelOutputRead ();
-			p.CancelErrorRead ();
-
-			p.OutputDataReceived  -= output;
-			p.ErrorDataReceived   -= error;
-
-			p.OutputDataReceived  += WriteProcessOutputMessage;
-			p.ErrorDataReceived   += WriteProcessErrorMessage;
-
-			p.BeginOutputReadLine ();
-			p.BeginErrorReadLine ();
-
-			EmulatorProcessId = p.Id;
-		}
-
-		static void WriteProcessOutputMessage (object sender, DataReceivedEventArgs e)
-		{
-			Console.WriteLine (e.Data);
+			return true;
 		}
 
-		static void WriteProcessErrorMessage (object sender, DataReceivedEventArgs e)
-		{
-			Console.Error.WriteLine (e.Data);
-		}
 	}
 }

build-tools/automation/yaml-templates/run-dotnet-preview.yaml

@@ -9,6 +9,7 @@ parameters:
   condition: succeeded()
   continueOnError: true
   retryCountOnTaskFailure: 0
+  taskTimeoutInMinutes: 0
 
 steps:
 - powershell: |
@@ -31,3 +32,4 @@ steps:
   condition: ${{ parameters.condition }}
   continueOnError: ${{ parameters.continueOnError }}
   retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
+  timeoutInMinutes: ${{ parameters.taskTimeoutInMinutes }}

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -154,3 +154,55 @@ stages:
     - template: /build-tools/automation/yaml-templates/fail-on-issue.yaml
       parameters:
         condition: ${{ parameters.shouldFailOnIssue }}
+
+  - job: debug_windows_tests
+    displayName: Windows > Tests > Debugging
+    timeoutInMinutes: 180
+    pool:
+      name: Android-1ESPT
+      image: $(WindowsPoolImage1ESPT)
+      os: windows
+    workspace:
+      clean: all
+    steps:
+    - template: /build-tools/automation/yaml-templates/setup-test-environment.yaml
+      parameters:
+        installTestSlicer: true
+        installApkDiff: false
+        xaSourcePath: ${{ parameters.xaSourcePath }}
+        repositoryAlias: ${{ parameters.repositoryAlias }}
+        commit: ${{ parameters.commit }}
+        xaprepareScenario: EmulatorTestDependencies
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifactName: $(TestAssembliesArtifactName)
+        downloadPath: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)
+
+    - template: /build-tools/automation/yaml-templates/start-stop-emulator.yaml
+      parameters:
+        xaSourcePath: ${{ parameters.xaSourcePath }}
+        startContinueOnError: ${{ parameters.emulatorStartContinueOnError }}
+
+    - template: /build-tools/automation/yaml-templates/run-sliced-nunit-tests.yaml
+      parameters:
+        testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\MSBuildDeviceIntegration\$(DotNetStableTargetFramework)\MSBuildDeviceIntegration.dll
+        testFilter: name == DebuggingTest
+        testRunTitle: DebuggingTest tests on Windows
+
+    - ${{ if ne(parameters.usesCleanImages, true) }}:
+      - template: /build-tools/automation/yaml-templates/start-stop-emulator.yaml
+        parameters:
+          command: stop
+          xaSourcePath: ${{ parameters.xaSourcePath }}
+
+    - template: /build-tools/automation/yaml-templates/upload-results.yaml
+      parameters:
+        configuration: $(XA.Build.Configuration)
+        artifactName: Test Results - Emulator Debugging - Windows-$(System.JobPositionInPhase)
+        xaSourcePath: ${{ parameters.xaSourcePath }}
+        use1ESTemplate: ${{ parameters.use1ESTemplate }}
+
+    - template: /build-tools/automation/yaml-templates/fail-on-issue.yaml
+      parameters:
+        condition: ${{ parameters.shouldFailOnIssue }}

build-tools/automation/yaml-templates/start-stop-emulator.yaml

@@ -7,16 +7,18 @@ parameters:
   avdType:                # Device AVD, like 'android-wear', required if 'specificImage' is 'true'
   emulatorMSBuildArgs:    # Extra args to pass to the emulator configuration like '-p:TestAvdExtraBootArgs=-writable-system', optional
   launchTimeoutMin: 20    # Minutes to wait for the emulator to start
+  taskTimeoutInMinutes: 30 # Minutes to wait for the task to finish
   startContinueOnError: false    # If 'true', do not mark the step as failed if starting the emulator fails
   xaSourcePath: $(System.DefaultWorkingDirectory) # working directory
 
 steps:
 - ${{ if eq(parameters.command, 'start') }}:
-  - task: DotNetCoreCLI@2
-    displayName: Start emulator
-    continueOnError: ${{ parameters.startContinueOnError }}
-    inputs:
-      projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
+  - template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml
+    parameters:
+      displayName: Start emulator
+      continueOnError: ${{ parameters.startContinueOnError }}
+      taskTimeoutInMinutes: ${{ parameters.taskTimeoutInMinutes }}
+      project: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
       ${{ if eq(parameters.specificImage, true) }}:
         arguments: >-
           -c $(XA.Build.Configuration) -t:"InstallAvdImage;AcquireAndroidTarget"
@@ -35,12 +37,13 @@ steps:
           -bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
 
 - ${{ if eq(parameters.command, 'stop') }}:
-  - task: DotNetCoreCLI@2
-    displayName: Stop emulator
-    condition: always()
-    continueOnError: true
-    inputs:
-      projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
+  - template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml
+    parameters:
+      displayName: Stop emulator
+      condition: always()
+      continueOnError: true
+      taskTimeoutInMinutes: ${{ parameters.taskTimeoutInMinutes }}
+      project: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
       ${{ if eq(parameters.specificImage, true) }}:
         arguments: >-
           -c $(XA.Build.Configuration) -t:"AcquireAndroidTarget,ReleaseAndroidTarget"

build-tools/scripts/TestApks.targets

@@ -79,7 +79,7 @@
         AvdManagerHome="$(AvdManagerHome)"
         Arguments="$(TestAvdExtraBootArgs)"
         ImageName="$(TestAvdName)"
-        LogcatFile="$(_LogcatFilenameBase)-full.txt"
+        LogcatFile="$(_LogcatFilenameBase)-start-emu.txt"
         Port="$(_AdbEmulatorPort)"
         ToolExe="$(EmulatorToolExe)"
         ToolPath="$(EmulatorToolPath)">