[XABT] Move scanning for ACW map JLOs to FindJavaObjectsStep. (#9930)

Context: ea399ed
Context: 25d1f00
Context: https://github.com/xamarin/monodroid/commit/a04b73b30e5bbd81b3773865c313a8084f0a21d1
Context: dotnet/java-interop@bc44f08

Building on ea399ed, move the process of scanning for
`Java.Lang.Object` and `Java.Lang.Throwable` subclasses needed for
the `acw-map.txt` generation task to the `FindJavaObjectsStep`
"linker step".

For an example of what `acw-map.txt` is for, see 25d1f00 and see the
"`<R8/>` and `acw-map.txt` parsing" section.

This process currently captures `interface` types, which
`JavaCallableWrappers` doesn't support.  Expanding the Java Callable
Wrapper serialization format (dotnet/java-interop@bc44f085) to allow
interfaces and other `acw-map.txt`-needed data it didn't already
support resulted in a lot of extra processing and waste.

Instead, expand the `.jlo.xml` file to have two sections, one for JCW
needed data and one for `acw-map.txt` needed data:

	<api>
	  <jcw-types>
	    <type name="MainActivity"
	        package="crc645107ba1b8b6ee4d3"
	        application_java_class="android.app.Application"
	        mono_runtime_initialization="mono.MonoPackageManager.LoadApplication (context);"
	        extends_type="android.app.Activity"
	        partial_assembly_qualified_name="tempbuild.MainActivity, tempbuild">
	      <constructors>
	        <constructor name="MainActivity"
	            method="n_.ctor:()V:"
	            jni_signature="()V"
	            managed_parameters=""
	            retval="void"
	            is_dynamically_registered="True" />
	      </constructors>
	      <methods>
	        <method name="clone"
	            method="n_clone:()Ljava/lang/Object;:GetCloneHandler"
	            jni_signature="()Ljava/lang/Object;"
	            retval="java.lang.Object" />
	        <method name="onCreate"
	            method="n_onCreate:(Landroid/os/Bundle;)V:GetOnCreate_Landroid_os_Bundle_Handler"\
	            jni_signature="(Landroid/os/Bundle;)V"
	            params="android.os.Bundle p0"
	            retval="void"
	            super_call="p0"
	            activate_call="p0" />
	      </methods>
	    </type>
	  </jcw-types>
	  <acw-types partial-assembly-name="tempbuild" module-name="tempbuild.dll">
	    <type
	        assembly-qualified-name="tempbuild.MainActivity, tempbuild, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null"
	        compat-jni-name="tempbuild.MainActivity"
	        java-key="crc645107ba1b8b6ee4d3.MainActivity"
	        managed-key="tempbuild.MainActivity"
	        partial-assembly-qualified-name="tempbuild.MainActivity, tempbuild" />
	  </acw-types>
	</api>

Additionally, for assemblies that cannot contain `Java.Lang.Object`
subclasses that we do not need to scan (e.g. `System.Runtime.dll`),
we no longer write an "empty" XML file like this:

	<types was_scanned="False" />

Instead, we now write an actual empty (0 byte) file to disk, which
saves build time when deserializing.  

As with ea399ed, this temporarily leaves the old `acw-map.txt`
generation code in place, guarded behind the
`$(_AndroidJLOCheckedBuild)` MSBuild property.  This property generates
the `acw-map.txt` file in both the new and old way, and emits a build
error if there are differences.  (A consistent sort was added so that
both maps are sorted the same.)

TODO: re-evaluate `acw-map.txt` semantics: does it *need* interfaces?

Author: jpobst

external/Java.Interop

@@ -40,23 +40,20 @@ public bool ProcessAssembly (AssemblyDefinition assembly, string destinationJLOX
 		var initial_count = types.Count;
 
 		// Filter out Java types we don't care about
-		types = types.Where (t => !t.IsInterface && !JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (t, Context)).ToList ();
+		types = types.Where (t => !JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (t, Context)).ToList ();
 
 		Log.LogDebugMessage ($"{assembly.Name.Name} - Found {initial_count} Java types, filtered to {types.Count}");
 
-		var wrappers = ConvertToCallableWrappers (types);
+		var xml = new JavaObjectsXmlFile ();
 
-		using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-			XmlExporter.Export (sw, wrappers, true);
-			Files.CopyIfStreamChanged (sw.BaseStream, destinationJLOXml);
-		}
+		xml.ACWMapEntries.AddRange (types.Select (t => ACWMapEntry.Create (t, Context)));
+		xml.JavaCallableWrappers.AddRange (ConvertToCallableWrappers (types.Where (t => !t.IsInterface).ToList ()));
 
-		return true;
-	}
+		xml.Export (destinationJLOXml);
 
-	public static void WriteEmptyXmlFile (string destination)
-	{
-		XmlExporter.Export (destination, [], false);
+		Log.LogDebugMessage ($"Wrote '{destinationJLOXml}', {xml.JavaCallableWrappers.Count} JCWs, {xml.ACWMapEntries.Count} ACWs");
+
+		return true;
 	}
 
 	List<TypeDefinition> ScanForJavaTypes (AssemblyDefinition assembly)

src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/FindJavaObjectsStep.cs

@@ -147,11 +147,11 @@ protected virtual void CreateRunState (RunState runState, MSBuildLinkContext con
 
 	protected virtual void RunPipeline (ITaskItem source, ITaskItem destination, RunState runState, WriterParameters writerParameters)
 	{
-		var destinationJLOXml = Path.ChangeExtension (destination.ItemSpec, ".jlo.xml");
+		var destinationJLOXml = JavaObjectsXmlFile.GetJavaObjectsXmlFilePath (destination.ItemSpec);
 
 		if (!TryScanForJavaObjects (source, destination, runState, writerParameters)) {
 			// Even if we didn't scan for Java objects, we still write an empty .xml file for later steps
-			FindJavaObjectsStep.WriteEmptyXmlFile (destinationJLOXml);
+			JavaObjectsXmlFile.WriteEmptyFile (destinationJLOXml, Log);
 		}
 	}
 
@@ -160,7 +160,7 @@ bool TryScanForJavaObjects (ITaskItem source, ITaskItem destination, RunState ru
 		if (!ShouldScanAssembly (source))
 			return false;
 
-		var destinationJLOXml = Path.ChangeExtension (destination.ItemSpec, ".jlo.xml");
+		var destinationJLOXml = JavaObjectsXmlFile.GetJavaObjectsXmlFilePath (destination.ItemSpec);
 		var assemblyDefinition = runState.resolver!.GetAssembly (source.ItemSpec);
 
 		var scanned = runState.findJavaObjectsStep!.ProcessAssembly (assemblyDefinition, destinationJLOXml);

src/Xamarin.Android.Build.Tasks/Tasks/AssemblyModifierPipeline.cs

@@ -1,5 +1,7 @@
 #nullable enable
 using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using Microsoft.Android.Build.Tasks;
 using Microsoft.Build.Framework;
@@ -17,29 +19,68 @@ public class GenerateACWMap : AndroidTask
 	[Required]
 	public string IntermediateOutputDirectory { get; set; } = "";
 
+	[Required]
+	public ITaskItem [] ResolvedAssemblies { get; set; } = [];
+
+	// This property is temporary and is used to ensure that the new "linker step"
+	// JLO scanning produces the same results as the old process. It will be removed
+	// once the process is complete.
+	public bool RunCheckedBuild { get; set; }
+
+	[Required]
+	public string [] SupportedAbis { get; set; } = [];
+
 	public override bool RunTask ()
 	{
-		// Retrieve the stored NativeCodeGenState
-		var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
-			MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
-			RegisteredTaskObjectLifetime.Build
-		);
+		// Temporarily used to ensure we still generate the same as the old code
+		if (RunCheckedBuild) {
+			// Retrieve the stored NativeCodeGenState
+			var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState>> (
+				MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory),
+				RegisteredTaskObjectLifetime.Build
+			);
 
-		// We only need the first architecture, since this task is architecture-agnostic
-		var templateCodeGenState = nativeCodeGenStates.First ().Value;
+			// We only need the first architecture, since this task is architecture-agnostic
+			var templateCodeGenState = nativeCodeGenStates.First ().Value;
 
-		var acwMapGen = new ACWMapGenerator (Log);
+			var acwMapGen = new ACWMapGenerator (Log);
 
-		if (!acwMapGen.Generate (templateCodeGenState, AcwMapFile)) {
-			Log.LogDebugMessage ("ACW map generation failed");
-		}
+			if (!acwMapGen.Generate (templateCodeGenState, AcwMapFile)) {
+				Log.LogDebugMessage ("ACW map generation failed");
+			}
 
-		if (Log.HasLoggedErrors) {
-			// Ensure that on a rebuild, we don't *skip* the `_GenerateJavaStubs` target,
-			// by ensuring that the target outputs have been deleted.
-			Files.DeleteFile (AcwMapFile, Log);
+			return !Log.HasLoggedErrors;
 		}
 
+		GenerateMap ();
+
 		return !Log.HasLoggedErrors;
 	}
+
+	void GenerateMap ()
+	{
+		// Get the set of assemblies for the "first" ABI. The ACW map is
+		// not ABI-specific, so we can use any ABI to generate the wrappers.
+		var allAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, SupportedAbis, validate: true);
+		var singleArchAssemblies = allAssembliesPerArch.First ().Value.Values.ToList ();
+
+		var entries = new List<ACWMapEntry> ();
+
+		foreach (var assembly in singleArchAssemblies) {
+			var wrappersPath = JavaObjectsXmlFile.GetJavaObjectsXmlFilePath (assembly.ItemSpec);
+
+			if (!File.Exists (wrappersPath)) {
+				Log.LogError ($"'{wrappersPath}' not found.");
+				return;
+			}
+
+			var xml = JavaObjectsXmlFile.Import (wrappersPath, JavaObjectsXmlFileReadType.AndroidResourceFixups);
+
+			entries.AddRange (xml.ACWMapEntries);
+		}
+
+		var acwMapGen = new ACWMapGenerator (Log);
+
+		acwMapGen.Generate (entries, AcwMapFile);
+	}
 }

src/Xamarin.Android.Build.Tasks/Tasks/GenerateACWMap.cs

@@ -5,7 +5,6 @@
 using System.IO;
 using System.Linq;
 using Java.Interop.Tools.JavaCallableWrappers;
-using Java.Interop.Tools.JavaCallableWrappers.Adapters;
 using Java.Interop.Tools.JavaCallableWrappers.CallableWrapperMembers;
 using Java.Interop.Tools.TypeNameMappings;
 using Microsoft.Android.Build.Tasks;
@@ -63,16 +62,21 @@ void GenerateWrappers (List<ITaskItem> assemblies)
 		var sw = Stopwatch.StartNew ();
 
 		foreach (var assembly in assemblies) {
-			var assemblyPath = assembly.ItemSpec;
-			var assemblyName = Path.GetFileNameWithoutExtension (assemblyPath);
-			var wrappersPath = Path.Combine (Path.GetDirectoryName (assemblyPath), $"{assemblyName}.jlo.xml");
+			var wrappersPath = JavaObjectsXmlFile.GetJavaObjectsXmlFilePath (assembly.ItemSpec);
 
 			if (!File.Exists (wrappersPath)) {
 				Log.LogError ($"'{wrappersPath}' not found.");
 				return;
 			}
 
-			wrappers.AddRange (XmlImporter.Import (wrappersPath, out var _));
+			var xml = JavaObjectsXmlFile.Import (wrappersPath, JavaObjectsXmlFileReadType.JavaCallableWrappers);
+
+			if (xml.JavaCallableWrappers.Count == 0) {
+				Log.LogDebugMessage ($"'{wrappersPath}' is empty, skipping.");
+				continue;
+			}
+
+			wrappers.AddRange (xml.JavaCallableWrappers);
 		}
 
 		Log.LogDebugMessage ($"Deserialized {wrappers.Count} Java callable wrappers in {sw.ElapsedMilliseconds}ms");

src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaCallableWrappers.cs

@@ -277,15 +277,14 @@ void CompareScannedAssemblies ()
 			// Find every assembly that was scanned by the linker by looking at the .jlo.xml files
 			foreach (var assembly in ResolvedAssemblies) {
 				var assemblyPath = assembly.ItemSpec;
-				var assemblyName = Path.GetFileNameWithoutExtension (assemblyPath);
-				var wrappersPath = Path.Combine (Path.GetDirectoryName (assemblyPath), $"{assemblyName}.jlo.xml");
+				var wrappersPath = JavaObjectsXmlFile.GetJavaObjectsXmlFilePath (assembly.ItemSpec);
 
 				if (!File.Exists (wrappersPath))
 					Log.LogError ($"'{wrappersPath}' not found.");
 
-				XmlImporter.Import (wrappersPath, out var wasScanned);
+				var xml = JavaObjectsXmlFile.Import (wrappersPath, JavaObjectsXmlFileReadType.None);
 
-				if (wasScanned) {
+				if (xml.WasScanned) {
 					Log.LogDebugMessage ($"CompareScannedAssemblies: Found scanned assembly .jlo.xml '{assemblyPath}'");
 					linker_scanned_assemblies.Add (assembly.ItemSpec);
 				}

src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
-
+using System.IO;
+using System.Linq;
+using System.Xml.Linq;
 using Java.Interop.Tools.Cecil;
 using Java.Interop.Tools.TypeNameMappings;
 using Microsoft.Android.Build.Tasks;
@@ -33,7 +35,7 @@ public bool Generate (NativeCodeGenState codeGenState, string acwMapFile)
 		bool success = true;
 
 		using var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ();
-		foreach (TypeDefinition type in javaTypes) {
+		foreach (TypeDefinition type in javaTypes.OrderBy (t => t.FullName.Replace ('/', '.'))) {
 			string managedKey = type.FullName.Replace ('/', '.');
 			string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.');
 
@@ -79,7 +81,19 @@ public bool Generate (NativeCodeGenState codeGenState, string acwMapFile)
 		}
 
 		acw_map.Flush ();
-		Files.CopyIfStreamChanged (acw_map.BaseStream, acwMapFile);
+
+		// If there's conflicts, the "new way" file never got written, and will show up as
+		// "changed" in our comparison test, so skip it.
+		if (javaConflicts.Count > 0) {
+			return false;
+		}
+
+		if (Files.HasStreamChanged (acw_map.BaseStream, acwMapFile)) {
+			log.LogError ($"ACW map file '{acwMapFile}' changed");
+			Files.CopyIfStreamChanged (acw_map.BaseStream, acwMapFile + "2");
+		} else {
+			log.LogDebugMessage ($"ACW map file '{acwMapFile}' unchanged");
+		}
 
 		foreach (var kvp in managedConflicts) {
 			log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value));
@@ -99,4 +113,121 @@ public bool Generate (NativeCodeGenState codeGenState, string acwMapFile)
 
 		return success;
 	}
+
+	public void Generate (List<ACWMapEntry> javaTypes, string acwMapFile)
+	{
+		// We need to save a map of .NET type -> ACW type for resource file fixups
+		var managed = new Dictionary<string, ACWMapEntry> (javaTypes.Count, StringComparer.Ordinal);
+		var java = new Dictionary<string, ACWMapEntry> (javaTypes.Count, StringComparer.Ordinal);
+
+		var managedConflicts = new Dictionary<string, List<string>> (0, StringComparer.Ordinal);
+		var javaConflicts = new Dictionary<string, List<string>> (0, StringComparer.Ordinal);
+
+		using var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ();
+
+		foreach (var type in javaTypes.OrderBy (t => t.ManagedKey)) {
+			string managedKey = type.ManagedKey;
+			string javaKey = type.JavaKey;
+
+			acw_map.Write (type.PartialAssemblyQualifiedName);
+			acw_map.Write (';');
+			acw_map.Write (javaKey);
+			acw_map.WriteLine ();
+
+			ACWMapEntry conflict;
+			bool hasConflict = false;
+
+			if (managed.TryGetValue (managedKey, out conflict)) {
+				if (!conflict.ModuleName.Equals (type.ModuleName)) {
+					if (!managedConflicts.TryGetValue (managedKey, out var list))
+						managedConflicts.Add (managedKey, list = new List<string> { conflict.PartialAssemblyName });
+					list.Add (type.PartialAssemblyName);
+				}
+				hasConflict = true;
+			}
+
+			if (java.TryGetValue (javaKey, out conflict)) {
+				if (!conflict.ModuleName.Equals(type.ModuleName)) {
+					if (!javaConflicts.TryGetValue (javaKey, out var list))
+						javaConflicts.Add (javaKey, list = new List<string> { conflict.AssemblyQualifiedName });
+					list.Add (type.AssemblyQualifiedName);
+				}
+				hasConflict = true;
+			}
+
+			if (!hasConflict) {
+				managed.Add (managedKey, type);
+				java.Add (javaKey, type);
+
+				acw_map.Write (managedKey);
+				acw_map.Write (';');
+				acw_map.Write (javaKey);
+				acw_map.WriteLine ();
+
+				acw_map.Write (type.CompatJniName);
+				acw_map.Write (';');
+				acw_map.Write (javaKey);
+				acw_map.WriteLine ();
+			}
+		}
+
+		acw_map.Flush ();
+
+		foreach (var kvp in managedConflicts) {
+			log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value));
+			log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]);
+		}
+
+		foreach (var kvp in javaConflicts) {
+			log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key);
+
+			foreach (var typeName in kvp.Value) {
+				log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName);
+			}
+		}
+
+		// Don't write the output file if there are any errors so that
+		// future incremental builds will try again.
+		if (javaConflicts.Count > 0)
+			return;
+
+		Files.CopyIfStreamChanged (acw_map.BaseStream, acwMapFile);
+	}
+}
+
+class ACWMapEntry
+{
+	public string AssemblyQualifiedName { get; set; }
+	public string CompatJniName { get; set; }
+	public string JavaKey { get; set; }
+	public string ManagedKey { get; set; }
+	public string ModuleName { get; set; }
+	public string PartialAssemblyName { get; set; }
+	public string PartialAssemblyQualifiedName { get; set; }
+
+	public static ACWMapEntry Create (TypeDefinition type, TypeDefinitionCache cache)
+	{
+		return new ACWMapEntry {
+			AssemblyQualifiedName = type.GetAssemblyQualifiedName (cache),
+			CompatJniName = JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.'),
+			JavaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'),
+			ManagedKey = type.FullName.Replace ('/', '.'),
+			ModuleName = type.Module.Name,
+			PartialAssemblyName = type.GetPartialAssemblyName (cache),
+			PartialAssemblyQualifiedName = type.GetPartialAssemblyQualifiedName (cache),
+		};
+	}
+
+	public static ACWMapEntry Create (XElement type, string partialAssemblyName, string moduleName)
+	{
+		return new ACWMapEntry {
+			AssemblyQualifiedName = type.GetAttributeOrDefault ("assembly-qualified-name", string.Empty),
+			CompatJniName = type.GetAttributeOrDefault ("compat-jni-name", string.Empty),
+			JavaKey = type.GetAttributeOrDefault ("java-key", string.Empty),
+			ManagedKey = type.GetAttributeOrDefault ("managed-key", string.Empty),
+			ModuleName = moduleName,
+			PartialAssemblyName = partialAssemblyName,
+			PartialAssemblyQualifiedName = type.GetAttributeOrDefault ("partial-assembly-qualified-name", string.Empty),
+		};
+	}
 }