From 309c61ab64b070904d3bd48da88bd6bb222d5c05 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Mon, 1 Dec 2025 16:48:53 -0500 Subject: [PATCH 01/11] Provide the downloadable metal toolchain to swiftbuild --- Fixtures/Metal/SimpleLibrary/Package.swift | 19 +++++ .../Sources/MyRenderer/Renderer.swift | 4 ++ .../Sources/MyRenderer/Shaders.metal | 12 ++++ .../MySharedTypes/include/SharedTypes.h | 14 ++++ .../MyRendererTests/MyRendererTests.swift | 6 ++ Package.swift | 7 ++ Sources/PackageModel/Toolchain.swift | 3 + .../PackageModel/ToolchainConfiguration.swift | 10 ++- Sources/PackageModel/UserToolchain.swift | 38 +++++++++- .../SwiftBuildSupport/SwiftBuildSystem.swift | 9 +++ .../MockBuildTestHelper.swift | 2 + Tests/BuildMetalTests/BuildMetalTests.swift | 69 +++++++++++++++++++ 12 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 Fixtures/Metal/SimpleLibrary/Package.swift create mode 100644 Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Renderer.swift create mode 100644 Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Shaders.metal create mode 100644 Fixtures/Metal/SimpleLibrary/Sources/MySharedTypes/include/SharedTypes.h create mode 100644 Fixtures/Metal/SimpleLibrary/Tests/MyRendererTests/MyRendererTests.swift create mode 100644 Tests/BuildMetalTests/BuildMetalTests.swift diff --git a/Fixtures/Metal/SimpleLibrary/Package.swift b/Fixtures/Metal/SimpleLibrary/Package.swift new file mode 100644 index 00000000000..36228ca965d --- /dev/null +++ b/Fixtures/Metal/SimpleLibrary/Package.swift @@ -0,0 +1,19 @@ +// swift-tools-version: 6.2 +// The swift-tools-version declares the minimum version of Swift required to build this package. +import PackageDescription + +let package = Package( + name: "MyRenderer", + products: [ + .library( + name: "MyRenderer", + targets: ["MyRenderer"]), + ], + targets: [ + .target( + name: "MyRenderer", + dependencies: ["MySharedTypes"]), + + .target(name: "MySharedTypes") + ] +) diff --git a/Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Renderer.swift b/Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Renderer.swift new file mode 100644 index 00000000000..816eb50119b --- /dev/null +++ b/Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Renderer.swift @@ -0,0 +1,4 @@ +import MySharedTypes + + +let vertex = AAPLVertex(position: .init(250, -250), color: .init(1, 0, 0, 1)) diff --git a/Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Shaders.metal b/Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Shaders.metal new file mode 100644 index 00000000000..491edf63048 --- /dev/null +++ b/Fixtures/Metal/SimpleLibrary/Sources/MyRenderer/Shaders.metal @@ -0,0 +1,12 @@ +// A relative path to SharedTypes.h. +#import "../MySharedTypes/include/SharedTypes.h" + +#include +using namespace metal; + +vertex float4 simpleVertexShader(const device AAPLVertex *vertices [[buffer(0)]], + uint vertexID [[vertex_id]]) { + AAPLVertex in = vertices[vertexID]; + return float4(in.position.x, in.position.y, 0.0, 1.0); +} + diff --git a/Fixtures/Metal/SimpleLibrary/Sources/MySharedTypes/include/SharedTypes.h b/Fixtures/Metal/SimpleLibrary/Sources/MySharedTypes/include/SharedTypes.h new file mode 100644 index 00000000000..ea51fd839d4 --- /dev/null +++ b/Fixtures/Metal/SimpleLibrary/Sources/MySharedTypes/include/SharedTypes.h @@ -0,0 +1,14 @@ +#ifndef SharedTypes_h +#define SharedTypes_h + + +#import + + +typedef struct { + vector_float2 position; + vector_float4 color; +} AAPLVertex; + + +#endif /* SharedTypes_h */ diff --git a/Fixtures/Metal/SimpleLibrary/Tests/MyRendererTests/MyRendererTests.swift b/Fixtures/Metal/SimpleLibrary/Tests/MyRendererTests/MyRendererTests.swift new file mode 100644 index 00000000000..fad1d528b79 --- /dev/null +++ b/Fixtures/Metal/SimpleLibrary/Tests/MyRendererTests/MyRendererTests.swift @@ -0,0 +1,6 @@ +import Testing +@testable import MyRenderer + +@Test func example() async throws { + // Write your test here and use APIs like `#expect(...)` to check expected conditions. +} diff --git a/Package.swift b/Package.swift index aa32ba1e384..a1b09a03ae8 100644 --- a/Package.swift +++ b/Package.swift @@ -991,6 +991,13 @@ let package = Package( name: "SwiftBuildSupportTests", dependencies: ["SwiftBuildSupport", "_InternalTestSupport", "_InternalBuildTestSupport"] ), + .testTarget( + name: "BuildMetalTests", + dependencies: [ + "_InternalTestSupport", + "Basics" + ] + ), // Examples (These are built to ensure they stay up to date with the API.) .executableTarget( name: "package-info", diff --git a/Sources/PackageModel/Toolchain.swift b/Sources/PackageModel/Toolchain.swift index fb2fbce9f17..d1f6e241311 100644 --- a/Sources/PackageModel/Toolchain.swift +++ b/Sources/PackageModel/Toolchain.swift @@ -47,6 +47,9 @@ public protocol Toolchain { /// The manifest and library locations used by this toolchain. var swiftPMLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation { get } + /// Path to the Metal toolchain if available.. + var metalToolchainPath: AbsolutePath? { get } + /// Path of the `clang` compiler. func getClangCompiler() throws -> AbsolutePath diff --git a/Sources/PackageModel/ToolchainConfiguration.swift b/Sources/PackageModel/ToolchainConfiguration.swift index 79d49a1aa04..26655dd7b1c 100644 --- a/Sources/PackageModel/ToolchainConfiguration.swift +++ b/Sources/PackageModel/ToolchainConfiguration.swift @@ -46,6 +46,10 @@ public struct ToolchainConfiguration { /// Currently computed only for Windows. public var swiftTestingPath: AbsolutePath? + /// Path to the Metal toolchain. + /// This is optional and only available on Darwin platforms. + public var metalToolchainPath: AbsolutePath? + /// Creates the set of manifest resources associated with a `swiftc` executable. /// /// - Parameters: @@ -56,6 +60,8 @@ public struct ToolchainConfiguration { /// - swiftPMLibrariesRootPath: Custom path for SwiftPM libraries. Computed based on the compiler path by default. /// - sdkRootPath: Optional path to SDK root. /// - xctestPath: Optional path to XCTest. + /// - swiftTestingPath: Optional path to swift-testing. + /// - metalToolchainPath: Optional path to Metal toolchain. public init( librarianPath: AbsolutePath, swiftCompilerPath: AbsolutePath, @@ -64,7 +70,8 @@ public struct ToolchainConfiguration { swiftPMLibrariesLocation: SwiftPMLibrariesLocation? = nil, sdkRootPath: AbsolutePath? = nil, xctestPath: AbsolutePath? = nil, - swiftTestingPath: AbsolutePath? = nil + swiftTestingPath: AbsolutePath? = nil, + metalToolchainPath: AbsolutePath? = nil ) { let swiftPMLibrariesLocation = swiftPMLibrariesLocation ?? { return .init(swiftCompilerPath: swiftCompilerPath) @@ -78,6 +85,7 @@ public struct ToolchainConfiguration { self.sdkRootPath = sdkRootPath self.xctestPath = xctestPath self.swiftTestingPath = swiftTestingPath + self.metalToolchainPath = metalToolchainPath } } diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 69edefd7132..10a54865b5b 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -901,6 +901,8 @@ public final class UserToolchain: Toolchain { ) } + let metalToolchainPath = try? Self.deriveMetalToolchainPath(triple: triple, environment: environment) + self.configuration = .init( librarianPath: librarianPath, swiftCompilerPath: swiftCompilers.manifest, @@ -909,7 +911,8 @@ public final class UserToolchain: Toolchain { swiftPMLibrariesLocation: swiftPMLibrariesLocation, sdkRootPath: self.swiftSDK.pathsConfiguration.sdkRootPath, xctestPath: xctestPath, - swiftTestingPath: swiftTestingPath + swiftTestingPath: swiftTestingPath, + metalToolchainPath: metalToolchainPath ) self.fileSystem = fileSystem @@ -1041,6 +1044,35 @@ public final class UserToolchain: Toolchain { return (platform, info) } + private static func deriveMetalToolchainPath( + triple: Basics.Triple, + environment: Environment + ) throws -> AbsolutePath? { + guard triple.isDarwin() else { + return nil + } + + let xcodebuildArgs = ["/usr/bin/xcodebuild", "-showComponent", "metalToolchain", "-json"] + guard let output = try? AsyncProcess.checkNonZeroExit(arguments: xcodebuildArgs, environment: environment) + .spm_chomp() else { + return nil + } + + guard let json = try? JSON(string: output) else { + return nil + } + + guard let status = try? json.get("status") as String, status == "installed" else { + return nil + } + + guard let toolchainSearchPath = try? json.get("toolchainSearchPath") as String else { + return nil + } + + return try AbsolutePath(validating: toolchainSearchPath).appending(component: "Metal.xctoolchain") + } + // TODO: We should have some general utility to find tools. private static func deriveXCTestPath( swiftSDK: SwiftSDK, @@ -1224,6 +1256,10 @@ public final class UserToolchain: Toolchain { configuration.sdkRootPath } + public var metalToolchainPath: AbsolutePath? { + configuration.metalToolchainPath + } + public var swiftCompilerEnvironment: Environment { configuration.swiftCompilerEnvironment } diff --git a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift index bf0e98161cf..5779419be63 100644 --- a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift +++ b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift @@ -885,6 +885,15 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { settings["SWIFT_EXEC"] = buildParameters.toolchain.swiftCompilerPath.pathString } + if let metalToolchainPath = buildParameters.toolchain.metalToolchainPath { + let metalToolchainID = try await session.registerToolchain(at: metalToolchainPath.pathString) + if let toolChains = settings["TOOLCHAINS"] { + settings["TOOLCHAINS"] = "\(metalToolchainID) " + toolChains + } else { + settings["TOOLCHAINS"] = "\(metalToolchainID)" + } + } + // FIXME: workaround for old Xcode installations such as what is in CI settings["LM_SKIP_METADATA_EXTRACTION"] = "YES" if let symbolGraphOptions { diff --git a/Sources/_InternalTestSupport/MockBuildTestHelper.swift b/Sources/_InternalTestSupport/MockBuildTestHelper.swift index 024726107c9..871670fd219 100644 --- a/Sources/_InternalTestSupport/MockBuildTestHelper.swift +++ b/Sources/_InternalTestSupport/MockBuildTestHelper.swift @@ -20,6 +20,7 @@ import SPMBuildCore import TSCUtility public struct MockToolchain: PackageModel.Toolchain { + public let metalToolchainPath: Basics.AbsolutePath? #if os(Windows) public let librarianPath = AbsolutePath("/fake/path/to/link.exe") #elseif canImport(Darwin) @@ -54,6 +55,7 @@ public struct MockToolchain: PackageModel.Toolchain { public init(swiftResourcesPath: AbsolutePath? = nil) { self.swiftResourcesPath = swiftResourcesPath + self.metalToolchainPath = nil } } diff --git a/Tests/BuildMetalTests/BuildMetalTests.swift b/Tests/BuildMetalTests/BuildMetalTests.swift new file mode 100644 index 00000000000..fe48acd6d7b --- /dev/null +++ b/Tests/BuildMetalTests/BuildMetalTests.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import _InternalTestSupport +import Testing +import Basics +import Foundation +import Metal + +@Suite +struct BuildMetalTests { + + @Test( + .tags( + .TestSize.large + ), + .requireHostOS(.macOS), + arguments: getBuildData(for: [.swiftbuild]), + ) + func simpleLibrary(data: BuildData) async throws { + let buildSystem = data.buildSystem + let configuration = data.config + + try await fixture(name: "Metal/SimpleLibrary") { fixturePath in + + // Build the package + let (_, _) = try await executeSwiftBuild( + fixturePath, + configuration: configuration, + buildSystem: buildSystem, + throwIfCommandFails: true + ) + + // Get the bin path + let (binPathOutput, _) = try await executeSwiftBuild( + fixturePath, + configuration: configuration, + extraArgs: ["--show-bin-path"], + buildSystem: buildSystem, + throwIfCommandFails: true + ) + + let binPath = try AbsolutePath(validating: binPathOutput.trimmingCharacters(in: .whitespacesAndNewlines)) + + // Check that default.metallib exists + let metallibPath = binPath.appending(components:["MyRenderer_MyRenderer.bundle", "Contents", "Resources", "default.metallib"]) + #expect( + localFileSystem.exists(metallibPath), + "Expected default.metallib to exist at \(metallibPath)" + ) + + // Verify we can load the metal library + let device = MTLCreateSystemDefaultDevice()! + let library = try device.makeLibrary(URL: URL(fileURLWithPath: metallibPath.pathString)) + + #expect(library.functionNames.contains("simpleVertexShader")) + } + } + +} From a1425086dbf9e2f292e83bc15ddc19046d63c830 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Wed, 3 Dec 2025 10:20:52 -0500 Subject: [PATCH 02/11] Register metal toolchain using EXTERNAL_TOOLCHAINS_DIR --- Sources/PackageModel/Toolchain.swift | 5 +++- .../PackageModel/ToolchainConfiguration.swift | 8 +++++- Sources/PackageModel/UserToolchain.swift | 19 +++++++++----- .../SwiftBuildSupport/SwiftBuildSystem.swift | 26 ++++++++++++------- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/Sources/PackageModel/Toolchain.swift b/Sources/PackageModel/Toolchain.swift index d1f6e241311..b6d5a3f478e 100644 --- a/Sources/PackageModel/Toolchain.swift +++ b/Sources/PackageModel/Toolchain.swift @@ -47,9 +47,12 @@ public protocol Toolchain { /// The manifest and library locations used by this toolchain. var swiftPMLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation { get } - /// Path to the Metal toolchain if available.. + /// Path to the Metal toolchain if available. var metalToolchainPath: AbsolutePath? { get } + // Metal toolchain ID if available. + var metalToolchainId: String? { get } + /// Path of the `clang` compiler. func getClangCompiler() throws -> AbsolutePath diff --git a/Sources/PackageModel/ToolchainConfiguration.swift b/Sources/PackageModel/ToolchainConfiguration.swift index 26655dd7b1c..4d0e4e7dba5 100644 --- a/Sources/PackageModel/ToolchainConfiguration.swift +++ b/Sources/PackageModel/ToolchainConfiguration.swift @@ -50,6 +50,10 @@ public struct ToolchainConfiguration { /// This is optional and only available on Darwin platforms. public var metalToolchainPath: AbsolutePath? + /// Metal toolchain identifier + /// This is optional and only available on Darwin platforms. + public var metalToolchainId: String? + /// Creates the set of manifest resources associated with a `swiftc` executable. /// /// - Parameters: @@ -71,7 +75,8 @@ public struct ToolchainConfiguration { sdkRootPath: AbsolutePath? = nil, xctestPath: AbsolutePath? = nil, swiftTestingPath: AbsolutePath? = nil, - metalToolchainPath: AbsolutePath? = nil + metalToolchainPath: AbsolutePath? = nil, + metalToolchainId: String? = nil ) { let swiftPMLibrariesLocation = swiftPMLibrariesLocation ?? { return .init(swiftCompilerPath: swiftCompilerPath) @@ -86,6 +91,7 @@ public struct ToolchainConfiguration { self.xctestPath = xctestPath self.swiftTestingPath = swiftTestingPath self.metalToolchainPath = metalToolchainPath + self.metalToolchainId = metalToolchainId } } diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 10a54865b5b..a600b330c86 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -901,7 +901,7 @@ public final class UserToolchain: Toolchain { ) } - let metalToolchainPath = try? Self.deriveMetalToolchainPath(triple: triple, environment: environment) + let metalToolchain = try? Self.deriveMetalToolchainPath(triple: triple, environment: environment) self.configuration = .init( librarianPath: librarianPath, @@ -912,7 +912,8 @@ public final class UserToolchain: Toolchain { sdkRootPath: self.swiftSDK.pathsConfiguration.sdkRootPath, xctestPath: xctestPath, swiftTestingPath: swiftTestingPath, - metalToolchainPath: metalToolchainPath + metalToolchainPath: metalToolchain?.path, + metalToolchainId: metalToolchain?.identifier ) self.fileSystem = fileSystem @@ -1047,12 +1048,12 @@ public final class UserToolchain: Toolchain { private static func deriveMetalToolchainPath( triple: Basics.Triple, environment: Environment - ) throws -> AbsolutePath? { + ) throws -> (path: AbsolutePath, identifier: String)? { guard triple.isDarwin() else { return nil } - let xcodebuildArgs = ["/usr/bin/xcodebuild", "-showComponent", "metalToolchain", "-json"] + let xcodebuildArgs = ["/usr/bin/xcrun", "xcodebuild", "-showComponent", "metalToolchain", "-json"] guard let output = try? AsyncProcess.checkNonZeroExit(arguments: xcodebuildArgs, environment: environment) .spm_chomp() else { return nil @@ -1066,11 +1067,13 @@ public final class UserToolchain: Toolchain { return nil } - guard let toolchainSearchPath = try? json.get("toolchainSearchPath") as String else { + guard let toolchainSearchPath = try? json.get("toolchainSearchPath") as String, + let toolchainIdentifier = try? json.get("toolchainIdentifier") as String else { return nil } - return try AbsolutePath(validating: toolchainSearchPath).appending(component: "Metal.xctoolchain") + let path = try AbsolutePath(validating: toolchainSearchPath) + return (path: path, identifier: toolchainIdentifier) } // TODO: We should have some general utility to find tools. @@ -1260,6 +1263,10 @@ public final class UserToolchain: Toolchain { configuration.metalToolchainPath } + public var metalToolchainId: String? { + configuration.metalToolchainId + } + public var swiftCompilerEnvironment: Environment { configuration.swiftCompilerEnvironment } diff --git a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift index 5779419be63..4e7eca3f280 100644 --- a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift +++ b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift @@ -64,14 +64,21 @@ func withService( public func createSession( service: SWBBuildService, name: String, - toolchainPath: Basics.AbsolutePath, + toolchain: Toolchain, packageManagerResourcesDirectory: Basics.AbsolutePath? ) async throws-> (SWBBuildServiceSession, [SwiftBuildMessage.DiagnosticInfo]) { + + var buildSessionEnv: [String: String]? = nil + if let metalToolchainPath = toolchain.metalToolchainPath { + buildSessionEnv = ["EXTERNAL_TOOLCHAINS_DIR": metalToolchainPath.pathString] + } + let toolchainPath = try toolchain.toolchainDir + // SWIFT_EXEC and SWIFT_EXEC_MANIFEST may need to be overridden in debug scenarios in order to pick up Open Source toolchains let sessionResult = if toolchainPath.components.contains(where: { $0.hasSuffix(".app") }) { - await service.createSession(name: name, developerPath: nil, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: nil) + await service.createSession(name: name, developerPath: nil, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: buildSessionEnv) } else { - await service.createSession(name: name, swiftToolchainPath: toolchainPath.pathString, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: nil) + await service.createSession(name: name, swiftToolchainPath: toolchainPath.pathString, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: buildSessionEnv) } switch sessionResult { case (.success(let session), let diagnostics): @@ -84,14 +91,14 @@ public func createSession( func withSession( service: SWBBuildService, name: String, - toolchainPath: Basics.AbsolutePath, + toolchain: Toolchain, packageManagerResourcesDirectory: Basics.AbsolutePath?, body: @escaping ( _ session: SWBBuildServiceSession, _ diagnostics: [SwiftBuild.SwiftBuildMessage.DiagnosticInfo] ) async throws -> Void ) async throws { - let (session, diagnostics) = try await createSession(service: service, name: name, toolchainPath: toolchainPath, packageManagerResourcesDirectory: packageManagerResourcesDirectory) + let (session, diagnostics) = try await createSession(service: service, name: name, toolchain: toolchain, packageManagerResourcesDirectory: packageManagerResourcesDirectory) do { try await body(session, diagnostics) } catch let bodyError { @@ -544,7 +551,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { var serializedDiagnosticPathsByTargetName: [String: [Basics.AbsolutePath]] = [:] do { - try await withSession(service: service, name: self.buildParameters.pifManifest.pathString, toolchainPath: self.buildParameters.toolchain.toolchainDir, packageManagerResourcesDirectory: self.packageManagerResourcesDirectory) { session, _ in + try await withSession(service: service, name: self.buildParameters.pifManifest.pathString, toolchain: self.buildParameters.toolchain, packageManagerResourcesDirectory: self.packageManagerResourcesDirectory) { session, _ in self.outputStream.send("Building for \(self.buildParameters.configuration == .debug ? "debugging" : "production")...\n") // Load the workspace, and set the system information to the default @@ -885,12 +892,11 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { settings["SWIFT_EXEC"] = buildParameters.toolchain.swiftCompilerPath.pathString } - if let metalToolchainPath = buildParameters.toolchain.metalToolchainPath { - let metalToolchainID = try await session.registerToolchain(at: metalToolchainPath.pathString) + if let metalToolchainId = buildParameters.toolchain.metalToolchainId { if let toolChains = settings["TOOLCHAINS"] { - settings["TOOLCHAINS"] = "\(metalToolchainID) " + toolChains + settings["TOOLCHAINS"] = "\(metalToolchainId) " + toolChains } else { - settings["TOOLCHAINS"] = "\(metalToolchainID)" + settings["TOOLCHAINS"] = "\(metalToolchainId)" } } From ca6ff3dbe78d3bc55f704a34c0c958d0cc6a6ba3 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Wed, 3 Dec 2025 10:33:12 -0500 Subject: [PATCH 03/11] Update metal tests to only run on macOS --- Sources/_InternalTestSupport/MockBuildTestHelper.swift | 1 + Tests/BuildMetalTests/BuildMetalTests.swift | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/_InternalTestSupport/MockBuildTestHelper.swift b/Sources/_InternalTestSupport/MockBuildTestHelper.swift index 871670fd219..23f604c6db7 100644 --- a/Sources/_InternalTestSupport/MockBuildTestHelper.swift +++ b/Sources/_InternalTestSupport/MockBuildTestHelper.swift @@ -20,6 +20,7 @@ import SPMBuildCore import TSCUtility public struct MockToolchain: PackageModel.Toolchain { + public let metalToolchainId: String? public let metalToolchainPath: Basics.AbsolutePath? #if os(Windows) public let librarianPath = AbsolutePath("/fake/path/to/link.exe") diff --git a/Tests/BuildMetalTests/BuildMetalTests.swift b/Tests/BuildMetalTests/BuildMetalTests.swift index fe48acd6d7b..8bb47020899 100644 --- a/Tests/BuildMetalTests/BuildMetalTests.swift +++ b/Tests/BuildMetalTests/BuildMetalTests.swift @@ -19,6 +19,7 @@ import Metal @Suite struct BuildMetalTests { +#if os(macOS) @Test( .tags( .TestSize.large @@ -65,5 +66,5 @@ struct BuildMetalTests { #expect(library.functionNames.contains("simpleVertexShader")) } } - +#endif } From 2aa9f967219f8e26d4df70e292ab228fe7b5926a Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Wed, 3 Dec 2025 10:59:00 -0500 Subject: [PATCH 04/11] Fix missing initializer --- Sources/_InternalTestSupport/MockBuildTestHelper.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/_InternalTestSupport/MockBuildTestHelper.swift b/Sources/_InternalTestSupport/MockBuildTestHelper.swift index 23f604c6db7..c2da73d6092 100644 --- a/Sources/_InternalTestSupport/MockBuildTestHelper.swift +++ b/Sources/_InternalTestSupport/MockBuildTestHelper.swift @@ -57,6 +57,7 @@ public struct MockToolchain: PackageModel.Toolchain { public init(swiftResourcesPath: AbsolutePath? = nil) { self.swiftResourcesPath = swiftResourcesPath self.metalToolchainPath = nil + self.metalToolchainId = nil } } From 971c31698c7b1c5227397ac93105d26beabfa1b7 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Thu, 4 Dec 2025 09:39:13 -0500 Subject: [PATCH 05/11] Use xcrun --find to discover metal toolchain --- Sources/PackageModel/UserToolchain.swift | 38 +++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index a600b330c86..1fe36965a88 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -901,7 +901,7 @@ public final class UserToolchain: Toolchain { ) } - let metalToolchain = try? Self.deriveMetalToolchainPath(triple: triple, environment: environment) + let metalToolchain = try? Self.deriveMetalToolchainPath(fileSystem: fileSystem, triple: triple, environment: environment) self.configuration = .init( librarianPath: librarianPath, @@ -1046,6 +1046,7 @@ public final class UserToolchain: Toolchain { } private static func deriveMetalToolchainPath( + fileSystem: FileSystem, triple: Basics.Triple, environment: Environment ) throws -> (path: AbsolutePath, identifier: String)? { @@ -1053,27 +1054,44 @@ public final class UserToolchain: Toolchain { return nil } - let xcodebuildArgs = ["/usr/bin/xcrun", "xcodebuild", "-showComponent", "metalToolchain", "-json"] - guard let output = try? AsyncProcess.checkNonZeroExit(arguments: xcodebuildArgs, environment: environment) - .spm_chomp() else { + let xcrunCmd = ["/usr/bin/xcrun", "--find", "metal"] + guard let output = try? AsyncProcess.checkNonZeroExit(arguments: xcrunCmd, environment: environment).spm_chomp() else { return nil } - guard let json = try? JSON(string: output) else { + guard let metalPath = try? AbsolutePath(validating: output) else { return nil } - guard let status = try? json.get("status") as String, status == "installed" else { + guard let toolchainPath: AbsolutePath = { + var currentPath = metalPath + while currentPath != currentPath.parentDirectory { + if currentPath.basename == "Metal.xctoolchain" { + return currentPath + } + currentPath = currentPath.parentDirectory + } return nil + }() else { + return nil + } + + let toolchainInfoPlist = toolchainPath.appending(component: "ToolchainInfo.plist") + + struct MetalToolchainInfo: Decodable { + let Identifier: String } - guard let toolchainSearchPath = try? json.get("toolchainSearchPath") as String, - let toolchainIdentifier = try? json.get("toolchainIdentifier") as String else { + let toolchainIdentifier: String + do { + let data: Data = try fileSystem.readFileContents(toolchainInfoPlist) + let info = try PropertyListDecoder().decode(MetalToolchainInfo.self, from: data) + toolchainIdentifier = info.Identifier + } catch { return nil } - let path = try AbsolutePath(validating: toolchainSearchPath) - return (path: path, identifier: toolchainIdentifier) + return (path: toolchainPath.parentDirectory, identifier: toolchainIdentifier) } // TODO: We should have some general utility to find tools. From 7a94da3315a57fa0a22d9eb14d6d4d81fd70d828 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Thu, 4 Dec 2025 09:51:28 -0500 Subject: [PATCH 06/11] Clean up how we update the TOOLCHAINS build setting --- Sources/SwiftBuildSupport/SwiftBuildSystem.swift | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift index 4e7eca3f280..cd141844b46 100644 --- a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift +++ b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift @@ -881,7 +881,8 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { // If the SwiftPM toolchain corresponds to a toolchain registered with the lower level build system, add it to the toolchain stack. // Otherwise, apply overrides for each component of the SwiftPM toolchain. - if let toolchainID = try await session.lookupToolchain(at: buildParameters.toolchain.toolchainDir.pathString) { + let toolchainID = try await session.lookupToolchain(at: buildParameters.toolchain.toolchainDir.pathString) + if let toolchainID { settings["TOOLCHAINS"] = "\(toolchainID.rawValue) $(inherited)" } else { // FIXME: This list of overrides is incomplete. @@ -892,12 +893,9 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { settings["SWIFT_EXEC"] = buildParameters.toolchain.swiftCompilerPath.pathString } - if let metalToolchainId = buildParameters.toolchain.metalToolchainId { - if let toolChains = settings["TOOLCHAINS"] { - settings["TOOLCHAINS"] = "\(metalToolchainId) " + toolChains - } else { - settings["TOOLCHAINS"] = "\(metalToolchainId)" - } + let overrideToolchains = [buildParameters.toolchain.metalToolchainId, toolchainID?.rawValue].compactMap { $0 } + if !overrideToolchains.isEmpty { + settings["TOOLCHAINS"] = (overrideToolchains + ["$(inherited)"]).joined(separator: " ") } // FIXME: workaround for old Xcode installations such as what is in CI From f8cf09d740e724845afbd5883ec0795bfb5018c8 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Thu, 4 Dec 2025 09:55:38 -0500 Subject: [PATCH 07/11] Clean up how we update the TOOLCHAINS build setting --- Sources/SwiftBuildSupport/SwiftBuildSystem.swift | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift index cd141844b46..e197b1be330 100644 --- a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift +++ b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift @@ -882,9 +882,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { // If the SwiftPM toolchain corresponds to a toolchain registered with the lower level build system, add it to the toolchain stack. // Otherwise, apply overrides for each component of the SwiftPM toolchain. let toolchainID = try await session.lookupToolchain(at: buildParameters.toolchain.toolchainDir.pathString) - if let toolchainID { - settings["TOOLCHAINS"] = "\(toolchainID.rawValue) $(inherited)" - } else { + if toolchainID == nil { // FIXME: This list of overrides is incomplete. // An error with determining the override should not be fatal here. settings["CC"] = try? buildParameters.toolchain.getClangCompiler().pathString From 7ed2c5c6e960768bdcadc5197b5fa1b38ffa2312 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Thu, 4 Dec 2025 10:07:59 -0500 Subject: [PATCH 08/11] Require downloadable Metal toolchain in CI in order to enable Metal tests. --- Tests/BuildMetalTests/BuildMetalTests.swift | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Tests/BuildMetalTests/BuildMetalTests.swift b/Tests/BuildMetalTests/BuildMetalTests.swift index 8bb47020899..49796c565b4 100644 --- a/Tests/BuildMetalTests/BuildMetalTests.swift +++ b/Tests/BuildMetalTests/BuildMetalTests.swift @@ -21,11 +21,10 @@ struct BuildMetalTests { #if os(macOS) @Test( - .tags( - .TestSize.large - ), + .disabled("Require downloadable Metal toolchain"), + .tags(.TestSize.large), .requireHostOS(.macOS), - arguments: getBuildData(for: [.swiftbuild]), + arguments: getBuildData(for: [.swiftbuild]) ) func simpleLibrary(data: BuildData) async throws { let buildSystem = data.buildSystem From d2e70e8bbf45a6d217646f99c5bbc88e2a921d2f Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Thu, 4 Dec 2025 10:10:04 -0500 Subject: [PATCH 09/11] Use #require when creating the MTLCreateSystemDefaultDevice --- Tests/BuildMetalTests/BuildMetalTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/BuildMetalTests/BuildMetalTests.swift b/Tests/BuildMetalTests/BuildMetalTests.swift index 49796c565b4..efad05f9627 100644 --- a/Tests/BuildMetalTests/BuildMetalTests.swift +++ b/Tests/BuildMetalTests/BuildMetalTests.swift @@ -59,7 +59,7 @@ struct BuildMetalTests { ) // Verify we can load the metal library - let device = MTLCreateSystemDefaultDevice()! + let device = try #require(MTLCreateSystemDefaultDevice()) let library = try device.makeLibrary(URL: URL(fileURLWithPath: metallibPath.pathString)) #expect(library.functionNames.contains("simpleVertexShader")) From 66e402ac895e29993a7b9f74408b0ff951c000e1 Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Thu, 4 Dec 2025 12:52:43 -0500 Subject: [PATCH 10/11] import Metal only on macOS --- Tests/BuildMetalTests/BuildMetalTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/BuildMetalTests/BuildMetalTests.swift b/Tests/BuildMetalTests/BuildMetalTests.swift index efad05f9627..d33481571f1 100644 --- a/Tests/BuildMetalTests/BuildMetalTests.swift +++ b/Tests/BuildMetalTests/BuildMetalTests.swift @@ -14,7 +14,9 @@ import _InternalTestSupport import Testing import Basics import Foundation +#if os(macOS) import Metal +#endif @Suite struct BuildMetalTests { From e1def1f494fc823c8bccd0c1e6f68c90e31a25db Mon Sep 17 00:00:00 2001 From: Robert Connell Date: Fri, 5 Dec 2025 22:13:24 -0500 Subject: [PATCH 11/11] Resolve merge conflict --- Sources/SwiftBuildSupport/SwiftBuildSystem.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift index 29d4c9b7e72..9e10380367d 100644 --- a/Sources/SwiftBuildSupport/SwiftBuildSystem.swift +++ b/Sources/SwiftBuildSupport/SwiftBuildSystem.swift @@ -1261,7 +1261,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem { package func createLongLivedSession(name: String) async throws -> LongLivedBuildServiceSession { let service = try await SWBBuildService(connectionMode: .inProcessStatic(swiftbuildServiceEntryPoint)) do { - let (session, diagnostics) = try await createSession(service: service, name: name, toolchainPath: buildParameters.toolchain.toolchainDir, packageManagerResourcesDirectory: packageManagerResourcesDirectory) + let (session, diagnostics) = try await createSession(service: service, name: name, toolchain: buildParameters.toolchain, packageManagerResourcesDirectory: packageManagerResourcesDirectory) let teardownHandler = { try await session.close() await service.close()