From e3d3571bc7fd900ca369f1f4d7dc34cd1ae6db18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 20 Jan 2025 18:41:33 +0100 Subject: [PATCH 01/30] [android] Run tests for libdispatch in x64 emulator (wip) --- utils/android/utils/ctest_mock.sh | 58 +++++++++++++++++++++++++++++++ utils/build.ps1 | 57 +++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 utils/android/utils/ctest_mock.sh diff --git a/utils/android/utils/ctest_mock.sh b/utils/android/utils/ctest_mock.sh new file mode 100644 index 0000000000000..cb823bebb8cd4 --- /dev/null +++ b/utils/android/utils/ctest_mock.sh @@ -0,0 +1,58 @@ +#!/system/bin/sh + +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +SUB_DIR=$1 +if [ ! -d "$SUB_DIR" ]; then + echo "Error: Directory $SUB_DIR does not exist" + exit 1 +fi + +CTEST_FILE="$SUB_DIR/CTestTestfile.cmake" +if [ ! -f "$CTEST_FILE" ]; then + echo "Error: $CTEST_FILE not found in $SUB_DIR" + exit 1 +fi + +# TODO: Parse them from CTEST_FILE +TESTS="dispatch_apply dispatch_api dispatch_debug dispatch_queue_finalizer dispatch_overcommit dispatch_context_for_key dispatch_after dispatch_timer dispatch_timer_short dispatch_timer_timeout dispatch_sema dispatch_timer_bit31 dispatch_timer_bit63 dispatch_timer_set_time dispatch_data dispatch_io_muxed dispatch_io_net dispatch_io_pipe dispatch_io_pipe_close dispatch_select dispatch_c99 dispatch_plusplus" + +COUNT=$(echo "$TESTS" | tr ' ' '\n' | wc -l) +echo "Found $COUNT test-cases in $SUB_DIR" + +UTILITY="$SUB_DIR/bsdtestharness" +if [ ! -f "$UTILITY" ]; then + echo "Utility not found: $UTILITY." + exit 1 +fi + +chmod +x "$UTILITY" + +# We need libdispatch.so and libBlocksRuntime.so +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SUB_DIR" + +# TODO: Parse the timeout as well from CTestTestfile.cmake? +TIMOUT=120 +FAILURE=0 + +for TEST in $TESTS; do + chmod +x "$SUB_DIR/$TEST" + OUTPUT=$(timeout "${TIMOUT}s" "$UTILITY" "$SUB_DIR/$TEST" 2>&1) + EC=$? + if [ $EC -eq 124 ]; then + echo "Error: $TEST timed out after $TIMOUT seconds" + echo "************\nOutput:\n$OUTPUT\n************" + FAILURE=1 + elif [ $EC -ne 0 ]; then + echo "Error: $TEST failed" + echo "************\nOutput:\n$OUTPUT\n************" + FAILURE=1 + else + echo "$TEST passed" + fi +done + +exit $FAILURE diff --git a/utils/build.ps1 b/utils/build.ps1 index 3281c79885cab..1a6043ed35d9a 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -723,7 +723,7 @@ function Fetch-Dependencies { DownloadAndVerify $WixURL "$BinaryCache\WiX-$WiXVersion.zip" $WiXHash Extract-ZipFile WiX-$WiXVersion.zip $BinaryCache WiX-$WiXVersion - if ($SkipBuild) { return } + if ($SkipBuild -and -not $Test) { return } DownloadAndVerify $PinnedBuild "$BinaryCache\$PinnedToolchain.exe" $PinnedSHA256 @@ -823,6 +823,30 @@ function Fetch-Dependencies { DownloadAndVerify $NDKURL "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip" $NDKHash Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" -CreateExtractPath $false + + if ($Test -and -not(Test-Path "$NDKDir\licenses")) { + $NDKDir = "$BinaryCache\android-ndk-r26b" + $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" + $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" + DownloadAndVerify $CLToolsURL "$BinaryCache\android-cmdline-tools.zip" $CLToolsHash + + # Install cmdline-tools + New-Item -Type Directory -Path "$NDKDir\licenses" -ErrorAction Ignore | Out-Null + Set-Content -Path "$NDKDir\licenses\android-sdk-license" -Value "24333f8a63b6825ea9c5514f83c2829b004d1fee" + Set-Content -Path "$NDKDir\licenses\android-sdk-preview-license" -Value "84831b9409646a918e30573bab4c9c91346d8abd" + Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp" + Invoke-Program "$NDKDir\.temp\cmdline-tools\bin\sdkmanager.bat" --sdk_root="$NDKDir" "cmdline-tools;latest" "--channel=3" + + # Install packages + $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" + foreach ($Package in @("emulator", "platforms;android-29", "system-images;android-29;default;x86_64", "platform-tools")) { + Invoke-Program $AndroidSdkMgr $Package + } + + # Create test device + $AndroidAvdMgr = "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" + Invoke-Program $AndroidAvdMgr create avd --name "swift-test-device" --package "system-images;android-29;default;x86_64" + } } if ($IncludeDS2) { @@ -2033,6 +2057,33 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { } } +function Test-Dispatch([Platform]$Platform) { + # TODO: Make it global + $NDKDir = "$BinaryCache\android-ndk-r26b" + + # TODO: Start once for all tests and make it shuts down + $emulator = "$NDKDir\emulator\emulator.exe" + Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" ` + -RedirectStandardOutput "$NDKDir\.temp\emulator.out" ` + -RedirectStandardError "$NDKDir\.temp\emulator.err" + + # TODO: Find a better indicator than a upper-bound delay + Start-Sleep -Seconds 5 + + # This is just a hack for now + $LocalBin = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) + $CacheName = Split-Path $LocalBin -Leaf + $RemoteBin = "/data/local/tmp/$CacheName" + + # TODO: On my local machine I have to grant network access once. How to do that in CI? + $adb = "$NDKDir\platform-tools\adb.exe" + Invoke-Program $adb "shell" "rm -rf $RemoteBin" + Invoke-Program $adb "shell" "mkdir $RemoteBin" + Invoke-Program $adb "push" "$LocalBin/." $RemoteBin + Invoke-Program $adb "push" "S:/ctest_mock.sh" "/data/local/tmp" + Invoke-Program $adb "shell" "sh /data/local/tmp/ctest_mock.sh $RemoteBin" +} + function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { if ($Test) { # Foundation tests build via swiftpm rather than CMake @@ -3073,6 +3124,10 @@ if (-not $IsCrossCompiling) { if ($Test -contains "dispatch") { Build-Dispatch Windows $HostArch -Test + # TODO: This is a hack. We need different devices for different arches. + if (AndroidSDKs -contains "x86_64") { + Test-Dispatch Android + } } if ($Test -contains "foundation") { Build-Foundation Windows $HostArch -Test From 00a6f90142dc35fcee99118d73b325dabcf205b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 22 Jan 2025 12:30:19 +0100 Subject: [PATCH 02/30] fixup! [android] Run tests for libdispatch in x64 emulator (wip) --- utils/android/utils/ctest_mock.sh | 116 +++++++++++++++--------------- utils/build.ps1 | 21 ++++-- 2 files changed, 72 insertions(+), 65 deletions(-) diff --git a/utils/android/utils/ctest_mock.sh b/utils/android/utils/ctest_mock.sh index cb823bebb8cd4..aed55b9912d41 100644 --- a/utils/android/utils/ctest_mock.sh +++ b/utils/android/utils/ctest_mock.sh @@ -1,58 +1,58 @@ -#!/system/bin/sh - -if [ -z "$1" ]; then - echo "Usage: $0 " - exit 1 -fi - -SUB_DIR=$1 -if [ ! -d "$SUB_DIR" ]; then - echo "Error: Directory $SUB_DIR does not exist" - exit 1 -fi - -CTEST_FILE="$SUB_DIR/CTestTestfile.cmake" -if [ ! -f "$CTEST_FILE" ]; then - echo "Error: $CTEST_FILE not found in $SUB_DIR" - exit 1 -fi - -# TODO: Parse them from CTEST_FILE -TESTS="dispatch_apply dispatch_api dispatch_debug dispatch_queue_finalizer dispatch_overcommit dispatch_context_for_key dispatch_after dispatch_timer dispatch_timer_short dispatch_timer_timeout dispatch_sema dispatch_timer_bit31 dispatch_timer_bit63 dispatch_timer_set_time dispatch_data dispatch_io_muxed dispatch_io_net dispatch_io_pipe dispatch_io_pipe_close dispatch_select dispatch_c99 dispatch_plusplus" - -COUNT=$(echo "$TESTS" | tr ' ' '\n' | wc -l) -echo "Found $COUNT test-cases in $SUB_DIR" - -UTILITY="$SUB_DIR/bsdtestharness" -if [ ! -f "$UTILITY" ]; then - echo "Utility not found: $UTILITY." - exit 1 -fi - -chmod +x "$UTILITY" - -# We need libdispatch.so and libBlocksRuntime.so -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SUB_DIR" - -# TODO: Parse the timeout as well from CTestTestfile.cmake? -TIMOUT=120 -FAILURE=0 - -for TEST in $TESTS; do - chmod +x "$SUB_DIR/$TEST" - OUTPUT=$(timeout "${TIMOUT}s" "$UTILITY" "$SUB_DIR/$TEST" 2>&1) - EC=$? - if [ $EC -eq 124 ]; then - echo "Error: $TEST timed out after $TIMOUT seconds" - echo "************\nOutput:\n$OUTPUT\n************" - FAILURE=1 - elif [ $EC -ne 0 ]; then - echo "Error: $TEST failed" - echo "************\nOutput:\n$OUTPUT\n************" - FAILURE=1 - else - echo "$TEST passed" - fi -done - -exit $FAILURE +#!/system/bin/sh + +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +SUB_DIR=$1 +if [ ! -d "$SUB_DIR" ]; then + echo "Error: Directory $SUB_DIR does not exist" + exit 1 +fi + +CTEST_FILE="$SUB_DIR/CTestTestfile.cmake" +if [ ! -f "$CTEST_FILE" ]; then + echo "Error: $CTEST_FILE not found in $SUB_DIR" + exit 1 +fi + +# TODO: Parse them from CTEST_FILE +TESTS="dispatch_apply dispatch_api dispatch_debug dispatch_queue_finalizer dispatch_overcommit dispatch_context_for_key dispatch_after dispatch_timer dispatch_timer_short dispatch_timer_timeout dispatch_sema dispatch_timer_bit31 dispatch_timer_bit63 dispatch_timer_set_time dispatch_data dispatch_io_muxed dispatch_io_net dispatch_io_pipe dispatch_io_pipe_close dispatch_select dispatch_c99 dispatch_plusplus" + +COUNT=$(echo "$TESTS" | tr ' ' '\n' | wc -l) +echo "Found $COUNT test-cases in $SUB_DIR" + +UTILITY="$SUB_DIR/bsdtestharness" +if [ ! -f "$UTILITY" ]; then + echo "Utility not found: $UTILITY." + exit 1 +fi + +chmod +x "$UTILITY" + +# We need libdispatch.so and libBlocksRuntime.so +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SUB_DIR" + +# TODO: Parse the timeout as well from CTestTestfile.cmake? +TIMOUT=120 +FAILURE=0 + +for TEST in $TESTS; do + chmod +x "$SUB_DIR/$TEST" + OUTPUT=$(timeout "${TIMOUT}s" "$UTILITY" "$SUB_DIR/$TEST" 2>&1) + EC=$? + if [ $EC -eq 124 ]; then + echo "Error: $TEST timed out after $TIMOUT seconds" + echo "************\nOutput:\n$OUTPUT\n************" + FAILURE=1 + elif [ $EC -ne 0 ]; then + echo "Error: $TEST failed" + echo "************\nOutput:\n$OUTPUT\n************" + FAILURE=1 + else + echo "$TEST passed" + fi +done + +exit $FAILURE diff --git a/utils/build.ps1 b/utils/build.ps1 index 1a6043ed35d9a..e0b6b52a75a07 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -824,8 +824,8 @@ function Fetch-Dependencies { Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" -CreateExtractPath $false + $NDKDir = "$BinaryCache\android-ndk-r26b" if ($Test -and -not(Test-Path "$NDKDir\licenses")) { - $NDKDir = "$BinaryCache\android-ndk-r26b" $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" DownloadAndVerify $CLToolsURL "$BinaryCache\android-cmdline-tools.zip" $CLToolsHash @@ -2077,11 +2077,18 @@ function Test-Dispatch([Platform]$Platform) { # TODO: On my local machine I have to grant network access once. How to do that in CI? $adb = "$NDKDir\platform-tools\adb.exe" - Invoke-Program $adb "shell" "rm -rf $RemoteBin" - Invoke-Program $adb "shell" "mkdir $RemoteBin" - Invoke-Program $adb "push" "$LocalBin/." $RemoteBin - Invoke-Program $adb "push" "S:/ctest_mock.sh" "/data/local/tmp" - Invoke-Program $adb "shell" "sh /data/local/tmp/ctest_mock.sh $RemoteBin" + Write-Host "$adb shell rm -rf $RemoteBin" + Invoke-Program $adb shell "rm -rf $RemoteBin" + Write-Host "$adb shell mkdir $RemoteBin" + Invoke-Program $adb shell "mkdir $RemoteBin" + Write-Host "$adb push $LocalBin/. $RemoteBin" + Invoke-Program $adb push "$LocalBin/." $RemoteBin + Write-Host "$adb push $SourceCache/swift/utils/android/utils/ctest_mock.sh /data/local/tmp" + Invoke-Program $adb push "$SourceCache/swift/utils/android/utils/ctest_mock.sh" "/data/local/tmp" + Write-Host "$adb shell chmod +x /data/local/tmp/ctest_mock.sh" + Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest_mock.sh" + Write-Host "$adb shell sh /data/local/tmp/ctest_mock.sh $RemoteBin" + Invoke-Program $adb shell "sh /data/local/tmp/ctest_mock.sh $RemoteBin" } function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { @@ -3125,7 +3132,7 @@ if (-not $IsCrossCompiling) { if ($Test -contains "dispatch") { Build-Dispatch Windows $HostArch -Test # TODO: This is a hack. We need different devices for different arches. - if (AndroidSDKs -contains "x86_64") { + if ($AndroidSDKs -contains "x86_64") { Test-Dispatch Android } } From d025ab2be3ff86afd8b4332b6bb8bfa020d9660a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 22 Jan 2025 13:05:58 +0100 Subject: [PATCH 03/30] [dev] Temporarily enable Android x86_64 in CI and test only dispatch --- utils/build-windows-toolchain.bat | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/build-windows-toolchain.bat b/utils/build-windows-toolchain.bat index 131c283eddc1e..d831a23778958 100644 --- a/utils/build-windows-toolchain.bat +++ b/utils/build-windows-toolchain.bat @@ -60,7 +60,7 @@ set TMPDIR=%BuildRoot%\tmp set NINJA_STATUS=[%%f/%%t][%%p][%%es] :: Build the -Test argument, if any, by subtracting skipped tests -set TestArg=-Test lld,lldb,swift,dispatch,foundation,xctest,swift-format,sourcekit-lsp, +set TestArg=-Test dispatch, for %%I in (%SKIP_TESTS%) do (call set TestArg=%%TestArg:%%I,=%%) if "%TestArg:~-1%"=="," (set TestArg=%TestArg:~0,-1%) else (set TestArg= ) @@ -77,6 +77,8 @@ powershell.exe -ExecutionPolicy RemoteSigned -File %~dp0build.ps1 ^ -ImageRoot %BuildRoot% ^ %SkipPackagingArg% ^ %TestArg% ^ + -AndroidSDKs x86_64 ^ + -WindowsSDKs X64 ^ -Stage %PackageRoot% ^ -Summary || (exit /b 1) From e256dff57d4287054fdfbdfefd26b94d47a22497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 22 Jan 2025 13:28:51 +0100 Subject: [PATCH 04/30] Properly spawn and shutdown both, adb and the emulator --- utils/build.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index e0b6b52a75a07..aa5069da49182 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2067,9 +2067,6 @@ function Test-Dispatch([Platform]$Platform) { -RedirectStandardOutput "$NDKDir\.temp\emulator.out" ` -RedirectStandardError "$NDKDir\.temp\emulator.err" - # TODO: Find a better indicator than a upper-bound delay - Start-Sleep -Seconds 5 - # This is just a hack for now $LocalBin = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) $CacheName = Split-Path $LocalBin -Leaf @@ -2077,6 +2074,7 @@ function Test-Dispatch([Platform]$Platform) { # TODO: On my local machine I have to grant network access once. How to do that in CI? $adb = "$NDKDir\platform-tools\adb.exe" + Invoke-Program $adb "wait-for-device" Write-Host "$adb shell rm -rf $RemoteBin" Invoke-Program $adb shell "rm -rf $RemoteBin" Write-Host "$adb shell mkdir $RemoteBin" @@ -2089,6 +2087,8 @@ function Test-Dispatch([Platform]$Platform) { Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest_mock.sh" Write-Host "$adb shell sh /data/local/tmp/ctest_mock.sh $RemoteBin" Invoke-Program $adb shell "sh /data/local/tmp/ctest_mock.sh $RemoteBin" + Invoke-Program $adb emu kill + Invoke-Program $adb kill-server } function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { From 39c6b90736bc8efb0b199e7bbe53b9861cf2996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 22 Jan 2025 16:52:12 +0100 Subject: [PATCH 05/30] Switch to cross-compiled ctest driver --- utils/build.ps1 | 93 ++++++++++++++++--- .../update-checkout-config.json | 2 +- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index aa5069da49182..fd4bef0881c13 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -824,7 +824,7 @@ function Fetch-Dependencies { Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" -CreateExtractPath $false - $NDKDir = "$BinaryCache\android-ndk-r26b" + $NDKDir = Get-AndroidNDKPath if ($Test -and -not(Test-Path "$NDKDir\licenses")) { $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" @@ -2057,11 +2057,50 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { } } -function Test-Dispatch([Platform]$Platform) { - # TODO: Make it global - $NDKDir = "$BinaryCache\android-ndk-r26b" +function Build-CTest([Platform]$Platform, $Arch) { + if (-not(Test-Path $SourceCache\libarchive)) { + Invoke-Program git clone "https://github.com/libarchive/libarchive" $SourceCache\libarchive + } + Build-CMakeProject ` + -Src $SourceCache\libarchive ` + -Bin "$($Arch.BinaryCache)\$Platform\libarchive" ` + -InstallTo "$($Arch.SDKInstallRoot)\usr" ` + -Arch $Arch ` + -Platform $Platform ` + -UseBuiltCompilers C,CXX - # TODO: Start once for all tests and make it shuts down + if (-not(Test-Path $SourceCache\libuv)) { + Invoke-Program git clone "https://github.com/libuv/libuv" $SourceCache\libuv + } + Build-CMakeProject ` + -Src $SourceCache\libuv ` + -Bin "$($Arch.BinaryCache)\$Platform\libuv" ` + -Arch $Arch ` + -Platform $Platform ` + -UseBuiltCompilers C,CXX ` + -BuildTargets "libuv.a" + + Build-CMakeProject ` + -Src $SourceCache\cmake ` + -Bin "$($Arch.BinaryCache)\$Platform\cmake" ` + -Arch $Arch ` + -Platform $Platform ` + -UseBuiltCompilers C,CXX ` + -BuildTargets ctest ` + -Defines (@{ + CMAKE_USE_SYSTEM_LIBUV = "YES"; + CMAKE_USE_SYSTEM_LIBARCHIVE = "YES"; + LibArchive_INCLUDE_DIR = "$($Arch.SDKInstallRoot)\usr\include"; + LibArchive_LIBRARY = "$($Arch.SDKInstallRoot)\usr\lib\libarchive.a"; + LibUV_INCLUDE_DIR = "$SourceCache\libuv\include"; + LibUV_LIBRARY = "$($Arch.BinaryCache)\$Platform\libuv\libuv.a"; + }) +} + +function Test-Dispatch([Platform]$Platform, $Arch) { + $NDKDir = Get-AndroidNDKPath + + # TODO: One emulator instance for all tests? $emulator = "$NDKDir\emulator\emulator.exe" Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" ` -RedirectStandardOutput "$NDKDir\.temp\emulator.out" ` @@ -2072,21 +2111,40 @@ function Test-Dispatch([Platform]$Platform) { $CacheName = Split-Path $LocalBin -Leaf $RemoteBin = "/data/local/tmp/$CacheName" - # TODO: On my local machine I have to grant network access once. How to do that in CI? + # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? $adb = "$NDKDir\platform-tools\adb.exe" Invoke-Program $adb "wait-for-device" + + # Add binary directory to emulator Write-Host "$adb shell rm -rf $RemoteBin" Invoke-Program $adb shell "rm -rf $RemoteBin" Write-Host "$adb shell mkdir $RemoteBin" Invoke-Program $adb shell "mkdir $RemoteBin" - Write-Host "$adb push $LocalBin/. $RemoteBin" - Invoke-Program $adb push "$LocalBin/." $RemoteBin - Write-Host "$adb push $SourceCache/swift/utils/android/utils/ctest_mock.sh /data/local/tmp" - Invoke-Program $adb push "$SourceCache/swift/utils/android/utils/ctest_mock.sh" "/data/local/tmp" - Write-Host "$adb shell chmod +x /data/local/tmp/ctest_mock.sh" - Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest_mock.sh" - Write-Host "$adb shell sh /data/local/tmp/ctest_mock.sh $RemoteBin" - Invoke-Program $adb shell "sh /data/local/tmp/ctest_mock.sh $RemoteBin" + Write-Host "$adb push $LocalBin\. $RemoteBin" + Invoke-Program $adb push "$LocalBin\." $RemoteBin + + # Replace absolute paths in config file + Write-Host "$adb shell sed -i 's|$($LocalBin.Replace('\', '/'))|$RemoteBin|g' $RemoteBin/tests/CTestTestfile.cmake" + Invoke-Program $adb shell "sed -i 's|$($LocalBin.Replace('\', '/'))|$RemoteBin|g' $RemoteBin/tests/CTestTestfile.cmake" + Invoke-Program $adb shell "cat $RemoteBin/tests/CTestTestfile.cmake" + + # Set executable flag for all tests (and bsdtestharness utility) + $Executables = @("bsdtestharness","dispatch_apply","dispatch_api","dispatch_debug","dispatch_queue_finalizer","dispatch_overcommit","dispatch_context_for_key","dispatch_after","dispatch_timer","dispatch_timer_short","dispatch_timer_timeout","dispatch_sema","dispatch_timer_bit31","dispatch_timer_bit63","dispatch_timer_set_time","dispatch_data","dispatch_io_muxed","dispatch_io_net","dispatch_io_pipe","dispatch_io_pipe_close","dispatch_select","dispatch_c99","dispatch_plusplus") + foreach ($Exe in $Executables) { + Write-Host "$adb shell chmod +x $RemoteBin/$Exe" + Invoke-Program $adb shell "chmod +x $RemoteBin/$Exe" + } + + # Add ctest driver to emulator and make it executable + Write-Host "$adb push $($Arch.BinaryCache)\$Platform\cmake\bin\ctest /data/local/tmp" + Invoke-Program $adb push "$($Arch.BinaryCache)\$Platform\cmake\bin\ctest" "/data/local/tmp" + Write-Host "$adb shell chmod +x /data/local/tmp/ctest" + Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest" + + # Run tests + Write-Host "$adb shell /data/local/tmp/ctest --test-dir $RemoteBin" + Invoke-Program $adb shell "/data/local/tmp/ctest --test-dir $RemoteBin" + Invoke-Program $adb emu kill Invoke-Program $adb kill-server } @@ -3133,7 +3191,8 @@ if (-not $IsCrossCompiling) { Build-Dispatch Windows $HostArch -Test # TODO: This is a hack. We need different devices for different arches. if ($AndroidSDKs -contains "x86_64") { - Test-Dispatch Android + Invoke-BuildStep Build-CTest Android AndroidX64 + Test-Dispatch Android AndroidX64 } } if ($Test -contains "foundation") { @@ -3178,6 +3237,10 @@ if (-not $IsCrossCompiling) { exit 1 } finally { + $adb = "$(Get-AndroidNDKPath)\platform-tools\adb.exe" + Invoke-Program $adb emu kill + Invoke-Program $adb kill-server + if ($Summary) { $TimingData | Select-Object Platform,Arch,Checkout,"Elapsed Time" | Sort-Object -Descending -Property "Elapsed Time" | Format-Table -AutoSize } diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index b01cb5aee0854..adfeb2b3006c4 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -65,7 +65,7 @@ }, "cmake": { "remote": { "id": "KitWare/CMake" }, - "platforms": [ "Linux" ] + "platforms": [ "Linux", "Windows" ] }, "swift-cmark-gfm": { "remote": { "id": "swiftlang/swift-cmark" } }, From b9f18c0b1c03b056683d7e45afc2b9e520ee5f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Fri, 24 Jan 2025 15:26:57 +0100 Subject: [PATCH 06/30] For the moment install Java Runtime on the fly --- utils/build.ps1 | 51 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index fd4bef0881c13..0b4feaaab5d71 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -716,7 +716,7 @@ function Fetch-Dependencies { } } - if ($SkipBuild -and $SkipPackaging) { return } + if ($SkipBuild -and $SkipPackaging -and -not $Test) { return } $WiXURL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion" $WiXHash = "DF9BDB347183716F82EFE2CECB8C54BB3554AA907A69F47A41741D6FA4D0A754" @@ -824,28 +824,51 @@ function Fetch-Dependencies { Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" -CreateExtractPath $false + # FIXME: Both Java and Android emulator must be available in the environment. + # This is a terrible workaround. It's a waste of time and resources. $NDKDir = Get-AndroidNDKPath if ($Test -and -not(Test-Path "$NDKDir\licenses")) { + # Download Java Runtime + switch ($BuildArchName) { + "AMD64" { + $JavaURL = "https://aka.ms/download-jdk/microsoft-jdk-17.0.14-windows-x64.zip" + $JavaHash = "3619082f4a667f52c97cce983364a517993f798ae248c411765becfd9705767f" + DownloadAndVerify $JavaURL "$BinaryCache\android-cmdline-tools.zip" $JavaHash + } + "ARM64" { + $JavaURL = "https://aka.ms/download-jdk/microsoft-jdk-17.0.14-windows-aarch64.zip" + $JavaHash = "2a12c7b3d46712de9671f5f011a3cae9ee53d5ff3b0604136ee079a906146448" + } + default { throw "Unsupported processor architecture" } + } + DownloadAndVerify $JavaURL "$BinaryCache\microsoft-jdk.zip" $JavaHash + Extract-ZipFile -ZipFileName "microsoft-jdk.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp" + + # Download cmdline-tools $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" DownloadAndVerify $CLToolsURL "$BinaryCache\android-cmdline-tools.zip" $CLToolsHash + Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp" - # Install cmdline-tools + # Accept licenses New-Item -Type Directory -Path "$NDKDir\licenses" -ErrorAction Ignore | Out-Null Set-Content -Path "$NDKDir\licenses\android-sdk-license" -Value "24333f8a63b6825ea9c5514f83c2829b004d1fee" Set-Content -Path "$NDKDir\licenses\android-sdk-preview-license" -Value "84831b9409646a918e30573bab4c9c91346d8abd" - Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp" - Invoke-Program "$NDKDir\.temp\cmdline-tools\bin\sdkmanager.bat" --sdk_root="$NDKDir" "cmdline-tools;latest" "--channel=3" - # Install packages - $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" - foreach ($Package in @("emulator", "platforms;android-29", "system-images;android-29;default;x86_64", "platform-tools")) { - Invoke-Program $AndroidSdkMgr $Package - } + # Install packages and create test device + Isolate-EnvVars { + $env:JAVA_HOME = "$NDKDir\.temp\jdk-17.0.14+7" + $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" - # Create test device - $AndroidAvdMgr = "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" - Invoke-Program $AndroidAvdMgr create avd --name "swift-test-device" --package "system-images;android-29;default;x86_64" + Invoke-Program "$NDKDir\.temp\cmdline-tools\bin\sdkmanager.bat" --sdk_root="$NDKDir" "cmdline-tools;latest" "--channel=3" + $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" + foreach ($Package in @("emulator", "platforms;android-29", "system-images;android-29;default;x86_64", "platform-tools")) { + Invoke-Program $AndroidSdkMgr $Package + } + + $AndroidAvdMgr = "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" + Invoke-Program $AndroidAvdMgr create avd --name "swift-test-device" --package "system-images;android-29;default;x86_64" + } } } @@ -3191,8 +3214,8 @@ if (-not $IsCrossCompiling) { Build-Dispatch Windows $HostArch -Test # TODO: This is a hack. We need different devices for different arches. if ($AndroidSDKs -contains "x86_64") { - Invoke-BuildStep Build-CTest Android AndroidX64 - Test-Dispatch Android AndroidX64 + Invoke-BuildStep Build-CTest Android $AndroidX64 + Test-Dispatch Android $AndroidX64 } } if ($Test -contains "foundation") { From e2a091225c555fe50e9c6f16466a97576b36997f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 27 Jan 2025 11:41:51 +0100 Subject: [PATCH 07/30] Fix invalid 'zip is already extracted and up to date' --- utils/build.ps1 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 0b4feaaab5d71..2ca717d3e80bf 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -833,7 +833,6 @@ function Fetch-Dependencies { "AMD64" { $JavaURL = "https://aka.ms/download-jdk/microsoft-jdk-17.0.14-windows-x64.zip" $JavaHash = "3619082f4a667f52c97cce983364a517993f798ae248c411765becfd9705767f" - DownloadAndVerify $JavaURL "$BinaryCache\android-cmdline-tools.zip" $JavaHash } "ARM64" { $JavaURL = "https://aka.ms/download-jdk/microsoft-jdk-17.0.14-windows-aarch64.zip" @@ -842,13 +841,13 @@ function Fetch-Dependencies { default { throw "Unsupported processor architecture" } } DownloadAndVerify $JavaURL "$BinaryCache\microsoft-jdk.zip" $JavaHash - Extract-ZipFile -ZipFileName "microsoft-jdk.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp" + Extract-ZipFile -ZipFileName "microsoft-jdk.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp\jdk" # Download cmdline-tools $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" DownloadAndVerify $CLToolsURL "$BinaryCache\android-cmdline-tools.zip" $CLToolsHash - Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp" + Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp\cmdline-tools" # Accept licenses New-Item -Type Directory -Path "$NDKDir\licenses" -ErrorAction Ignore | Out-Null @@ -857,10 +856,10 @@ function Fetch-Dependencies { # Install packages and create test device Isolate-EnvVars { - $env:JAVA_HOME = "$NDKDir\.temp\jdk-17.0.14+7" + $env:JAVA_HOME = "$NDKDir\.temp\jdk\jdk-17.0.14+7" $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" - Invoke-Program "$NDKDir\.temp\cmdline-tools\bin\sdkmanager.bat" --sdk_root="$NDKDir" "cmdline-tools;latest" "--channel=3" + Invoke-Program "$NDKDir\.temp\cmdline-tools\cmdline-tools\bin\sdkmanager.bat" --sdk_root="$NDKDir" "cmdline-tools;latest" "--channel=3" $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" foreach ($Package in @("emulator", "platforms;android-29", "system-images;android-29;default;x86_64", "platform-tools")) { Invoke-Program $AndroidSdkMgr $Package From ff8a9fac23fb9daf57966b793b269962d5d6ec10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 27 Jan 2025 11:43:35 +0100 Subject: [PATCH 08/30] Add timer for FetchDependencies --- utils/build.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index 2ca717d3e80bf..c479a09589b26 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -718,6 +718,11 @@ function Fetch-Dependencies { if ($SkipBuild -and $SkipPackaging -and -not $Test) { return } + $Stopwatch = [Diagnostics.Stopwatch]::StartNew() + if ($ToBatch) { + Write-Host -ForegroundColor Cyan "[$([DateTime]::Now.ToString("yyyy-MM-dd HH:mm:ss"))] Fetch-Dependencies..." + } + $WiXURL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion" $WiXHash = "DF9BDB347183716F82EFE2CECB8C54BB3554AA907A69F47A41741D6FA4D0A754" DownloadAndVerify $WixURL "$BinaryCache\WiX-$WiXVersion.zip" $WiXHash @@ -905,6 +910,11 @@ function Fetch-Dependencies { } } } + + if (-not $ToBatch) { + Write-Host -ForegroundColor Cyan "[$([DateTime]::Now.ToString("yyyy-MM-dd HH:mm:ss"))] Fetch-Dependencies took $($Stopwatch.Elapsed)" + Write-Host "" + } } function Get-PinnedToolchainTool() { From 366071a7e9091e43d2582966a207ee1aeeb27ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 27 Jan 2025 11:45:12 +0100 Subject: [PATCH 09/30] utils: trim whitespace in build.ps1 (NFC) --- utils/build.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index c479a09589b26..e4b94f144e846 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -600,7 +600,7 @@ function Invoke-VsDevShell($Arch) { if ($ToBatch) { Write-Output "call `"$VSInstallRoot\Common7\Tools\VsDevCmd.bat`" $DevCmdArguments" } else { - # This dll path is valid for VS2019 and VS2022, but it was under a vsdevcmd subfolder in VS2017 + # This dll path is valid for VS2019 and VS2022, but it was under a vsdevcmd subfolder in VS2017 Import-Module "$VSInstallRoot\Common7\Tools\Microsoft.VisualStudio.DevShell.dll" Enter-VsDevShell -VsInstallPath $VSInstallRoot -SkipAutomaticLocation -DevCmdArguments $DevCmdArguments @@ -682,7 +682,6 @@ function Fetch-Dependencies { Expand-Archive -Path $source -DestinationPath $destination -Force } - function Extract-Toolchain { param ( @@ -1449,7 +1448,7 @@ function Build-WiXProject() { $ProductVersionArg = $ProductVersion if (-not $Bundle) { # WiX v4 will accept a semantic version string for Bundles, - # but Packages still require a purely numerical version number, + # but Packages still require a purely numerical version number, # so trim any semantic versioning suffixes $ProductVersionArg = [regex]::Replace($ProductVersion, "[-+].*", "") } @@ -2740,7 +2739,7 @@ function Test-Format { "-Xswiftc", "-I$(Get-HostProjectBinaryCache Format)\swift", "-Xlinker", "-L$(Get-HostProjectBinaryCache Format)\lib" ) - + Isolate-EnvVars { $env:SWIFTFORMAT_BUILD_ONLY_TESTS=1 # Testing swift-format is faster in serial mode than in parallel mode, probably because parallel test execution From bbc7c011da34cd5f2a4a71b646fd879af2e0fd4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 27 Jan 2025 15:38:10 +0100 Subject: [PATCH 10/30] Don't mix up release NDK with the one installed via cmdline-tools --- utils/build.ps1 | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index e4b94f144e846..38863301ce9de 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -830,7 +830,7 @@ function Fetch-Dependencies { # FIXME: Both Java and Android emulator must be available in the environment. # This is a terrible workaround. It's a waste of time and resources. - $NDKDir = Get-AndroidNDKPath + $NDKDir = "$BinaryCache\android-sdk" if ($Test -and -not(Test-Path "$NDKDir\licenses")) { # Download Java Runtime switch ($BuildArchName) { @@ -845,13 +845,13 @@ function Fetch-Dependencies { default { throw "Unsupported processor architecture" } } DownloadAndVerify $JavaURL "$BinaryCache\microsoft-jdk.zip" $JavaHash - Extract-ZipFile -ZipFileName "microsoft-jdk.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp\jdk" + Extract-ZipFile -ZipFileName "microsoft-jdk.zip" -BinaryCache $BinaryCache -ExtractPath "android-sdk-jdk" # Download cmdline-tools $CLToolsURL = "https://dl.google.com/android/repository/commandlinetools-win-11076708_latest.zip" $CLToolsHash = "4d6931209eebb1bfb7c7e8b240a6a3cb3ab24479ea294f3539429574b1eec862" DownloadAndVerify $CLToolsURL "$BinaryCache\android-cmdline-tools.zip" $CLToolsHash - Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-r26b\.temp\cmdline-tools" + Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-sdk-cmdline-tools" # Accept licenses New-Item -Type Directory -Path "$NDKDir\licenses" -ErrorAction Ignore | Out-Null @@ -860,17 +860,18 @@ function Fetch-Dependencies { # Install packages and create test device Isolate-EnvVars { - $env:JAVA_HOME = "$NDKDir\.temp\jdk\jdk-17.0.14+7" + $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" - Invoke-Program "$NDKDir\.temp\cmdline-tools\cmdline-tools\bin\sdkmanager.bat" --sdk_root="$NDKDir" "cmdline-tools;latest" "--channel=3" + Invoke-Program "$BinaryCache\android-sdk-cmdline-tools\cmdline-tools\bin\sdkmanager.bat" "--sdk_root=$NDKDir" '"cmdline-tools;latest"' '--channel=3' $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" - foreach ($Package in @("emulator", "platforms;android-29", "system-images;android-29;default;x86_64", "platform-tools")) { - Invoke-Program $AndroidSdkMgr $Package + foreach ($Package in @('"ndk;26.2.11394342"', '"system-images;android-29;default;x86_64"', '"platforms;android-29"', '"platform-tools"')) { + Write-Host "$AndroidSdkMgr $Package" + Invoke-Program -OutNull $AndroidSdkMgr $Package } $AndroidAvdMgr = "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" - Invoke-Program $AndroidAvdMgr create avd --name "swift-test-device" --package "system-images;android-29;default;x86_64" + Invoke-Program $AndroidAvdMgr create avd --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' } } } @@ -2143,7 +2144,7 @@ function Test-Dispatch([Platform]$Platform, $Arch) { $RemoteBin = "/data/local/tmp/$CacheName" # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? - $adb = "$NDKDir\platform-tools\adb.exe" + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" Invoke-Program $adb "wait-for-device" # Add binary directory to emulator @@ -3268,7 +3269,7 @@ if (-not $IsCrossCompiling) { exit 1 } finally { - $adb = "$(Get-AndroidNDKPath)\platform-tools\adb.exe" + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" Invoke-Program $adb emu kill Invoke-Program $adb kill-server From 1d00e4bac645048f3925a1d9dc4a6950549faab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Tue, 28 Jan 2025 17:02:30 +0100 Subject: [PATCH 11/30] Disable interactive mode in avdmanager --- utils/build.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 38863301ce9de..7dace5d6a776f 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -870,8 +870,8 @@ function Fetch-Dependencies { Invoke-Program -OutNull $AndroidSdkMgr $Package } - $AndroidAvdMgr = "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" - Invoke-Program $AndroidAvdMgr create avd --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' + # There is no way to disable interactive mode in avdmanager + "no" | & "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" create avd --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' } } } From 69bb836658c640c1647362853cce860a93a46dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Tue, 28 Jan 2025 17:30:58 +0100 Subject: [PATCH 12/30] Fix emulator path, drop ndk package, add comments --- utils/build.ps1 | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 7dace5d6a776f..12788c3e6b825 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -863,15 +863,16 @@ function Fetch-Dependencies { $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" - Invoke-Program "$BinaryCache\android-sdk-cmdline-tools\cmdline-tools\bin\sdkmanager.bat" "--sdk_root=$NDKDir" '"cmdline-tools;latest"' '--channel=3' + # Let cmdline-tools install itself. This is idiomatic for installing the Android SDK. + Invoke-Program "$BinaryCache\android-sdk-cmdline-tools\cmdline-tools\bin\sdkmanager.bat" -OutNull "--sdk_root=$NDKDir" '"cmdline-tools;latest"' '--channel=3' $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" - foreach ($Package in @('"ndk;26.2.11394342"', '"system-images;android-29;default;x86_64"', '"platforms;android-29"', '"platform-tools"')) { + foreach ($Package in @('"system-images;android-29;default;x86_64"', '"platforms;android-29"', '"platform-tools"')) { Write-Host "$AndroidSdkMgr $Package" Invoke-Program -OutNull $AndroidSdkMgr $Package } # There is no way to disable interactive mode in avdmanager - "no" | & "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" create avd --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' + "no" | & "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" create avd --force --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' } } } @@ -2130,13 +2131,11 @@ function Build-CTest([Platform]$Platform, $Arch) { } function Test-Dispatch([Platform]$Platform, $Arch) { - $NDKDir = Get-AndroidNDKPath - # TODO: One emulator instance for all tests? - $emulator = "$NDKDir\emulator\emulator.exe" + $emulator = "$BinaryCache\android-sdk\emulator\emulator.exe" Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" ` - -RedirectStandardOutput "$NDKDir\.temp\emulator.out" ` - -RedirectStandardError "$NDKDir\.temp\emulator.err" + -RedirectStandardOutput "$BinaryCache\android-sdk\.temp\emulator.out" ` + -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" # This is just a hack for now $LocalBin = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) From f9dd2f380e68a48475853786df728a019685a6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Tue, 28 Jan 2025 17:32:03 +0100 Subject: [PATCH 13/30] Rename: NDKDir -> SDKDir --- utils/build.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 12788c3e6b825..30b83e8fefa03 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -830,8 +830,8 @@ function Fetch-Dependencies { # FIXME: Both Java and Android emulator must be available in the environment. # This is a terrible workaround. It's a waste of time and resources. - $NDKDir = "$BinaryCache\android-sdk" - if ($Test -and -not(Test-Path "$NDKDir\licenses")) { + $SDKDir = "$BinaryCache\android-sdk" + if ($Test -and -not(Test-Path "$SDKDir\licenses")) { # Download Java Runtime switch ($BuildArchName) { "AMD64" { @@ -854,9 +854,9 @@ function Fetch-Dependencies { Extract-ZipFile -ZipFileName "android-cmdline-tools.zip" -BinaryCache $BinaryCache -ExtractPath "android-sdk-cmdline-tools" # Accept licenses - New-Item -Type Directory -Path "$NDKDir\licenses" -ErrorAction Ignore | Out-Null - Set-Content -Path "$NDKDir\licenses\android-sdk-license" -Value "24333f8a63b6825ea9c5514f83c2829b004d1fee" - Set-Content -Path "$NDKDir\licenses\android-sdk-preview-license" -Value "84831b9409646a918e30573bab4c9c91346d8abd" + New-Item -Type Directory -Path "$SDKDir\licenses" -ErrorAction Ignore | Out-Null + Set-Content -Path "$SDKDir\licenses\android-sdk-license" -Value "24333f8a63b6825ea9c5514f83c2829b004d1fee" + Set-Content -Path "$SDKDir\licenses\android-sdk-preview-license" -Value "84831b9409646a918e30573bab4c9c91346d8abd" # Install packages and create test device Isolate-EnvVars { @@ -864,15 +864,15 @@ function Fetch-Dependencies { $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" # Let cmdline-tools install itself. This is idiomatic for installing the Android SDK. - Invoke-Program "$BinaryCache\android-sdk-cmdline-tools\cmdline-tools\bin\sdkmanager.bat" -OutNull "--sdk_root=$NDKDir" '"cmdline-tools;latest"' '--channel=3' - $AndroidSdkMgr = "$NDKDir\cmdline-tools\latest\bin\sdkmanager.bat" + Invoke-Program "$BinaryCache\android-sdk-cmdline-tools\cmdline-tools\bin\sdkmanager.bat" -OutNull "--sdk_root=$SDKDir" '"cmdline-tools;latest"' '--channel=3' + $AndroidSdkMgr = "$SDKDir\cmdline-tools\latest\bin\sdkmanager.bat" foreach ($Package in @('"system-images;android-29;default;x86_64"', '"platforms;android-29"', '"platform-tools"')) { Write-Host "$AndroidSdkMgr $Package" Invoke-Program -OutNull $AndroidSdkMgr $Package } # There is no way to disable interactive mode in avdmanager - "no" | & "$NDKDir\cmdline-tools\latest\bin\avdmanager.bat" create avd --force --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' + "no" | & "$SDKDir\cmdline-tools\latest\bin\avdmanager.bat" create avd --force --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' } } } From 37b97bbe662c277ed3cd9d999d5dd2c80a4fb07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 29 Jan 2025 14:43:39 +0100 Subject: [PATCH 14/30] Revert "Switch to cross-compiled ctest driver" This reverts commit 6ceb869500f28b0979f635debd82e4e38927f921. --- utils/build.ps1 | 78 +++---------------- .../update-checkout-config.json | 2 +- 2 files changed, 10 insertions(+), 70 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 30b83e8fefa03..18ce680219cc9 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2090,47 +2090,7 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { } } -function Build-CTest([Platform]$Platform, $Arch) { - if (-not(Test-Path $SourceCache\libarchive)) { - Invoke-Program git clone "https://github.com/libarchive/libarchive" $SourceCache\libarchive - } - Build-CMakeProject ` - -Src $SourceCache\libarchive ` - -Bin "$($Arch.BinaryCache)\$Platform\libarchive" ` - -InstallTo "$($Arch.SDKInstallRoot)\usr" ` - -Arch $Arch ` - -Platform $Platform ` - -UseBuiltCompilers C,CXX - - if (-not(Test-Path $SourceCache\libuv)) { - Invoke-Program git clone "https://github.com/libuv/libuv" $SourceCache\libuv - } - Build-CMakeProject ` - -Src $SourceCache\libuv ` - -Bin "$($Arch.BinaryCache)\$Platform\libuv" ` - -Arch $Arch ` - -Platform $Platform ` - -UseBuiltCompilers C,CXX ` - -BuildTargets "libuv.a" - - Build-CMakeProject ` - -Src $SourceCache\cmake ` - -Bin "$($Arch.BinaryCache)\$Platform\cmake" ` - -Arch $Arch ` - -Platform $Platform ` - -UseBuiltCompilers C,CXX ` - -BuildTargets ctest ` - -Defines (@{ - CMAKE_USE_SYSTEM_LIBUV = "YES"; - CMAKE_USE_SYSTEM_LIBARCHIVE = "YES"; - LibArchive_INCLUDE_DIR = "$($Arch.SDKInstallRoot)\usr\include"; - LibArchive_LIBRARY = "$($Arch.SDKInstallRoot)\usr\lib\libarchive.a"; - LibUV_INCLUDE_DIR = "$SourceCache\libuv\include"; - LibUV_LIBRARY = "$($Arch.BinaryCache)\$Platform\libuv\libuv.a"; - }) -} - -function Test-Dispatch([Platform]$Platform, $Arch) { +function Test-Dispatch([Platform]$Platform) { # TODO: One emulator instance for all tests? $emulator = "$BinaryCache\android-sdk\emulator\emulator.exe" Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" ` @@ -2145,37 +2105,18 @@ function Test-Dispatch([Platform]$Platform, $Arch) { # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" Invoke-Program $adb "wait-for-device" - - # Add binary directory to emulator Write-Host "$adb shell rm -rf $RemoteBin" Invoke-Program $adb shell "rm -rf $RemoteBin" Write-Host "$adb shell mkdir $RemoteBin" Invoke-Program $adb shell "mkdir $RemoteBin" - Write-Host "$adb push $LocalBin\. $RemoteBin" - Invoke-Program $adb push "$LocalBin\." $RemoteBin - - # Replace absolute paths in config file - Write-Host "$adb shell sed -i 's|$($LocalBin.Replace('\', '/'))|$RemoteBin|g' $RemoteBin/tests/CTestTestfile.cmake" - Invoke-Program $adb shell "sed -i 's|$($LocalBin.Replace('\', '/'))|$RemoteBin|g' $RemoteBin/tests/CTestTestfile.cmake" - Invoke-Program $adb shell "cat $RemoteBin/tests/CTestTestfile.cmake" - - # Set executable flag for all tests (and bsdtestharness utility) - $Executables = @("bsdtestharness","dispatch_apply","dispatch_api","dispatch_debug","dispatch_queue_finalizer","dispatch_overcommit","dispatch_context_for_key","dispatch_after","dispatch_timer","dispatch_timer_short","dispatch_timer_timeout","dispatch_sema","dispatch_timer_bit31","dispatch_timer_bit63","dispatch_timer_set_time","dispatch_data","dispatch_io_muxed","dispatch_io_net","dispatch_io_pipe","dispatch_io_pipe_close","dispatch_select","dispatch_c99","dispatch_plusplus") - foreach ($Exe in $Executables) { - Write-Host "$adb shell chmod +x $RemoteBin/$Exe" - Invoke-Program $adb shell "chmod +x $RemoteBin/$Exe" - } - - # Add ctest driver to emulator and make it executable - Write-Host "$adb push $($Arch.BinaryCache)\$Platform\cmake\bin\ctest /data/local/tmp" - Invoke-Program $adb push "$($Arch.BinaryCache)\$Platform\cmake\bin\ctest" "/data/local/tmp" - Write-Host "$adb shell chmod +x /data/local/tmp/ctest" - Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest" - - # Run tests - Write-Host "$adb shell /data/local/tmp/ctest --test-dir $RemoteBin" - Invoke-Program $adb shell "/data/local/tmp/ctest --test-dir $RemoteBin" - + Write-Host "$adb push $LocalBin/. $RemoteBin" + Invoke-Program $adb push "$LocalBin/." $RemoteBin + Write-Host "$adb push $SourceCache/swift/utils/android/utils/ctest_mock.sh /data/local/tmp" + Invoke-Program $adb push "$SourceCache/swift/utils/android/utils/ctest_mock.sh" "/data/local/tmp" + Write-Host "$adb shell chmod +x /data/local/tmp/ctest_mock.sh" + Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest_mock.sh" + Write-Host "$adb shell sh /data/local/tmp/ctest_mock.sh $RemoteBin" + Invoke-Program $adb shell "sh /data/local/tmp/ctest_mock.sh $RemoteBin" Invoke-Program $adb emu kill Invoke-Program $adb kill-server } @@ -3222,7 +3163,6 @@ if (-not $IsCrossCompiling) { Build-Dispatch Windows $HostArch -Test # TODO: This is a hack. We need different devices for different arches. if ($AndroidSDKs -contains "x86_64") { - Invoke-BuildStep Build-CTest Android $AndroidX64 Test-Dispatch Android $AndroidX64 } } diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index adfeb2b3006c4..b01cb5aee0854 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -65,7 +65,7 @@ }, "cmake": { "remote": { "id": "KitWare/CMake" }, - "platforms": [ "Linux", "Windows" ] + "platforms": [ "Linux" ] }, "swift-cmark-gfm": { "remote": { "id": "swiftlang/swift-cmark" } }, From b3f8b3e4ec549b024045a57e73f82f8c659fd6a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 29 Jan 2025 15:14:23 +0100 Subject: [PATCH 15/30] Polish script-based driver --- utils/android/emulator/ctest-libdispatch.sh | 49 +++++++++++++++++ utils/android/utils/ctest_mock.sh | 58 --------------------- utils/build.ps1 | 18 ++++--- 3 files changed, 59 insertions(+), 66 deletions(-) create mode 100644 utils/android/emulator/ctest-libdispatch.sh delete mode 100644 utils/android/utils/ctest_mock.sh diff --git a/utils/android/emulator/ctest-libdispatch.sh b/utils/android/emulator/ctest-libdispatch.sh new file mode 100644 index 0000000000000..4ebe1eb554ab7 --- /dev/null +++ b/utils/android/emulator/ctest-libdispatch.sh @@ -0,0 +1,49 @@ +#!/system/bin/sh + +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +SUB_DIR=$1 +if [ ! -d "$SUB_DIR" ]; then + echo "Error: Directory $SUB_DIR does not exist" + exit 1 +fi + +UTILITY="$SUB_DIR/bsdtestharness" +if [ ! -f "$UTILITY" ]; then + echo "Utility not found: $UTILITY." + exit 1 +fi + +# FIXME: Add this script to libdispatch and let CMake inject actual test at configuration time +TESTS="apply api debug queue_finalizer overcommit context_for_key after timer timer_short timer_timeout sema timer_bit31 timer_bit63 timer_set_time data io_muxed io_net io_pipe io_pipe_close select c99 plusplus" +COUNT=$(echo "$TESTS" | tr ' ' '\n' | wc -l) +echo "Found $COUNT test-cases in $SUB_DIR" + +# Tests depend on libdispatch.so and libBlocksRuntime.so +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SUB_DIR" + +TIMOUT=120 +FAILURE=0 +chmod +x "$UTILITY" + +for TEST in $TESTS; do + chmod +x "$SUB_DIR/dispatch_$TEST" + OUTPUT=$(timeout "${TIMOUT}s" "$UTILITY" "$SUB_DIR/dispatch_$TEST" 2>&1) + EC=$? + if [ $EC -eq 124 ]; then + echo "Error: dispatch_$TEST timed out after $TIMOUT seconds" + echo "************\nOutput:\n$OUTPUT\n************" + FAILURE=1 + elif [ $EC -ne 0 ]; then + echo "Error: dispatch_$TEST failed" + echo "************\nOutput:\n$OUTPUT\n************" + FAILURE=1 + else + echo "dispatch_$TEST passed" + fi +done + +exit $FAILURE diff --git a/utils/android/utils/ctest_mock.sh b/utils/android/utils/ctest_mock.sh deleted file mode 100644 index aed55b9912d41..0000000000000 --- a/utils/android/utils/ctest_mock.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/system/bin/sh - -if [ -z "$1" ]; then - echo "Usage: $0 " - exit 1 -fi - -SUB_DIR=$1 -if [ ! -d "$SUB_DIR" ]; then - echo "Error: Directory $SUB_DIR does not exist" - exit 1 -fi - -CTEST_FILE="$SUB_DIR/CTestTestfile.cmake" -if [ ! -f "$CTEST_FILE" ]; then - echo "Error: $CTEST_FILE not found in $SUB_DIR" - exit 1 -fi - -# TODO: Parse them from CTEST_FILE -TESTS="dispatch_apply dispatch_api dispatch_debug dispatch_queue_finalizer dispatch_overcommit dispatch_context_for_key dispatch_after dispatch_timer dispatch_timer_short dispatch_timer_timeout dispatch_sema dispatch_timer_bit31 dispatch_timer_bit63 dispatch_timer_set_time dispatch_data dispatch_io_muxed dispatch_io_net dispatch_io_pipe dispatch_io_pipe_close dispatch_select dispatch_c99 dispatch_plusplus" - -COUNT=$(echo "$TESTS" | tr ' ' '\n' | wc -l) -echo "Found $COUNT test-cases in $SUB_DIR" - -UTILITY="$SUB_DIR/bsdtestharness" -if [ ! -f "$UTILITY" ]; then - echo "Utility not found: $UTILITY." - exit 1 -fi - -chmod +x "$UTILITY" - -# We need libdispatch.so and libBlocksRuntime.so -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SUB_DIR" - -# TODO: Parse the timeout as well from CTestTestfile.cmake? -TIMOUT=120 -FAILURE=0 - -for TEST in $TESTS; do - chmod +x "$SUB_DIR/$TEST" - OUTPUT=$(timeout "${TIMOUT}s" "$UTILITY" "$SUB_DIR/$TEST" 2>&1) - EC=$? - if [ $EC -eq 124 ]; then - echo "Error: $TEST timed out after $TIMOUT seconds" - echo "************\nOutput:\n$OUTPUT\n************" - FAILURE=1 - elif [ $EC -ne 0 ]; then - echo "Error: $TEST failed" - echo "************\nOutput:\n$OUTPUT\n************" - FAILURE=1 - else - echo "$TEST passed" - fi -done - -exit $FAILURE diff --git a/utils/build.ps1 b/utils/build.ps1 index 18ce680219cc9..fe827a46e6198 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2111,12 +2111,12 @@ function Test-Dispatch([Platform]$Platform) { Invoke-Program $adb shell "mkdir $RemoteBin" Write-Host "$adb push $LocalBin/. $RemoteBin" Invoke-Program $adb push "$LocalBin/." $RemoteBin - Write-Host "$adb push $SourceCache/swift/utils/android/utils/ctest_mock.sh /data/local/tmp" - Invoke-Program $adb push "$SourceCache/swift/utils/android/utils/ctest_mock.sh" "/data/local/tmp" - Write-Host "$adb shell chmod +x /data/local/tmp/ctest_mock.sh" - Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest_mock.sh" - Write-Host "$adb shell sh /data/local/tmp/ctest_mock.sh $RemoteBin" - Invoke-Program $adb shell "sh /data/local/tmp/ctest_mock.sh $RemoteBin" + Write-Host "$adb push $SourceCache/swift/utils/android/emulator/ctest-libdispatch.sh /data/local/tmp" + Invoke-Program $adb push "$SourceCache/swift/utils/android/emulator/ctest-libdispatch.sh" "/data/local/tmp" + Write-Host "$adb shell chmod +x /data/local/tmp/ctest-libdispatch.sh" + Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest-libdispatch.sh" + Write-Host "$adb shell sh /data/local/tmp/ctest-libdispatch.sh $RemoteBin" + Invoke-Program $adb shell "sh /data/local/tmp/ctest-libdispatch.sh $RemoteBin" Invoke-Program $adb emu kill Invoke-Program $adb kill-server } @@ -3209,8 +3209,10 @@ if (-not $IsCrossCompiling) { exit 1 } finally { $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" - Invoke-Program $adb emu kill - Invoke-Program $adb kill-server + if (Test-Path $adb) { + & $adb emu kill | Out-Null + & $adb kill-server | Out-Null + } if ($Summary) { $TimingData | Select-Object Platform,Arch,Checkout,"Elapsed Time" | Sort-Object -Descending -Property "Elapsed Time" | Format-Table -AutoSize From caddf1347b5cecfc4ac0b910281981ab03a0a166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 29 Jan 2025 17:24:10 +0100 Subject: [PATCH 16/30] Use proposed ctest-android.sh from libdispatch --- utils/build.ps1 | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index fe827a46e6198..daacfc3cbd485 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2098,25 +2098,17 @@ function Test-Dispatch([Platform]$Platform) { -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" # This is just a hack for now - $LocalBin = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) - $CacheName = Split-Path $LocalBin -Leaf - $RemoteBin = "/data/local/tmp/$CacheName" + $CachePath = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) + $CacheName = Split-Path $CachePath -Leaf + $RemoteRoot = "/data/local/tmp" # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" Invoke-Program $adb "wait-for-device" - Write-Host "$adb shell rm -rf $RemoteBin" - Invoke-Program $adb shell "rm -rf $RemoteBin" - Write-Host "$adb shell mkdir $RemoteBin" - Invoke-Program $adb shell "mkdir $RemoteBin" - Write-Host "$adb push $LocalBin/. $RemoteBin" - Invoke-Program $adb push "$LocalBin/." $RemoteBin - Write-Host "$adb push $SourceCache/swift/utils/android/emulator/ctest-libdispatch.sh /data/local/tmp" - Invoke-Program $adb push "$SourceCache/swift/utils/android/emulator/ctest-libdispatch.sh" "/data/local/tmp" - Write-Host "$adb shell chmod +x /data/local/tmp/ctest-libdispatch.sh" - Invoke-Program $adb shell "chmod +x /data/local/tmp/ctest-libdispatch.sh" - Write-Host "$adb shell sh /data/local/tmp/ctest-libdispatch.sh $RemoteBin" - Invoke-Program $adb shell "sh /data/local/tmp/ctest-libdispatch.sh $RemoteBin" + Write-Host "$adb push $CachePath $RemoteRoot" + Invoke-Program $adb push $CachePath $RemoteRoot + Write-Host "$adb shell sh $RemoteRoot/$CacheName/tests/ctest-android.sh" + Invoke-Program $adb shell "sh $RemoteRoot/$CacheName/tests/ctest-android.sh" Invoke-Program $adb emu kill Invoke-Program $adb kill-server } From b23e1716b67ce552cdffd7257f97a2472a360dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 29 Jan 2025 20:59:05 +0100 Subject: [PATCH 17/30] [dev] Add version dumps to see what runs and what hangs --- utils/build.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index daacfc3cbd485..09ecadb8df5ab 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2093,6 +2093,9 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { function Test-Dispatch([Platform]$Platform) { # TODO: One emulator instance for all tests? $emulator = "$BinaryCache\android-sdk\emulator\emulator.exe" + Write-Host "$emulator -version" + Invoke-Program $emulator "-version" + Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" ` -RedirectStandardOutput "$BinaryCache\android-sdk\.temp\emulator.out" ` -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" @@ -2104,6 +2107,8 @@ function Test-Dispatch([Platform]$Platform) { # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + Write-Host "$adb version" + Invoke-Program $adb "version" Invoke-Program $adb "wait-for-device" Write-Host "$adb push $CachePath $RemoteRoot" Invoke-Program $adb push $CachePath $RemoteRoot @@ -3002,6 +3007,9 @@ try { Fetch-Dependencies +Test-Dispatch Android $AndroidX64 +exit(1) + if ($Clean) { 10..[HostComponent].getEnumValues()[-1] | ForEach-Object { Remove-Item -Force -Recurse "$BinaryCache\$_" -ErrorAction Ignore } # In case of a previous test run, clear out the swiftmodules as they are not a stable format. From 06ef13482734e47bd9d0d465e6987ac2aad17558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Wed, 29 Jan 2025 21:09:55 +0100 Subject: [PATCH 18/30] [dev] Actually log device info before we run into wait-for-device for debugging --- utils/build.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index 09ecadb8df5ab..fddf738724ba6 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2100,6 +2100,8 @@ function Test-Dispatch([Platform]$Platform) { -RedirectStandardOutput "$BinaryCache\android-sdk\.temp\emulator.out" ` -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" + Start-Sleep -Seconds 20 + # This is just a hack for now $CachePath = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) $CacheName = Split-Path $CachePath -Leaf @@ -2109,6 +2111,10 @@ function Test-Dispatch([Platform]$Platform) { $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" Write-Host "$adb version" Invoke-Program $adb "version" + Write-Host "$adb get-state @swift-test-device" + Invoke-Program $adb get-state "@swift-test-device" + Write-Host "$adb logcat -d @swift-test-device" + Invoke-Program $adb logcat -d "@swift-test-device" Invoke-Program $adb "wait-for-device" Write-Host "$adb push $CachePath $RemoteRoot" Invoke-Program $adb push $CachePath $RemoteRoot From 46c9954442e02fff39334e2103916e129280236c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 30 Jan 2025 10:57:50 +0100 Subject: [PATCH 19/30] [dev] Avoid exit if get-state fails and dump even more logs --- utils/build.ps1 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index fddf738724ba6..23afa12ec9ac7 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2100,7 +2100,7 @@ function Test-Dispatch([Platform]$Platform) { -RedirectStandardOutput "$BinaryCache\android-sdk\.temp\emulator.out" ` -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" - Start-Sleep -Seconds 20 + Start-Sleep -Seconds 30 # This is just a hack for now $CachePath = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) @@ -2112,9 +2112,13 @@ function Test-Dispatch([Platform]$Platform) { Write-Host "$adb version" Invoke-Program $adb "version" Write-Host "$adb get-state @swift-test-device" - Invoke-Program $adb get-state "@swift-test-device" + & $adb get-state "@swift-test-device" Write-Host "$adb logcat -d @swift-test-device" - Invoke-Program $adb logcat -d "@swift-test-device" + & $adb logcat -d "@swift-test-device" + Write-Host "$BinaryCache\android-sdk\.temp\emulator.out log:" + Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.out") + Write-Host "$BinaryCache\android-sdk\.temp\emulator.err log:" + Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.err") Invoke-Program $adb "wait-for-device" Write-Host "$adb push $CachePath $RemoteRoot" Invoke-Program $adb push $CachePath $RemoteRoot From 61e96ca7be607ade95efa5624559607877a58e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 30 Jan 2025 11:45:19 +0100 Subject: [PATCH 20/30] [dev] Temporarily drop stdout/stderr redirects from Start-Process --- utils/build.ps1 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 23afa12ec9ac7..9313e25bded9b 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2096,10 +2096,7 @@ function Test-Dispatch([Platform]$Platform) { Write-Host "$emulator -version" Invoke-Program $emulator "-version" - Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" ` - -RedirectStandardOutput "$BinaryCache\android-sdk\.temp\emulator.out" ` - -RedirectStandardError "$BinaryCache\android-sdk\.temp\emulator.err" - + Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" Start-Sleep -Seconds 30 # This is just a hack for now From 4d0d2c7603f6b3de64dd669eb28c9e1a9fcfba71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 30 Jan 2025 11:47:40 +0100 Subject: [PATCH 21/30] [dev] Temporarily drop log dumps as well --- utils/build.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index dc39053057977..5ff6ea1fdf521 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2164,10 +2164,10 @@ function Test-Dispatch([Platform]$Platform) { & $adb get-state "@swift-test-device" Write-Host "$adb logcat -d @swift-test-device" & $adb logcat -d "@swift-test-device" - Write-Host "$BinaryCache\android-sdk\.temp\emulator.out log:" - Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.out") - Write-Host "$BinaryCache\android-sdk\.temp\emulator.err log:" - Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.err") + #Write-Host "$BinaryCache\android-sdk\.temp\emulator.out log:" + #Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.out") + #Write-Host "$BinaryCache\android-sdk\.temp\emulator.err log:" + #Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.err") Invoke-Program $adb "wait-for-device" Write-Host "$adb push $CachePath $RemoteRoot" Invoke-Program $adb push $CachePath $RemoteRoot From cc59b11014da268f2fd9e7e7cf216a02fcf9a9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 30 Jan 2025 14:00:01 +0100 Subject: [PATCH 22/30] [dev] Dump virtual devices with avdmanager to confirm that swift-test-device exists --- utils/build.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index 5ff6ea1fdf521..a9e97ddca0619 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -915,6 +915,9 @@ function Fetch-Dependencies { # There is no way to disable interactive mode in avdmanager "no" | & "$SDKDir\cmdline-tools\latest\bin\avdmanager.bat" create avd --force --name '"swift-test-device"' --package '"system-images;android-29;default;x86_64"' + + # Dump virtual devices to confirm that swift-test-device exists + Invoke-Program "$SDKDir\cmdline-tools\latest\bin\avdmanager.bat" list avd } } } @@ -2148,6 +2151,9 @@ function Test-Dispatch([Platform]$Platform) { Write-Host "$emulator -version" Invoke-Program $emulator "-version" + # Dump virtual devices (again) to confirm that swift-test-device exists + Invoke-Program "$BinaryCache\android-sdk\cmdline-tools\latest\bin\avdmanager.bat" list avd + Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" Start-Sleep -Seconds 30 From d328dcb2a012e77da7c7653510c9a6b1d8f416ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 30 Jan 2025 14:34:23 +0100 Subject: [PATCH 23/30] Set Java environment again when starting the emulator --- utils/build.ps1 | 76 ++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index a9e97ddca0619..7056f5f22e7a4 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -2146,41 +2146,47 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { } function Test-Dispatch([Platform]$Platform) { - # TODO: One emulator instance for all tests? - $emulator = "$BinaryCache\android-sdk\emulator\emulator.exe" - Write-Host "$emulator -version" - Invoke-Program $emulator "-version" - - # Dump virtual devices (again) to confirm that swift-test-device exists - Invoke-Program "$BinaryCache\android-sdk\cmdline-tools\latest\bin\avdmanager.bat" list avd - - Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" - Start-Sleep -Seconds 30 - - # This is just a hack for now - $CachePath = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) - $CacheName = Split-Path $CachePath -Leaf - $RemoteRoot = "/data/local/tmp" - - # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? - $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" - Write-Host "$adb version" - Invoke-Program $adb "version" - Write-Host "$adb get-state @swift-test-device" - & $adb get-state "@swift-test-device" - Write-Host "$adb logcat -d @swift-test-device" - & $adb logcat -d "@swift-test-device" - #Write-Host "$BinaryCache\android-sdk\.temp\emulator.out log:" - #Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.out") - #Write-Host "$BinaryCache\android-sdk\.temp\emulator.err log:" - #Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.err") - Invoke-Program $adb "wait-for-device" - Write-Host "$adb push $CachePath $RemoteRoot" - Invoke-Program $adb push $CachePath $RemoteRoot - Write-Host "$adb shell sh $RemoteRoot/$CacheName/tests/ctest-android.sh" - Invoke-Program $adb shell "sh $RemoteRoot/$CacheName/tests/ctest-android.sh" - Invoke-Program $adb emu kill - Invoke-Program $adb kill-server + # Install packages and create test device + Isolate-EnvVars { + $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" + $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" + + # TODO: One emulator instance for all tests? + $emulator = "$BinaryCache\android-sdk\emulator\emulator.exe" + Write-Host "$emulator -version" + Invoke-Program $emulator "-version" + + # Dump virtual devices (again) to confirm that swift-test-device exists + Invoke-Program "$BinaryCache\android-sdk\cmdline-tools\latest\bin\avdmanager.bat" list avd + + Start-Process -FilePath $emulator -ArgumentList "@swift-test-device" + Start-Sleep -Seconds 30 + + # This is just a hack for now + $CachePath = (Get-TargetProjectBinaryCache $AndroidX64 Dispatch) + $CacheName = Split-Path $CachePath -Leaf + $RemoteRoot = "/data/local/tmp" + + # TODO: On my local machine I have to grant adb.exe network access once. How to do that in CI? + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + Write-Host "$adb version" + Invoke-Program $adb "version" + Write-Host "$adb get-state @swift-test-device" + & $adb get-state "@swift-test-device" + Write-Host "$adb logcat -d @swift-test-device" + & $adb logcat -d "@swift-test-device" + #Write-Host "$BinaryCache\android-sdk\.temp\emulator.out log:" + #Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.out") + #Write-Host "$BinaryCache\android-sdk\.temp\emulator.err log:" + #Write-Host (Get-Content -Path "$BinaryCache\android-sdk\.temp\emulator.err") + Invoke-Program $adb "wait-for-device" + Write-Host "$adb push $CachePath $RemoteRoot" + Invoke-Program $adb push $CachePath $RemoteRoot + Write-Host "$adb shell sh $RemoteRoot/$CacheName/tests/ctest-android.sh" + Invoke-Program $adb shell "sh $RemoteRoot/$CacheName/tests/ctest-android.sh" + Invoke-Program $adb emu kill + Invoke-Program $adb kill-server + } } function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { From 26cfa08a6d66dd9b8f962a49b2de7f7b63ad991b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Mon, 3 Feb 2025 14:02:19 +0100 Subject: [PATCH 24/30] [dev] Run and teardown the emulator with debug output and multipe attempts --- utils/build.ps1 | 98 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 7056f5f22e7a4..0721d978ed426 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -971,6 +971,101 @@ function Fetch-Dependencies { } } +$AndroidEmulatorPid = $null +$AndroidEmulatorArchName = $null + +function AndroidEmulator-CreateDevice($ArchName) { + $DeviceName = "swift-test-device-$ArchName" + $Packages = "system-images;android-29;default;$ArchName" + $AvdTool = "$BinaryCache\android-sdk\cmdline-tools\latest\bin\avdmanager.bat" + + $Output = & $AvdTool list avd + if ($Output -match $DeviceName) { + Write-Host "Found Android virtual device for arch $ArchName" + } else { + Write-Host "Create Android virtual device for arch $ArchName" + "no" | & $AvdTool create avd --force --name $DeviceName --package $Packages + } + return $DeviceName +} + +function AndroidEmulator-Run($ArchName) { + if ($AndroidEmulatorArchName -ne $ArchName) { + AndroidEmulator-TearDown + } + if ($AndroidEmulatorPid -eq $null) { + Isolate-EnvVars { + $env:ANDROID_SDK_HOME = "$BinaryCache\android-sdk" + $env:JAVA_HOME = "$BinaryCache\android-sdk-jdk\jdk-17.0.14+7" + $env:Path = "${env:JAVA_HOME}\bin;${env:Path}" + + Write-Host "ANDROID_SDK_HOME = $env:ANDROID_SDK_HOME" + $AvdTool = "$BinaryCache\android-sdk\cmdline-tools\latest\bin\avdmanager.bat" + Write-Host "$AvdTool list avd reports:" + Invoke-Program $AvdTool list avd + + $Device = (AndroidEmulator-CreateDevice $ArchName) + + Write-Host "$AvdTool list avd reports:" + Invoke-Program $AvdTool list avd + + $EmuTool = "$BinaryCache\android-sdk\emulator\emulator.exe" + Write-Host "$EmuTool -list-avds reports:" + Invoke-Program $EmuTool -list-avds + + Write-Host "Start Android emulator for arch $ArchName" + foreach($Attempt in 1..5) { + $Process = Start-Process -PassThru $EmuTool "@$Device" + #@("-avd", "$Device") + try { + Write-Host "Waiting for process $($Process.Id) to start" + Start-Sleep -Seconds 1 + $_ = Get-Process -Id $Process.Id + break + } catch { + if ($Attempt -lt 5) { + Write-Host "Process $($Process.Id) failed to start, trying again..." + Start-Sleep -Seconds 3 + } else { + throw "Android emulator process $($Process.Id) failed to start." + } + } + } + + $global:AndroidEmulatorPid = $Process.Id + $global:AndroidEmulatorArchName = $ArchName + + Write-Host "Waiting while Android emulator boots in process $AndroidEmulatorPid" + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + Invoke-Program $adb "wait-for-device" + + Write-Host "SUCCESS: Emulator ready" + } + } +} + +function AndroidEmulator-TearDown() { + if ($AndroidEmulatorPid -ne $null) { + if (Get-Process -Id $AndroidEmulatorPid -ErrorAction SilentlyContinue) { + Write-Host "Tear down Android emulator for arch $AndroidEmulatorArchName" + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + & $adb emu kill | Out-Null + & $adb kill-server | Out-Null + + try { + Write-Host "Waiting for process $AndroidEmulatorPid to exit" + Wait-Process -Id $AndroidEmulatorPid -Timeout 1 + } catch { + Write-Host "Process $AndroidEmulatorPid failed to exit. Shutting it down." + Stop-Process -Force -Id $AndroidEmulatorPid + } + + $global:AndroidEmulatorPid = $null + $global:AndroidEmulatorArchName = $null + } + } +} + function Get-PinnedToolchainTool() { if (Test-Path "$BinaryCache\toolchains\${PinnedToolchain}\LocalApp\Programs\Swift\Toolchains\$(Get-PinnedToolchainVersion)+Asserts\usr\bin") { return "$BinaryCache\toolchains\${PinnedToolchain}\LocalApp\Programs\Swift\Toolchains\$(Get-PinnedToolchainVersion)+Asserts\usr\bin" @@ -3082,7 +3177,8 @@ try { Fetch-Dependencies -Test-Dispatch Android $AndroidX64 +AndroidEmulator-Run $AndroidX64.LLVMName +AndroidEmulator-TearDown exit(1) if ($Clean) { From c2d9b340b7d865282026d0832dbad6e250435e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Tue, 4 Feb 2025 12:31:44 +0100 Subject: [PATCH 25/30] Running avdmanager via Start-Process fixes the emulator crashes --- utils/build.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 0721d978ed426..04e5354ae2e19 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -984,7 +984,10 @@ function AndroidEmulator-CreateDevice($ArchName) { Write-Host "Found Android virtual device for arch $ArchName" } else { Write-Host "Create Android virtual device for arch $ArchName" - "no" | & $AvdTool create avd --force --name $DeviceName --package $Packages + # We had issues with not closing file handles to the created AVD, which + # caused the emulator to crash at startup. Using `Start-Process` instead of + # the `&` operator prevents that. + Start-Process "cmd.exe" -ArgumentList "/c echo no | $AvdTool create avd --force --name $DeviceName --package $Packages" -NoNewWindow -Wait } return $DeviceName } From f93328809067c8b45ce85321d83fb3939ca17f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Tue, 4 Feb 2025 14:57:09 +0100 Subject: [PATCH 26/30] Let's grant adb a few more attempts --- utils/build.ps1 | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 04e5354ae2e19..9528cb44b0d48 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1019,7 +1019,6 @@ function AndroidEmulator-Run($ArchName) { Write-Host "Start Android emulator for arch $ArchName" foreach($Attempt in 1..5) { $Process = Start-Process -PassThru $EmuTool "@$Device" - #@("-avd", "$Device") try { Write-Host "Waiting for process $($Process.Id) to start" Start-Sleep -Seconds 1 @@ -1040,7 +1039,22 @@ function AndroidEmulator-Run($ArchName) { Write-Host "Waiting while Android emulator boots in process $AndroidEmulatorPid" $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" - Invoke-Program $adb "wait-for-device" + foreach($Attempt in 1..5) { + Start-Sleep -Seconds 10 + $Process = Start-Process $adb "wait-for-device" -PassThru -NoNewWindow + try { + Write-Host "adb wait-for-device running in process $($Process.Id)" + Wait-Process -Id $Process.Id -Timeout 20 + break + } catch { + if ($Attempt -lt 5) { + Write-Host "adb failed to connect. Shutting it down." + Stop-Process -Force -Id $Process.Id + } else { + throw "Android Debug Bridge process $($Process.Id) failed to connect." + } + } + } Write-Host "SUCCESS: Emulator ready" } From 984688d73b35082d4afed0b9d5f1ac86ce71f42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Thu, 20 Feb 2025 18:48:35 +0100 Subject: [PATCH 27/30] Try to grant network access permission to adb --- utils/build.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index 19c9d788876cf..013c5fc294a75 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -919,6 +919,11 @@ function Fetch-Dependencies { # Dump virtual devices to confirm that swift-test-device exists Invoke-Program "$SDKDir\cmdline-tools\latest\bin\avdmanager.bat" list avd + + # Grant network access permission to CLI tool + $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" + New-NetFirewallRule -DisplayName "abd inbound" -Direction Inbound -Program $adb -RemoteAddress LocalSubnet -Action Allow + New-NetFirewallRule -DisplayName "abd outbound" -Direction Outbound -Program $adb -RemoteAddress LocalSubnet -Action Allow } } } From a3595d7243f370f01a30ee920c82c7ecc312e580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Fri, 21 Feb 2025 17:27:34 +0100 Subject: [PATCH 28/30] [dev] More emulator params and dump verbose outputs --- utils/build.ps1 | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 013c5fc294a75..12d83050b659c 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1023,8 +1023,11 @@ function AndroidEmulator-Run($ArchName) { Invoke-Program $EmuTool -list-avds Write-Host "Start Android emulator for arch $ArchName" + $Args = "-verbose -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -avd $Device" foreach($Attempt in 1..5) { - $Process = Start-Process -PassThru $EmuTool "@$Device" + $Process = Start-Process -PassThru -NoNewWindow $EmuTool -ArgumentList $Args ` + -RedirectStandardError "$BinaryCache\emulator-stderr.log" ` + -RedirectStandardOutput "$BinaryCache\emulator-stdout.log" try { Write-Host "Waiting for process $($Process.Id) to start" Start-Sleep -Seconds 1 @@ -3311,6 +3314,12 @@ Fetch-Dependencies AndroidEmulator-Run $AndroidX64.LLVMName AndroidEmulator-TearDown + +Write-Host "******** emulator-stderr.log ********" +Get-Content -Path "$BinaryCache\emulator-stderr.log" + +Write-Host "******** emulator-stdout.log ********" +Get-Content -Path "$BinaryCache\emulator-stdout.log" exit(1) if ($Clean) { From f12cf68cf286b359d4d35464fbdd04f2a2bf1616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Fri, 21 Feb 2025 17:39:55 +0100 Subject: [PATCH 29/30] [dev] Run -accel-check and dump the output --- utils/build.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index 12d83050b659c..3b84c13a99156 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1022,6 +1022,9 @@ function AndroidEmulator-Run($ArchName) { Write-Host "$EmuTool -list-avds reports:" Invoke-Program $EmuTool -list-avds + Write-Host "$EmuTool -accel-check reports:" + Invoke-Program $EmuTool -accel-check + Write-Host "Start Android emulator for arch $ArchName" $Args = "-verbose -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -avd $Device" foreach($Attempt in 1..5) { From c63f01625297893997046530d1e5a572c3924a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Fri, 21 Feb 2025 18:35:37 +0100 Subject: [PATCH 30/30] Dump logs in erro case as well --- utils/build.ps1 | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 3b84c13a99156..a821d043604dd 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -3317,12 +3317,6 @@ Fetch-Dependencies AndroidEmulator-Run $AndroidX64.LLVMName AndroidEmulator-TearDown - -Write-Host "******** emulator-stderr.log ********" -Get-Content -Path "$BinaryCache\emulator-stderr.log" - -Write-Host "******** emulator-stdout.log ********" -Get-Content -Path "$BinaryCache\emulator-stdout.log" exit(1) if ($Clean) { @@ -3541,11 +3535,13 @@ if (-not $IsCrossCompiling) { exit 1 } finally { - $adb = "$BinaryCache\android-sdk\platform-tools\adb.exe" - if (Test-Path $adb) { - & $adb emu kill | Out-Null - & $adb kill-server | Out-Null - } + AndroidEmulator-TearDown + + Write-Host "******** emulator-stderr.log ********" + Get-Content -Path "$BinaryCache\emulator-stderr.log" + + Write-Host "******** emulator-stdout.log ********" + Get-Content -Path "$BinaryCache\emulator-stdout.log" if ($Summary) { $TimingData | Select-Object Platform,Arch,Checkout,"Elapsed Time" | Sort-Object -Descending -Property "Elapsed Time" | Format-Table -AutoSize