diff --git a/.github/bin/pip-install-requirements.ps1 b/.github/bin/pip-install-requirements.ps1 new file mode 100644 index 0000000000..829f53aeec --- /dev/null +++ b/.github/bin/pip-install-requirements.ps1 @@ -0,0 +1,4 @@ +./emsdk.ps1 activate tot +Set-Location emscripten +Invoke-Expression "$env:EMSDK_PYTHON -m pip install --upgrade pip" +Invoke-Expression "$env:EMSDK_PYTHON -m pip install -r requirements-dev.txt" diff --git a/.github/bin/run-tests.ps1 b/.github/bin/run-tests.ps1 new file mode 100644 index 0000000000..8ddfd16505 --- /dev/null +++ b/.github/bin/run-tests.ps1 @@ -0,0 +1,3 @@ +./emsdk.ps1 activate tot +Set-Location emscripten +Invoke-Expression "$env:EMSDK_PYTHON .\test\runner.py ${args[0]} --failfast" diff --git a/.github/workflows/build-deps.yml b/.github/workflows/build-deps.yml index 7e3fe0291f..1a31e89285 100644 --- a/.github/workflows/build-deps.yml +++ b/.github/workflows/build-deps.yml @@ -3,20 +3,291 @@ name: Build emsdk dependencies on: workflow_dispatch: inputs: - branch: - description: 'LLVM Webassembly arm64' + llvm_build: + description: 'LLVM WebAssembly Arm64' + required: false + default: false + type: boolean + llvm_build_cross_compilation: + description: 'LLVM WebAssembly Arm64 (cross compilation)' + required: false + default: false + type: boolean + python_build: + description: 'Python Arm64' + required: false + default: false + type: boolean + binaryen_build: + description: 'Binaryen Arm64' + required: false + default: false + type: boolean + emscripten_build: + description: 'Emscripten Arm64' + required: false + default: false + type: boolean + nodejs_build: + description: 'Node.js Arm64' + required: false + default: false + type: boolean + wasm_binaries_build: + description: 'WASM binaries Arm64' required: false default: false type: boolean jobs: build-llvm-webassembly-arm64: + if: ${{ inputs.llvm_build || inputs.wasm_binaries_build }} + + runs-on: [self-hosted, Windows, ARM64, WASM] + timeout-minutes: 600 + + steps: + + - name: Build LLVM WebAssembly for Arm64 + shell: cmd + run: | + git clone --config core.autocrlf=false https://github.com/llvm/llvm-project.git --single-branch --depth 1 + cd llvm-project + if exist build_arm64 rmdir /s /q build_arm64 + call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\VC\Auxiliary\Build\vcvarsall.bat" arm64 + cmake -G Ninja -S llvm -B build_arm64 ^ + -DCLANG_ENABLE_ARCMT=OFF ^ + -DCLANG_ENABLE_STATIC_ANALYZER=OFF ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON ^ + -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' ^ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ^ + -DCMAKE_LINKER="C:\Program Files\Microsoft Visual Studio\2022\Preview\VC\Tools\Llvm\x64\bin\lld-link.exe" ^ + -DLLVM_BUILD_LLVM_DYLIB=OFF ^ + -DLLVM_LINK_LLVM_DYLIB=OFF ^ + -DLLVM_DISTRIBUTION_COMPONENTS='clang;lld;llvm-ar;llvm-addr2line;llvm-cxxfilt;llvm-dwarfdump;llvm-dwp;llvm-nm;llvm-objcopy;llvm-objdump;llvm-ranlib;llvm-readobj;llvm-size;llvm-strings;llvm-strip;llvm-symbolizer;clang-resource-headers' ^ + -DLLVM_ENABLE_ASSERTIONS=OFF ^ + -DLLVM_ENABLE_BINDINGS=OFF ^ + -DLLVM_ENABLE_LIBXML2=OFF ^ + -DLLVM_ENABLE_PROJECTS='lld;clang' ^ + -DLLVM_ENABLE_TERMINFO=ON ^ + -DLLVM_INCLUDE_EXAMPLES=OFF ^ + -DLLVM_INCLUDE_TESTS=OFF ^ + -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON ^ + -DLLVM_TARGETS_TO_BUILD='host;WebAssembly' ^ + -DLLVM_TOOL_LTO_BUILD=OFF ^ + -DLLVM_TOOLCHAIN_TOOLS='clang;lld;llvm-ar;llvm-addr2line;llvm-cxxfilt;llvm-dwarfdump;llvm-dwp;llvm-nm;llvm-objcopy;llvm-objdump;llvm-ranlib;llvm-readobj;llvm-size;llvm-strings;llvm-strip;llvm-symbolizer;clang-resource-headers' ^ + -DLLVM_USE_CRT_RELEASE=MT ^ + -DLLVM_USE_CRT_DEBUG=MTd + cd build_arm64 + ninja -v install-distribution + + - name: Pack llvm-arm64.zip + run: | + cd llvm-project/build_arm64 + $unneeded_tool = 'clang-check', 'clang-cl', 'clang-cpp', + 'clang-extdef-mapping', 'clang-format', + 'clang-func-mapping', 'clang-import-test', + 'clang-linker-wrapper', 'clang-offload-bundler', + 'clang-offload-packager', 'clang-refactor', + 'clang-rename', 'clang-repl', 'clang-scan-deps', + 'diagtool', 'git-clang-format', 'hmaptool', 'ld.lld', + 'ld64.lld', 'ld64.lld.darwinnew', 'ld64.lld.darwinold', + 'lld-link', 'libclang.dll', 'llvm-cov', 'llvm-ml', + 'llvm-lib', 'llvm-pdbutil', 'llvm-profdata', + 'llvm-rc' + foreach ($tool in $unneeded_tool) { + $tool = "bin" + $tool + ".exe" + if (Test-Path $tool) { + "Remove " + $tool + Remove-Item $tool + } + } + Compress-Archive -Path bin,lib\clang -DestinationPath llvm-arm64 + + - name: Archive llvm-arm64.zip + uses: actions/upload-artifact@v3 + with: + name: llvm-arm64 + path: llvm-project/build_arm64/llvm-arm64.zip + retention-days: 1 + + build-llvm-webassembly-arm64_cross_compilation: + if: ${{ inputs.llvm_build_llvm_build_cross_compilation }} runs-on: windows-latest steps: - - name: Build LLVM WebAssembly arm64 + - name: Build LLVM WebAssembly for Arm64 (Cross Compilation) + run: | + Set-PSDebug -Trace 1 + git clone --config core.autocrlf=false https://github.com/llvm/llvm-project.git --single-branch --branch main --depth 1 + cd llvm-project + cmake -S llvm -B build_host -DLLVM_ENABLE_PROJECTS='lld;clang' -DLLVM_TARGETS_TO_BUILD="" + cmake --build build_host --target llvm-tblgen llvm-nm clang-tblgen --config MinSizeRel + $llvm_root = Get-Location + cmake -S llvm -B build_arm64 -A ARM64 ` + -DLLVM_TABLEGEN="$llvm_root\build_host\MinSizeRel\bin\llvm-tblgen.exe" ` + -DCLANG_TABLEGEN="$llvm_root\build_host\MinSizeRel\bin\clang-tblgen.exe" ` + -DLLVM_NM="$llvm_root\build_host\MinSizeRel\bin\llvm-nm.exe" ` + -DLLVM_ENABLE_PROJECTS='lld;clang' -DLLVM_TARGETS_TO_BUILD="host;WebAssembly" ` + -DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF + cmake --build build_arm64 --config MinSizeRel + + build-python: + if: ${{ inputs.python_build }} + + runs-on: [self-hosted, Windows, ARM64, WASM] + + steps: + - name: Git checkout + uses: actions/checkout@v3 + with: + path: emsdk + + - name: Build Emscripten Python package for Arm64 + working-directory: emsdk + shell: cmd + run: python scripts\update_python.py + + - name: Archive python-3.9.2-arm64-4+pywin32.zip + uses: actions/upload-artifact@v3 + with: + name: python-3.9.10-arm64-4+pywin32 + path: emsdk/python-3.9.10-arm64-4+pywin32.zip + retention-days: 3 + + build-binaryen: + if: ${{ inputs.binaryen_build || inputs.wasm_binaries_build}} + + runs-on: [self-hosted, Windows, ARM64, WASM] + + steps: + - name: Build Binaryen for Arm64 + shell: cmd + run: | + git clone https://github.com/WebAssembly/binaryen.git --single-branch --depth 1 + cd binaryen + git submodule init + git submodule update + if exist bin rmdir /s /q bin + call "%ProgramFiles%\Microsoft Visual Studio\2022\Preview\VC\Auxiliary\Build\vcvarsall.bat" arm64 + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER='ccache' . + ninja + + + - name: Archive binaryen-arm64.zip + uses: actions/upload-artifact@v3 + with: + name: binaryen-arm64 + path: binaryen/binaryen-arm64.zip + retention-days: 1 + + build-nodejs: + if: ${{ inputs.nodejs_build }} + + runs-on: [self-hosted, Windows, ARM64, WASM] + + steps: + + - name: Add 7-Zip to PATH + run: | + echo "PATH=$env:PATH;C:\Program Files\7-Zip" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Check 7-Zip installed + run: | + 7z + + - name: Build Node.js for Arm64 + run: | + git clone https://github.com/Windows-on-ARM-Experiments/node.git --single-branch -b fix-arm64-compilation --depth 1 + cd node + .\vcbuild.bat arm64 package + + - name: Prepare Node.js package + run: | + cd .\node\out\Release + Remove-Item -Path .\tmp -Recurse -Force -ErrorAction Ignore + New-Item -ItemType Directory -Force -Path .\tmp\node-v20.0.0-win-arm64\bin + cp .\node-v20.0.0-win-arm64\* .\tmp\node-v20.0.0-win-arm64\bin + ls .\tmp\node-v20.0.0-win-arm64\bin + ls .\tmp\node-v20.0.0-win-arm64\bin\node_modules + Compress-Archive -Path .\tmp\node-v20.0.0-win-arm64 -DestinationPath node-v20.0.0-win-arm64-2.zip + + - name: Archive node-v20.0.0-win-arm64.zip + uses: actions/upload-artifact@v3 + with: + name: node-v20.0.0-win-arm64 + path: node/out/Release/tmp/* + + - name: Archive node-v20.0.0-win-arm64-2.zip + uses: actions/upload-artifact@v3 + with: + name: node-v20.0.0-win-arm64-2 + path: node/out/Release/node-v20.0.0-win-arm64-2.zip + + - name: Cleanup + run: | + Remove-Item -Path .\node\out\Release\tmp -Recurse -Force -ErrorAction Ignore + + build-emscripten: + if: ${{ inputs.emscripten_build || inputs.wasm_binaries_build }} + + runs-on: [ubuntu-latest] + + steps: + + - name: Build Emscripten + run: | + git clone https://github.com/emscripten-core/emscripten.git --single-branch --depth 1 + cd emscripten + ./tools/install.py ../emscripten_package + npm ci --production --no-optional ../emscripten_package + cd ../emscripten_package + zip -r ../emscripten-package.zip . + + - name: Archive emscripten-package.zip + uses: actions/upload-artifact@v3 + with: + name: emscripten-package + path: emscripten-package.zip + retention-days: 1 + + build-wasm-binaries: + if: ${{ inputs.wasm_binaries_build }} + needs: [build-emscripten, build-binaryen, build-llvm-webassembly-arm64] + + runs-on: [windows-latest] + + steps: + + - uses: actions/download-artifact@v3 + with: + name: emscripten-package + + - uses: actions/download-artifact@v3 + with: + name: llvm-arm64 + + - uses: actions/download-artifact@v3 + with: + name: binaryen-arm64 + + - name: Pack wasm-binaries for Arm64 run: | - set ls + New-Item -ItemType Directory -Force -Path emsdk/emscripten + Expand-Archive -LiteralPath emscripten-package.zip -DestinationPath emsdk\emscripten + Expand-Archive -LiteralPath llvm-arm64.zip -DestinationPath emsdk + Expand-Archive -LiteralPath binaryen-arm64.zip -DestinationPath emsdk + cd emsdk + Compress-Archive -Path * -DestinationPath wasm-binaries-arm64 + + - name: Archive wasm-binaries-arm64.zip + uses: actions/upload-artifact@v3 + with: + name: wasm-binaries-arm64 + path: emsdk/wasm-binaries-arm64.zip + retention-days: 3 + diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml new file mode 100644 index 0000000000..da207cd52b --- /dev/null +++ b/.github/workflows/test-package.yml @@ -0,0 +1,71 @@ +name: Test the release package + +on: + workflow_dispatch: + inputs: + emscripten_repository: + description: 'Emscripten repository' + required: true + default: 'Windows-on-ARM-Experiments/emscripten' + type: string + emscripten_branch: + description: 'Emscripten repository branch' + required: true + default: 'fix-unit-tests-arm64' + type: string + emscripten_test_suite: + description: 'Emscripten test suite' + required: true + default: 'test_hello_world' + type: string + +defaults: + run: + working-directory: test-package + +jobs: + test-binary-package: + runs-on: windows-latest + timeout-minutes: 240 + + steps: + - name: Create working directory + working-directory: ${{ github.workspace }} + run: New-Item -ItemType Directory -Force -Path test-package + + - name: Git checkout + uses: actions/checkout@v3 + with: + path: test-package + + - name: List Emscripten packages + run: .\emsdk.ps1 list + + - name: Install latest release package + run: .\emsdk.ps1 install tot + + - name: Activate latest release package + run: .\emsdk.ps1 activate tot + + - name: List Emscripten packages + run: .\emsdk.ps1 list + + - name: Git checkout Emscripten + uses: actions/checkout@v3 + with: + repository: ${{ inputs.emscripten_repository }} + ref: ${{ inputs.emscripten_branch }} + path: test-package/emscripten + submodules: 'recursive' + + - name: Install Node packages + working-directory: test-package/emscripten + run: npm ci + + - name: Install Python packages to emsdk Python + run: ${{ github.workspace }}\test-package\.github\bin\pip-install-requirements.ps1 + + - name: Run unit tests with emsdk Python + run: ${{ github.workspace }}\test-package\.github\bin\run-tests.ps1 ${{ inputs.emscripten_test_suite }} + env: + EMCC_DEBUG: 1 # Disables parallel tests execution which leads to race conditions. diff --git a/emsdk.py b/emsdk.py index 1666869e3a..f807e883b8 100644 --- a/emsdk.py +++ b/emsdk.py @@ -1452,7 +1452,7 @@ def emscripten_npm_install(tool, directory): closure_compiler_native = 'google-closure-compiler-linux' if MACOS and ARCH in ('x86', 'x86_64'): closure_compiler_native = 'google-closure-compiler-osx' - if WINDOWS and ARCH in ('x86_64', 'aarch64'): + if WINDOWS and ARCH in ('x86_64'): closure_compiler_native = 'google-closure-compiler-windows' if closure_compiler_native: diff --git a/emsdk_manifest.json b/emsdk_manifest.json index d2fc362622..a9fd3a3029 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -109,7 +109,7 @@ "arch": "aarch64", "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries-arm64.tbz2", "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries-arm64.tbz2", - "windows_url": "https://github.com/Windows-on-ARM-Experiments/emsdk/releases/download/v0.1.0-alpha/emsdk-arm64.zip", + "windows_url": "https://github.com/Windows-on-ARM-Experiments/emsdk/releases/download/v0.1.0-alpha/wasm-binaries-arm64.zip", "zipfile_prefix": "%releases-tag%-", "install_path": "upstream", "activated_path": "%installation_dir%/emscripten", @@ -350,6 +350,16 @@ "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, + { + "id": "node", + "version": "20.0.0-pre", + "arch": "aarch64", + "bitness": 64, + "windows_url": "https://github.com/Windows-on-ARM-Experiments/emsdk/releases/download/v0.1.0-alpha/node-v20.0.0-pre-win-arm64.zip", + "activated_path": "%installation_dir%/bin", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, { @@ -433,10 +443,10 @@ }, { "id": "python", - "version": "3.11.1", + "version": "3.9.10-nuget", "bitness": 64, "arch": "aarch64", - "windows_url": "https://www.python.org/ftp/python/3.11.1/python-3.11.1-embed-arm64.zip", + "windows_url": "https://github.com/Windows-on-ARM-Experiments/emsdk/releases/download/v0.1.0-alpha/python-3.9.10-arm64-4+pywin32.zip", "activated_cfg": "PYTHON='%installation_dir%/python.exe'", "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe" }, @@ -754,7 +764,7 @@ { "version": "releases-upstream-%releases-tag%", "bitness": 64, - "uses": ["python-3.11.1-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-20.0.0-pre-64bit", "python-3.9.10-nuget-64bit", "releases-upstream-%releases-tag%-64bit"], "os": "win", "arch": "aarch64", "custom_install_script": "emscripten_npm_install" diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6c3eed9e65..5593e92f4c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,30 +3,30 @@ - + https://github.com/dotnet/arcade - 691fd54681d10eef3c2681fceb8b09b9f9ba9bb0 + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - 691fd54681d10eef3c2681fceb8b09b9f9ba9bb0 + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - 691fd54681d10eef3c2681fceb8b09b9f9ba9bb0 + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - 691fd54681d10eef3c2681fceb8b09b9f9ba9bb0 + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - 691fd54681d10eef3c2681fceb8b09b9f9ba9bb0 + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 - + https://github.com/dotnet/arcade - 691fd54681d10eef3c2681fceb8b09b9f9ba9bb0 + 96d8be74c39a4765ec919ff9bebf9e0c875fc195 diff --git a/eng/Versions.props b/eng/Versions.props index a0477709e2..536521aa09 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -17,10 +17,10 @@ release - 8.0.0-beta.23077.1 - 8.0.0-beta.23077.1 - 8.0.0-beta.23077.1 - 8.0.0-beta.23077.1 + 8.0.0-beta.23110.3 + 8.0.0-beta.23110.3 + 8.0.0-beta.23110.3 + 8.0.0-beta.23110.3 1.1.87-gba258badda diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 69e65eeae7..bcb579e37a 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -137,6 +137,7 @@ $locJson = @{ @{ LanguageSet = $LanguageSet CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) LocItems = @( $macosHtmlFiles | ForEach-Object { $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index fbc67effc3..27ccdb9ecc 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -83,7 +83,8 @@ try { Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { if ($PathPromotion -eq $True) { - if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" + if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value @@ -93,11 +94,6 @@ try { if ($ToolVersion -eq "latest") { $ToolVersion = "" } - $ArcadeToolsDirectory = "C:\arcade-tools" - if (-not (Test-Path $ArcadeToolsDirectory)) { - Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." - exit 1 - } $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) if ($ToolDirectories -eq $null) { Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." @@ -125,6 +121,7 @@ try { if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." } } exit 0 diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 0000000000..6661fed566 Binary files /dev/null and b/eng/common/loc/P22DotNetHtmlLocalization.lss differ diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index c670cb7968..7aee4213e1 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index f0af425d9f..61914a1fbc 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -134,7 +134,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@0 - - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 inputs: buildType: current @@ -171,7 +171,7 @@ jobs: TeamName: $(_TeamName) - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - task: CopyFiles@2 displayName: Gather binaries for publish to artifacts inputs: @@ -192,7 +192,7 @@ jobs: ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true condition: always() - - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - publish: artifacts/log artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} displayName: Publish logs diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index e40bf35203..8a3deef2b7 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -46,20 +46,12 @@ jobs: # source-build builds run in Docker, including the default managed platform. # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: - # Main environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore-Public + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - # Servicing build environments - ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: - name: NetCore1ESPool-Svc-Internal + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 1b820b4160..9cc5c550d3 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -26,23 +26,32 @@ # demands: ImageOverride -equals windows.vs2019.amd64 variables: -# Coalesce the target and source branches so we know when a PR targets a release branch -# If these variables are somehow missing, fall back to main (tends to have more capacity) -- name: BranchNameForPoolSelection - value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) -# Any new -Svc alternative pools should have variables added here to allow for splitting work - -# Main branch pools -- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}: + # Any new -Svc alternative pools should have variables added here to allow for splitting work - name: DncEngPublicBuildPool - value: NetCore-Public - - name: DncEngInternalBuildPool - value: NetCore1ESPool-Internal + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] -# Release branch pools -- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}: - - name: DncEngPublicBuildPool - value: NetCore-Svc-Public - name: DncEngInternalBuildPool - value: NetCore1ESPool-Svc-Internal + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/global.json b/global.json index 2048d3cccb..05b6f66845 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "dotnet": "8.0.100-alpha.1.23061.8" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23077.1", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23077.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23110.3", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23110.3" } } diff --git a/scripts/update_python.py b/scripts/update_python.py index b39597d1f9..8dfef06a8b 100755 --- a/scripts/update_python.py +++ b/scripts/update_python.py @@ -31,10 +31,13 @@ import sys from subprocess import check_call -version = '3.9.2' +version = '3.9.10' # 3.9.7 is the first available, but 3.9.10 is the latest in the 3.9 series major_minor_version = '.'.join(version.split('.')[:2]) # e.g. '3.9.2' -> '3.9' -download_url = 'https://www.nuget.org/api/v2/package/python/%s' % version revision = '4' +download_urls = { + 'amd64': 'https://www.nuget.org/api/v2/package/python/%s' % version, + 'arm64': 'https://www.nuget.org/api/v2/package/pythonarm64/%s' % version +} pywin32_version = '227' pywin32_base = 'https://github.com/mhammond/pywin32/releases/download/b%s/' % pywin32_version @@ -60,18 +63,19 @@ def zip_cmd(): return ['zip', '-rq'] -def make_python_patch(): - pywin32_filename = 'pywin32-%s.win-amd64-py%s.exe' % (pywin32_version, major_minor_version) - filename = 'python-%s-amd64.zip' % (version) - out_filename = 'python-%s-%s-amd64+pywin32.zip' % (version, revision) +def make_python_patch(arch): + # TODO: pywin32 does not support Arm64 yet (https://github.com/mhammond/pywin32/issues/1462) + pywin32_filename = 'pywin32-%s.win-%s-py%s.exe' % (pywin32_version, 'amd64', major_minor_version) + filename = 'python-%s-%s.zip' % (version, arch) + out_filename = 'python-%s-%s-%s+pywin32.zip' % (version, arch, revision) if not os.path.exists(pywin32_filename): url = pywin32_base + pywin32_filename print('Downloading pywin32: ' + url) urllib.request.urlretrieve(url, pywin32_filename) if not os.path.exists(filename): - print('Downloading python: ' + download_url) - urllib.request.urlretrieve(download_url, filename) + print('Downloading python: ' + download_urls[arch]) + urllib.request.urlretrieve(download_urls[arch], filename) os.mkdir('python-nuget') check_call(unzip_cmd() + [os.path.abspath(filename)], cwd='python-nuget') @@ -175,7 +179,11 @@ def build_python(): def main(): if sys.platform.startswith('win') or '--win32' in sys.argv: - make_python_patch() + if platform.machine() == 'ARM64' or '--arm64' in sys.argv: + arch = 'arm64' + else: + arch = 'amd64' + make_python_patch(arch) else: build_python() return 0