From 981e4889148abfbe442fb595fdc42a476e2f184f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 19 Apr 2024 16:25:55 -0400 Subject: [PATCH 01/35] add code just to test populating stable release stage yaml file for now --- .github/workflows/updateBuildYamls.yml | 55 ++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/updateBuildYamls.yml diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml new file mode 100644 index 000000000..ff80870bb --- /dev/null +++ b/.github/workflows/updateBuildYamls.yml @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +name: Update Channel Based ReleaseStage Yaml Files + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + stableReleaseTag: + description: 'release tag for stable' + required: true + +defaults: + run: + shell: pwsh +jobs: + update-yaml-stable: + name: Update ReleaseStageYaml for Stable Channel + timeout-minutes: 15 + runs-on: ubuntu-20.04 + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Validate Version Syntax + run: | + $stableValue = ${{ inputs.stableReleaseTag }} + if ($stableValue-notmatch '^v\d+\.\d+\.\d+$') { + throw "stable release tag is not for a stable build: '$stableValue'" + } + - name: Update ReleaseStageYaml file + run: | + $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + ./build.ps1 -GenerateMatrixJson -Channel stable -Verbose -Acr All -OsFilter All + - name: Create Pull Request + if: github.event_name == 'workflow_dispatch' + id: cpr + uses: peter-evans/create-pull-request@v4 + with: + token: "${{ secrets.PR_PAT }}" + commit-message: "Update the stableReleaseStage yaml file" + committer: PwshBot + author: PwshBot + title: "Update the stableReleaseStage json" + base: master + draft: false + branch: update-matrix-json + push-to-fork: pwshBot/PowerShell-Docker From d5ca4370eea13261830f23db07cf2b72c6866b41 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 19 Apr 2024 16:43:18 -0400 Subject: [PATCH 02/35] add pr trigger to test --- .github/workflows/updateBuildYamls.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index ff80870bb..58ea6e295 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -12,6 +12,7 @@ on: stableReleaseTag: description: 'release tag for stable' required: true + pull_request: defaults: run: From 9ecfa17b514b2a3deb93841aa0a5d46cda0056e8 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 19 Apr 2024 17:05:18 -0400 Subject: [PATCH 03/35] cast param as string --- .github/workflows/updateBuildYamls.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 58ea6e295..0c7b24fe2 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -29,8 +29,8 @@ jobs: uses: actions/checkout@v2 - name: Validate Version Syntax run: | - $stableValue = ${{ inputs.stableReleaseTag }} - if ($stableValue-notmatch '^v\d+\.\d+\.\d+$') { + $stableValue = '${{ inputs.stableReleaseTag }}' + if ($stableValue -notmatch '^v\d+\.\d+\.\d+$') { throw "stable release tag is not for a stable build: '$stableValue'" } - name: Update ReleaseStageYaml file From 6f5851f8750d8b94992b50cedfcfa63f88f95f45 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sun, 21 Apr 2024 21:02:46 -0400 Subject: [PATCH 04/35] test with my account --- .github/workflows/updateBuildYamls.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 0c7b24fe2..ec337ce4c 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -45,12 +45,12 @@ jobs: id: cpr uses: peter-evans/create-pull-request@v4 with: - token: "${{ secrets.PR_PAT }}" + token: "${{ secrets.MY_PR_PAT }}" commit-message: "Update the stableReleaseStage yaml file" - committer: PwshBot - author: PwshBot + committer: anamnavi + author: anamnavi title: "Update the stableReleaseStage json" base: master draft: false - branch: update-matrix-json - push-to-fork: pwshBot/PowerShell-Docker + branch: update-releasestage-files + push-to-fork: anamnavi/PowerShell-Docker From 850ce417b6428ca99f5fcff9a589bc95c5ce40e4 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sun, 21 Apr 2024 21:42:00 -0400 Subject: [PATCH 05/35] use pwshbot and appropriate creds to create pr --- .github/workflows/updateBuildYamls.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index ec337ce4c..30cea05ea 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -40,17 +40,21 @@ jobs: $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath ./build.ps1 -GenerateMatrixJson -Channel stable -Verbose -Acr All -OsFilter All + - name: View Updated Files Content + run: | + $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' + $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' - name: Create Pull Request if: github.event_name == 'workflow_dispatch' id: cpr uses: peter-evans/create-pull-request@v4 with: - token: "${{ secrets.MY_PR_PAT }}" + token: "${{ secrets.PR_PAT }}" commit-message: "Update the stableReleaseStage yaml file" - committer: anamnavi - author: anamnavi + committer: PwshBot + author: PwshBot title: "Update the stableReleaseStage json" base: master draft: false - branch: update-releasestage-files - push-to-fork: anamnavi/PowerShell-Docker + branch: update-build-yaml-files + push-to-fork: pwshBot/PowerShell-Docker From 7f8023be12abb730d08a616585148457108a05d6 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sun, 21 Apr 2024 23:49:26 -0400 Subject: [PATCH 06/35] add parameterset to build script to update yamls --- build.ps1 | 144 +++++++++++++++++++++++++---- tools/buildHelper/buildHelper.psm1 | 89 +++++++++++++++++- 2 files changed, 215 insertions(+), 18 deletions(-) diff --git a/build.ps1 b/build.ps1 index b98e2a5fd..e90cf82bb 100755 --- a/build.ps1 +++ b/build.ps1 @@ -108,6 +108,7 @@ param( $Version, [Parameter(ParameterSetName="GenerateMatrixJson")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateManifestLists")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -115,6 +116,7 @@ param( $StableVersion, [Parameter(ParameterSetName="GenerateMatrixJson")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateManifestLists")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -122,6 +124,7 @@ param( $LtsVersion, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -129,6 +132,7 @@ param( $PreviewVersion, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -147,12 +151,14 @@ param( $ForcePesterInstall, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [string] [ValidateSet('All','OnlyAcr','NoAcr')] $Acr, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [string] [ValidateSet('All','Linux','Windows')] @@ -203,6 +209,7 @@ DynamicParam { Add-ParameterAttribute -ParameterSetName 'GetTagsByChannel' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'DupeCheck' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'GenerateMatrixJson' -Attributes $channelAttributes -Mandatory $false + Add-ParameterAttribute -ParameterSetName "UpdateBuildYaml" -Attributes $channelAttributes -Mandatory $false Add-ParameterAttribute -ParameterSetName 'GenerateTagsYaml' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'GenerateManifestLists' -Attributes $channelAttributes @@ -227,7 +234,7 @@ Begin { $ENV:DOCKER_BUILDKIT = 1 - if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1) + if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'UpdateBuildYaml', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1) { throw "Multiple Channels are not supported in this parameter set" } @@ -404,7 +411,7 @@ End { } } } - elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) { + elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) { if($Acr -eq 'OnlyAcr' -and !$useAcr) { continue @@ -437,7 +444,7 @@ End { $osVersion = $allMeta.meta.osVersion $manifestLists = $allMeta.meta.ManifestLists - if($osVersion -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) + if($osVersion -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) { if ($osVersion) { $osVersion = $osVersion.replace('${fromTag}', $actualTagData.fromTag) @@ -525,8 +532,8 @@ End { Invoke-PesterWrapper -Script $testsPath -OutputFile $logPath -ExtraParams $extraParams } - Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose - if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') { + Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose + if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') { $dockerImagesToScan = '' # print local image names # used only with the -Build @@ -585,6 +592,87 @@ End { } } + $channelsUsed = @{} + if ($UpdateBuildYaml.IsPresent) { + $matrix = @{ } + foreach ($repo in $tagGroups.Keys | Sort-Object) { + Write-Verbose -Verbose "repo: $repo" + $channelGroups = $tagGroups.$repo | Group-Object -Property Channel + Write-Verbose -Verbose "$($channelGroups.Name) $($channelGroups.Values)" + foreach($channelGroup in $channelGroups) + { + $channelName = $channelGroup.Name + Write-Verbose "generating $channelName json" + $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' + $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" + Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" + + if (!$channelsUsed.Contains($channelName)) + { + Write-Verbose -Verbose "channel was not in there already" + # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. + # But, we want the releaseStage.yml to be 1 per channel, ie stableReleaseStage.yml + $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath + if ($channelReleaseStageFileExists) + { + Remove-Item -Path $channelReleaseStagePath + } + New-Item -Type File -Path $channelReleaseStagePath + + # Call method to write generic lines needed at start of releaseStage file + Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath + $channelsUsed.Add($channelName, $channelGroup.Values) + } + + $osGroups = $channelGroup.Group | Group-Object -Property os + foreach ($osGroup in $osGroups) { + $osName = $osGroup.Name + $architectureGroups = $osGroup.Group | Group-Object -Property Architecture + foreach ($architectureGroup in $architectureGroups) { + $architectureName = $architectureGroup.Name + + # Filter out subimages. We cannot directly build subimages. + foreach ($tag in $architectureGroup.Group | Where-Object { $_.Name -notlike '*/*' } | Sort-Object -Property dockerfile) { + if (-not $matrix.ContainsKey($channelName)) { + $matrix.Add($channelName, @{ }) + } + + if (-not $matrix.$channelName.ContainsKey($osName)) { + $matrix.$channelName.Add($osName, @{ }) + } + + if (-not $matrix.$channelName.$osName.ContainsKey($architectureName)) { + $matrix.$channelName.$osName.Add($architectureName, @{}) + } + + $jobName = $tag.Name -replace '-', '_' + if (-not $matrix.$channelName[$osName][$architectureName].ContainsKey($jobName) -and -not $tag.ContinueOnError) { + $matrix.$channelName[$osName][$architectureName].Add($jobName, (ConvertTo-SortedDictionary -Hashtable @{ + Channel = $tag.Channel + ImageName = $tag.Name + ArtifactSuffixName = $tag.Name.ToString().Replace("\", "_").Replace("-","_") + JobName = $jobName + ContinueOnError = $tag.ContinueOnError + EndOfLife = $tag.EndOfLife + DistributionState = $tag.DistributionState + OsVersion = $tag.OsVersion + # azDevOps doesn't support arrays + TagList = $tag.Tags -join ';' + IsLinux = $tag.IsLinux + UseInCI = $tag.UseInCI + Architecture = $tag.Architecture + })) + + #perhaps write here + Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag + } + } + } + } + } + } + } + if ($GenerateMatrixJson.IsPresent) { $matrix = @{ } foreach ($repo in $tagGroups.Keys | Sort-Object) { @@ -593,6 +681,27 @@ End { { $channelName = $channelGroup.Name Write-Verbose "generating $channelName json" + $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' + $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" + Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" + + if (!$channelsUsed.Contains($channelName)) + { + Write-Verbose -Verbose "channel was not in there already" + # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. + # But, we want the releaseStage.yml to be 1 per channel, ie stableReleaseStage.yml + $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath + if ($channelReleaseStageFileExists) + { + Remove-Item -Path $channelReleaseStagePath + } + New-Item -Type File -Path $channelReleaseStagePath + + # Call method to write generic lines needed at start of releaseStage file + Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath + $channelsUsed.Add($channelName, $channelGroup.Values) + } + $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { $osName = $osGroup.Name @@ -617,19 +726,22 @@ End { $jobName = $tag.Name -replace '-', '_' if (-not $matrix.$channelName[$osName][$architectureName].ContainsKey($jobName) -and -not $tag.ContinueOnError) { $matrix.$channelName[$osName][$architectureName].Add($jobName, (ConvertTo-SortedDictionary -Hashtable @{ - Channel = $tag.Channel - ImageName = $tag.Name - JobName = $jobName - ContinueOnError = $tag.ContinueOnError - EndOfLife = $tag.EndOfLife - DistributionState = $tag.DistributionState - OsVersion = $tag.OsVersion + Channel = $tag.Channel + ImageName = $tag.Name + ArtifactSuffixName = $tag.Name.ToString().Replace("\", "_").Replace("-","_") + JobName = $jobName + ContinueOnError = $tag.ContinueOnError + EndOfLife = $tag.EndOfLife + DistributionState = $tag.DistributionState + OsVersion = $tag.OsVersion # azDevOps doesn't support arrays - TagList = $tag.Tags -join ';' - IsLinux = $tag.IsLinux - UseInCI = $tag.UseInCI - Architecture = $tag.Architecture + TagList = $tag.Tags -join ';' + IsLinux = $tag.IsLinux + UseInCI = $tag.UseInCI + Architecture = $tag.Architecture })) + + Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } } diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index 1d63eb98c..3db64d876 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -192,8 +192,7 @@ class DockerImageMetaData { $this.OsVersion = $this.OsVersion + ' (In Validation)' } 'EndOfLife' { - # add this back if we get the EOL information to be accurate - # $this.OsVersion = $this.OsVersion + ' (End of Life)' + $this.OsVersion = $this.OsVersion + ' (End of Life)' } } } @@ -1030,3 +1029,89 @@ Function ConvertTo-SortedDictionary { } return $sortedDictionary } + +function Get-StartOfYamlPopulated { + param( + [string] + $Channel, + + [string] + $YamlFilePath + ) + + if (!$YamlFilePath) + { + throw "yaml file DNE" + } + + $doubleSpace = " "*2 + + Add-Content -Path $YamlFilePath -Value "parameters:" + Add-Content -Path $YamlFilePath -Value "- name: channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'" + Add-Content -Path $YamlFilePath -Value "- name: channelPath" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''" + Add-Content -Path $YamlFilePath -Value "- name: vmImage" #not sure if this is needed really + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure" + Add-Content -Path $YamlFilePath -Value "stages:" + Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)dependsOn: ['StageResolveVersionandYaml']" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)displayName: Build $Channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)jobs:" +} + +function Get-TemplatePopulatedYaml { + param( + [string] + $YamlFilePath, + + [psobject] + $ImageInfo + ) + + if (!$YamlFilePath) + { + throw "yaml file DNE" + } + + $doubleSpace = " "*2 + $fourSpace = " "*4 + $sixSpace = " "*6 + + $imageName = $ImageInfo.Name + $artifactSuffix = $ImageInfo.Name.ToString().Replace("\", "_").Replace("-","_") + $architecture = $ImageInfo.Architecture + $poolOS = $ImageInfo.IsLinux ? "linux" : "windows" + $archBasedJobName = "Build_$($poolOS)_$($architecture)" + + if ($architecture -eq "arm32") + { + $architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32 + } + + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/newReleasePhase.yml@self" #this is where for loop should begin + Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" #Build_Linux_arm32 + Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # differs from artifactSuffix for test deps image names + Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'" + if ($poolOS -eq "linux") + { + # only need to specify buildKitValue set to 1 for linux + Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1" + } + else + { + if ($imageName -contains "2022") + { + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostVersion: '1ESWindows2022'" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'" + } + + Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3"#do we really need? + } + + Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}" +} From 8accc754216fc26eff41bd0f9dc31a4496ad7d21 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 10:59:47 -0400 Subject: [PATCH 07/35] add parameter for UpdateBuildYaml switch --- build.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.ps1 b/build.ps1 index e90cf82bb..1e5611e00 100755 --- a/build.ps1 +++ b/build.ps1 @@ -34,6 +34,10 @@ param( [switch] $GenerateMatrixJson, + [Parameter(Mandatory, ParameterSetName="UpdateBuildYaml")] + [switch] + $UpdateBuildYaml, + [Parameter(ParameterSetName="GenerateMatrixJson")] [switch] $FullJson, From d32020958937bed5531d5988cd76dc815259a6ca Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 11:09:08 -0400 Subject: [PATCH 08/35] clean up build script --- build.ps1 | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/build.ps1 b/build.ps1 index 1e5611e00..6d37ad769 100755 --- a/build.ps1 +++ b/build.ps1 @@ -600,9 +600,7 @@ End { if ($UpdateBuildYaml.IsPresent) { $matrix = @{ } foreach ($repo in $tagGroups.Keys | Sort-Object) { - Write-Verbose -Verbose "repo: $repo" $channelGroups = $tagGroups.$repo | Group-Object -Property Channel - Write-Verbose -Verbose "$($channelGroups.Name) $($channelGroups.Values)" foreach($channelGroup in $channelGroups) { $channelName = $channelGroup.Name @@ -613,9 +611,8 @@ End { if (!$channelsUsed.Contains($channelName)) { - Write-Verbose -Verbose "channel was not in there already" # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. - # But, we want the releaseStage.yml to be 1 per channel, ie stableReleaseStage.yml + # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath if ($channelReleaseStageFileExists) { @@ -623,7 +620,7 @@ End { } New-Item -Type File -Path $channelReleaseStagePath - # Call method to write generic lines needed at start of releaseStage file + # Note: only populate the start of the ReleaseStage.yml file once per channel Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath $channelsUsed.Add($channelName, $channelGroup.Values) } @@ -667,7 +664,7 @@ End { Architecture = $tag.Architecture })) - #perhaps write here + # Call method to write the part of the yaml file that is unique for each image based template call. Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } From 8c5a3e722ea64755375f6329284b59d63f6c7588 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 11:22:53 -0400 Subject: [PATCH 09/35] remove changes from GenerateMatrixJson parameterset --- build.ps1 | 76 ++++++++++++++++++++----------------------------------- 1 file changed, 27 insertions(+), 49 deletions(-) diff --git a/build.ps1 b/build.ps1 index 6d37ad769..541c423f2 100755 --- a/build.ps1 +++ b/build.ps1 @@ -672,6 +672,33 @@ End { } } } + + foreach ($channelName in $matrix.Keys | Sort-Object) { + $fullMatrix[$channelName] = @() + foreach ($osName in $matrix.$channelName.Keys | Sort-Object) { + $osMatrix = $matrix.$channelName.$osName + foreach ($architectureName in $osMatrix.Keys | Sort-Object) { + $architectureMatrix = $osMatrix.$architectureName + + $channelMatrix = [System.Collections.ArrayList]::new() + $architectureMatrix.Values | Sort-Object -Property ImageName | ForEach-Object { + $null = $channelMatrix.Add($_) + } + $fullMatrix[$channelName] += $channelMatrix + $matrixJson = $architectureMatrix | ConvertTo-Json -Compress + $variableName = "matrix_${channelName}_${osName}_${architectureName}" + if (!$FullJson) { + Set-BuildVariable -Name $variableName -Value $matrixJson -IsOutput + Write-Verbose -Verbose "*********END of JSON*********************" + } + } + } + } + + if($FullJson) { + $matrixJson = $fullMatrix | ConvertTo-Json -Depth 100 + Write-Output $matrixJson + } } if ($GenerateMatrixJson.IsPresent) { @@ -682,26 +709,6 @@ End { { $channelName = $channelGroup.Name Write-Verbose "generating $channelName json" - $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' - $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" - Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" - - if (!$channelsUsed.Contains($channelName)) - { - Write-Verbose -Verbose "channel was not in there already" - # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. - # But, we want the releaseStage.yml to be 1 per channel, ie stableReleaseStage.yml - $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath - if ($channelReleaseStageFileExists) - { - Remove-Item -Path $channelReleaseStagePath - } - New-Item -Type File -Path $channelReleaseStagePath - - # Call method to write generic lines needed at start of releaseStage file - Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath - $channelsUsed.Add($channelName, $channelGroup.Values) - } $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { @@ -741,41 +748,12 @@ End { UseInCI = $tag.UseInCI Architecture = $tag.Architecture })) - - Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } } } } } - - foreach ($channelName in $matrix.Keys | Sort-Object) { - $fullMatrix[$channelName] = @() - foreach ($osName in $matrix.$channelName.Keys | Sort-Object) { - $osMatrix = $matrix.$channelName.$osName - foreach ($architectureName in $osMatrix.Keys | Sort-Object) { - $architectureMatrix = $osMatrix.$architectureName - - $channelMatrix = [System.Collections.ArrayList]::new() - $architectureMatrix.Values | Sort-Object -Property ImageName | ForEach-Object { - $null = $channelMatrix.Add($_) - } - $fullMatrix[$channelName] += $channelMatrix - $matrixJson = $architectureMatrix | ConvertTo-Json -Compress - $variableName = "matrix_${channelName}_${osName}_${architectureName}" - if (!$FullJson) { - Set-BuildVariable -Name $variableName -Value $matrixJson -IsOutput - Write-Verbose -Verbose "*********END of JSON*********************" - } - } - } - } - - if($FullJson) { - $matrixJson = $fullMatrix | ConvertTo-Json -Depth 100 - Write-Output $matrixJson - } } if ($GenerateManifestLists.IsPresent) { From 49e8e1fd3dc28c5ae2c40f7add04df11b63fdad5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 11:44:51 -0400 Subject: [PATCH 10/35] use most up to date version of build helper --- tools/buildHelper/buildHelper.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index 3db64d876..b211a9453 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -192,7 +192,8 @@ class DockerImageMetaData { $this.OsVersion = $this.OsVersion + ' (In Validation)' } 'EndOfLife' { - $this.OsVersion = $this.OsVersion + ' (End of Life)' + # add this back if we get the EOL information to be accurate + # $this.OsVersion = $this.OsVersion + ' (End of Life)' } } } From ff972d8084b33b474ff3be69ecc8fc305f9a32c2 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 11:49:56 -0400 Subject: [PATCH 11/35] move condition below --- build.ps1 | 104 +++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/build.ps1 b/build.ps1 index 541c423f2..888ecbe34 100755 --- a/build.ps1 +++ b/build.ps1 @@ -596,8 +596,7 @@ End { } } - $channelsUsed = @{} - if ($UpdateBuildYaml.IsPresent) { + if ($GenerateMatrixJson.IsPresent) { $matrix = @{ } foreach ($repo in $tagGroups.Keys | Sort-Object) { $channelGroups = $tagGroups.$repo | Group-Object -Property Channel @@ -605,25 +604,6 @@ End { { $channelName = $channelGroup.Name Write-Verbose "generating $channelName json" - $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' - $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" - Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" - - if (!$channelsUsed.Contains($channelName)) - { - # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. - # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) - $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath - if ($channelReleaseStageFileExists) - { - Remove-Item -Path $channelReleaseStagePath - } - New-Item -Type File -Path $channelReleaseStagePath - - # Note: only populate the start of the ReleaseStage.yml file once per channel - Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath - $channelsUsed.Add($channelName, $channelGroup.Values) - } $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { @@ -663,45 +643,16 @@ End { UseInCI = $tag.UseInCI Architecture = $tag.Architecture })) - - # Call method to write the part of the yaml file that is unique for each image based template call. - Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } } } } } - - foreach ($channelName in $matrix.Keys | Sort-Object) { - $fullMatrix[$channelName] = @() - foreach ($osName in $matrix.$channelName.Keys | Sort-Object) { - $osMatrix = $matrix.$channelName.$osName - foreach ($architectureName in $osMatrix.Keys | Sort-Object) { - $architectureMatrix = $osMatrix.$architectureName - - $channelMatrix = [System.Collections.ArrayList]::new() - $architectureMatrix.Values | Sort-Object -Property ImageName | ForEach-Object { - $null = $channelMatrix.Add($_) - } - $fullMatrix[$channelName] += $channelMatrix - $matrixJson = $architectureMatrix | ConvertTo-Json -Compress - $variableName = "matrix_${channelName}_${osName}_${architectureName}" - if (!$FullJson) { - Set-BuildVariable -Name $variableName -Value $matrixJson -IsOutput - Write-Verbose -Verbose "*********END of JSON*********************" - } - } - } - } - - if($FullJson) { - $matrixJson = $fullMatrix | ConvertTo-Json -Depth 100 - Write-Output $matrixJson - } } - if ($GenerateMatrixJson.IsPresent) { + $channelsUsed = @{} + if ($UpdateBuildYaml.IsPresent) { $matrix = @{ } foreach ($repo in $tagGroups.Keys | Sort-Object) { $channelGroups = $tagGroups.$repo | Group-Object -Property Channel @@ -709,6 +660,25 @@ End { { $channelName = $channelGroup.Name Write-Verbose "generating $channelName json" + $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' + $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" + Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" + + if (!$channelsUsed.Contains($channelName)) + { + # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. + # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) + $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath + if ($channelReleaseStageFileExists) + { + Remove-Item -Path $channelReleaseStagePath + } + New-Item -Type File -Path $channelReleaseStagePath + + # Note: only populate the start of the ReleaseStage.yml file once per channel + Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath + $channelsUsed.Add($channelName, $channelGroup.Values) + } $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { @@ -748,12 +718,42 @@ End { UseInCI = $tag.UseInCI Architecture = $tag.Architecture })) + + # Call method to write the part of the yaml file that is unique for each image based template call. + Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } } } } } + + foreach ($channelName in $matrix.Keys | Sort-Object) { + $fullMatrix[$channelName] = @() + foreach ($osName in $matrix.$channelName.Keys | Sort-Object) { + $osMatrix = $matrix.$channelName.$osName + foreach ($architectureName in $osMatrix.Keys | Sort-Object) { + $architectureMatrix = $osMatrix.$architectureName + + $channelMatrix = [System.Collections.ArrayList]::new() + $architectureMatrix.Values | Sort-Object -Property ImageName | ForEach-Object { + $null = $channelMatrix.Add($_) + } + $fullMatrix[$channelName] += $channelMatrix + $matrixJson = $architectureMatrix | ConvertTo-Json -Compress + $variableName = "matrix_${channelName}_${osName}_${architectureName}" + if (!$FullJson) { + Set-BuildVariable -Name $variableName -Value $matrixJson -IsOutput + Write-Verbose -Verbose "*********END of JSON*********************" + } + } + } + } + + if($FullJson) { + $matrixJson = $fullMatrix | ConvertTo-Json -Depth 100 + Write-Output $matrixJson + } } if ($GenerateManifestLists.IsPresent) { From 872c2b9cdbe659712186c928dd521e21618ef224 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:01:45 -0400 Subject: [PATCH 12/35] update workflow for other channels and to use version value if passed in, otherwise use channels.json value --- .github/workflows/updateBuildYamls.yml | 97 ++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 30cea05ea..2115a2fda 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -11,7 +11,13 @@ on: inputs: stableReleaseTag: description: 'release tag for stable' - required: true + required: false + previewReleaseTag: + description: 'release tag for preview' + required: false + ltsReleaseTag: + description: 'release tag for lts' + required: false pull_request: defaults: @@ -28,22 +34,103 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Validate Version Syntax + id: validateversion run: | $stableValue = '${{ inputs.stableReleaseTag }}' - if ($stableValue -notmatch '^v\d+\.\d+\.\d+$') { - throw "stable release tag is not for a stable build: '$stableValue'" + if (!($stableValue -not -eq "")) + { + if ($stableValue -notmatch '^v\d+\.\d+\.\d+$') { + throw "stable release tag is not for a stable build: '$stableValue'" + } + + echo "stableVersion=$stableValue" >> $GITHUB_OUTPUT + } + else + { + $stableValue = '' + echo "stableVersion=$stableValue" >> $GITHUB_OUTPUT + } + + $previewValue = '${{ inputs.previewReleaseTag }}' + if (!($previewValue -not -eq "")) + { + if ($previewValue -notmatch '^v\d+\.\d+\.\d+$') { + throw "preview release tag is not for a preview build: '$previewValue'" + } + + echo "previewVersion=$previewValue" >> $GITHUB_OUTPUT + } + else + { + $previewValue = '' + echo "previewVersion=$previewValue" >> $GITHUB_OUTPUT + } + + $ltsValue = '${{ inputs.ltsReleaseTag }}' + if (!($ltsValue -not -eq "")) + { + if ($ltsValue -notmatch '^v\d+\.\d+\.\d+$') { + throw "lts release tag is not for a lts build: '$ltsValue'" + } + + echo "ltsVersion=$ltsValue" >> $GITHUB_OUTPUT + } + else + { + $ltsValue = '' + echo "ltsVersion=$ltsValue" >> $GITHUB_OUTPUT + } + - name: Update ReleaseStageYaml file for Stable + run: | + $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + if (!${{ steps.validateversion.outputs.stableVersion }}) + { + ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion ${{ steps.validateversion.outputs.stableVersion }} -Verbose -Acr All -OsFilter All + } + else + { + # Use the version from channels.json when no version is provided + ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All } - - name: Update ReleaseStageYaml file + - name: Update ReleaseStageYaml file for Preview run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - ./build.ps1 -GenerateMatrixJson -Channel stable -Verbose -Acr All -OsFilter All + if (!${{ steps.validateversion.outputs.previewVersion }}) + { + ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion ${{ steps.validateversion.outputs.previewVersion }} -Verbose -Acr All -OsFilter All + } + else + { + # Use the version from channels.json when no version is provided + ./build.ps1 -UpdateBuildYaml -Channel preview -Verbose -Acr All -OsFilter All + } + - name: Update ReleaseStageYaml file for LTS + run: | + $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + if (!${{ steps.validateversion.outputs.ltsVersion }}) + { + ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion ${{ steps.validateversion.outputs.ltsVersion }} -Verbose -Acr All -OsFilter All + } + else + { + # Use the version from channels.json when no version is provided + ./build.ps1 -UpdateBuildYaml -Channel lts -Verbose -Acr All -OsFilter All + } - name: View Updated Files Content run: | $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' + $content = Get-Content -Path $releaseStageFilePath + Write-Verbose -Verbose $content - name: Create Pull Request if: github.event_name == 'workflow_dispatch' id: cpr From 2736081161bf872575be42403884eefd0e02c144 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:09:15 -0400 Subject: [PATCH 13/35] fix condition negation --- .github/workflows/updateBuildYamls.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 2115a2fda..d83eb7e37 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -37,7 +37,7 @@ jobs: id: validateversion run: | $stableValue = '${{ inputs.stableReleaseTag }}' - if (!($stableValue -not -eq "")) + if (!($stableValue -eq "")) { if ($stableValue -notmatch '^v\d+\.\d+\.\d+$') { throw "stable release tag is not for a stable build: '$stableValue'" @@ -52,7 +52,7 @@ jobs: } $previewValue = '${{ inputs.previewReleaseTag }}' - if (!($previewValue -not -eq "")) + if (!($previewValue -eq "")) { if ($previewValue -notmatch '^v\d+\.\d+\.\d+$') { throw "preview release tag is not for a preview build: '$previewValue'" @@ -67,7 +67,7 @@ jobs: } $ltsValue = '${{ inputs.ltsReleaseTag }}' - if (!($ltsValue -not -eq "")) + if (!($ltsValue -eq "")) { if ($ltsValue -notmatch '^v\d+\.\d+\.\d+$') { throw "lts release tag is not for a lts build: '$ltsValue'" From ab82261fe88b44c144840105dee055b2e4df7f7c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:15:14 -0400 Subject: [PATCH 14/35] fix match expression for previewReleaseTag --- .github/workflows/updateBuildYamls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index d83eb7e37..0259bf2ca 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -54,7 +54,7 @@ jobs: $previewValue = '${{ inputs.previewReleaseTag }}' if (!($previewValue -eq "")) { - if ($previewValue -notmatch '^v\d+\.\d+\.\d+$') { + if ($previewValue -notmatch '^v\d+\.\d+\.\d+-(preview|rc)\.\d+$') { throw "preview release tag is not for a preview build: '$previewValue'" } From b19e36858e73fa01dab8f6571c71998c47e9675f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:21:12 -0400 Subject: [PATCH 15/35] Fix condition --- .github/workflows/updateBuildYamls.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 0259bf2ca..8d463644a 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -86,7 +86,8 @@ jobs: $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - if (!${{ steps.validateversion.outputs.stableVersion }}) + $stableVersion = '${{ steps.validateversion.outputs.stableVersion }}' + if (!($stableVersion -eq "")) { ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion ${{ steps.validateversion.outputs.stableVersion }} -Verbose -Acr All -OsFilter All } From d912398cc0f7d4b5c690216c270a4f1db8d98140 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:25:54 -0400 Subject: [PATCH 16/35] Fix condition --- .github/workflows/updateBuildYamls.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 8d463644a..4921e74b4 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -86,8 +86,8 @@ jobs: $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - $stableVersion = '${{ steps.validateversion.outputs.stableVersion }}' - if (!($stableVersion -eq "")) + $stableVersionValue = '${{ steps.validateversion.outputs.stableVersion }}' + if (!($stableVersionValue -eq "")) { ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion ${{ steps.validateversion.outputs.stableVersion }} -Verbose -Acr All -OsFilter All } @@ -102,7 +102,8 @@ jobs: $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - if (!${{ steps.validateversion.outputs.previewVersion }}) + $previewVersionValue = '${{ steps.validateversion.outputs.previewVersion }}' + if (!($previewVersionValue -eq "")) { ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion ${{ steps.validateversion.outputs.previewVersion }} -Verbose -Acr All -OsFilter All } @@ -117,7 +118,8 @@ jobs: $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - if (!${{ steps.validateversion.outputs.ltsVersion }}) + $ltsVersionValue = '${{ steps.validateversion.outputs.ltsVersion }}' + if (!($ltsVersionValue -eq "")) { ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion ${{ steps.validateversion.outputs.ltsVersion }} -Verbose -Acr All -OsFilter All } From bd0d6fc0ed02d6e022c93dd3dd819d0a0f14ba53 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:31:01 -0400 Subject: [PATCH 17/35] fix reading file content --- .github/workflows/updateBuildYamls.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 4921e74b4..131f67878 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -133,7 +133,10 @@ jobs: $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' $content = Get-Content -Path $releaseStageFilePath - Write-Verbose -Verbose $content + for($i=0; $i<$content.Length;$i++) + { + Write-Verbose -Verbose $content[$i] + } - name: Create Pull Request if: github.event_name == 'workflow_dispatch' id: cpr From a6120060499b82b4ccc87e7dee8dafc5f9815c8a Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 13:33:59 -0400 Subject: [PATCH 18/35] fix operator --- .github/workflows/updateBuildYamls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 131f67878..aed7ddd6b 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -133,7 +133,7 @@ jobs: $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' $content = Get-Content -Path $releaseStageFilePath - for($i=0; $i<$content.Length;$i++) + for($i=0; $i -lt $content.Length;$i++) { Write-Verbose -Verbose $content[$i] } From 7f7ff766d66f315b9f2aef91a14497369d643287 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 14:52:53 -0400 Subject: [PATCH 19/35] add debugging verbose statements --- .github/workflows/updateBuildYamls.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index aed7ddd6b..34e248655 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -89,11 +89,13 @@ jobs: $stableVersionValue = '${{ steps.validateversion.outputs.stableVersion }}' if (!($stableVersionValue -eq "")) { + Write-Verbose -Verbose "using stable version: $stableVersionValue" ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion ${{ steps.validateversion.outputs.stableVersion }} -Verbose -Acr All -OsFilter All } else { # Use the version from channels.json when no version is provided + Write-Verbose -Verbose "not using stable version from input" ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for Preview @@ -133,6 +135,7 @@ jobs: $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' $content = Get-Content -Path $releaseStageFilePath + Write-Verbose -Verbose "length of content: $content.Length" for($i=0; $i -lt $content.Length;$i++) { Write-Verbose -Verbose $content[$i] From 4734b3821be67ee171f66e3d7f238492a4c34dcd Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 14:57:19 -0400 Subject: [PATCH 20/35] Fix typos --- .github/workflows/updateBuildYamls.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 34e248655..8355047c6 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -95,7 +95,7 @@ jobs: else { # Use the version from channels.json when no version is provided - Write-Verbose -Verbose "not using stable version from input" + Write-Verbose -Verbose "not using stable version from input as value was $stableVersionValue" ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for Preview @@ -135,7 +135,7 @@ jobs: $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' $content = Get-Content -Path $releaseStageFilePath - Write-Verbose -Verbose "length of content: $content.Length" + Write-Verbose -Verbose "length of content: $($content.Length)" for($i=0; $i -lt $content.Length;$i++) { Write-Verbose -Verbose $content[$i] From ce67b37f43e07d55a47508b6b0d8b5f79adce784 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 15:16:28 -0400 Subject: [PATCH 21/35] fix how output variable is retrieved --- .github/workflows/updateBuildYamls.yml | 39 ++++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 8355047c6..f980c0a08 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -34,31 +34,31 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Validate Version Syntax - id: validateversion + id: validate-version run: | $stableValue = '${{ inputs.stableReleaseTag }}' if (!($stableValue -eq "")) { - if ($stableValue -notmatch '^v\d+\.\d+\.\d+$') { + if ($stableValue -notmatch '\d+\.\d+\.\d+$') { throw "stable release tag is not for a stable build: '$stableValue'" } - echo "stableVersion=$stableValue" >> $GITHUB_OUTPUT + echo "stableVersion=$stableValue" >> "$GITHUB_OUTPUT" } else { $stableValue = '' - echo "stableVersion=$stableValue" >> $GITHUB_OUTPUT + echo "stableVersion=$stableValue" >> "$GITHUB_OUTPUT" } $previewValue = '${{ inputs.previewReleaseTag }}' if (!($previewValue -eq "")) { - if ($previewValue -notmatch '^v\d+\.\d+\.\d+-(preview|rc)\.\d+$') { + if ($previewValue -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') { throw "preview release tag is not for a preview build: '$previewValue'" } - echo "previewVersion=$previewValue" >> $GITHUB_OUTPUT + echo "previewVersion=$previewValue" >> "$GITHUB_OUTPUT" } else { @@ -69,7 +69,7 @@ jobs: $ltsValue = '${{ inputs.ltsReleaseTag }}' if (!($ltsValue -eq "")) { - if ($ltsValue -notmatch '^v\d+\.\d+\.\d+$') { + if ($ltsValue -notmatch '\d+\.\d+\.\d+$') { throw "lts release tag is not for a lts build: '$ltsValue'" } @@ -81,33 +81,35 @@ jobs: echo "ltsVersion=$ltsValue" >> $GITHUB_OUTPUT } - name: Update ReleaseStageYaml file for Stable + env: + STABLE_VERSION: ${{ steps.validate-version.outputs.stableVersion }} run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - $stableVersionValue = '${{ steps.validateversion.outputs.stableVersion }}' - if (!($stableVersionValue -eq "")) + if (!($STABLE_VERSION -eq "")) { - Write-Verbose -Verbose "using stable version: $stableVersionValue" - ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion ${{ steps.validateversion.outputs.stableVersion }} -Verbose -Acr All -OsFilter All + Write-Verbose -Verbose "using stable version: $STABLE_VERSION" + ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion $STABLE_VERSION -Verbose -Acr All -OsFilter All } else { # Use the version from channels.json when no version is provided - Write-Verbose -Verbose "not using stable version from input as value was $stableVersionValue" + Write-Verbose -Verbose "not using stable version from input as value was $STABLE_VERSION" ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for Preview + env: + PREVIEW_VERSION: ${{ steps.validateversion.outputs.previewVersion }} run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - $previewVersionValue = '${{ steps.validateversion.outputs.previewVersion }}' - if (!($previewVersionValue -eq "")) + if (!($PREVIEW_VERSION -eq "")) { - ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion ${{ steps.validateversion.outputs.previewVersion }} -Verbose -Acr All -OsFilter All + ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion $PREVIEW_VERSION -Verbose -Acr All -OsFilter All } else { @@ -115,15 +117,16 @@ jobs: ./build.ps1 -UpdateBuildYaml -Channel preview -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for LTS + env: + LTS_VERSION: ${{ steps.validateversion.outputs.ltsVersion }} run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - $ltsVersionValue = '${{ steps.validateversion.outputs.ltsVersion }}' - if (!($ltsVersionValue -eq "")) + if (!($LTS_VERSION -eq "")) { - ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion ${{ steps.validateversion.outputs.ltsVersion }} -Verbose -Acr All -OsFilter All + ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion $LTS_VERSION -Verbose -Acr All -OsFilter All } else { From dc83b9f2f08e3eec5beda841eef6b40a135b14ae Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 15:29:52 -0400 Subject: [PATCH 22/35] fix name of output variable --- .github/workflows/updateBuildYamls.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index f980c0a08..41f4317ac 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -43,12 +43,13 @@ jobs: throw "stable release tag is not for a stable build: '$stableValue'" } - echo "stableVersion=$stableValue" >> "$GITHUB_OUTPUT" + Write-Verbose -Verbose "echo here" + echo "STABLE_VERSION=$stableValue" >> "$GITHUB_OUTPUT" } else { $stableValue = '' - echo "stableVersion=$stableValue" >> "$GITHUB_OUTPUT" + echo "STABLE_VERSION=$stableValue" >> "$GITHUB_OUTPUT" } $previewValue = '${{ inputs.previewReleaseTag }}' @@ -58,12 +59,12 @@ jobs: throw "preview release tag is not for a preview build: '$previewValue'" } - echo "previewVersion=$previewValue" >> "$GITHUB_OUTPUT" + echo "PREVIEW_VERSION=$previewValue" >> "$GITHUB_OUTPUT" } else { $previewValue = '' - echo "previewVersion=$previewValue" >> $GITHUB_OUTPUT + echo "PREVIEW_VERSION=$previewValue" >> $GITHUB_OUTPUT } $ltsValue = '${{ inputs.ltsReleaseTag }}' @@ -73,16 +74,16 @@ jobs: throw "lts release tag is not for a lts build: '$ltsValue'" } - echo "ltsVersion=$ltsValue" >> $GITHUB_OUTPUT + echo "LTS_VERSION=$ltsValue" >> $GITHUB_OUTPUT } else { $ltsValue = '' - echo "ltsVersion=$ltsValue" >> $GITHUB_OUTPUT + echo "LTS_VERSION=$ltsValue" >> $GITHUB_OUTPUT } - name: Update ReleaseStageYaml file for Stable env: - STABLE_VERSION: ${{ steps.validate-version.outputs.stableVersion }} + STABLE_VERSION: ${{ steps.validate-version.outputs.STABLE_VERSION }} run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' @@ -101,7 +102,7 @@ jobs: } - name: Update ReleaseStageYaml file for Preview env: - PREVIEW_VERSION: ${{ steps.validateversion.outputs.previewVersion }} + PREVIEW_VERSION: ${{ steps.validateversion.outputs.PREVIEW_VERSION }} run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' @@ -118,7 +119,7 @@ jobs: } - name: Update ReleaseStageYaml file for LTS env: - LTS_VERSION: ${{ steps.validateversion.outputs.ltsVersion }} + LTS_VERSION: ${{ steps.validateversion.outputs.LTS_VERSION }} run: | $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' From 3e5cfdc2b008491c3ab931d6644d29e59630e03c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 15:37:22 -0400 Subject: [PATCH 23/35] use env var syntax for github_output since it's a powershell runner --- .github/workflows/updateBuildYamls.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 41f4317ac..d715009a3 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -44,12 +44,12 @@ jobs: } Write-Verbose -Verbose "echo here" - echo "STABLE_VERSION=$stableValue" >> "$GITHUB_OUTPUT" + echo "STABLE_VERSION=$stableValue" >> $env:GITHUB_OUTPUT } else { $stableValue = '' - echo "STABLE_VERSION=$stableValue" >> "$GITHUB_OUTPUT" + echo "STABLE_VERSION=$stableValue" >> $env:GITHUB_OUTPUT } $previewValue = '${{ inputs.previewReleaseTag }}' @@ -59,12 +59,12 @@ jobs: throw "preview release tag is not for a preview build: '$previewValue'" } - echo "PREVIEW_VERSION=$previewValue" >> "$GITHUB_OUTPUT" + echo "PREVIEW_VERSION=$previewValue" >> $env:GITHUB_OUTPUT } else { $previewValue = '' - echo "PREVIEW_VERSION=$previewValue" >> $GITHUB_OUTPUT + echo "PREVIEW_VERSION=$previewValue" >> $env:GITHUB_OUTPUT } $ltsValue = '${{ inputs.ltsReleaseTag }}' @@ -74,12 +74,12 @@ jobs: throw "lts release tag is not for a lts build: '$ltsValue'" } - echo "LTS_VERSION=$ltsValue" >> $GITHUB_OUTPUT + echo "LTS_VERSION=$ltsValue" >> $env:GITHUB_OUTPUT } else { $ltsValue = '' - echo "LTS_VERSION=$ltsValue" >> $GITHUB_OUTPUT + echo "LTS_VERSION=$ltsValue" >> $env:GITHUB_OUTPUT } - name: Update ReleaseStageYaml file for Stable env: From 4d33c28ec500c0af49a0dc2b53c5c31d2ea71811 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 16:07:18 -0400 Subject: [PATCH 24/35] simplify- no need for output variables --- .github/workflows/updateBuildYamls.yml | 49 +++++++------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index d715009a3..748706a88 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -24,7 +24,7 @@ defaults: run: shell: pwsh jobs: - update-yaml-stable: + update-yamls: name: Update ReleaseStageYaml for Stable Channel timeout-minutes: 15 runs-on: ubuntu-20.04 @@ -42,14 +42,6 @@ jobs: if ($stableValue -notmatch '\d+\.\d+\.\d+$') { throw "stable release tag is not for a stable build: '$stableValue'" } - - Write-Verbose -Verbose "echo here" - echo "STABLE_VERSION=$stableValue" >> $env:GITHUB_OUTPUT - } - else - { - $stableValue = '' - echo "STABLE_VERSION=$stableValue" >> $env:GITHUB_OUTPUT } $previewValue = '${{ inputs.previewReleaseTag }}' @@ -58,13 +50,6 @@ jobs: if ($previewValue -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') { throw "preview release tag is not for a preview build: '$previewValue'" } - - echo "PREVIEW_VERSION=$previewValue" >> $env:GITHUB_OUTPUT - } - else - { - $previewValue = '' - echo "PREVIEW_VERSION=$previewValue" >> $env:GITHUB_OUTPUT } $ltsValue = '${{ inputs.ltsReleaseTag }}' @@ -73,44 +58,35 @@ jobs: if ($ltsValue -notmatch '\d+\.\d+\.\d+$') { throw "lts release tag is not for a lts build: '$ltsValue'" } - - echo "LTS_VERSION=$ltsValue" >> $env:GITHUB_OUTPUT - } - else - { - $ltsValue = '' - echo "LTS_VERSION=$ltsValue" >> $env:GITHUB_OUTPUT } - name: Update ReleaseStageYaml file for Stable - env: - STABLE_VERSION: ${{ steps.validate-version.outputs.STABLE_VERSION }} run: | + $stableValue = '${{ inputs.stableReleaseTag }}' $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - if (!($STABLE_VERSION -eq "")) + if (!($stableValue -eq "")) { - Write-Verbose -Verbose "using stable version: $STABLE_VERSION" - ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion $STABLE_VERSION -Verbose -Acr All -OsFilter All + Write-Verbose -Verbose "using stable version: $stableValue" + ./build.ps1 -UpdateBuildYaml -Channel stable -StableVersion $stableValue -Verbose -Acr All -OsFilter All } else { # Use the version from channels.json when no version is provided - Write-Verbose -Verbose "not using stable version from input as value was $STABLE_VERSION" + Write-Verbose -Verbose "not using stable version from input as value was $stableValue" ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for Preview - env: - PREVIEW_VERSION: ${{ steps.validateversion.outputs.PREVIEW_VERSION }} run: | + $previewValue = '${{ inputs.previewReleaseTag }}' $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - if (!($PREVIEW_VERSION -eq "")) + if (!($previewValue -eq "")) { - ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion $PREVIEW_VERSION -Verbose -Acr All -OsFilter All + ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion $previewValue -Verbose -Acr All -OsFilter All } else { @@ -118,16 +94,15 @@ jobs: ./build.ps1 -UpdateBuildYaml -Channel preview -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for LTS - env: - LTS_VERSION: ${{ steps.validateversion.outputs.LTS_VERSION }} run: | + $ltsValue = '${{ inputs.ltsReleaseTag }}' $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath - if (!($LTS_VERSION -eq "")) + if (!($ltsValue -eq "")) { - ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion $LTS_VERSION -Verbose -Acr All -OsFilter All + ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion $ltsValue -Verbose -Acr All -OsFilter All } else { From 96d43072d2b248196ba2250564f3235f7ab874c3 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 16:18:14 -0400 Subject: [PATCH 25/35] revert changes to generateMatrixJson pset --- build.ps1 | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/build.ps1 b/build.ps1 index 888ecbe34..2e1a4f452 100755 --- a/build.ps1 +++ b/build.ps1 @@ -604,7 +604,6 @@ End { { $channelName = $channelGroup.Name Write-Verbose "generating $channelName json" - $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { $osName = $osGroup.Name @@ -629,19 +628,18 @@ End { $jobName = $tag.Name -replace '-', '_' if (-not $matrix.$channelName[$osName][$architectureName].ContainsKey($jobName) -and -not $tag.ContinueOnError) { $matrix.$channelName[$osName][$architectureName].Add($jobName, (ConvertTo-SortedDictionary -Hashtable @{ - Channel = $tag.Channel - ImageName = $tag.Name - ArtifactSuffixName = $tag.Name.ToString().Replace("\", "_").Replace("-","_") - JobName = $jobName - ContinueOnError = $tag.ContinueOnError - EndOfLife = $tag.EndOfLife - DistributionState = $tag.DistributionState - OsVersion = $tag.OsVersion + Channel = $tag.Channel + ImageName = $tag.Name + JobName = $jobName + ContinueOnError = $tag.ContinueOnError + EndOfLife = $tag.EndOfLife + DistributionState = $tag.DistributionState + OsVersion = $tag.OsVersion # azDevOps doesn't support arrays - TagList = $tag.Tags -join ';' - IsLinux = $tag.IsLinux - UseInCI = $tag.UseInCI - Architecture = $tag.Architecture + TagList = $tag.Tags -join ';' + IsLinux = $tag.IsLinux + UseInCI = $tag.UseInCI + Architecture = $tag.Architecture })) } } @@ -649,6 +647,33 @@ End { } } } + + foreach ($channelName in $matrix.Keys | Sort-Object) { + $fullMatrix[$channelName] = @() + foreach ($osName in $matrix.$channelName.Keys | Sort-Object) { + $osMatrix = $matrix.$channelName.$osName + foreach ($architectureName in $osMatrix.Keys | Sort-Object) { + $architectureMatrix = $osMatrix.$architectureName + + $channelMatrix = [System.Collections.ArrayList]::new() + $architectureMatrix.Values | Sort-Object -Property ImageName | ForEach-Object { + $null = $channelMatrix.Add($_) + } + $fullMatrix[$channelName] += $channelMatrix + $matrixJson = $architectureMatrix | ConvertTo-Json -Compress + $variableName = "matrix_${channelName}_${osName}_${architectureName}" + if (!$FullJson) { + Set-BuildVariable -Name $variableName -Value $matrixJson -IsOutput + Write-Verbose -Verbose "*********END of JSON*********************" + } + } + } + } + + if($FullJson) { + $matrixJson = $fullMatrix | ConvertTo-Json -Depth 100 + Write-Output $matrixJson + } } $channelsUsed = @{} From 33d1e6b9b285b4026d032e8ca748843a3a14c7d7 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 16:27:28 -0400 Subject: [PATCH 26/35] add debugging verbose statements --- build.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 2e1a4f452..34d699c07 100755 --- a/build.ps1 +++ b/build.ps1 @@ -684,7 +684,7 @@ End { foreach($channelGroup in $channelGroups) { $channelName = $channelGroup.Name - Write-Verbose "generating $channelName json" + Write-Verbose "generating $channelName json for $($channelGroup.Values)" $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" @@ -745,6 +745,7 @@ End { })) # Call method to write the part of the yaml file that is unique for each image based template call. + Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values)" Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } From bbfad3213e460447f3ef782a7f48788f4ee718ed Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 16:32:23 -0400 Subject: [PATCH 27/35] add debugging verbose statements --- build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 34d699c07..199121c5e 100755 --- a/build.ps1 +++ b/build.ps1 @@ -684,7 +684,7 @@ End { foreach($channelGroup in $channelGroups) { $channelName = $channelGroup.Name - Write-Verbose "generating $channelName json for $($channelGroup.Values)" + Write-Verbose "generating $channelName json for $($channelGroup.Values) for $repo" $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" From 10179a79f9b9fc9515729b15b18fa412531b956f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 16:41:21 -0400 Subject: [PATCH 28/35] add debugging verbose statements --- build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 199121c5e..7a1226dae 100755 --- a/build.ps1 +++ b/build.ps1 @@ -745,7 +745,7 @@ End { })) # Call method to write the part of the yaml file that is unique for each image based template call. - Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values)" + Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } } From 6b6154e1bdc7bad950d5ae8b9f5fb55285f4a514 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 16:44:23 -0400 Subject: [PATCH 29/35] add debugging verbose statements --- build.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.ps1 b/build.ps1 index 7a1226dae..3447f96b1 100755 --- a/build.ps1 +++ b/build.ps1 @@ -748,6 +748,10 @@ End { Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag } + else + { + Write-Verbose -Verbose "NOT calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" + } } } } From 0a4a82e376a1f45ea6c1ceec70124c87e6a299a6 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 17:20:32 -0400 Subject: [PATCH 30/35] add debugging verbose statements --- build.ps1 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.ps1 b/build.ps1 index 3447f96b1..bc493e0da 100755 --- a/build.ps1 +++ b/build.ps1 @@ -691,6 +691,7 @@ End { if (!$channelsUsed.Contains($channelName)) { + Write-Verbose "newly seen channel" # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath @@ -708,12 +709,18 @@ End { $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { $osName = $osGroup.Name + Write-Verbose -Verbose "osName: $osName" $architectureGroups = $osGroup.Group | Group-Object -Property Architecture foreach ($architectureGroup in $architectureGroups) { $architectureName = $architectureGroup.Name + foreach ($tag in $architectureGroup.Group | Where-Object { $_.Name -like '*/*' } | Sort-Object -Property dockerfile) { + Write-Verbose -Verbose "tag name: $($tag.Name)" + } + # Filter out subimages. We cannot directly build subimages. foreach ($tag in $architectureGroup.Group | Where-Object { $_.Name -notlike '*/*' } | Sort-Object -Property dockerfile) { + Write-Verbose -Verbose "filtered in name: $($tag.Name)" if (-not $matrix.ContainsKey($channelName)) { $matrix.Add($channelName, @{ }) } From 23a9e4a49cb21c9d1e0fd0ff23f97eab2116bb67 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 17:50:52 -0400 Subject: [PATCH 31/35] simplify- no need to remove subimages as those can be built directly now --- build.ps1 | 90 +++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/build.ps1 b/build.ps1 index bc493e0da..05f2a6aba 100755 --- a/build.ps1 +++ b/build.ps1 @@ -712,53 +712,53 @@ End { Write-Verbose -Verbose "osName: $osName" $architectureGroups = $osGroup.Group | Group-Object -Property Architecture foreach ($architectureGroup in $architectureGroups) { - $architectureName = $architectureGroup.Name - - foreach ($tag in $architectureGroup.Group | Where-Object { $_.Name -like '*/*' } | Sort-Object -Property dockerfile) { - Write-Verbose -Verbose "tag name: $($tag.Name)" - } + # $architectureName = $architectureGroup.Name # Filter out subimages. We cannot directly build subimages. - foreach ($tag in $architectureGroup.Group | Where-Object { $_.Name -notlike '*/*' } | Sort-Object -Property dockerfile) { - Write-Verbose -Verbose "filtered in name: $($tag.Name)" - if (-not $matrix.ContainsKey($channelName)) { - $matrix.Add($channelName, @{ }) - } - - if (-not $matrix.$channelName.ContainsKey($osName)) { - $matrix.$channelName.Add($osName, @{ }) - } - - if (-not $matrix.$channelName.$osName.ContainsKey($architectureName)) { - $matrix.$channelName.$osName.Add($architectureName, @{}) - } - - $jobName = $tag.Name -replace '-', '_' - if (-not $matrix.$channelName[$osName][$architectureName].ContainsKey($jobName) -and -not $tag.ContinueOnError) { - $matrix.$channelName[$osName][$architectureName].Add($jobName, (ConvertTo-SortedDictionary -Hashtable @{ - Channel = $tag.Channel - ImageName = $tag.Name - ArtifactSuffixName = $tag.Name.ToString().Replace("\", "_").Replace("-","_") - JobName = $jobName - ContinueOnError = $tag.ContinueOnError - EndOfLife = $tag.EndOfLife - DistributionState = $tag.DistributionState - OsVersion = $tag.OsVersion - # azDevOps doesn't support arrays - TagList = $tag.Tags -join ';' - IsLinux = $tag.IsLinux - UseInCI = $tag.UseInCI - Architecture = $tag.Architecture - })) - - # Call method to write the part of the yaml file that is unique for each image based template call. - Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" - Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag - } - else - { - Write-Verbose -Verbose "NOT calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" - } + foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) { + # Call method to write the part of the yaml file that is unique for each image based template call. + Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" + Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag + + # Write-Verbose -Verbose "filtered in name: $($tag.Name)" + # if (-not $matrix.ContainsKey($channelName)) { + # $matrix.Add($channelName, @{ }) + # } + + # if (-not $matrix.$channelName.ContainsKey($osName)) { + # $matrix.$channelName.Add($osName, @{ }) + # } + + # if (-not $matrix.$channelName.$osName.ContainsKey($architectureName)) { + # $matrix.$channelName.$osName.Add($architectureName, @{}) + # } + + # $jobName = $tag.Name -replace '-', '_' + # if (-not $matrix.$channelName[$osName][$architectureName].ContainsKey($jobName) -and -not $tag.ContinueOnError) { + # $matrix.$channelName[$osName][$architectureName].Add($jobName, (ConvertTo-SortedDictionary -Hashtable @{ + # Channel = $tag.Channel + # ImageName = $tag.Name + # ArtifactSuffixName = $tag.Name.ToString().Replace("\", "_").Replace("-","_") + # JobName = $jobName + # ContinueOnError = $tag.ContinueOnError + # EndOfLife = $tag.EndOfLife + # DistributionState = $tag.DistributionState + # OsVersion = $tag.OsVersion + # # azDevOps doesn't support arrays + # TagList = $tag.Tags -join ';' + # IsLinux = $tag.IsLinux + # UseInCI = $tag.UseInCI + # Architecture = $tag.Architecture + # })) + + # # Call method to write the part of the yaml file that is unique for each image based template call. + # Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" + # Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag + #} + # else + # { + # Write-Verbose -Verbose "NOT calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" + # } } } } From fd836cfac12c4a3be8e1d99300b1a167a829be55 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 18:07:06 -0400 Subject: [PATCH 32/35] clean up code --- build.ps1 | 84 ++---------------------------- tools/buildHelper/buildHelper.psm1 | 18 +++---- 2 files changed, 13 insertions(+), 89 deletions(-) diff --git a/build.ps1 b/build.ps1 index 05f2a6aba..7f44828e3 100755 --- a/build.ps1 +++ b/build.ps1 @@ -678,119 +678,43 @@ End { $channelsUsed = @{} if ($UpdateBuildYaml.IsPresent) { - $matrix = @{ } foreach ($repo in $tagGroups.Keys | Sort-Object) { $channelGroups = $tagGroups.$repo | Group-Object -Property Channel foreach($channelGroup in $channelGroups) { $channelName = $channelGroup.Name - Write-Verbose "generating $channelName json for $($channelGroup.Values) for $repo" $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" - Write-Verbose -Verbose "releaseStage file: $channelReleaseStagePath" if (!$channelsUsed.Contains($channelName)) { - Write-Verbose "newly seen channel" # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. - # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) + # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) so only populate start of yaml file once per channel. $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath if ($channelReleaseStageFileExists) { Remove-Item -Path $channelReleaseStagePath } + New-Item -Type File -Path $channelReleaseStagePath - # Note: only populate the start of the ReleaseStage.yml file once per channel Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath $channelsUsed.Add($channelName, $channelGroup.Values) } $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { - $osName = $osGroup.Name - Write-Verbose -Verbose "osName: $osName" $architectureGroups = $osGroup.Group | Group-Object -Property Architecture foreach ($architectureGroup in $architectureGroups) { - # $architectureName = $architectureGroup.Name - - # Filter out subimages. We cannot directly build subimages. + # Note: For each image to be built (including test-deps images) the ReleaseStage.yml file needs to have a template call for the image. foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) { - # Call method to write the part of the yaml file that is unique for each image based template call. - Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" + Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)" Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag - - # Write-Verbose -Verbose "filtered in name: $($tag.Name)" - # if (-not $matrix.ContainsKey($channelName)) { - # $matrix.Add($channelName, @{ }) - # } - - # if (-not $matrix.$channelName.ContainsKey($osName)) { - # $matrix.$channelName.Add($osName, @{ }) - # } - - # if (-not $matrix.$channelName.$osName.ContainsKey($architectureName)) { - # $matrix.$channelName.$osName.Add($architectureName, @{}) - # } - - # $jobName = $tag.Name -replace '-', '_' - # if (-not $matrix.$channelName[$osName][$architectureName].ContainsKey($jobName) -and -not $tag.ContinueOnError) { - # $matrix.$channelName[$osName][$architectureName].Add($jobName, (ConvertTo-SortedDictionary -Hashtable @{ - # Channel = $tag.Channel - # ImageName = $tag.Name - # ArtifactSuffixName = $tag.Name.ToString().Replace("\", "_").Replace("-","_") - # JobName = $jobName - # ContinueOnError = $tag.ContinueOnError - # EndOfLife = $tag.EndOfLife - # DistributionState = $tag.DistributionState - # OsVersion = $tag.OsVersion - # # azDevOps doesn't support arrays - # TagList = $tag.Tags -join ';' - # IsLinux = $tag.IsLinux - # UseInCI = $tag.UseInCI - # Architecture = $tag.Architecture - # })) - - # # Call method to write the part of the yaml file that is unique for each image based template call. - # Write-Verbose -Verbose "calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" - # Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag - #} - # else - # { - # Write-Verbose -Verbose "NOT calling method to populate yaml for $channelName for $($channelGroup.Values) for $($tag.Name)" - # } } } } } } - - foreach ($channelName in $matrix.Keys | Sort-Object) { - $fullMatrix[$channelName] = @() - foreach ($osName in $matrix.$channelName.Keys | Sort-Object) { - $osMatrix = $matrix.$channelName.$osName - foreach ($architectureName in $osMatrix.Keys | Sort-Object) { - $architectureMatrix = $osMatrix.$architectureName - - $channelMatrix = [System.Collections.ArrayList]::new() - $architectureMatrix.Values | Sort-Object -Property ImageName | ForEach-Object { - $null = $channelMatrix.Add($_) - } - $fullMatrix[$channelName] += $channelMatrix - $matrixJson = $architectureMatrix | ConvertTo-Json -Compress - $variableName = "matrix_${channelName}_${osName}_${architectureName}" - if (!$FullJson) { - Set-BuildVariable -Name $variableName -Value $matrixJson -IsOutput - Write-Verbose -Verbose "*********END of JSON*********************" - } - } - } - } - - if($FullJson) { - $matrixJson = $fullMatrix | ConvertTo-Json -Depth 100 - Write-Output $matrixJson - } } if ($GenerateManifestLists.IsPresent) { diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index b211a9453..7ed0a4daf 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -1042,7 +1042,7 @@ function Get-StartOfYamlPopulated { if (!$YamlFilePath) { - throw "yaml file DNE" + throw "Yaml file $YamlFilePath provided as parameter cannot be found." } $doubleSpace = " "*2 @@ -1052,7 +1052,7 @@ function Get-StartOfYamlPopulated { Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'" Add-Content -Path $YamlFilePath -Value "- name: channelPath" Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''" - Add-Content -Path $YamlFilePath -Value "- name: vmImage" #not sure if this is needed really + Add-Content -Path $YamlFilePath -Value "- name: vmImage" Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure" Add-Content -Path $YamlFilePath -Value "stages:" Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel" @@ -1072,7 +1072,7 @@ function Get-TemplatePopulatedYaml { if (!$YamlFilePath) { - throw "yaml file DNE" + throw "Yaml file $YamlFilePath provided as parameter cannot be found." } $doubleSpace = " "*2 @@ -1090,16 +1090,16 @@ function Get-TemplatePopulatedYaml { $architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32 } - Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/newReleasePhase.yml@self" #this is where for loop should begin + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/newReleasePhase.yml@self" Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:" - Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" #Build_Linux_arm32 - Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # differs from artifactSuffix for test deps image names - Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32 + Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param) + Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" # i.e artifactSuffix: alpine317_test_deps Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'" Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'" if ($poolOS -eq "linux") { - # only need to specify buildKitValue set to 1 for linux + # only need to specify buildKitValue=1 for linux Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1" } else @@ -1110,7 +1110,7 @@ function Get-TemplatePopulatedYaml { Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'" } - Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3"#do we really need? + Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3" } Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}" From 0e9dc66e3880fca40f934a02a4c323c0c9653962 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 18:13:58 -0400 Subject: [PATCH 33/35] clean up workflow yaml --- .github/workflows/updateBuildYamls.yml | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index 748706a88..a6efc71c3 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -9,14 +9,14 @@ permissions: on: workflow_dispatch: inputs: - stableReleaseTag: - description: 'release tag for stable' + stableVersion: + description: 'stable release version (i.e: 7.4.1)' required: false - previewReleaseTag: - description: 'release tag for preview' + previewVersion: + description: 'preview release version (i.e: 7.4.0-preview.5)' required: false - ltsReleaseTag: - description: 'release tag for lts' + ltsVersion: + description: 'lts release version (i.e: 7.2.12)' required: false pull_request: @@ -36,7 +36,7 @@ jobs: - name: Validate Version Syntax id: validate-version run: | - $stableValue = '${{ inputs.stableReleaseTag }}' + $stableValue = '${{ inputs.stableVersion }}' if (!($stableValue -eq "")) { if ($stableValue -notmatch '\d+\.\d+\.\d+$') { @@ -44,7 +44,7 @@ jobs: } } - $previewValue = '${{ inputs.previewReleaseTag }}' + $previewValue = '${{ inputs.previewVersion }}' if (!($previewValue -eq "")) { if ($previewValue -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') { @@ -52,7 +52,7 @@ jobs: } } - $ltsValue = '${{ inputs.ltsReleaseTag }}' + $ltsValue = '${{ inputs.ltsVersion }}' if (!($ltsValue -eq "")) { if ($ltsValue -notmatch '\d+\.\d+\.\d+$') { @@ -61,7 +61,7 @@ jobs: } - name: Update ReleaseStageYaml file for Stable run: | - $stableValue = '${{ inputs.stableReleaseTag }}' + $stableValue = '${{ inputs.stableVersion }}' $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' @@ -74,34 +74,34 @@ jobs: else { # Use the version from channels.json when no version is provided - Write-Verbose -Verbose "not using stable version from input as value was $stableValue" ./build.ps1 -UpdateBuildYaml -Channel stable -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for Preview run: | - $previewValue = '${{ inputs.previewReleaseTag }}' + $previewValue = '${{ inputs.previewVersion }}' $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath if (!($previewValue -eq "")) { + Write-Verbose -Verbose "using preview version: $previewValue" ./build.ps1 -UpdateBuildYaml -Channel preview -PreviewVersion $previewValue -Verbose -Acr All -OsFilter All } else { - # Use the version from channels.json when no version is provided ./build.ps1 -UpdateBuildYaml -Channel preview -Verbose -Acr All -OsFilter All } - name: Update ReleaseStageYaml file for LTS run: | - $ltsValue = '${{ inputs.ltsReleaseTag }}' + $ltsValue = '${{ inputs.ltsVersion }}' $toolsFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath 'tools' $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' Import-Module $buildHelperModulePath if (!($ltsValue -eq "")) { + Write-Verbose -Verbose "using lts version: $ltsValue" ./build.ps1 -UpdateBuildYaml -Channel lts -LtsVersion $ltsValue -Verbose -Acr All -OsFilter All } else From 73eca4c426ed666116a1f2efd727c4817e05bc4a Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 18:17:32 -0400 Subject: [PATCH 34/35] remove step and workflow dispatch used for testing --- .github/workflows/updateBuildYamls.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/updateBuildYamls.yml b/.github/workflows/updateBuildYamls.yml index a6efc71c3..7d760e2cd 100644 --- a/.github/workflows/updateBuildYamls.yml +++ b/.github/workflows/updateBuildYamls.yml @@ -18,7 +18,6 @@ on: ltsVersion: description: 'lts release version (i.e: 7.2.12)' required: false - pull_request: defaults: run: @@ -109,16 +108,6 @@ jobs: # Use the version from channels.json when no version is provided ./build.ps1 -UpdateBuildYaml -Channel lts -Verbose -Acr All -OsFilter All } - - name: View Updated Files Content - run: | - $ciFolderPath = Join-Path -Path ${{ github.workspace }} -ChildPath '.vsts-ci' - $releaseStageFilePath = Join-Path -Path $ciFolderPath -ChildPath 'stableReleaseStage.yml' - $content = Get-Content -Path $releaseStageFilePath - Write-Verbose -Verbose "length of content: $($content.Length)" - for($i=0; $i -lt $content.Length;$i++) - { - Write-Verbose -Verbose $content[$i] - } - name: Create Pull Request if: github.event_name == 'workflow_dispatch' id: cpr From 6d5b03265da5aa0dc1940158b36a9775e8638bc0 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 22 Apr 2024 18:20:07 -0400 Subject: [PATCH 35/35] fix name of releasePhase.yml file that will be called --- tools/buildHelper/buildHelper.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index 7ed0a4daf..85bc26d1b 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -1090,7 +1090,7 @@ function Get-TemplatePopulatedYaml { $architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32 } - Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/newReleasePhase.yml@self" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/releasePhase.yml@self" Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:" Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32 Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param)