Batch generation

.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

@@ -0,0 +1,44 @@
+param (
+    [string]$MatrixKey,
+    [string]$Target,
+    [string]$RepoRoot,
+    [string]$ArtifactRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+Write-Host "Building targets: $Target"
+
+$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
+$modules = $Target -split ','
+$results = @()
+
+foreach ($module in $modules) {
+    $startTime = Get-Date
+    $result = @{
+        Module = $module
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+    }
+    try {
+        Write-Host "Building module: $module"
+        & $buildModulesPath -TargetModule $module
+    } catch {
+        Write-Warning "Failed to build module: $module"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    } finally {
+        $endTime = Get-Date
+        $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+git add .
+$patchPath = Join-Path $ArtifactRoot "changed.patch"
+git diff --cached > $patchPath
+
+$reportPath = Join-Path $ArtifactRoot "BuildReport-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8
+
+Write-Host "Build report written to $reportPath"

.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1

@@ -0,0 +1,22 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$Owner,
+    [string]$Repo,
+    [string]$BaseBranch,
+    [string]$NewBranch,
+    [string]$Token
+)
+
+$headers = @{ Authorization = "Bearer $Token"; "User-Agent" = "ADO-Pipeline" }
+$branchInfo = Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/ref/heads/$BaseBranch" -Headers $headers
+$sha = $branchInfo.object.sha
+
+$body = @{
+    ref = "refs/heads/$NewBranch"
+    sha = $sha
+} | ConvertTo-Json
+
+Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
+                -Method Post -Headers $headers -Body $body -ContentType "application/json"
+
+Write-Host "Created branch '$NewBranch' from '$BBaseBranch'"

.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

@@ -0,0 +1,86 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [int]$MaxParalleAnalyzeJobs = 3,
+    [int]$MaxParalleTestWindowsJobs = 3,
+    [int]$MaxParalleTestLinuxJobs = 3,
+    [int]$MaxParalleTestMacJobs = 3,
+    [string[]]$ChangedFiles
+)
+
+$autorestFolders = @{}
+for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+    if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+        $parent = $Matches[2]
+        $child = $Matches[3]
+        $key = "$parent/$child"
+
+        $autorestFolders[$key] = $true
+    }
+}
+
+$changedSubModules = $autorestFolders.Keys
+# TODO(Bernard) Remove test data after test
+# $changedSubModules = @("A", "B", "C", "D", "E", "F", "G")
+Write-Host "Chagned sub modules: "
+foreach ($subModule in $changedSubModules) {
+    Write-Host $subModule
+}
+
+function Split-List {
+    param (
+        [array]$subModules,
+        [int]$maxParallelJobs
+    )
+
+    $count = $subModules.Count
+    $n = [Math]::Min($count, $maxParallelJobs)
+
+    if ($n -eq 0) {
+        return @()
+    }
+
+    $result = @()
+
+    for ($i = 0; $i -lt $n; $i++) {
+        $result += ,@()
+    }
+
+    for ($i = 0; $i -lt $count; $i++) {
+        $groupIndex = $i % $n
+        $result[$groupIndex] += $subModules[$i]
+    }
+
+    return ,$result
+}
+
+function Write-Matrix {
+    param (
+        [string]$variableName,
+        [array]$groupedSubModules
+    )
+
+    $index = 0
+    foreach ($subModules in $groupedSubModules) {
+        $moduleNamesStr = $subModules -join ','
+        $key = ($index + 1).ToString() + "-" + $subModules.Count
+        $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
+        $index++
+    }
+
+    if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+        $MatrixStr = $MatrixStr.Substring(1)
+    }
+    Write-Host "##vso[task.setVariable variable=$variableName;isOutput=true]{$MatrixStr}"
+    }
+
+$groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleAnalyzeJobs
+Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
+
+$groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestWindowsJobs
+Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
+
+$groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestLinuxJobs
+Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
+
+$groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestMacJobs
+Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules

.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

@@ -0,0 +1,83 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [int]$MaxParallelJobs = 3
+)
+
+$moduleRoot = Join-Path $RepoRoot 'src'
+$subModules = @()
+
+Get-ChildItem -Path $moduleRoot -Directory | ForEach-Object {
+    $module = $_
+    Get-ChildItem -Path $module.FullName -Directory | Where-Object {
+        $_.Name -like '*.autorest'
+    } | ForEach-Object {
+        $sub_module = $_
+        $subModules += ,@($module.Name, $sub_module.Name)
+    }
+}
+
+$subModules = @(
+    # V3
+    @("Cdn","Cdn.Autorest"),
+    @("ImageBuilder", "ImageBuilder.Autorest"),
+
+    # V4
+    @("Chaos", "Chaos.Autorest"),
+    @("DeviceRegistry", "DeviceRegistry.Autorest"),
+    @("Astro", "Astro.Autorest"),
+
+    # V4 Multi sub-modules
+    @("Communication","EmailService.Autorest")
+    # @("Communication", "EmailServicedata.Autorest")
+)
+
+Write-Host "Total matched sub modules: $($subModules.Count)"
+
+function Split-List {
+    param (
+        [array]$subModules,
+        [int]$maxParallelJobs
+    )
+
+    $count = $subModules.Count
+    $n = [Math]::Min($count, $maxParallelJobs)
+    if ($n -eq 0) {
+        return @()
+    }
+
+    $result = @()
+    $sizePerGroup = [Math]::Ceiling($count / $n)
+
+    for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
+        $group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+        $result += ,$group
+    }
+
+    return $result
+}
+
+$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
+
+Write-Host "Total matched devides: $($devidedSubModules.Count)"
+
+$index = 0
+foreach ($subModules in $devidedSubModules) {
+    Write-Host "Outer Group ${index}:"
+    $subIndex = 0
+    foreach ($subModule in $subModules) {
+        Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
+        $subIndex++
+    }
+
+    $moduleNames = $subModules | ForEach-Object { $_[0] }
+    $moduleNamesStr = $moduleNames -join ','
+    $key = ($index + 1).ToString() + "-" + $subModules.Count
+    $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
+    $index++
+}
+
+if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+    $MatrixStr = $MatrixStr.Substring(1)
+}
+Write-Host "##vso[task.setVariable variable=buildTargets;isOutput=true]{$MatrixStr}"

.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1

@@ -0,0 +1,45 @@
+param (
+    [string]$MatrixKey,
+    [string]$Target,
+    [string]$TestEnvName,
+    [string]$RepoRoot,
+    [string]$ArtifactRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+Write-Host "Test $($TestEnvName): $Target"
+
+$modules = $Target -split ','
+$results = @()  
+
+foreach ($module in $modules) {
+    $startTime = Get-Date
+    $result = @{
+        Module = $module
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+    }
+
+    try {
+        Write-Host "Testing module: $module"
+        $subModulePath = Join-Path $RepoRoot 'src' $module
+        # TODO(Bernard) Remove log after test
+        Write-Host "Sub module path: $subModulePath"
+        Set-Location -Path $subModulePath
+        & ".\test-module.ps1"  
+    }
+    catch {
+        Write-Warning "Failed to test module: $module"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    }
+    finally {
+        $endTime = Get-Date
+        $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+$reportPath = Join-Path $ArtifactRoot "Test$($TestEnvName)Report-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8