Skip to content

Commit 56711e1

Browse files
committed
Use -Zpackage-workspace for packing and publishing
1 parent 08a1e55 commit 56711e1

File tree

11 files changed

+215
-136
lines changed

11 files changed

+215
-136
lines changed

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ members = [
1818
]
1919
exclude = [
2020
"eng/scripts",
21+
"target",
2122
]
2223

2324
[workspace.package]

eng/pipelines/templates/jobs/pack.yml

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,6 @@ jobs:
3434
parameters:
3535
Toolchain: nightly
3636

37-
- template: /eng/pipelines/templates/steps/use-rust.yml@self
38-
parameters:
39-
Toolchain: stable
40-
4137
- ${{ if eq(parameters.TestPipeline, 'true') }}:
4238
- template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
4339
parameters:
@@ -61,11 +57,17 @@ jobs:
6157
pwsh: true
6258
filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
6359
arguments: >
64-
-OutputPath '$(Build.ArtifactStagingDirectory)'
60+
-OutputPath '$(Build.ArtifactStagingDirectory)/Packages'
6561
-PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
6662
6763
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
6864
parameters:
6965
ArtifactPath: $(Build.ArtifactStagingDirectory)
7066
ArtifactName: ${{ parameters.PipelineArtifactName }}
7167
CustomCondition: and(succeededOrFailed(), ne(variables['NoPackagesChanged'],'true'))
68+
69+
- template: /eng/common/pipelines/templates/steps/create-apireview.yml
70+
parameters:
71+
Artifacts: ${{ parameters.Artifacts }}
72+
73+
- template: /eng/common/pipelines/templates/steps/detect-api-changes.yml

eng/pipelines/templates/stages/archetype-rust-release.yml

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -84,25 +84,48 @@ stages:
8484
runOnce:
8585
deploy:
8686
steps:
87-
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
88-
parameters:
89-
skipCheckoutNone: true
90-
9187
- template: /eng/pipelines/templates/steps/use-rust.yml@self
9288
parameters:
9389
Toolchain: stable
9490

95-
- task: PowerShell@2
91+
- pwsh: |
92+
$additionalOwners = @('heaths', 'hallipr')
93+
$token = $env:CARGO_REGISTRY_TOKEN
94+
$crateName = '${{artifact.name}}'
95+
$artifactPath = "$(Pipeline.Workspace)/drop/$crateName"
96+
97+
Write-Host "Dry-run verifying '$crateName'"
98+
99+
# remove Cargo.toml.orig before calling dry-run as it causes a reserved file name error
100+
Remove-Item -Path "$artifactPath/contents/Cargo.toml.orig" -Force -ErrorAction Ignore
101+
102+
$command = "cargo publish --manifest-path '$artifactPath/contents/Cargo.toml' --dry-run"
103+
Write-Host "> $command"
104+
Invoke-Expression $command
105+
106+
if (!$?) {
107+
Write-Error "Dry-run failed for '$crateName'"
108+
exit 1
109+
}
110+
111+
Write-Host "Publishing package '$crateName'"
112+
113+
# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish
114+
Invoke-WebRequest -Method Put -Uri 'https://crates.io/api/v1/crates/new' `
115+
-Headers @{ Accept = 'application/json'; Authorization = $token } `
116+
-ContentType 'application/json' `
117+
-InFile "$artifactPath/$crateName.bin"
118+
119+
$existingOwners = (cargo owner --list $crateName) -replace " \(.*", ""
120+
$missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ }
121+
122+
foreach ($owner in $missingOwners) {
123+
Write-Host "> cargo owner --add $owner $crateName"
124+
cargo owner --add $owner $crateName
125+
}
96126
displayName: Publish Crate
97127
env:
98128
CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
99-
inputs:
100-
targetType: filePath
101-
filePath: $(Build.SourcesDirectory)/eng/scripts/Publish-Crates.ps1
102-
arguments: >
103-
-PackagesPath '$(Pipeline.Workspace)/drop'
104-
-CrateNames '${{artifact.name}}'
105-
-AdditionalOwners 'heaths','hallipr'
106129
107130
# - job: CreateApiReview
108131
# displayName: "Api Review"

eng/scripts/Language-Settings.ps1

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,3 +167,21 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working
167167
ReadmeContent = $readmeContent
168168
}
169169
}
170+
171+
function Find-rust-Artifacts-For-Apireview([string]$ArtifactPath, [string]$packageName) {
172+
[array]$files = Get-ChildItem -Path $ArtifactPath -Recurse -Filter "$packageName`_rust.json"
173+
174+
if (!$files) {
175+
Write-Host "$($packageName) does not have api review json"
176+
return $null
177+
}
178+
elseif ($files.Count -ne 1) {
179+
Write-Host "$($artifactPath) should contain only one api review for $($packageName)"
180+
Write-Host "No of files $($files.Count)"
181+
return $null
182+
}
183+
$packages = @{
184+
$files[0].Name = $files[0].FullName
185+
}
186+
return $packages
187+
}

eng/scripts/Pack-Crates.ps1

Lines changed: 115 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ param(
88
[string[]]$PackageNames,
99
[Parameter(ParameterSetName = 'PackageInfo')]
1010
[string]$PackageInfoDirectory,
11-
[switch]$NoVerify
11+
[switch]$Verify
1212
)
1313

1414
$ErrorActionPreference = 'Stop'
@@ -80,8 +80,8 @@ function Get-CargoPackages() {
8080
}
8181

8282
function Get-PackagesToBuild() {
83-
$packages = Get-CargoPackages
84-
$outputPackageNames = Get-OutputPackageNames $packages
83+
[array]$packages = Get-CargoPackages
84+
[array]$outputPackageNames = Get-OutputPackageNames $packages
8585

8686
# We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built
8787
[array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) }
@@ -126,105 +126,151 @@ function Get-PackagesToBuild() {
126126
return $buildOrder
127127
}
128128

129-
function Initialize-VendorDirectory() {
130-
$path = "$RepoRoot/target/vendor"
131-
Invoke-LoggedCommand "cargo vendor $path" -GroupOutput | Out-Host
132-
return $path
133-
}
129+
# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish
130+
# https://github.com/rust-lang/cargo/blob/5c87c14f9a162daf10d4133fdaab35c72d67b018/crates/crates-io/lib.rs#L42
131+
function Get-ApiMetadata($package) {
132+
$packagePath = Split-Path -Path $package.manifest_path -Parent
133+
$readmePath = $package.readme ? (Join-Path -Path $packagePath -ChildPath $package.readme) : $null
134+
135+
$jsonBody = [ordered]@{
136+
'name' = $package.name
137+
'vers' = $package.version
138+
'deps' = @()
139+
'features' = $package.features
140+
'authors' = $package.authors
141+
'description' = $package.description
142+
'documentation' = $package.documentation
143+
'homepage' = $package.homepage
144+
'readme' = if ($readmePath -and (Test-Path -Path $readmePath)) {
145+
Get-Content -Path $readmePath -Raw
146+
}
147+
else {
148+
$null
149+
}
150+
'readme_file' = $package.readme
151+
'keywords' = $package.keywords
152+
'categories' = $package.categories
153+
'license' = $package.license
154+
'license_file' = $package.license_file
155+
'repository' = $package.repository
156+
'links' = $package.links
157+
'rust_version' = $package.rust_version
158+
}
134159

135-
function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) {
136-
$packageName = $Package.name
137-
$packageVersion = $Package.version
160+
foreach ($dependency in $package.dependencies) {
161+
$jsonBody.deps += @{
162+
'name' = $dependency.name
163+
'version_req' = $dependency.req
164+
'features' = $dependency.features
165+
'optional' = $dependency.optional
166+
'default_features' = $dependency.default_features
167+
'target' = $dependency.target
168+
'kind' = $dependency.kind
169+
'explicit_name_in_toml' = $dependency.rename
170+
}
171+
}
172+
173+
return $jsonBody
174+
}
138175

139-
# create an index entry for the package
140-
$packagePath = "$RepoRoot/target/package/$packageName-$packageVersion"
176+
function New-ApiPutFile($crateMetadata, $crateFilePath) {
177+
$metadataBytes = [Text.Encoding]::Utf8.GetBytes($crateMetadata)
178+
$metadataLengthBytes = [BitConverter]::GetBytes([UInt32]$metadataBytes.Length)
179+
$crateBytes = [IO.File]::ReadAllBytes($crateFilePath)
180+
$crateLengthBytes = [BitConverter]::GetBytes([UInt32]$crateBytes.Length)
141181

142-
Write-Host "Copying package '$packageName' to vendor directory '$LocalRegistryPath'"
143-
Copy-Item -Path $packagePath -Destination $LocalRegistryPath -Recurse
182+
$bytes += $metadataLengthBytes + $metadataBytes + $crateLengthBytes + $crateBytes
144183

145-
#write an empty checksum file
146-
'{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8
184+
return $bytes
147185
}
148186

149-
# For all dependencies with paths, but no versions, add the version from the path
150-
function Add-PathVersions($packages) {
151-
# Install PSToml if it's not already installed
152-
if (-not (PowerShellGet\Get-InstalledModule -Name PSToml -ErrorAction SilentlyContinue)) {
153-
PowerShellGet\Install-Module -Name PSToml -Scope CurrentUser -Force
154-
}
187+
function Create-ApiReview($package) {
188+
$command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $($package.name)"
189+
Invoke-LoggedCommand $command -GroupOutput | Out-Host
155190

156-
foreach ($package in $packages) {
157-
$dirty = $false
158-
$toml = Get-Content -Path $Package.manifest_path -Raw | ConvertFrom-Toml
159-
160-
foreach ($name in $toml.dependencies.Keys) {
161-
# we want to look at the dependency as it was resolved by `cargo metadata`
162-
# this will resolve workspace depdencies, but retain their path/no-version state
163-
$dependency = $package.dependencies | Where-Object -Property name -EQ -Value $name | Select-Object -First 1
164-
# If the dependency is a path dependency, set the version to the version of the package in the workspace
165-
if ($dependency.path -and !$dependency.version) {
166-
$tomlDependency = $toml.dependencies.$name
167-
$dependencyVersion = $packages | Where-Object -Property name -EQ -Value $name | Select-Object -ExpandProperty version -First 1
168-
169-
$tomlDependency.version = $dependencyVersion
170-
$dirty = $true
171-
}
172-
}
173-
if ($dirty) {
174-
$toml | ConvertTo-Toml -Depth 10 | Set-Content -Path $Package.manifest_path -Encoding utf8
175-
}
176-
}
191+
$packagePath = Split-Path -Path $package.manifest_path -Parent
192+
193+
"$packagePath/review/$($package.name).rust.json"
177194
}
178195

179196
Push-Location $RepoRoot
180197
try {
181-
$localRegistryPath = Initialize-VendorDirectory
182-
183198
[array]$packages = Get-PackagesToBuild
184199

185-
Write-Host "Building packages in the following order:"
200+
$command = "cargo +nightly -Zpackage-workspace package --allow-dirty --locked"
201+
202+
Write-Host "Building packages:"
186203
foreach ($package in $packages) {
187204
$packageName = $package.name
188205
$type = if ($package.OutputPackage) { "output" } else { "dependency" }
189206
Write-Host " $packageName ($type)"
207+
$command += " --package $packageName"
190208
}
191209

192-
foreach ($package in $packages) {
193-
Write-Host ""
210+
if (!$Verify) {
211+
$command += " --no-verify"
212+
}
213+
214+
if ($env:SYSTEM_DEBUG -eq 'true') {
215+
Write-Host "##[group] $RepoRoot/Cargo.lock"
216+
Get-Content "$RepoRoot/Cargo.lock"
217+
Write-Host "##[endgroup]"
218+
}
219+
220+
Invoke-LoggedCommand -Command $command -GroupOutput
221+
222+
if ($env:SYSTEM_DEBUG -eq 'true') {
223+
Write-Host "##[group] $RepoRoot/Cargo.lock"
224+
Get-Content "$RepoRoot/Cargo.lock"
225+
Write-Host "##[endgroup]"
226+
}
194227

228+
foreach ($package in $packages) {
195229
$packageName = $package.name
196230
$packageVersion = $package.version
197231

198-
$command = "cargo publish --locked --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
232+
if ($OutputPath -and $package.OutputPackage) {
233+
Write-Host "`nProcessing package '$($package.name)'"
199234

200-
if ($NoVerify) {
201-
$command += " --no-verify"
202-
}
235+
$sourceCrateFile = "$RepoRoot/target/package/$packageName-$packageVersion.crate"
203236

204-
Invoke-LoggedCommand -Command $command -GroupOutput
237+
New-Item -ItemType Directory -Path "$OutputPath/$packageName" -Force | Out-Null
205238

239+
$targetExpandedDirectory = "$OutputPath/$packageName/contents"
240+
$targetCrateFile = "$OutputPath/$packageName/$packageName-$packageVersion.crate"
241+
$targetJsonFile = "$OutputPath/$packageName/$packageName-$packageVersion.json"
242+
$targetBinFile = "$OutputPath/$packageName/$packageName.bin"
243+
$targetApiReviewFile = "$OutputPath/$packageName/$packageName`_rust.json"
206244

207-
# copy the package to the local registry
208-
Add-CrateToLocalRegistry `
209-
-LocalRegistryPath $localRegistryPath `
210-
-Package $package
245+
Write-Host "Copying crate file to '$targetCrateFile'"
246+
Copy-Item -Path $sourceCrateFile -Destination $targetCrateFile -Force
211247

212-
if ($OutputPath -and $package.OutputPackage) {
213-
$packageOutputPath = "$OutputPath/$packageName"
214-
$targetPackagePath = "$RepoRoot/target/package/$packageName-$packageVersion"
248+
$crateMetadata = Get-ApiMetadata $package | ConvertTo-Json -Depth 10
215249

216-
if (Test-Path -Path $packageOutputPath) {
217-
Remove-Item -Path $packageOutputPath -Recurse -Force
250+
Write-Host "Writing crates.io request metadata to '$targetJsonFile'"
251+
$crateMetadata | Out-File -FilePath "$targetJsonFile" -Encoding utf8
252+
253+
$uploadBytes = New-ApiPutFile $crateMetadata $sourceCrateFile
254+
Write-Host "Writing crates.io request bundle to '$targetBinFile'"
255+
[IO.File]::WriteAllBytes($targetBinFile, $uploadBytes)
256+
257+
if (Test-Path $targetExpandedDirectory) {
258+
Write-Host "Removing existing directory '$targetExpandedDirectory'"
259+
Remove-Item -Path $targetExpandedDirectory -Recurse -Force
218260
}
219261

220-
Write-Host "Copying package '$packageName' to '$packageOutputPath'"
221-
New-Item -ItemType Directory -Path $packageOutputPath -Force | Out-Null
222-
Copy-Item -Path $targetPackagePath/* -Destination $packageOutputPath -Recurse -Exclude "Cargo.toml.orig"
262+
Write-Host "Exctracting crate file to '$targetExpandedDirectory'"
263+
New-Item -ItemType Directory -Path $targetExpandedDirectory -Force | Out-Null
264+
tar -xf $sourceCrateFile --directory $targetExpandedDirectory --strip-components=1
265+
266+
267+
Write-Host "Creating API review file"
268+
$apiReviewFile = Create-ApiReview $package
269+
270+
Write-Host "Copying API review file to '$targetApiReviewFile'"
271+
Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force
223272
}
224273
}
225-
226-
Write-Host "Removing local registry"
227-
Remove-Item -Path $localRegistryPath -Recurse -Force | Out-Null
228274
}
229275
finally {
230276
Pop-Location

eng/scripts/Publish-Crates.ps1

Lines changed: 0 additions & 31 deletions
This file was deleted.

0 commit comments

Comments
 (0)