Skip to content

Commit 0c3d24a

Browse files
authored
Use native cargo commands for crates.io work (#2140)
1 parent 17aa7f2 commit 0c3d24a

File tree

9 files changed

+60
-154
lines changed

9 files changed

+60
-154
lines changed

eng/pipelines/templates/jobs/ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ jobs:
155155
filePath: $(Build.SourcesDirectory)/eng/scripts/Analyze-Code.ps1
156156
arguments: >
157157
-Toolchain '$(Toolchain)'
158-
-PackageInfoPath '$(Build.ArtifactStagingDirectory)/PackageInfo'
158+
-PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
159159
-SkipPackageAnalysis:('$(NoPackagesChanged)' -eq 'true')
160160
161161
- template: /eng/common/pipelines/templates/steps/check-spelling.yml

eng/pipelines/templates/stages/archetype-rust-release.yml

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,13 @@ stages:
6363
- deployment: PublishPackage
6464
displayName: "Publish to Crates.io"
6565
condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true'))
66+
templateContext:
67+
type: releaseJob # Required, this indicates this deployment job is a release job
68+
isProduction: true # Required, must be 'true' or 'false'
69+
inputs: # All input build artifacts must be declared here
70+
- input: pipelineArtifact # Required, type of the input artifact
71+
artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact
72+
targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to
6673
environment: ${{parameters.Environment}}
6774
# This timeout shouldn't be necessary once we're able to parallelize better. Right now,
6875
# this is here to ensure larger areas (30+) libraries don't time out.
@@ -77,18 +84,20 @@ stages:
7784
deploy:
7885
steps:
7986
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
87+
parameters:
88+
skipCheckoutNone: true
8089

81-
- download: current
82-
displayName: Download ${{parameters.PipelineArtifactName}} artifact
83-
artifact: ${{parameters.PipelineArtifactName}}
90+
- template: /eng/pipelines/templates/steps/use-rust.yml@self
91+
parameters:
92+
Toolchain: stable
8493

8594
- task: PowerShell@2
8695
displayName: Publish Crate
8796
inputs:
8897
targetType: filePath
8998
filePath: $(Build.SourcesDirectory)/eng/scripts/Publish-Crates.ps1
9099
arguments: >
91-
-PackagesPath '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}'
100+
-PackagesPath '$(Pipeline.Workspace)/drop'
92101
-CrateNames '${{artifact.name}}'
93102
-Token $(azure-sdk-cratesio-token)
94103
-AdditionalOwners 'heaths','hallipr'

eng/pipelines/templates/steps/test-packages.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ steps:
3030
pwsh: true
3131
filePath: $(Build.SourcesDirectory)/eng/scripts/Test-Packages.ps1
3232
arguments: >
33-
-PackageInfoPath '$(Pipeline.Workspace)/${{parameters.BuildArtifactName}}/PackageInfo'
33+
-PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.BuildArtifactName}}/PackageInfo'
3434
-Toolchain '$(Toolchain)'
3535
-UnitTests $${{ parameters.UnitTests }}
3636
-FunctionalTests $${{ parameters.FunctionalTests }}

eng/scripts/Analyze-Code.ps1

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#Requires -Version 7.0
44
param(
55
[string]$Toolchain = 'stable',
6-
[string]$PackageInfoPath,
6+
[string]$PackageInfoDirectory,
77
[switch]$CheckWasm = $true,
88
[switch]$SkipPackageAnalysis
99
)
@@ -12,7 +12,6 @@ $ErrorActionPreference = 'Stop'
1212
Set-StrictMode -Version 2.0
1313

1414
. (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1')
15-
. (Join-Path $EngCommonScriptsDir 'Helpers' 'CommandInvocation-Helpers.ps1')
1615

1716
Write-Host "Analyzing code with
1817
Toolchain: '$Toolchain'`n"
@@ -41,12 +40,12 @@ Invoke-LoggedCommand "cargo +$Toolchain doc --workspace --no-deps --all-features
4140
$verifyDependenciesScript = Join-Path $RepoRoot 'eng' 'scripts' 'verify-dependencies.rs' -Resolve
4241

4342
if (!$SkipPackageAnalysis) {
44-
if (!(Test-Path $PackageInfoPath)) {
45-
Write-Error "Package info path '$PackageInfoPath' does not exist."
43+
if (!(Test-Path $PackageInfoDirectory)) {
44+
Write-Error "Package info path '$PackageInfoDirectory' does not exist."
4645
exit 1
4746
}
4847

49-
$packagesToTest = Get-ChildItem $PackageInfoPath -Filter "*.json" -Recurse
48+
$packagesToTest = Get-ChildItem $PackageInfoDirectory -Filter "*.json" -Recurse
5049
| Get-Content -Raw
5150
| ConvertFrom-Json
5251

eng/scripts/Language-Settings.ps1

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
$Language = "rust"
22
$LanguageDisplayName = "Rust"
33
$PackageRepository = "crates.io"
4-
$packagePattern = "cargo-metadata.json"
4+
$packagePattern = "Cargo.toml"
55
#$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv"
66
$GithubUri = "https://github.com/Azure/azure-sdk-for-rust"
77
$PackageRepositoryUri = "https://crates.io/crates"
88

9+
. (Join-Path $EngCommonScriptsDir "Helpers" "PSModule-Helpers.ps1")
10+
911
function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate, $ReplaceLatestEntryTitle = $true) {
1012
if ($null -eq $ReleaseDate) {
1113
$ReleaseDate = Get-Date -Format "yyyy-MM-dd"
@@ -141,28 +143,19 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang
141143
}
142144

143145
function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) {
144-
#$pkg will be a FileInfo object for the cargo-metadata.json file in a package artifact directory
145-
$package = Get-Content -Path $pkg.FullName -Raw | ConvertFrom-Json
146-
$packageName = $package.name
147-
$packageVersion = $package.vers
148-
149-
$crateFile = Get-ChildItem $pkg.DirectoryName -Filter '*.crate'
146+
#$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory
147+
$package = cargo read-manifest --manifest-path $pkg.FullName
150148

151-
New-Item -Path $workingDirectory -ItemType Directory -Force | Out-Null
152-
$workFolder = Join-Path $workingDirectory $crateFile.BaseName
153-
if (Test-Path $workFolder) {
154-
Remove-item $workFolder -Recurse -Force | Out-Null
155-
}
149+
$packageName = $package.name
150+
$packageVersion = $package.version
156151

157-
# This will extract the contents of the crate file into a folder matching the file name
158-
tar -xvzf $crateFile.FullName -C $workingDirectory | Out-Null
152+
$changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1
153+
$readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1
159154

160-
$changeLogLoc = Get-ChildItem -Path $workFolder -Filter "CHANGELOG.md" | Select-Object -First 1
161155
if ($changeLogLoc) {
162156
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion
163157
}
164158

165-
$readmeContentLoc = Get-ChildItem -Path $workFolder -Filter "README.md" | Select-Object -First 1
166159
if ($readmeContentLoc) {
167160
$readmeContent = Get-Content -Raw $readmeContentLoc
168161
}

eng/scripts/Pack-Crates.ps1

Lines changed: 4 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ param(
1212
$ErrorActionPreference = 'Stop'
1313

1414
. (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1')
15-
. (Join-Path $EngCommonScriptsDir 'Helpers' 'CommandInvocation-Helpers.ps1')
1615

1716
if ($OutputPath) {
1817
$OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName
@@ -168,61 +167,6 @@ function Add-PathVersions($packages) {
168167
}
169168
}
170169

171-
function Get-ApiMetadata($package) {
172-
$packagePath = Split-Path -Path $package.manifest_path -Parent
173-
$readmePath = Join-Path -Path $packagePath -ChildPath $package.readme
174-
$jsonBody = [ordered]@{
175-
'name' = $package.name
176-
'vers' = $package.version
177-
'deps' = @()
178-
'features' = $package.features
179-
'authors' = $package.authors
180-
'description' = $package.description
181-
'documentation' = $package.documentation
182-
'homepage' = $package.homepage
183-
'readme' = if ($package.readme -and (Test-Path -Path $readmePath)) {
184-
Get-Content -Path $readmePath -Raw
185-
}
186-
else {
187-
$null
188-
}
189-
'readme_file' = $package.readme
190-
'keywords' = $package.keywords
191-
'categories' = $package.categories
192-
'license' = $package.license
193-
'license_file' = $package.license_file
194-
'repository' = $package.repository
195-
'links' = $package.links
196-
'rust_version' = $package.rust_version
197-
}
198-
199-
foreach ($dependency in $package.dependencies) {
200-
$jsonBody.deps += @{
201-
'name' = $dependency.name
202-
'version_req' = $dependency.req
203-
'features' = $dependency.features
204-
'optional' = $dependency.optional
205-
'default_features' = $dependency.default_features
206-
'target' = $dependency.target
207-
'kind' = $dependency.kind
208-
'explicit_name_in_toml' = $dependency.rename
209-
}
210-
}
211-
212-
return $jsonBody
213-
}
214-
215-
function New-ApiPutFile($crateMetadata, $CrateFilePath) {
216-
$metadataBytes = [Text.Encoding]::Utf8.GetBytes($crateMetadata)
217-
$metadataLengthBytes = [BitConverter]::GetBytes([UInt32]$metadataBytes.Length)
218-
$crateBytes = [IO.File]::ReadAllBytes($CrateFilePath)
219-
$crateLengthBytes = [BitConverter]::GetBytes([UInt32]$crateBytes.Length)
220-
221-
$bytes += $metadataLengthBytes + $metadataBytes + $crateLengthBytes + $crateBytes
222-
223-
return $bytes
224-
}
225-
226170
Push-Location $RepoRoot
227171
try {
228172
$localRegistryPath = Initialize-VendorDirectory
@@ -245,14 +189,7 @@ try {
245189

246190
Invoke-LoggedCommand `
247191
-GroupOutput `
248-
-Command "cargo package --package $packageName --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
249-
250-
$crateFile = "$RepoRoot/target/package/$packageName-$packageVersion.crate"
251-
252-
if (-not (Test-Path -Path $crateFile)) {
253-
Write-Error "Building the package '$packageName' didn't produce a crate file in the expected location: '$crateFile'"
254-
exit 1
255-
}
192+
-Command "cargo publish --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
256193

257194
# copy the package to the local registry
258195
Add-CrateToLocalRegistry `
@@ -261,31 +198,15 @@ try {
261198

262199
if ($OutputPath -and $package.OutputPackage) {
263200
$packageOutputPath = "$OutputPath/$packageName"
201+
$targetPackagePath = "$RepoRoot/target/package/$packageName-$packageVersion"
202+
264203
if (Test-Path -Path $packageOutputPath) {
265204
Remove-Item -Path $packageOutputPath -Recurse -Force
266205
}
267206

268207
Write-Host "Copying package '$packageName' to '$packageOutputPath'"
269-
270208
New-Item -ItemType Directory -Path $packageOutputPath -Force | Out-Null
271-
Copy-Item -Path $crateFile -Destination $packageOutputPath
272-
# Copy package's Cargo.toml to the output directory
273-
Copy-Item -Path "$RepoRoot/target/package/$packageName-$packageVersion/Cargo.toml" -Destination $packageOutputPath
274-
# Write package metadata to the output directory
275-
276-
$metadataFile = "$packageOutputPath/cargo-metadata.json"
277-
$uploadFile = "$packageOutputPath/cargo-put.bin"
278-
279-
$crateMetadata = Get-ApiMetadata $package | ConvertTo-Json -Depth 10
280-
281-
Write-Host "Writing crates.io request metadata to '$metadataFile'"
282-
$crateMetadata | Out-File -FilePath "$metadataFile" -Encoding utf8
283-
284-
$uploadBytes = New-ApiPutFile $crateMetadata $crateFile
285-
Write-Host "Writing crates.io request bundle to '$uploadFile'"
286-
[IO.File]::WriteAllBytes($uploadFile, $uploadBytes)
287-
288-
Get-ApiMetadata $package | ConvertTo-Json -Depth 100 | Out-File -FilePath "$packageOutputPath/cargo-metadata.json" -Encoding utf8
209+
Copy-Item -Path $targetPackagePath -Destination $packageOutputPath -Recurse -Exclude "Cargo.toml.orig"
289210
}
290211
}
291212

eng/scripts/Publish-Crates.ps1

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,34 +11,22 @@ param(
1111
$ErrorActionPreference = 'Stop'
1212
Set-StrictMode -Version 2.0
1313

14-
function TryAddOwners($crateName) {
15-
foreach ($owner in $AdditionalOwners) {
16-
Write-Host "Adding owner: '$owner' to crate: '$crateName'"
17-
# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#owners-add
18-
# ignore errors is owner already exists
19-
$body = @{ users = @($owner) } | ConvertTo-Json
20-
21-
$response = Invoke-WebRequest -Method Put -Uri "https://crates.io/api/v1/crates/$crateName/owners" `
22-
-Headers @{ Accept = 'application/json'; Authorization = $Token } `
23-
-ContentType 'application/json' `
24-
-Body $body`
25-
-SkipHttpErrorCheck
26-
27-
if ($response.StatusCode -ge 400 -and $response.Content -notmatch 'already an owner') {
28-
Write-Host "Failed to add owner: '$owner' to crate: '$crateName'"
29-
Write-Host "Response: $($response.Content)"
30-
exit 1
31-
}
32-
}
33-
}
34-
3514
foreach ($crateName in $CrateNames) {
3615
Write-Host "Publishing packae: '$crateName'"
16+
$manifestPath = "$PackagesPath/$crateName/Cargo.toml"
3717
# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish
38-
Invoke-WebRequest -Method Put -Uri 'https://crates.io/api/v1/crates/new' `
39-
-Headers @{ Accept = 'application/json'; Authorization = $Token } `
40-
-ContentType 'application/json' `
41-
-InFile "$PackagesPath/$crateName/cargo-put.bin"
18+
Write-Host "> cargo publish --manifest-path `"$manifestPath`" --token <TOKEN>"
19+
cargo publish --manifest-path $manifestPath --token $Token
20+
if (!$?) {
21+
Write-Error "Failed to publish package: '$crateName'"
22+
exit 1
23+
}
4224

43-
TryAddOwners $crateName
25+
$existingOwners = (Invoke-LoggedCommand cargo owner --list $crateName) -replace " \(.*", ""
26+
$missingOwners = $AdditionalOwners | Where-Object { $existingOwners -notcontains $_ }
27+
28+
foreach ($owner in $missingOwners) {
29+
Write-Host "> cargo owner --add $owner $crateName --token <TOKEN>"
30+
cargo owner --add $owner $crateName --token $Token
31+
}
4432
}

eng/scripts/Test-Packages.ps1

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,27 @@ param(
55
[string]$Toolchain = 'stable',
66
[bool]$UnitTests = $true,
77
[bool]$FunctionalTests = $true,
8-
[string]$PackageInfoPath
8+
[string]$PackageInfoDirectory
99
)
1010

1111
$ErrorActionPreference = 'Stop'
1212
Set-StrictMode -Version 2.0
1313

1414
. "$PSScriptRoot\..\common\scripts\common.ps1"
15-
. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1)
1615

1716
Write-Host "Testing packages with
1817
Toolchain: '$Toolchain'
1918
UnitTests: '$UnitTests'
2019
FunctionalTests: '$FunctionalTests'
21-
PackageInfoPath: '$PackageInfoPath'"
20+
PackageInfoDirectory: '$PackageInfoDirectory'"
2221

23-
if ($PackageInfoPath) {
24-
if (!(Test-Path $PackageInfoPath)) {
25-
Write-Error "Package info path '$PackageInfoPath' does not exist."
22+
if ($PackageInfoDirectory) {
23+
if (!(Test-Path $PackageInfoDirectory)) {
24+
Write-Error "Package info path '$PackageInfoDirectory' does not exist."
2625
exit 1
2726
}
2827

29-
$packagesToTest = Get-ChildItem $PackageInfoPath -Filter "*.json" -Recurse
28+
$packagesToTest = Get-ChildItem $PackageInfoDirectory -Filter "*.json" -Recurse
3029
| Get-Content -Raw
3130
| ConvertFrom-Json
3231
}

eng/scripts/Yank-Crates.ps1

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,26 @@
22

33
#Requires -Version 7.0
44
param(
5-
[string]$PackagesPath,
5+
[string]$PackageInfoDirectory,
66
[string[]]$CrateNames,
77
[string]$Token
88
)
99

1010
$ErrorActionPreference = 'Stop'
11-
#Set-StrictMode -Version 2.0
11+
Set-StrictMode -Version 2.0
1212

1313
$hasErrors = $false
1414
foreach ($crateName in $crateNames) {
15-
$crate = Get-Content "$PackagesPath/$crateName/cargo-metadata.json" -Raw | ConvertFrom-Json
16-
$crateVersion = $crate.vers
15+
$crate = Get-Content "$PackageInfoDirectory/$crateName.json" -Raw | ConvertFrom-Json
16+
$crateVersion = $crate.Version
1717

1818
Write-Host "Yanking crate: '$crateName@$crateVersion'"
1919

20-
# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#yank
21-
$response = Invoke-WebRequest -Method Delete -Uri "https://crates.io/api/v1/crates/$crateName/$crateVersion/yank" `
22-
-Headers @{ Accept = 'application/json'; Authorization = $Token } `
23-
-SkipHttpErrorCheck
24-
25-
if ($response.StatusCode -ge 400) {
20+
Write-Host "cargo yank $crateName --version $crateVersion --token <TOKEN>"
21+
cargo yank $crateName --version $crateVersion --token $Token
22+
23+
if (!$?) {
2624
Write-Host "Failed to yank crate: '$crateName@$crateVersion'"
27-
Write-Host "Response: $($response.Content)"
2825
$hasErrors = $true
2926
}
3027
}

0 commit comments

Comments
 (0)