-
Notifications
You must be signed in to change notification settings - Fork 285
Use -Zpackage-workspace
for packing
#2349
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -20,6 +20,7 @@ members = [ | |
] | ||
exclude = [ | ||
"eng/scripts", | ||
"target", | ||
] | ||
|
||
[workspace.package] | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,7 +8,7 @@ param( | |
[string[]]$PackageNames, | ||
[Parameter(ParameterSetName = 'PackageInfo')] | ||
[string]$PackageInfoDirectory, | ||
[switch]$NoVerify | ||
[switch]$Verify | ||
) | ||
|
||
$ErrorActionPreference = 'Stop' | ||
|
@@ -80,8 +80,8 @@ function Get-CargoPackages() { | |
} | ||
|
||
function Get-PackagesToBuild() { | ||
$packages = Get-CargoPackages | ||
$outputPackageNames = Get-OutputPackageNames $packages | ||
[array]$packages = Get-CargoPackages | ||
[array]$outputPackageNames = Get-OutputPackageNames $packages | ||
|
||
# We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built | ||
[array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) } | ||
|
@@ -126,26 +126,65 @@ function Get-PackagesToBuild() { | |
return $buildOrder | ||
} | ||
|
||
function Initialize-VendorDirectory() { | ||
$path = "$RepoRoot/target/vendor" | ||
Invoke-LoggedCommand "cargo vendor $path" -GroupOutput | Out-Host | ||
return $path | ||
} | ||
# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish | ||
# https://github.com/rust-lang/cargo/blob/5c87c14f9a162daf10d4133fdaab35c72d67b018/crates/crates-io/lib.rs#L42 | ||
function Get-ApiMetadata($package) { | ||
$packagePath = Split-Path -Path $package.manifest_path -Parent | ||
$readmePath = $package.readme ? (Join-Path -Path $packagePath -ChildPath $package.readme) : $null | ||
|
||
$jsonBody = [ordered]@{ | ||
heaths marked this conversation as resolved.
Show resolved
Hide resolved
|
||
'name' = $package.name | ||
'vers' = $package.version | ||
'deps' = @() | ||
'features' = $package.features | ||
'authors' = $package.authors | ||
'description' = $package.description | ||
'documentation' = $package.documentation | ||
'homepage' = $package.homepage | ||
'readme' = if ($readmePath -and (Test-Path -Path $readmePath)) { | ||
Get-Content -Path $readmePath -Raw | ||
} | ||
else { | ||
$null | ||
} | ||
'readme_file' = $package.readme | ||
'keywords' = $package.keywords | ||
'categories' = $package.categories | ||
'license' = $package.license | ||
'license_file' = $package.license_file | ||
'repository' = $package.repository | ||
'links' = $package.links | ||
'rust_version' = $package.rust_version | ||
} | ||
|
||
function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) { | ||
$packageName = $Package.name | ||
$packageVersion = $Package.version | ||
foreach ($dependency in $package.dependencies) { | ||
$jsonBody.deps += @{ | ||
'name' = $dependency.name | ||
'version_req' = $dependency.req | ||
'features' = $dependency.features | ||
'optional' = $dependency.optional | ||
'default_features' = $dependency.default_features | ||
'target' = $dependency.target | ||
'kind' = $dependency.kind | ||
'explicit_name_in_toml' = $dependency.rename | ||
} | ||
} | ||
|
||
# create an index entry for the package | ||
$packagePath = "$RepoRoot/target/package/$packageName-$packageVersion" | ||
return $jsonBody | ||
} | ||
|
||
Write-Host "Copying package '$packageName' to vendor directory '$LocalRegistryPath'" | ||
Copy-Item -Path $packagePath -Destination $LocalRegistryPath -Recurse | ||
function New-ApiPutFile($crateMetadata, $crateFilePath) { | ||
$metadataBytes = [Text.Encoding]::Utf8.GetBytes($crateMetadata) | ||
$metadataLengthBytes = [BitConverter]::GetBytes([UInt32]$metadataBytes.Length) | ||
$crateBytes = [IO.File]::ReadAllBytes($crateFilePath) | ||
$crateLengthBytes = [BitConverter]::GetBytes([UInt32]$crateBytes.Length) | ||
|
||
#write an empty checksum file | ||
'{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8 | ||
$bytes += $metadataLengthBytes + $metadataBytes + $crateLengthBytes + $crateBytes | ||
|
||
return $bytes | ||
} | ||
Comment on lines
+176
to
185
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why not just use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this isn't computing a hash. Its constructing a binary file for posting to the crates.io api There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ...which comes back to my original question: why are we doing that? I get why we crack open |
||
|
||
|
||
function Create-ApiViewFile($package) { | ||
$packageName = $package.name | ||
$command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName" | ||
|
@@ -158,61 +197,77 @@ function Create-ApiViewFile($package) { | |
|
||
Push-Location $RepoRoot | ||
try { | ||
$localRegistryPath = Initialize-VendorDirectory | ||
|
||
[array]$packages = Get-PackagesToBuild | ||
|
||
Write-Host "Building packages in the following order:" | ||
$command = "cargo +nightly -Zpackage-workspace package --allow-dirty --locked" | ||
|
||
Write-Host "Building packages:" | ||
foreach ($package in $packages) { | ||
$packageName = $package.name | ||
$type = if ($package.OutputPackage) { "output" } else { "dependency" } | ||
Write-Host " $packageName ($type)" | ||
$command += " --package $packageName" | ||
} | ||
|
||
foreach ($package in $packages) { | ||
Write-Host "" | ||
|
||
$packageName = $package.name | ||
$packageVersion = $package.version | ||
|
||
$command = "cargo publish --locked --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty" | ||
if (!$Verify) { | ||
$command += " --no-verify" | ||
} | ||
|
||
if ($NoVerify) { | ||
$command += " --no-verify" | ||
} | ||
if ($env:SYSTEM_DEBUG -eq 'true') { | ||
Write-Host "##[group] $RepoRoot/Cargo.lock" | ||
Get-Content "$RepoRoot/Cargo.lock" | ||
Write-Host "##[endgroup]" | ||
} | ||
|
||
Invoke-LoggedCommand -Command $command -GroupOutput | ||
Invoke-LoggedCommand -Command $command -GroupOutput | ||
|
||
if ($env:SYSTEM_DEBUG -eq 'true') { | ||
Write-Host "##[group] $RepoRoot/Cargo.lock" | ||
Get-Content "$RepoRoot/Cargo.lock" | ||
Write-Host "##[endgroup]" | ||
} | ||
heaths marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
# copy the package to the local registry | ||
Add-CrateToLocalRegistry ` | ||
-LocalRegistryPath $localRegistryPath ` | ||
-Package $package | ||
if ($OutputPath) { | ||
foreach ($package in $packages | Where-Object { $_.OutputPackage }) { | ||
$packageName = $package.name | ||
$packageVersion = $package.version | ||
|
||
Write-Host "`nProcessing package '$packageName'" | ||
|
||
if ($OutputPath -and $package.OutputPackage) { | ||
$sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion" | ||
$sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion.crate" | ||
$targetPath = "$OutputPath/$packageName" | ||
$targetContentsPath = "$targetPath/contents" | ||
$targetApiReviewFile = "$targetPath/$packageName.rust.json" | ||
$targetCrateFile = "$targetPath/$packageName-$packageVersion.crate" | ||
$targetJsonFile = "$targetPath/$packageName-$packageVersion.json" | ||
$targetBinFile = "$targetPath/$packageName.bin" | ||
|
||
if (Test-Path -Path $targetContentsPath) { | ||
Remove-Item -Path $targetContentsPath -Recurse -Force | ||
} | ||
|
||
Write-Host "Copying package '$packageName' to '$targetContentsPath'" | ||
Remove-Item -Path $targetPath -Recurse -Force -ErrorAction SilentlyContinue | ||
New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null | ||
Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig" | ||
|
||
Write-Host "Copying crate file to '$targetCrateFile'" | ||
Copy-Item -Path $sourcePath -Destination $targetCrateFile -Force | ||
|
||
$crateMetadata = Get-ApiMetadata $package | ConvertTo-Json -Depth 10 | ||
|
||
Write-Host "Writing crates.io request metadata to '$targetJsonFile'" | ||
$crateMetadata | Out-File -FilePath "$targetJsonFile" -Encoding utf8 | ||
|
||
$uploadBytes = New-ApiPutFile $crateMetadata $sourcePath | ||
Write-Host "Writing crates.io request bundle to '$targetBinFile'" | ||
[IO.File]::WriteAllBytes($targetBinFile, $uploadBytes) | ||
|
||
Write-Host "Exctracting crate file to '$targetContentsPath'" | ||
New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null | ||
tar -xf $sourcePath --directory $targetContentsPath --strip-components=1 | Out-Null | ||
|
||
Write-Host "Creating API review file" | ||
$apiReviewFile = Create-ApiViewFile $package | ||
|
||
Write-Host "Copying API review file to '$targetApiReviewFile'" | ||
Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force | ||
} | ||
} | ||
|
||
Write-Host "Removing local registry" | ||
Remove-Item -Path $localRegistryPath -Recurse -Force | Out-Null | ||
} | ||
finally { | ||
Pop-Location | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -31,6 +31,4 @@ foreach ($crateName in $crateNames) { | |
} | ||
} | ||
|
||
if ($hasErrors) { | ||
exit 1 | ||
} | ||
exit $hasErrors ? 1 : 0 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is always ignored. Why did you add it? This is going to raise a lot of eyebrows.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
it's not ignored 😬. The targets folder will get package folder in it. If you run any
cargo
commands on the packages in /targets, it complains about the package thinking its running in a workspace but not being a member or excludedThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why are you running commands against anything
targets
? That's not our place space. For package analysis - though I maintain there's nothing to analyze (it's zipped source) - that should probably be done outside the source repo in the temp working dir AzDO, GHA, etc. give us. Let's not unnecessarily complicate things.