Use -Zpackage-workspace for packing

Cargo.toml

@@ -20,6 +20,7 @@ members = [
 ]
 exclude = [
   "eng/scripts",
+  "target",
 ]
 
 [workspace.package]

eng/pipelines/templates/jobs/pack.yml

@@ -35,10 +35,6 @@ jobs:
       WorkingDirectory: $(System.DefaultWorkingDirectory)/eng/tools/generate_api_report
       SetDefault: false
 
-  - template: /eng/pipelines/templates/steps/use-rust.yml@self
-    parameters:
-      Toolchain: stable
-
   - ${{ if eq(parameters.TestPipeline, 'true') }}:
     - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
       parameters:

eng/pipelines/templates/stages/archetype-rust-release.yml

@@ -92,15 +92,30 @@ stages:
                     $additionalOwners = @('heaths', 'hallipr')
                     $token = $env:CARGO_REGISTRY_TOKEN
                     $crateName = '${{artifact.name}}'
+                    $artifactPath = "$(Pipeline.Workspace)/drop/$crateName"
+
+                    Write-Host "Dry-run verifying '$crateName'"
+
+                    # remove Cargo.toml.orig before calling dry-run as it causes a reserved file name error
+                    Remove-Item -Path "$artifactPath/contents/Cargo.toml.orig" -Force -ErrorAction Ignore
+
+                    $command = "cargo publish --manifest-path '$artifactPath/contents/Cargo.toml' --dry-run"
+                    Write-Host "> $command"
+                    Invoke-Expression $command
 
-                    $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml"
-                    Write-Host "> cargo publish --manifest-path `"$manifestPath`""
-                    cargo publish --manifest-path $manifestPath
                     if (!$?) {
-                      Write-Error "Failed to publish package: '$crateName'"
+                      Write-Error "Dry-run failed for '$crateName'"
                       exit 1
                     }
 
+                    Write-Host "Publishing package '$crateName'"
+
+                    # https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish
+                    Invoke-WebRequest -Method Put -Uri 'https://crates.io/api/v1/crates/new' `
+                      -Headers @{ Accept = 'application/json'; Authorization = $token } `
+                      -ContentType 'application/json' `
+                      -InFile "$artifactPath/$crateName.bin"
+
                     $existingOwners = (cargo owner --list $crateName) -replace " \(.*", ""
                     $missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ }
 

eng/scripts/Pack-Crates.ps1

@@ -8,7 +8,7 @@ param(
   [string[]]$PackageNames,
   [Parameter(ParameterSetName = 'PackageInfo')]
   [string]$PackageInfoDirectory,
-  [switch]$NoVerify
+  [switch]$Verify
 )
 
 $ErrorActionPreference = 'Stop'
@@ -80,8 +80,8 @@ function Get-CargoPackages() {
 }
 
 function Get-PackagesToBuild() {
-  $packages = Get-CargoPackages
-  $outputPackageNames = Get-OutputPackageNames $packages
+  [array]$packages = Get-CargoPackages
+  [array]$outputPackageNames = Get-OutputPackageNames $packages
 
   # We start with output packages, then recursively add unreleased dependencies to the list of packages that need to be built
   [array]$packagesToBuild = $packages | Where-Object { $outputPackageNames.Contains($_.name) }
@@ -126,26 +126,65 @@ function Get-PackagesToBuild() {
   return $buildOrder
 }
 
-function Initialize-VendorDirectory() {
-  $path = "$RepoRoot/target/vendor"
-  Invoke-LoggedCommand "cargo vendor $path" -GroupOutput | Out-Host
-  return $path
-}
+# https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish
+# https://github.com/rust-lang/cargo/blob/5c87c14f9a162daf10d4133fdaab35c72d67b018/crates/crates-io/lib.rs#L42
+function Get-ApiMetadata($package) {
+  $packagePath = Split-Path -Path $package.manifest_path -Parent
+  $readmePath = $package.readme ? (Join-Path -Path $packagePath -ChildPath $package.readme) : $null
+
+  $jsonBody = [ordered]@{
+    'name'          = $package.name
+    'vers'          = $package.version
+    'deps'          = @()
+    'features'      = $package.features
+    'authors'       = $package.authors
+    'description'   = $package.description
+    'documentation' = $package.documentation
+    'homepage'      = $package.homepage
+    'readme'        = if ($readmePath -and (Test-Path -Path $readmePath)) {
+      Get-Content -Path $readmePath -Raw
+    }
+    else {
+      $null
+    }
+    'readme_file'   = $package.readme
+    'keywords'      = $package.keywords
+    'categories'    = $package.categories
+    'license'       = $package.license
+    'license_file'  = $package.license_file
+    'repository'    = $package.repository
+    'links'         = $package.links
+    'rust_version'  = $package.rust_version
+  }
 
-function Add-CrateToLocalRegistry($LocalRegistryPath, $Package) {
-  $packageName = $Package.name
-  $packageVersion = $Package.version
+  foreach ($dependency in $package.dependencies) {
+    $jsonBody.deps += @{
+      'name'                  = $dependency.name
+      'version_req'           = $dependency.req
+      'features'              = $dependency.features
+      'optional'              = $dependency.optional
+      'default_features'      = $dependency.default_features
+      'target'                = $dependency.target
+      'kind'                  = $dependency.kind
+      'explicit_name_in_toml' = $dependency.rename
+    }
+  }
 
-  # create an index entry for the package
-  $packagePath = "$RepoRoot/target/package/$packageName-$packageVersion"
+  return $jsonBody
+}
 
-  Write-Host "Copying package '$packageName' to vendor directory '$LocalRegistryPath'"
-  Copy-Item -Path $packagePath -Destination $LocalRegistryPath -Recurse
+function New-ApiPutFile($crateMetadata, $crateFilePath) {
+  $metadataBytes = [Text.Encoding]::Utf8.GetBytes($crateMetadata)
+  $metadataLengthBytes = [BitConverter]::GetBytes([UInt32]$metadataBytes.Length)
+  $crateBytes = [IO.File]::ReadAllBytes($crateFilePath)
+  $crateLengthBytes = [BitConverter]::GetBytes([UInt32]$crateBytes.Length)
 
-  #write an empty checksum file
-  '{"files":{}}' | Out-File -FilePath "$LocalRegistryPath/$packageName-$packageVersion/.cargo-checksum.json" -Encoding utf8
+  $bytes += $metadataLengthBytes + $metadataBytes + $crateLengthBytes + $crateBytes
+
+  return $bytes
 }
 
+
 function Create-ApiViewFile($package) {
   $packageName = $package.name
   $command = "cargo run --manifest-path $RepoRoot/eng/tools/generate_api_report/Cargo.toml -- --package $packageName"
@@ -158,61 +197,77 @@ function Create-ApiViewFile($package) {
 
 Push-Location $RepoRoot
 try {
-  $localRegistryPath = Initialize-VendorDirectory
-
   [array]$packages = Get-PackagesToBuild
 
-  Write-Host "Building packages in the following order:"
+  $command = "cargo +nightly -Zpackage-workspace package --allow-dirty --locked"
+
+  Write-Host "Building packages:"
   foreach ($package in $packages) {
     $packageName = $package.name
     $type = if ($package.OutputPackage) { "output" } else { "dependency" }
     Write-Host "  $packageName ($type)"
+    $command += " --package $packageName"
   }
 
-  foreach ($package in $packages) {
-    Write-Host ""
-
-    $packageName = $package.name
-    $packageVersion = $package.version
-
-    $command = "cargo publish --locked --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
+  if (!$Verify) {
+    $command += " --no-verify"
+  }
 
-    if ($NoVerify) {
-      $command += " --no-verify"
-    }
+  if ($env:SYSTEM_DEBUG -eq 'true') {
+    Write-Host "##[group] $RepoRoot/Cargo.lock"
+    Get-Content "$RepoRoot/Cargo.lock"
+    Write-Host "##[endgroup]"
+  }
 
-    Invoke-LoggedCommand -Command $command -GroupOutput
+  Invoke-LoggedCommand -Command $command -GroupOutput
 
+  if ($env:SYSTEM_DEBUG -eq 'true') {
+    Write-Host "##[group] $RepoRoot/Cargo.lock"
+    Get-Content "$RepoRoot/Cargo.lock"
+    Write-Host "##[endgroup]"
+  }
 
-    # copy the package to the local registry
-    Add-CrateToLocalRegistry `
-      -LocalRegistryPath $localRegistryPath `
-      -Package $package
+  if ($OutputPath) {
+    foreach ($package in $packages | Where-Object { $_.OutputPackage }) {
+      $packageName = $package.name
+      $packageVersion = $package.version
+
+      Write-Host "`nProcessing package '$packageName'"
 
-    if ($OutputPath -and $package.OutputPackage) {
-      $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion"
+      $sourcePath = "$RepoRoot/target/package/$packageName-$packageVersion.crate"
       $targetPath = "$OutputPath/$packageName"
       $targetContentsPath = "$targetPath/contents"
       $targetApiReviewFile = "$targetPath/$packageName.rust.json"
+      $targetCrateFile = "$targetPath/$packageName-$packageVersion.crate"
+      $targetJsonFile = "$targetPath/$packageName-$packageVersion.json"
+      $targetBinFile = "$targetPath/$packageName.bin"
 
-      if (Test-Path -Path $targetContentsPath) {
-        Remove-Item -Path $targetContentsPath -Recurse -Force
-      }
-
-      Write-Host "Copying package '$packageName' to '$targetContentsPath'"
+      Remove-Item -Path $targetPath -Recurse -Force -ErrorAction SilentlyContinue
       New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null
-      Copy-Item -Path $sourcePath/* -Destination $targetContentsPath -Recurse -Exclude "Cargo.toml.orig"
 
+      Write-Host "Copying crate file to '$targetCrateFile'"
+      Copy-Item -Path $sourcePath -Destination $targetCrateFile -Force
+
+      $crateMetadata = Get-ApiMetadata $package | ConvertTo-Json -Depth 10
+
+      Write-Host "Writing crates.io request metadata to '$targetJsonFile'"
+      $crateMetadata | Out-File -FilePath "$targetJsonFile" -Encoding utf8
+
+      $uploadBytes = New-ApiPutFile $crateMetadata $sourcePath
+      Write-Host "Writing crates.io request bundle to '$targetBinFile'"
+      [IO.File]::WriteAllBytes($targetBinFile, $uploadBytes)
+
+      Write-Host "Exctracting crate file to '$targetContentsPath'"
+      New-Item -ItemType Directory -Path $targetContentsPath -Force | Out-Null
+      tar -xf $sourcePath --directory $targetContentsPath --strip-components=1 | Out-Null
+
       Write-Host "Creating API review file"
       $apiReviewFile = Create-ApiViewFile $package
 
       Write-Host "Copying API review file to '$targetApiReviewFile'"
       Copy-Item -Path $apiReviewFile -Destination $targetApiReviewFile -Force
     }
   }
-
-  Write-Host "Removing local registry"
-  Remove-Item -Path $localRegistryPath -Recurse -Force | Out-Null
 }
 finally {
   Pop-Location

eng/scripts/Update-PackageVersion.ps1

@@ -84,7 +84,7 @@ if ($content -ne $updated) {
   Write-Host "Updated version in $tomlPath from $($pkgProperties.Version) to $packageSemVer."
 
   Write-Host "Updaging dependencies in Cargo.toml files."
-  Invoke-LoggedCommand "cargo +nightly -Zscript '$RepoRoot/eng/scripts/update-pathversions.rs' update" | Out-Null
+  Invoke-LoggedCommand "cargo +nightly -Zscript '$RepoRoot/eng/scripts/update-pathversions.rs' update"
 
   Write-Host "Updating Cargo.lock using 'cargo update --workspace'."
   Invoke-LoggedCommand "cargo update --workspace" | Out-Null

eng/scripts/Yank-Crates.ps1

@@ -31,6 +31,4 @@ foreach ($crateName in $crateNames) {
   }
 }
 
-if ($hasErrors) {
-  exit 1
-}
+exit $hasErrors ? 1 : 0

eng/scripts/update-pathversions.rs

@@ -24,7 +24,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .expect("requires 'add' or 'update' mode argument");
 
     let script_root = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
-    let repo_root = script_root.join("../../..").canonicalize()?;
+    let repo_root = script_root.join("../..").canonicalize()?;
 
     // find all Cargo.toml files in the repo_root directory
     let exclude_dirs = vec![
@@ -38,13 +38,13 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     for mut toml_file in toml_files {
         let should_add = add_mode && !toml_file.is_publish_disabled;
-
-        update_package_versions(toml_file.document.as_table_mut(), &package_versions, should_add);
+        println!("Processing {}", toml_file.path.display());
+        update_package_versions(toml_file.document.as_table_mut(), &package_versions, None, should_add);
 
         // if the toml file has a workspace table, update the workspace table
         if let Some(workspace) = toml_file.document.get_mut("workspace") {
             if let Some(table) = workspace.as_table_mut() {
-                update_package_versions(table, &package_versions, should_add);
+                update_package_versions(table, &package_versions, Some("workspace"), should_add);
             }
         }
 
@@ -57,6 +57,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 }
 
 fn load_cargo_toml_files(repo_root: &PathBuf, exclude_dirs: &Vec<PathBuf>) -> Result<Vec<TomlInfo>, Box<dyn Error>> {
+    println!("Loading Cargo.toml files in {}", repo_root.display());
     let mut toml_paths = Vec::new();
     find_cargo_toml_files(repo_root, exclude_dirs, &mut toml_paths)?;
 
@@ -109,7 +110,7 @@ fn get_package_versions(toml_files: &Vec<TomlInfo>) -> Vec<(String, String, bool
     package_versions
 }
 
-fn update_package_versions(toml: &mut Table, package_versions: &Vec<(String, String, bool)>, add: bool) {
+fn update_package_versions(toml: &mut Table, package_versions: &Vec<(String, String, bool)>, prefix: Option<&str>, add: bool) {
     // for each dependency table, for each package in package_versions
     // if the package is in the dependency table
     //   if the dependency has both path and version properties, update the version property
@@ -121,16 +122,26 @@ fn update_package_versions(toml: &mut Table, package_versions: &Vec<(String, Str
     let dependency_tables = get_dependency_tables(toml);
 
     for (table_name, table) in dependency_tables {
+        let table_display_name = if prefix.is_some() {
+            format!("{}.{}", prefix.unwrap(), table_name)
+        } else {
+            table_name.clone()
+        };
         for (package, version, is_publish_disabled) in package_versions {
             if let Some(dependency) = table.get_mut(package) {
                 // azure_idenentity will only be a transitive dev-dependency
                 let should_add = add && table_name != "dev-dependencies" && !is_publish_disabled && package != "azure_identity";
 
-                let has_path_property = dependency.get("path").is_some();
-                let has_version_property = dependency.get("version").is_some();
-
-                if has_path_property && (has_version_property || should_add) {
-                    dependency["version"] = value(version);
+                if dependency.get("path").is_some() {
+                    if let Some(current_version) = dependency.get("version") {
+                        if current_version.as_str() != Some(version) {
+                            dependency["version"] = value(version);
+                            println!("  Updating {table_display_name}.{package} to {version}");
+                        }
+                    } else if should_add {
+                        dependency["version"] = value(version);
+                        println!("  Adding {table_display_name}.{package} version {version}");
+                    }
                 }
             }
         }