Auto merge of #149808 - Kobzol:opt-dist-reuse-pgo, r=<try>

rust-bors[bot] · web-flow · commit ccd98352c00c · 2025-12-09T10:57:40.000Z
WIP: Try to reuse PGO profiles in `opt-dist`
diff --git a/Cargo.lock b/Cargo.lock
@@ -266,6 +266,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "basic-toml"
 version = "0.1.10"
@@ -382,6 +388,12 @@ version = "0.6.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 
+[[package]]
+name = "bytes"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
+
 [[package]]
 name = "camino"
 version = "1.2.1"
@@ -1726,6 +1738,22 @@ dependencies = [
  "match_token",
 ]
 
+[[package]]
+name = "http"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
 [[package]]
 name = "humansize"
 version = "2.1.3"
@@ -2762,6 +2790,7 @@ dependencies = [
  "tabled",
  "tar",
  "tempfile",
+ "ureq",
  "xz2",
 ]
 
@@ -3311,6 +3340,20 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.16",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "run_make_support"
 version = "0.0.0"
@@ -4865,7 +4908,7 @@ version = "0.0.0"
 dependencies = [
  "arrayvec",
  "askama",
- "base64",
+ "base64 0.21.7",
  "expect-test",
  "indexmap",
  "itertools",
@@ -4982,6 +5025,41 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.23.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
+dependencies = [
+ "log",
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c"
+dependencies = [
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.22"
@@ -5389,6 +5467,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -6065,6 +6149,41 @@ dependencies = [
  "tidy",
 ]
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "ureq"
+version = "3.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d39cb1dbab692d82a977c0392ffac19e188bd9186a9f32806f0aaa859d75585a"
+dependencies = [
+ "base64 0.22.1",
+ "flate2",
+ "log",
+ "percent-encoding",
+ "rustls",
+ "rustls-pki-types",
+ "ureq-proto",
+ "utf-8",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
+dependencies = [
+ "base64 0.22.1",
+ "http",
+ "httparse",
+ "log",
+]
+
 [[package]]
 name = "url"
 version = "2.5.7"
@@ -6345,6 +6464,15 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e"
+dependencies = [
+ "rustls-pki-types",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -6526,6 +6654,15 @@ dependencies = [
  "windows-link 0.2.1",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.59.0"
@@ -6863,6 +7000,12 @@ dependencies = [
  "synstructure",
 ]
 
+[[package]]
+name = "zeroize"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+
 [[package]]
 name = "zerotrie"
 version = "0.2.3"
diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml
@@ -21,3 +21,4 @@ tempfile = "3.5"
 derive_builder = "0.20"
 clap = { version = "4", features = ["derive"] }
 tabled = { version = "0.15", default-features = false, features = ["std"] }
+ureq = "3.1.4"
diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs
@@ -1,7 +1,12 @@
+#![allow(unused, warnings)]
+
 use anyhow::Context;
+use build_helper::ci::CiEnv;
+use build_helper::git::{GitConfig, get_closest_upstream_commit};
 use camino::{Utf8Path, Utf8PathBuf};
 use clap::Parser;
 use log::LevelFilter;
+use ureq::http::StatusCode;
 use utils::io;
 
 use crate::bolt::{bolt_optimize, with_bolt_instrumented};
@@ -10,11 +15,11 @@ use crate::exec::{Bootstrap, cmd};
 use crate::tests::run_tests;
 use crate::timer::Timer;
 use crate::training::{
-    gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, llvm_benchmarks,
-    rustc_benchmarks,
+    BoltProfile, LlvmPGOProfile, RustcPGOProfile, gather_bolt_profiles, gather_llvm_profiles,
+    gather_rustc_profiles, llvm_benchmarks, rustc_benchmarks,
 };
 use crate::utils::artifact_size::print_binary_sizes;
-use crate::utils::io::{copy_directory, reset_directory};
+use crate::utils::io::{copy_directory, find_file_in_dir, reset_directory};
 use crate::utils::{
     clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group,
     write_timer_to_summary,
@@ -464,15 +469,55 @@ fn main() -> anyhow::Result<()> {
 
     let mut timer = Timer::new();
 
-    let result = execute_pipeline(&env, &mut timer, build_args);
+    let parent_sha = get_closest_upstream_commit(
+        None,
+        &GitConfig { nightly_branch: "main", git_merge_commit_email: "bors@rust-lang.org" },
+        CiEnv::GitHubActions,
+    )
+    .unwrap()
+    .unwrap();
+
+    let url = format!(
+        "https://ci-artifacts.rust-lang.org/rustc-builds/{parent_sha}/reproducible-artifacts-nightly-x86_64-unknown-linux-gnu.tar.xz"
+    );
+    let response = ureq::get(url).call()?;
+    if !response.status().is_success() {
+        return Err(anyhow::anyhow!("Error while downloading PGO: {}", response.status()));
+    }
+    let mut body = response.into_body();
+    let mut archive = tar::Archive::new(xz::read::XzDecoder::new(body.as_reader()));
+
+    let pgo_dir = Utf8Path::new("pgo-profiles");
+    archive.unpack(pgo_dir)?;
+    let dir = find_file_in_dir(pgo_dir, "reproducible-artifacts", "")?;
+    let dir = dir.join("reproducible-artifacts");
+    let llvm_pgo_profile = LlvmPGOProfile(dir.join("llvm-pgo.profdata"));
+    let rustc_pgo_profile = RustcPGOProfile(dir.join("rustc-pgo.profdata"));
+    let bolt_profiles =
+        [BoltProfile(dir.join("LLVM-bolt.profdata")), BoltProfile(dir.join("rustc-bolt.profdata"))];
+
+    let mut dist = Bootstrap::dist(&env, &build_args)
+        .llvm_pgo_optimize(Some(&llvm_pgo_profile))
+        .rustc_pgo_optimize(&rustc_pgo_profile);
+
+    // for bolt_profile in bolt_profiles {
+    //     dist = dist.with_bolt_profile(Some(bolt_profile));
+    // }
+    // dist = dist.with_llvm_bolt_ldflags().with_rustc_bolt_ldflags();
+
+    // Final stage: Assemble the dist artifacts
+    // The previous PGO optimized rustc build and PGO optimized LLVM builds should be reused.
+    timer.section("Stage 5 (final build)", |stage| dist.run(stage))?;
+
+    // let result = execute_pipeline(&env, &mut timer, build_args);
     log::info!("Timer results\n{}", timer.format_stats());
 
     if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") {
         write_timer_to_summary(&summary_path, &timer)?;
     }
 
     print_free_disk_space()?;
-    result.context("Optimized build pipeline has failed")?;
+    // result.context("Optimized build pipeline has failed")?;
     print_binary_sizes(&env)?;
 
     Ok(())
