feat: vendor crates required by opt-dist to collect profiles

weihanglo · weihanglo · commit 51a87fcc3f50 · 2024-05-23T17:10:36.000-04:00
These are the default package set required by opt-dist to correctly work, hence for people wanting to build a production grade of rustc in a sandboxed / air-gapped environment, these need to be vendored. The size of `rustc-src-nightly.tar.xz` before and after this change: * Before: 298M * After: 323M (+8%) These crates are the default set of packages required by opt-dist to correctly work, hence for people wanting to build a production grade of rustc in an sandboxed / air-gapped environment, these need to be vendored. The size of `rustc-src-nightly.tar.xz` before and after this change: * Before: 298M * After: 323M (+8%) Size change might or might not be a concern. See the previous discussion: #125166 (comment) Previous efforts on making: * #125125 * #125166 --- Note that extra works still need to be done to make it fully vendored. * The current pinned rustc-perf uses `tempfile::Tempdir` as the working directory when collecting profiles from some of these packages. This "tmp" working directory usage make it impossible for Cargo to pick up the correct vendor sources setting in `.cargo/config.toml` bundled in the rustc-src tarball. [^1] * opt-dist verifies the final built rustc against a subset of rustc test suite. However it rolls out its own `config.toml` without setting `vendor = true`, and that results in `./vendor/` directory removed. [^2] [^1]: https://github.com/rust-lang/rustc-perf/blob/4f313add609f43e928e98132358e8426ed3969ae/collector/src/compile/benchmark/mod.rs#L164-L173 [^2]: https://github.com/rust-lang/rust/blob/606afbb617a2949a4e35c4b0258ff94c980b9451/src/tools/opt-dist/src/tests.rs#L62-L77
diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
@@ -1041,6 +1041,21 @@ impl Step for PlainSourceTarball {
                 .env("RUSTC_BOOTSTRAP", "1")
                 .current_dir(plain_dst_src);
 
+            // Vendor packages that are required by opt-dist to collect PGO profiles.
+            let pkgs_for_pgo_training = build_helper::metrics::LLVM_PGO_CRATES
+                .iter()
+                .chain(build_helper::metrics::RUSTC_PGO_CRATES)
+                .map(|pkg| {
+                    let mut manifest_path =
+                        builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks");
+                    manifest_path.push(pkg);
+                    manifest_path.push("Cargo.toml");
+                    manifest_path
+                });
+            for manifest_path in pkgs_for_pgo_training {
+                cmd.arg("--sync").arg(manifest_path);
+            }
+
             let config = if !builder.config.dry_run() {
                 t!(String::from_utf8(t!(cmd.output()).stdout))
             } else {
diff --git a/src/tools/build_helper/src/metrics.rs b/src/tools/build_helper/src/metrics.rs
@@ -1,5 +1,28 @@
 use serde_derive::{Deserialize, Serialize};
 
+/// The default set of crates for opt-dist to collect LLVM profiles.
+pub const LLVM_PGO_CRATES: &[&str] = &[
+    "syn-1.0.89",
+    "cargo-0.60.0",
+    "serde-1.0.136",
+    "ripgrep-13.0.0",
+    "regex-1.5.5",
+    "clap-3.1.6",
+    "hyper-0.14.18",
+];
+
+/// The default set of crates for opt-dist to collect rustc profiles.
+pub const RUSTC_PGO_CRATES: &[&str] = &[
+    "externs",
+    "ctfe-stress-5",
+    "cargo-0.60.0",
+    "token-stream-stress",
+    "match-stress",
+    "tuple-stress",
+    "diesel-1.4.8",
+    "bitmaps-3.1.0",
+];
+
 #[derive(Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub struct JsonRoot {
diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs
@@ -3,30 +3,10 @@ use crate::exec::{cmd, CmdBuilder};
 use crate::utils::io::{count_files, delete_directory};
 use crate::utils::with_log_group;
 use anyhow::Context;
+use build_helper::metrics::{LLVM_PGO_CRATES, RUSTC_PGO_CRATES};
 use camino::{Utf8Path, Utf8PathBuf};
 use humansize::BINARY;
 
-const LLVM_PGO_CRATES: &[&str] = &[
-    "syn-1.0.89",
-    "cargo-0.60.0",
-    "serde-1.0.136",
-    "ripgrep-13.0.0",
-    "regex-1.5.5",
-    "clap-3.1.6",
-    "hyper-0.14.18",
-];
-
-const RUSTC_PGO_CRATES: &[&str] = &[
-    "externs",
-    "ctfe-stress-5",
-    "cargo-0.60.0",
-    "token-stream-stress",
-    "match-stress",
-    "tuple-stress",
-    "diesel-1.4.8",
-    "bitmaps-3.1.0",
-];
-
 fn init_compiler_benchmarks(
     env: &Environment,
     profiles: &[&str],
