Skip to content

Commit 47cde5e

Browse files
borsbors
committed
Auto merge of #122709 - onur-ozkan:use-precompiled-rustc-by-default, r=<try>
use precompiled rustc for non-dist builders Makes non-dist builders to use precompiled CI rustc by default if they are available for the target triple. As we are going to make `rust.download-rustc=if-unchanged` default option with #119899, we need to make sure `if-unchanged` logic never breaks and works as expected. As an addition, this will significantly improve the build times on CI when there's no change on the compiler. blocker for #119899 try-job: x86_64-gnu-nopt try-job: aarch64-apple
2 parents 68e4d96 + 4082f9f commit 47cde5e

File tree

11 files changed

+176
-13
lines changed

11 files changed

+176
-13
lines changed

‎src/bootstrap/src/core/builder.rs

+9
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,15 @@ impl StepDescription {
414414
.map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind)))
415415
.collect::<Vec<_>>();
416416

417+
if builder.download_rustc() && (builder.kind == Kind::Dist || builder.kind == Kind::Install)
418+
{
419+
eprintln!(
420+
"ERROR: '{}' subcommand is incompatible with `rust.download-rustc`.",
421+
builder.kind.as_str()
422+
);
423+
crate::exit!(1);
424+
}
425+
417426
// sanity checks on rules
418427
for (desc, should_run) in v.iter().zip(&should_runs) {
419428
assert!(

‎src/bootstrap/src/core/builder/tests.rs

+48
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use std::thread;
22

3+
use build_helper::git::get_closest_merge_commit;
4+
35
use super::*;
46
use crate::Flags;
57
use crate::core::build_steps::doc::DocumentationFormat;
@@ -212,6 +214,52 @@ fn alias_and_path_for_library() {
212214
assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]);
213215
}
214216

217+
#[test]
218+
fn ci_rustc_if_unchanged_logic() {
219+
let config = Config::parse_inner(
220+
Flags::parse(&[
221+
"build".to_owned(),
222+
"--dry-run".to_owned(),
223+
"--set=rust.download-rustc='if-unchanged'".to_owned(),
224+
]),
225+
|&_| Ok(Default::default()),
226+
);
227+
228+
if config.rust_info.is_from_tarball() {
229+
return;
230+
}
231+
232+
let build = Build::new(config.clone());
233+
let builder = Builder::new(&build);
234+
235+
if config.out.exists() {
236+
fs::remove_dir_all(&config.out).unwrap();
237+
}
238+
239+
builder.run_step_descriptions(&Builder::get_step_descriptions(config.cmd.kind()), &[]);
240+
241+
let compiler_path = build.src.join("compiler");
242+
let library_path = build.src.join("compiler");
243+
244+
let commit =
245+
get_closest_merge_commit(Some(&builder.config.src), &builder.config.git_config(), &[
246+
compiler_path.clone(),
247+
library_path.clone(),
248+
])
249+
.unwrap();
250+
251+
let has_changes = !helpers::git(Some(&builder.src))
252+
.args(["diff-index", "--quiet", &commit])
253+
.arg("--")
254+
.args([compiler_path, library_path])
255+
.as_command_mut()
256+
.status()
257+
.unwrap()
258+
.success();
259+
260+
assert!(has_changes == config.download_rustc_commit.is_none());
261+
}
262+
215263
mod defaults {
216264
use pretty_assertions::assert_eq;
217265

‎src/bootstrap/src/core/config/config.rs

+40-11
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use std::str::FromStr;
1313
use std::sync::OnceLock;
1414
use std::{cmp, env, fs};
1515

16+
use build_helper::ci::CiEnv;
1617
use build_helper::exit;
1718
use build_helper::git::{GitConfig, get_closest_merge_commit, output_result};
1819
use serde::{Deserialize, Deserializer};
@@ -22,6 +23,7 @@ use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX;
2223
use crate::core::build_steps::llvm;
2324
pub use crate::core::config::flags::Subcommand;
2425
use crate::core::config::flags::{Color, Flags, Warnings};
26+
use crate::core::download::is_download_ci_available;
2527
use crate::utils::cache::{INTERNER, Interned};
2628
use crate::utils::channel::{self, GitInfo};
2729
use crate::utils::helpers::{self, exe, output, t};
@@ -1627,9 +1629,11 @@ impl Config {
16271629
config.mandir = mandir.map(PathBuf::from);
16281630
}
16291631

1632+
config.llvm_assertions =
1633+
toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false));
1634+
16301635
// Store off these values as options because if they're not provided
16311636
// we'll infer default values for them later
1632-
let mut llvm_assertions = None;
16331637
let mut llvm_tests = None;
16341638
let mut llvm_enzyme = None;
16351639
let mut llvm_plugins = None;
@@ -1712,7 +1716,8 @@ impl Config {
17121716
is_user_configured_rust_channel = channel.is_some();
17131717
set(&mut config.channel, channel.clone());
17141718

1715-
config.download_rustc_commit = config.download_ci_rustc_commit(download_rustc);
1719+
config.download_rustc_commit =
1720+
config.download_ci_rustc_commit(download_rustc, config.llvm_assertions);
17161721

17171722
debug = debug_toml;
17181723
debug_assertions = debug_assertions_toml;
@@ -1848,7 +1853,7 @@ impl Config {
18481853
optimize: optimize_toml,
18491854
thin_lto,
18501855
release_debuginfo,
1851-
assertions,
1856+
assertions: _,
18521857
tests,
18531858
enzyme,
18541859
plugins,
@@ -1882,7 +1887,6 @@ impl Config {
18821887
Some(StringOrBool::Bool(false)) | None => {}
18831888
}
18841889
set(&mut config.ninja_in_file, ninja);
1885-
llvm_assertions = assertions;
18861890
llvm_tests = tests;
18871891
llvm_enzyme = enzyme;
18881892
llvm_plugins = plugins;
@@ -1911,8 +1915,8 @@ impl Config {
19111915
config.llvm_enable_warnings = enable_warnings.unwrap_or(false);
19121916
config.llvm_build_config = build_config.clone().unwrap_or(Default::default());
19131917

1914-
let asserts = llvm_assertions.unwrap_or(false);
1915-
config.llvm_from_ci = config.parse_download_ci_llvm(download_ci_llvm, asserts);
1918+
config.llvm_from_ci =
1919+
config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions);
19161920

19171921
if config.llvm_from_ci {
19181922
let warn = |option: &str| {
@@ -2080,7 +2084,6 @@ impl Config {
20802084
// Now that we've reached the end of our configuration, infer the
20812085
// default values for all options that we haven't otherwise stored yet.
20822086

2083-
config.llvm_assertions = llvm_assertions.unwrap_or(false);
20842087
config.llvm_tests = llvm_tests.unwrap_or(false);
20852088
config.llvm_enzyme = llvm_enzyme.unwrap_or(false);
20862089
config.llvm_plugins = llvm_plugins.unwrap_or(false);
@@ -2419,8 +2422,9 @@ impl Config {
24192422
ci_config_toml,
24202423
);
24212424

2422-
let disable_ci_rustc_if_incompatible =
2423-
env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
2425+
// Primarily used by CI runners to avoid handling download-rustc incompatible
2426+
// options one by one on shell scripts.
2427+
let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
24242428
.is_some_and(|s| s == "1" || s == "true");
24252429

24262430
if disable_ci_rustc_if_incompatible && res.is_err() {
@@ -2711,7 +2715,15 @@ impl Config {
27112715
}
27122716

27132717
/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
2714-
fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> {
2718+
fn download_ci_rustc_commit(
2719+
&self,
2720+
download_rustc: Option<StringOrBool>,
2721+
llvm_assertions: bool,
2722+
) -> Option<String> {
2723+
if !is_download_ci_available(&self.build.triple, llvm_assertions) {
2724+
return None;
2725+
}
2726+
27152727
// If `download-rustc` is not set, default to rebuilding.
27162728
let if_unchanged = match download_rustc {
27172729
None | Some(StringOrBool::Bool(false)) => return None,
@@ -2724,7 +2736,11 @@ impl Config {
27242736

27252737
// Look for a version to compare to based on the current commit.
27262738
// Only commits merged by bors will have CI artifacts.
2727-
let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap();
2739+
let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[
2740+
self.src.join("compiler"),
2741+
self.src.join("library"),
2742+
])
2743+
.unwrap();
27282744
if commit.is_empty() {
27292745
println!("ERROR: could not find commit hash for downloading rustc");
27302746
println!("HELP: maybe your repository history is too shallow?");
@@ -2733,6 +2749,19 @@ impl Config {
27332749
crate::exit!(1);
27342750
}
27352751

2752+
if CiEnv::is_ci() && {
2753+
let head_sha =
2754+
output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut());
2755+
let head_sha = head_sha.trim();
2756+
commit == head_sha
2757+
} {
2758+
eprintln!("CI rustc commit matches with HEAD and we are in CI.");
2759+
eprintln!(
2760+
"`rustc.download-ci` functionality will be skipped as artifacts are not available."
2761+
);
2762+
return None;
2763+
}
2764+
27362765
// Warn if there were changes to the compiler or standard library since the ancestor commit.
27372766
let has_changes = !t!(helpers::git(Some(&self.src))
27382767
.args(["diff-index", "--quiet", &commit])

‎src/bootstrap/src/core/config/tests.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use super::{ChangeIdWrapper, Config};
1212
use crate::core::build_steps::clippy::get_clippy_rules_in_order;
1313
use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig};
1414

15-
fn parse(config: &str) -> Config {
15+
pub(crate) fn parse(config: &str) -> Config {
1616
Config::parse_inner(
1717
Flags::parse(&["check".to_string(), "--config=/does/not/exist".to_string()]),
1818
|&_| toml::from_str(&config),

‎src/bootstrap/src/core/download.rs

+40
Original file line numberDiff line numberDiff line change
@@ -832,3 +832,43 @@ fn path_is_dylib(path: &Path) -> bool {
832832
// The .so is not necessarily the extension, it might be libLLVM.so.18.1
833833
path.to_str().map_or(false, |path| path.contains(".so"))
834834
}
835+
836+
/// Checks whether the CI rustc is available for the given target triple.
837+
pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
838+
// All tier 1 targets and tier 2 targets with host tools.
839+
const SUPPORTED_PLATFORMS: &[&str] = &[
840+
"aarch64-apple-darwin",
841+
"aarch64-pc-windows-msvc",
842+
"aarch64-unknown-linux-gnu",
843+
"aarch64-unknown-linux-musl",
844+
"arm-unknown-linux-gnueabi",
845+
"arm-unknown-linux-gnueabihf",
846+
"armv7-unknown-linux-gnueabihf",
847+
"i686-pc-windows-gnu",
848+
"i686-pc-windows-msvc",
849+
"i686-unknown-linux-gnu",
850+
"loongarch64-unknown-linux-gnu",
851+
"powerpc-unknown-linux-gnu",
852+
"powerpc64-unknown-linux-gnu",
853+
"powerpc64le-unknown-linux-gnu",
854+
"riscv64gc-unknown-linux-gnu",
855+
"s390x-unknown-linux-gnu",
856+
"x86_64-apple-darwin",
857+
"x86_64-pc-windows-gnu",
858+
"x86_64-pc-windows-msvc",
859+
"x86_64-unknown-freebsd",
860+
"x86_64-unknown-illumos",
861+
"x86_64-unknown-linux-gnu",
862+
"x86_64-unknown-linux-musl",
863+
"x86_64-unknown-netbsd",
864+
];
865+
866+
const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
867+
&["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];
868+
869+
if llvm_assertions {
870+
SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
871+
} else {
872+
SUPPORTED_PLATFORMS.contains(&target_triple)
873+
}
874+
}

‎src/ci/docker/host-x86_64/mingw-check/Dockerfile

+14-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,20 @@ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
4646
# Check library crates on all tier 1 targets.
4747
# We disable optimized compiler built-ins because that requires a C toolchain for the target.
4848
# We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
49-
ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
49+
ENV SCRIPT \
50+
# `core::builder::tests::ci_rustc_if_unchanged_logic` bootstrap test covers the `rust.download-rustc=if-unchanged` logic.
51+
# Here we are adding a dummy commit on compiler and running that test to ensure when there is a change on the compiler,
52+
# we never download ci rustc with `rust.download-rustc=if-unchanged` option.
53+
echo \"\" >> ../compiler/rustc/src/main.rs && \
54+
git config --global user.email \"[email protected]\" && \
55+
git config --global user.name \"dummy\" && \
56+
git add ../compiler/rustc/src/main.rs && \
57+
git commit -m \"test commit for rust.download-rustc=if-unchanged logic\" && \
58+
DISABLE_CI_RUSTC_IF_INCOMPATIBLE=0 python3 ../x.py test bootstrap -- core::builder::tests::ci_rustc_if_unchanged_logic && \
59+
# Revert the dummy commit
60+
git reset --hard HEAD~1 && \
61+
62+
python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
5063
/scripts/check-default-config-profiles.sh && \
5164
python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
5265
python3 ../x.py clippy bootstrap -Dwarnings && \

‎src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile

+4
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ RUN mkdir -p $RUST_INSTALL_DIR/etc
5858
# Fuchsia only supports LLVM.
5959
ENV CODEGEN_BACKENDS llvm
6060

61+
# download-rustc is not allowed for `x install`
62+
ENV NO_DOWNLOAD_CI_RUSTC 1
63+
6164
ENV RUST_CONFIGURE_ARGS \
6265
--prefix=$RUST_INSTALL_DIR \
6366
--sysconfdir=etc \
@@ -70,6 +73,7 @@ ENV RUST_CONFIGURE_ARGS \
7073
--set target.x86_64-unknown-fuchsia.ar=/usr/local/bin/llvm-ar \
7174
--set target.x86_64-unknown-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
7275
--set target.x86_64-unknown-fuchsia.linker=/usr/local/bin/ld.lld
76+
7377
ENV SCRIPT \
7478
python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
7579
bash ../src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh

‎src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ ENV RUST_CONFIGURE_ARGS \
8484
--enable-new-symbol-mangling
8585

8686
ENV HOST_TARGET x86_64-unknown-linux-gnu
87+
ENV FORCE_CI_RUSTC 1
8788

8889
COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
8990
COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/

‎src/ci/docker/run.sh

+1
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,7 @@ docker \
343343
--env PR_CI_JOB \
344344
--env OBJDIR_ON_HOST="$objdir" \
345345
--env CODEGEN_BACKENDS \
346+
--env DISABLE_CI_RUSTC_IF_INCOMPATIBLE="$DISABLE_CI_RUSTC_IF_INCOMPATIBLE" \
346347
--init \
347348
--rm \
348349
rust-ci \

‎src/ci/github-actions/jobs.yml

+5
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ envs:
8585
# it in each job definition.
8686
pr:
8787
- image: mingw-check
88+
env:
89+
# We are adding (temporarily) a dummy commit on the compiler
90+
READ_ONLY_SRC: "0"
8891
<<: *job-linux-4c
8992
- image: mingw-check-tidy
9093
continue_on_error: true
@@ -207,6 +210,8 @@ auto:
207210
<<: *job-linux-8c
208211

209212
- image: mingw-check
213+
env:
214+
READ_ONLY_SRC: 0
210215
<<: *job-linux-4c
211216

212217
- image: test-various

‎src/ci/run.sh

+13
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,13 @@ if [ "$CI" != "" ]; then
5252
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set change-id=99999999"
5353
fi
5454

55+
# If runner uses an incompatible option and `FORCE_CI_RUSTC` is not defined,
56+
# switch to in-tree rustc.
57+
if [ "$FORCE_CI_RUSTC" == "" ]; then
58+
echo "debug: `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` configured."
59+
DISABLE_CI_RUSTC_IF_INCOMPATIBLE=1
60+
fi
61+
5562
if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \
5663
isCiBranch automation/bors/try; then
5764
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
@@ -169,10 +176,16 @@ else
169176
if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
170177
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-unchanged"
171178
else
179+
# CI rustc requires CI LLVM to be enabled (see https://github.com/rust-lang/rust/issues/123586).
180+
NO_DOWNLOAD_CI_RUSTC=1
172181
# When building for CI we want to use the static C++ Standard library
173182
# included with LLVM, since a dynamic libstdcpp may not be available.
174183
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.static-libstdcpp"
175184
fi
185+
186+
if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then
187+
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
188+
fi
176189
fi
177190

178191
if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then

0 commit comments

Comments
 (0)