Skip to content

Commit 02538d4

Browse files
alexcrichtonalexcrichton
committed
mk: Bootstrap from stable instead of snapshots
This commit removes all infrastructure from the repository for our so-called snapshots to instead bootstrap the compiler from stable releases. Bootstrapping from a previously stable release is a long-desired feature of distros because they're not fans of downloading binary stage0 blobs from us. Additionally, this makes our own CI easier as we can decommission all of the snapshot builders and start having a regular cadence to when we update the stage0 compiler. A new `src/etc/get-stage0.py` script was added which shares some code with `src/bootstrap/bootstrap.py` to read a new file, `src/stage0.txt`, which lists the current stage0 compiler as well as cargo that we bootstrap from. This script will download the relevant `rustc` package an unpack it into `$target/stage0` as we do today. One problem of bootstrapping from stable releases is that we're not able to compile unstable code (e.g. all the `#![feature]` directives in libcore/libstd). To overcome this we employ two strategies: * The bootstrap key of the previous compiler is hardcoded into `src/stage0.txt` (enabled as a result of #32731) and exported by the build system. This enables nightly features in the compiler we download. * The standard library and compiler are pinned to a specific stage0, which doesn't change, so we're guaranteed that we'll continue compiling as we start from a known fixed source. The process for making a release will also need to be tweaked now to continue to cadence of bootstrapping from the previous release. This process looks like: 1. Merge `beta` to `stable` 2. Produce a new stable compiler. 3. Change `master` to bootstrap from this new stable compiler. 4. Merge `master` to `beta` 5. Produce a new beta compiler 6. Change `master` to bootstrap from this new beta compiler. Step 3 above should involve very few changes as `master` was previously bootstrapping from `beta` which is the same as `stable` at that point in time. Step 6, however, is where we benefit from removing lots of `#[cfg(stage0)]` and get to use new features. This also shouldn't slow the release too much as steps 1-5 requires little work other than waiting and step 6 just needs to happen at some point during a release cycle, it's not time sensitive. Closes #29555 Closes #29557
1 parent 478a33d commit 02538d4

23 files changed

+200
-2999
lines changed

‎Makefile.in

-7
Original file line numberDiff line numberDiff line change
@@ -214,13 +214,6 @@ include $(CFG_SRC_DIR)mk/debuggers.mk
214214
# Secondary makefiles, conditionalized for speed
215215
######################################################################
216216

217-
# Binary snapshots
218-
ifneq ($(strip $(findstring snap,$(MAKECMDGOALS)) \
219-
$(findstring clean,$(MAKECMDGOALS))),)
220-
CFG_INFO := $(info cfg: including snap rules)
221-
include $(CFG_SRC_DIR)mk/snap.mk
222-
endif
223-
224217
# The test suite
225218
ifneq ($(strip $(findstring check,$(MAKECMDGOALS)) \
226219
$(findstring test,$(MAKECMDGOALS)) \

‎mk/dist.mk

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ PKG_FILES := \
6161
rtstartup \
6262
rustllvm \
6363
rustc \
64-
snapshots.txt \
64+
stage0.txt \
6565
rust-installer \
6666
tools \
6767
test) \

‎mk/main.mk

+1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ CFG_FILENAME_EXTRA=$(shell printf '%s' $(CFG_RELEASE)$(CFG_EXTRA_FILENAME) | $(C
3434
# intentionally not "secure" by any definition, this is largely just a deterrent
3535
# from users enabling unstable features on the stable compiler.
3636
CFG_BOOTSTRAP_KEY=$(CFG_FILENAME_EXTRA)
37+
CFG_BOOTSTRAP_KEY_STAGE0=$(shell grep 'rustc_key' $(S)src/stage0.txt | sed 's/rustc_key: '//)
3738

3839
ifeq ($(CFG_RELEASE_CHANNEL),stable)
3940
# This is the normal semver version string, e.g. "0.12.0", "0.12.0-nightly"

‎mk/reconfig.mk

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,6 @@ else
3838
SREL_ROOT := $(SREL)
3939
endif
4040

41-
config.stamp: $(S)configure $(S)Makefile.in $(S)src/snapshots.txt
41+
config.stamp: $(S)configure $(S)Makefile.in $(S)src/stage0.txt
4242
@$(call E, cfg: reconfiguring)
4343
$(SREL_ROOT)configure $(CFG_CONFIGURE_ARGS)

‎mk/snap.mk

-28
This file was deleted.

‎mk/stage0.mk

+3-6
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,14 @@ $(HLIB0_H_$(CFG_BUILD))/:
1010
endif
1111

1212
$(SNAPSHOT_RUSTC_POST_CLEANUP): \
13-
$(S)src/snapshots.txt \
14-
$(S)src/etc/get-snapshot.py $(MKFILE_DEPS) \
13+
$(S)src/stage0.txt \
14+
$(S)src/etc/get-stage0.py $(MKFILE_DEPS) \
1515
| $(HBIN0_H_$(CFG_BUILD))/
16-
1716
@$(call E, fetch: $@)
18-
# Note: the variable "SNAPSHOT_FILE" is generally not set, and so
19-
# we generally only pass one argument to this script.
2017
ifdef CFG_ENABLE_LOCAL_RUST
2118
$(Q)$(S)src/etc/local_stage0.sh $(CFG_BUILD) $(CFG_LOCAL_RUST_ROOT) rustlib
2219
else
23-
$(Q)$(CFG_PYTHON) $(S)src/etc/get-snapshot.py $(CFG_BUILD) $(SNAPSHOT_FILE)
20+
$(Q)$(CFG_PYTHON) $(S)src/etc/get-stage0.py $(CFG_BUILD)
2421
endif
2522
$(Q)if [ -e "$@" ]; then touch "$@"; else echo "ERROR: snapshot $@ not found"; exit 1; fi
2623

‎mk/target.mk

+16-2
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ $(foreach host,$(CFG_HOST), \
6565
# $(4) is the crate name
6666
define RUST_TARGET_STAGE_N
6767

68+
ifeq ($(1),0)
69+
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
70+
export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
71+
endif
72+
6873
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): CFG_COMPILER_HOST_TRIPLE = $(2)
6974
$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
7075
$$(CRATEFILE_$(4)) \
@@ -113,6 +118,11 @@ endef
113118
# $(4) - name of the tool being built
114119
define TARGET_TOOL
115120

121+
ifeq ($(1),0)
122+
$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
123+
export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
124+
endif
125+
116126
$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
117127
$$(TOOL_SOURCE_$(4)) \
118128
$$(TOOL_INPUTS_$(4)) \
@@ -167,11 +177,15 @@ SNAPSHOT_RUSTC_POST_CLEANUP=$(HBIN0_H_$(CFG_BUILD))/rustc$(X_$(CFG_BUILD))
167177

168178
define TARGET_HOST_RULES
169179

170-
$$(TLIB$(1)_T_$(2)_H_$(3))/:
180+
$$(TLIB$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
181+
mkdir -p $$@
182+
183+
$$(TBIN$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
171184
mkdir -p $$@
172185

173186
$$(TLIB$(1)_T_$(2)_H_$(3))/%: $$(RT_OUTPUT_DIR_$(2))/% \
174-
| $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP)
187+
$$(SNAPSHOT_RUSTC_POST_CLEANUP) \
188+
| $$(TLIB$(1)_T_$(2)_H_$(3))/
175189
@$$(call E, cp: $$@)
176190
$$(Q)cp $$< $$@
177191
endef

‎mk/tests.mk

+5-2
Original file line numberDiff line numberDiff line change
@@ -241,13 +241,16 @@ cleantestlibs:
241241
######################################################################
242242

243243
.PHONY: tidy
244-
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD))
244+
tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
245+
$(SNAPSHOT_RUSTC_POST_CLEANUP)
245246
$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src
246247

247248
$(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \
248249
$(TSREQ0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) \
249250
$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))/stamp.std \
250-
$(call rwildcard,$(S)src/tools/tidy/src,*.rs)
251+
$(call rwildcard,$(S)src/tools/tidy/src,*.rs) \
252+
$(SNAPSHOT_RUSTC_POST_CLEANUP) | \
253+
$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))
251254
$(STAGE0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $(S)src/tools/tidy/src/main.rs \
252255
--out-dir $(@D) --crate-name tidy
253256

‎src/bootstrap/bootstrap.py

+84-67
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,22 @@ def run(args, verbose=False):
7979
raise RuntimeError(err)
8080
sys.exit(err)
8181

82+
def stage0_data(rust_root):
83+
nightlies = os.path.join(rust_root, "src/stage0.txt")
84+
with open(nightlies, 'r') as nightlies:
85+
data = {}
86+
for line in nightlies.read().split("\n"):
87+
if line.startswith("#") or line == '':
88+
continue
89+
a, b = line.split(": ", 1)
90+
data[a] = b
91+
return data
92+
8293
class RustBuild:
83-
def download_rust_nightly(self):
94+
def download_stage0(self):
8495
cache_dst = os.path.join(self.build_dir, "cache")
85-
rustc_cache = os.path.join(cache_dst, self.snap_rustc_date())
86-
cargo_cache = os.path.join(cache_dst, self.snap_cargo_date())
96+
rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
97+
cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
8798
if not os.path.exists(rustc_cache):
8899
os.makedirs(rustc_cache)
89100
if not os.path.exists(cargo_cache):
@@ -93,41 +104,49 @@ def download_rust_nightly(self):
93104
(not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
94105
if os.path.exists(self.bin_root()):
95106
shutil.rmtree(self.bin_root())
96-
filename = "rust-std-nightly-" + self.build + ".tar.gz"
97-
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
107+
channel = self.stage0_rustc_channel()
108+
filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
109+
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
98110
tarball = os.path.join(rustc_cache, filename)
99111
if not os.path.exists(tarball):
100112
get(url + "/" + filename, tarball, verbose=self.verbose)
101113
unpack(tarball, self.bin_root(),
102114
match="rust-std-" + self.build,
103115
verbose=self.verbose)
104116

105-
filename = "rustc-nightly-" + self.build + ".tar.gz"
106-
url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
117+
filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
118+
url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
107119
tarball = os.path.join(rustc_cache, filename)
108120
if not os.path.exists(tarball):
109121
get(url + "/" + filename, tarball, verbose=self.verbose)
110122
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
111123
with open(self.rustc_stamp(), 'w') as f:
112-
f.write(self.snap_rustc_date())
124+
f.write(self.stage0_rustc_date())
113125

114126
if self.cargo().startswith(self.bin_root()) and \
115127
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
116-
filename = "cargo-nightly-" + self.build + ".tar.gz"
117-
url = "https://static.rust-lang.org/cargo-dist/" + self.snap_cargo_date()
128+
channel = self.stage0_cargo_channel()
129+
filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
130+
url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
118131
tarball = os.path.join(cargo_cache, filename)
119132
if not os.path.exists(tarball):
120133
get(url + "/" + filename, tarball, verbose=self.verbose)
121134
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
122135
with open(self.cargo_stamp(), 'w') as f:
123-
f.write(self.snap_cargo_date())
136+
f.write(self.stage0_cargo_date())
124137

125-
def snap_cargo_date(self):
138+
def stage0_cargo_date(self):
126139
return self._cargo_date
127140

128-
def snap_rustc_date(self):
141+
def stage0_cargo_channel(self):
142+
return self._cargo_channel
143+
144+
def stage0_rustc_date(self):
129145
return self._rustc_date
130146

147+
def stage0_rustc_channel(self):
148+
return self._rustc_channel
149+
131150
def rustc_stamp(self):
132151
return os.path.join(self.bin_root(), '.rustc-stamp')
133152

@@ -138,13 +157,13 @@ def rustc_out_of_date(self):
138157
if not os.path.exists(self.rustc_stamp()):
139158
return True
140159
with open(self.rustc_stamp(), 'r') as f:
141-
return self.snap_rustc_date() != f.read()
160+
return self.stage0_rustc_date() != f.read()
142161

143162
def cargo_out_of_date(self):
144163
if not os.path.exists(self.cargo_stamp()):
145164
return True
146165
with open(self.cargo_stamp(), 'r') as f:
147-
return self.snap_cargo_date() != f.read()
166+
return self.stage0_cargo_date() != f.read()
148167

149168
def bin_root(self):
150169
return os.path.join(self.build_dir, self.build, "stage0")
@@ -187,15 +206,6 @@ def exe_suffix(self):
187206
else:
188207
return ''
189208

190-
def parse_nightly_dates(self):
191-
nightlies = os.path.join(self.rust_root, "src/nightlies.txt")
192-
with open(nightlies, 'r') as nightlies:
193-
rustc, cargo = nightlies.read().split("\n")[:2]
194-
assert rustc.startswith("rustc: ")
195-
assert cargo.startswith("cargo: ")
196-
self._rustc_date = rustc[len("rustc: "):]
197-
self._cargo_date = cargo[len("cargo: "):]
198-
199209
def build_bootstrap(self):
200210
env = os.environ.copy()
201211
env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
@@ -300,46 +310,53 @@ def build_triple(self):
300310

301311
return cputype + '-' + ostype
302312

303-
parser = argparse.ArgumentParser(description='Build rust')
304-
parser.add_argument('--config')
305-
parser.add_argument('-v', '--verbose', action='store_true')
306-
307-
args = [a for a in sys.argv if a != '-h']
308-
args, _ = parser.parse_known_args(args)
309-
310-
# Configure initial bootstrap
311-
rb = RustBuild()
312-
rb.config_toml = ''
313-
rb.config_mk = ''
314-
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
315-
rb.build_dir = os.path.join(os.getcwd(), "build")
316-
rb.verbose = args.verbose
317-
318-
try:
319-
with open(args.config or 'config.toml') as config:
320-
rb.config_toml = config.read()
321-
except:
322-
pass
323-
try:
324-
rb.config_mk = open('config.mk').read()
325-
except:
326-
pass
327-
328-
# Fetch/build the bootstrap
329-
rb.build = rb.build_triple()
330-
rb.parse_nightly_dates()
331-
rb.download_rust_nightly()
332-
sys.stdout.flush()
333-
rb.build_bootstrap()
334-
sys.stdout.flush()
335-
336-
# Run the bootstrap
337-
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
338-
args.append('--src')
339-
args.append(rb.rust_root)
340-
args.append('--build')
341-
args.append(rb.build)
342-
args.extend(sys.argv[1:])
343-
env = os.environ.copy()
344-
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
345-
rb.run(args, env)
313+
def main():
314+
parser = argparse.ArgumentParser(description='Build rust')
315+
parser.add_argument('--config')
316+
parser.add_argument('-v', '--verbose', action='store_true')
317+
318+
args = [a for a in sys.argv if a != '-h']
319+
args, _ = parser.parse_known_args(args)
320+
321+
# Configure initial bootstrap
322+
rb = RustBuild()
323+
rb.config_toml = ''
324+
rb.config_mk = ''
325+
rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
326+
rb.build_dir = os.path.join(os.getcwd(), "build")
327+
rb.verbose = args.verbose
328+
329+
try:
330+
with open(args.config or 'config.toml') as config:
331+
rb.config_toml = config.read()
332+
except:
333+
pass
334+
try:
335+
rb.config_mk = open('config.mk').read()
336+
except:
337+
pass
338+
339+
data = stage0_data(rb.rust_root)
340+
rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
341+
rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
342+
343+
# Fetch/build the bootstrap
344+
rb.build = rb.build_triple()
345+
rb.download_stage0()
346+
sys.stdout.flush()
347+
rb.build_bootstrap()
348+
sys.stdout.flush()
349+
350+
# Run the bootstrap
351+
args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
352+
args.append('--src')
353+
args.append(rb.rust_root)
354+
args.append('--build')
355+
args.append(rb.build)
356+
args.extend(sys.argv[1:])
357+
env = os.environ.copy()
358+
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
359+
rb.run(args, env)
360+
361+
if __name__ == '__main__':
362+
main()

‎src/bootstrap/build/channel.rs

+8
Original file line numberDiff line numberDiff line change
@@ -84,4 +84,12 @@ pub fn collect(build: &mut Build) {
8484
build.bootstrap_key = format!("{:02x}{:02x}{:02x}{:02x}",
8585
key[0], key[1], key[2], key[3]);
8686
env::set_var("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key);
87+
88+
let mut s = String::new();
89+
t!(t!(File::open(build.src.join("src/stage0.txt"))).read_to_string(&mut s));
90+
if let Some(line) = s.lines().find(|l| l.starts_with("rustc_key")) {
91+
if let Some(key) = line.split(": ").nth(1) {
92+
build.bootstrap_key_stage0 = key.to_string();
93+
}
94+
}
8795
}

‎src/bootstrap/build/compile.rs

-1
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,6 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
179179
.env("CFG_VERSION", &build.version)
180180
.env("CFG_BOOTSTRAP_KEY", &build.bootstrap_key)
181181
.env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(String::new()))
182-
.env("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key)
183182
.env("CFG_LIBDIR_RELATIVE", "lib");
184183

185184
if let Some(ref ver_date) = build.ver_date {

0 commit comments

Comments
 (0)