Skip to content

Commit ab45885

Browse files
borsbors
committed
Auto merge of #114843 - Zalathar:test-coverage-map, r=oli-obk
coverage: Explicitly test the coverage maps produced by codegen/LLVM Our existing coverage tests verify the output of end-to-end coverage reports, but we don't have any way to test the specific mapping information (code regions and their associated counters) that are emitted by `rustc_codegen_llvm` and LLVM. That makes it harder to to be confident in changes that would modify those mappings (whether deliberately or accidentally). This PR addresses that by adding a new `coverage-map` test suite that does the following: - Compiles test files to LLVM IR assembly (`.ll`) - Feeds those IR files to a custom tool (`src/tools/coverage-dump`) that extracts and decodes coverage mappings, and prints them in a more human-readable format - Checks the output of that tool against known-good snapshots --- I recommend excluding the last commit while reviewing the main changes, because that last commit is just ~40 test files copied over from `tests/run-coverage`, plus their blessed coverage-map snapshots and a readme file. Those snapshots aren't really intended to be checked by hand; they're mostly there to increase the chances that an unintended change to coverage maps will be observable (even if it requires relatively specific circumstances to manifest).
2 parents f222a2d + 3141177 commit ab45885

File tree

102 files changed

+6395
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+6395
-6
lines changed

‎Cargo.lock

+18
Original file line numberDiff line numberDiff line change
@@ -722,6 +722,18 @@ version = "0.8.4"
722722
source = "registry+https://github.com/rust-lang/crates.io-index"
723723
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
724724

725+
[[package]]
726+
name = "coverage-dump"
727+
version = "0.1.0"
728+
dependencies = [
729+
"anyhow",
730+
"leb128",
731+
"md-5",
732+
"miniz_oxide",
733+
"regex",
734+
"rustc-demangle",
735+
]
736+
725737
[[package]]
726738
name = "coverage_test_macros"
727739
version = "0.0.0"
@@ -2041,6 +2053,12 @@ version = "1.3.0"
20412053
source = "registry+https://github.com/rust-lang/crates.io-index"
20422054
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
20432055

2056+
[[package]]
2057+
name = "leb128"
2058+
version = "0.2.5"
2059+
source = "registry+https://github.com/rust-lang/crates.io-index"
2060+
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
2061+
20442062
[[package]]
20452063
name = "levenshtein"
20462064
version = "1.0.5"

‎Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ members = [
4343
"src/tools/generate-windows-sys",
4444
"src/tools/rustdoc-gui-test",
4545
"src/tools/opt-dist",
46+
"src/tools/coverage-dump",
4647
]
4748

4849
exclude = [

‎src/bootstrap/builder.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -703,7 +703,8 @@ impl<'a> Builder<'a> {
703703
llvm::Lld,
704704
llvm::CrtBeginEnd,
705705
tool::RustdocGUITest,
706-
tool::OptimizedDist
706+
tool::OptimizedDist,
707+
tool::CoverageDump,
707708
),
708709
Kind::Check | Kind::Clippy | Kind::Fix => describe!(
709710
check::Std,
@@ -725,6 +726,7 @@ impl<'a> Builder<'a> {
725726
test::Tidy,
726727
test::Ui,
727728
test::RunPassValgrind,
729+
test::CoverageMap,
728730
test::RunCoverage,
729731
test::MirOpt,
730732
test::Codegen,

‎src/bootstrap/test.rs

+14
Original file line numberDiff line numberDiff line change
@@ -1340,6 +1340,12 @@ host_test!(RunMakeFullDeps {
13401340

13411341
default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" });
13421342

1343+
default_test!(CoverageMap {
1344+
path: "tests/coverage-map",
1345+
mode: "coverage-map",
1346+
suite: "coverage-map"
1347+
});
1348+
13431349
host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" });
13441350
host_test!(RunCoverageRustdoc {
13451351
path: "tests/run-coverage-rustdoc",
@@ -1545,6 +1551,14 @@ note: if you're sure you want to do this, please open an issue as to why. In the
15451551
.arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
15461552
}
15471553

1554+
if mode == "coverage-map" {
1555+
let coverage_dump = builder.ensure(tool::CoverageDump {
1556+
compiler: compiler.with_stage(0),
1557+
target: compiler.host,
1558+
});
1559+
cmd.arg("--coverage-dump-path").arg(coverage_dump);
1560+
}
1561+
15481562
if mode == "run-make" || mode == "run-coverage" {
15491563
let rust_demangler = builder
15501564
.ensure(tool::RustDemangler {

‎src/bootstrap/tool.rs

+1
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ bootstrap_tool!(
306306
GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
307307
RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
308308
OptimizedDist, "src/tools/opt-dist", "opt-dist";
309+
CoverageDump, "src/tools/coverage-dump", "coverage-dump";
309310
);
310311

311312
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]

‎src/tools/compiletest/src/common.rs

+6
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ string_enum! {
6666
JsDocTest => "js-doc-test",
6767
MirOpt => "mir-opt",
6868
Assembly => "assembly",
69+
CoverageMap => "coverage-map",
6970
RunCoverage => "run-coverage",
7071
}
7172
}
@@ -161,6 +162,9 @@ pub struct Config {
161162
/// The rust-demangler executable.
162163
pub rust_demangler_path: Option<PathBuf>,
163164

165+
/// The coverage-dump executable.
166+
pub coverage_dump_path: Option<PathBuf>,
167+
164168
/// The Python executable to use for LLDB and htmldocck.
165169
pub python: String,
166170

@@ -639,6 +643,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
639643
UI_STDERR_32,
640644
UI_STDERR_16,
641645
UI_COVERAGE,
646+
UI_COVERAGE_MAP,
642647
];
643648
pub const UI_STDERR: &str = "stderr";
644649
pub const UI_STDOUT: &str = "stdout";
@@ -649,6 +654,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
649654
pub const UI_STDERR_32: &str = "32bit.stderr";
650655
pub const UI_STDERR_16: &str = "16bit.stderr";
651656
pub const UI_COVERAGE: &str = "coverage";
657+
pub const UI_COVERAGE_MAP: &str = "cov-map";
652658

653659
/// Absolute path to the directory where all output for all tests in the given
654660
/// `relative_dir` group should reside. Example:

‎src/tools/compiletest/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
4848
.reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
4949
.optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
5050
.optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
51+
.optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
5152
.reqopt("", "python", "path to python to use for doc tests", "PATH")
5253
.optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
5354
.optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
@@ -218,6 +219,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
218219
rustc_path: opt_path(matches, "rustc-path"),
219220
rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
220221
rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
222+
coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
221223
python: matches.opt_str("python").unwrap(),
222224
jsondocck_path: matches.opt_str("jsondocck-path"),
223225
jsondoclint_path: matches.opt_str("jsondoclint-path"),

‎src/tools/compiletest/src/runtest.rs

+66-5
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs
66
use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
77
use crate::common::{CompareMode, FailMode, PassMode};
88
use crate::common::{Config, TestPaths};
9-
use crate::common::{Pretty, RunCoverage, RunPassValgrind};
10-
use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT};
9+
use crate::common::{CoverageMap, Pretty, RunCoverage, RunPassValgrind};
10+
use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
1111
use crate::compute_diff::{write_diff, write_filtered_diff};
1212
use crate::errors::{self, Error, ErrorKind};
1313
use crate::header::TestProps;
@@ -254,6 +254,7 @@ impl<'test> TestCx<'test> {
254254
MirOpt => self.run_mir_opt_test(),
255255
Assembly => self.run_assembly_test(),
256256
JsDocTest => self.run_js_doc_test(),
257+
CoverageMap => self.run_coverage_map_test(),
257258
RunCoverage => self.run_coverage_test(),
258259
}
259260
}
@@ -467,6 +468,46 @@ impl<'test> TestCx<'test> {
467468
}
468469
}
469470

471+
fn run_coverage_map_test(&self) {
472+
let Some(coverage_dump_path) = &self.config.coverage_dump_path else {
473+
self.fatal("missing --coverage-dump");
474+
};
475+
476+
let proc_res = self.compile_test_and_save_ir();
477+
if !proc_res.status.success() {
478+
self.fatal_proc_rec("compilation failed!", &proc_res);
479+
}
480+
drop(proc_res);
481+
482+
let llvm_ir_path = self.output_base_name().with_extension("ll");
483+
484+
let mut dump_command = Command::new(coverage_dump_path);
485+
dump_command.arg(llvm_ir_path);
486+
let proc_res = self.run_command_to_procres(&mut dump_command);
487+
if !proc_res.status.success() {
488+
self.fatal_proc_rec("coverage-dump failed!", &proc_res);
489+
}
490+
491+
let kind = UI_COVERAGE_MAP;
492+
493+
let expected_coverage_dump = self.load_expected_output(kind);
494+
let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]);
495+
496+
let coverage_dump_errors = self.compare_output(
497+
kind,
498+
&actual_coverage_dump,
499+
&expected_coverage_dump,
500+
self.props.compare_output_lines_by_subset,
501+
);
502+
503+
if coverage_dump_errors > 0 {
504+
self.fatal_proc_rec(
505+
&format!("{coverage_dump_errors} errors occurred comparing coverage output."),
506+
&proc_res,
507+
);
508+
}
509+
}
510+
470511
fn run_coverage_test(&self) {
471512
let should_run = self.run_if_enabled();
472513
let proc_res = self.compile_test(should_run, Emit::None);
@@ -650,6 +691,10 @@ impl<'test> TestCx<'test> {
650691
let mut cmd = Command::new(tool_path);
651692
configure_cmd_fn(&mut cmd);
652693

694+
self.run_command_to_procres(&mut cmd)
695+
}
696+
697+
fn run_command_to_procres(&self, cmd: &mut Command) -> ProcRes {
653698
let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`"));
654699

655700
let proc_res = ProcRes {
@@ -2321,9 +2366,11 @@ impl<'test> TestCx<'test> {
23212366
}
23222367
}
23232368
DebugInfo => { /* debuginfo tests must be unoptimized */ }
2324-
RunCoverage => {
2325-
// Coverage reports are affected by optimization level, and
2326-
// the current snapshots assume no optimization by default.
2369+
CoverageMap | RunCoverage => {
2370+
// Coverage mappings and coverage reports are affected by
2371+
// optimization level, so they ignore the optimize-tests
2372+
// setting and set an optimization level in their mode's
2373+
// compile flags (below) or in per-test `compile-flags`.
23272374
}
23282375
_ => {
23292376
rustc.arg("-O");
@@ -2392,8 +2439,22 @@ impl<'test> TestCx<'test> {
23922439

23932440
rustc.arg(dir_opt);
23942441
}
2442+
CoverageMap => {
2443+
rustc.arg("-Cinstrument-coverage");
2444+
// These tests only compile to MIR, so they don't need the
2445+
// profiler runtime to be present.
2446+
rustc.arg("-Zno-profiler-runtime");
2447+
// Coverage mappings are sensitive to MIR optimizations, and
2448+
// the current snapshots assume `opt-level=2` unless overridden
2449+
// by `compile-flags`.
2450+
rustc.arg("-Copt-level=2");
2451+
}
23952452
RunCoverage => {
23962453
rustc.arg("-Cinstrument-coverage");
2454+
// Coverage reports are sometimes sensitive to optimizations,
2455+
// and the current snapshots assume no optimization unless
2456+
// overridden by `compile-flags`.
2457+
rustc.arg("-Copt-level=0");
23972458
}
23982459
RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake
23992460
| CodegenUnits | JsDocTest | Assembly => {

‎src/tools/coverage-dump/Cargo.toml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[package]
2+
name = "coverage-dump"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7+
8+
[dependencies]
9+
anyhow = "1.0.71"
10+
leb128 = "0.2.5"
11+
md5 = { package = "md-5" , version = "0.10.5" }
12+
miniz_oxide = "0.7.1"
13+
regex = "1.8.4"
14+
rustc-demangle = "0.1.23"

‎src/tools/coverage-dump/README.md

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
This tool extracts coverage mapping information from an LLVM IR assembly file
2+
(`.ll`), and prints it in a more human-readable form that can be used for
3+
snapshot tests.
4+
5+
The output format is mostly arbitrary, so it's OK to change the output as long
6+
as any affected tests are also re-blessed. However, the output should be
7+
consistent across different executions on different platforms, so avoid
8+
printing any information that is platform-specific or non-deterministic.

0 commit comments

Comments
 (0)