Skip to content

Commit 9d3b375

Browse files
fmeumcopybara-github
fmeum
authored and
copybara-github
committed
Fix split post-processing of LLVM-based coverage
`--experimental_split_coverage_postprocessing` now works in combination with `--experimental_generate_llvm_lcov`, which required adding the runtime objects as inputs to the separate coverage post-processing spawn. Previously, they were only added to the runfiles of the test action, which aren't (and shouldn't) be added to the coverage spawn. As a side effect, this provides a more natural fix for #15121 than bb6f1a7, which added an additional check to only add those runtime objects to the `llvm-cov` command line that were present in runfiles. Now, since all runtime objects are staged as inputs, they are known to be available. This improves coverage accuracy when e.g. runtime objects are packaged into jars and loaded at runtime. Along the way the common setup for LLVM-based coverage tests has been extracted into a helper function and the tests have been updated to work with a broader range of clang versions. Previously, tests weren't run at all due to the clang version on the runners not falling into the restrictive range. Now, the tests are executed on the Ubuntu 20.04 runner (but not on any other runners, including macOS). Closes #18634. PiperOrigin-RevId: 542055078 Change-Id: Id8851f8685bb7724891acca395bb91556ee2b29a
1 parent 3a31a6c commit 9d3b375

File tree

4 files changed

+228
-143
lines changed

4 files changed

+228
-143
lines changed

‎src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ def _add(ctx, linking_statically):
182182
def _get_file_content(objects):
183183
result = []
184184
for obj in objects:
185-
result.append(obj.short_path)
185+
result.append(obj.path)
186186
result.append("\n")
187187
return "".join(result)
188188

@@ -192,7 +192,7 @@ def _add_transitive_info_providers(ctx, cc_toolchain, cpp_config, feature_config
192192
runtime_objects_list = ctx.actions.declare_file(ctx.label.name + "runtime_objects_list.txt")
193193
file_content = _get_file_content(runtime_objects_for_coverage)
194194
ctx.actions.write(output = runtime_objects_list, content = file_content, is_executable = False)
195-
additional_meta_data = [runtime_objects_list]
195+
additional_meta_data = [runtime_objects_list] + runtime_objects_for_coverage
196196

197197
instrumented_files_provider = cc_helper.create_cc_instrumented_files_info(
198198
ctx = ctx,

‎src/test/shell/bazel/bazel_coverage_cc_test_llvm.sh‎

Lines changed: 81 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,41 @@ if [[ "${COVERAGE_GENERATOR_DIR}" != "released" ]]; then
2727
add_to_bazelrc "build --override_repository=remote_coverage_tools=${COVERAGE_GENERATOR_DIR}"
2828
fi
2929

30+
# Configures Bazel to emit coverage using LLVM tools, returning a non-zero exit
31+
# code if the tools are not available.
32+
function setup_llvm_coverage_tools_for_lcov() {
33+
local -r clang=$(which clang || true)
34+
if [[ ! -x "${clang}" ]]; then
35+
echo "clang not installed. Skipping test."
36+
return 1
37+
fi
38+
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
39+
if [ "$clang_version" -lt 9 ]; then
40+
# No lcov produced with <9.0.
41+
echo "clang versions <9.0 are not supported, got $clang_version. Skipping test."
42+
return 1
43+
fi
44+
45+
local -r llvm_profdata=$(which llvm-profdata || true)
46+
if [[ ! -x "${llvm_profdata}" ]]; then
47+
echo "llvm-profdata not installed. Skipping test."
48+
return 1
49+
fi
50+
51+
local -r llvm_cov=$(which llvm-cov || true)
52+
if [[ ! -x "${llvm_cov}" ]]; then
53+
echo "llvm-cov not installed. Skipping test."
54+
return 1
55+
fi
56+
57+
add_to_bazelrc "common --repo_env=BAZEL_LLVM_COV=${llvm_cov}"
58+
add_to_bazelrc "common --repo_env=BAZEL_LLVM_PROFDATA=${llvm_profdata}"
59+
add_to_bazelrc "common --repo_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1"
60+
add_to_bazelrc "common --repo_env=CC=${clang}"
61+
add_to_bazelrc "common --repo_env=GCOV=${llvm_profdata}"
62+
add_to_bazelrc "common --experimental_generate_llvm_lcov"
63+
}
64+
3065
# Writes the C++ source files and a corresponding BUILD file for which to
3166
# collect code coverage. The sources are a.cc, a.h and t.cc.
3267
function setup_a_cc_lib_and_t_cc_test() {
@@ -109,31 +144,37 @@ function test_cc_test_llvm_coverage_doesnt_fail() {
109144
}
110145

111146
function test_cc_test_llvm_coverage_produces_lcov_report() {
112-
local -r clang="/usr/bin/clang"
113-
if [[ ! -x ${clang} ]]; then
114-
return
115-
fi
116-
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
117-
if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then
118-
# No lcov produced with <9.0, no branch coverage with 10.0 and 11.0.
119-
echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return
120-
fi
147+
setup_llvm_coverage_tools_for_lcov || return 0
148+
setup_a_cc_lib_and_t_cc_test
121149

122-
local -r llvm_profdata="/usr/bin/llvm-profdata"
123-
if [[ ! -x ${llvm_profdata} ]]; then
124-
return
125-
fi
150+
bazel coverage --test_output=all //:t &>$TEST_log || fail "Coverage for //:t failed"
126151

127-
local -r llvm_cov="/usr/bin/llvm-cov"
128-
if [[ ! -x ${llvm_cov} ]]; then
129-
return
130-
fi
152+
local expected_result="SF:a.cc
153+
FN:3,_Z1ab
154+
FNDA:1,_Z1ab
155+
FNF:1
156+
FNH:1
157+
DA:3,1
158+
DA:4,1
159+
DA:5,1
160+
DA:6,1
161+
DA:7,0
162+
DA:8,0
163+
DA:9,1
164+
LH:5
165+
LF:7
166+
end_of_record"
131167

168+
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
169+
}
170+
171+
function test_cc_test_llvm_coverage_produces_lcov_report_with_split_postprocessing() {
172+
setup_llvm_coverage_tools_for_lcov || return 0
132173
setup_a_cc_lib_and_t_cc_test
133174

134-
BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \
135-
BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \
136-
--test_output=all //:t &>$TEST_log || fail "Coverage for //:t failed"
175+
bazel coverage \
176+
--experimental_split_coverage_postprocessing --experimental_fetch_all_coverage_outputs \
177+
--test_env=VERBOSE_COVERAGE=1 --test_output=all //:t &>$TEST_log || fail "Coverage for //:t failed"
137178

138179
local expected_result="SF:a.cc
139180
FN:3,_Z1ab
@@ -151,29 +192,11 @@ LH:5
151192
LF:7
152193
end_of_record"
153194

154-
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))"
195+
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
155196
}
156197

157198
function test_cc_test_with_runtime_objects_not_in_runfiles() {
158-
local -r clang="/usr/bin/clang"
159-
if [[ ! -x ${clang} ]]; then
160-
return
161-
fi
162-
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
163-
if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then
164-
# No lcov produced with <9.0, no branch coverage with 10.0 and 11.0.
165-
echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return
166-
fi
167-
168-
local -r llvm_profdata="/usr/bin/llvm-profdata"
169-
if [[ ! -x ${llvm_profdata} ]]; then
170-
return
171-
fi
172-
173-
local -r llvm_cov="/usr/bin/llvm-cov"
174-
if [[ ! -x ${llvm_cov} ]]; then
175-
return
176-
fi
199+
setup_llvm_coverage_tools_for_lcov || return 0
177200

178201
cat << EOF > BUILD
179202
cc_test(
@@ -206,10 +229,8 @@ int main(int argc, char const *argv[])
206229
EOF
207230

208231

209-
BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \
210-
BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \
211-
--test_output=all --instrument_test_targets \
212-
//:main &>$TEST_log || fail "Coverage for //:main failed"
232+
bazel coverage --test_output=all --instrument_test_targets \
233+
//:main &>$TEST_log || fail "Coverage for //:main failed"
213234

214235
local expected_result="SF:main.cpp
215236
FN:4,main
@@ -225,7 +246,7 @@ LH:5
225246
LF:5
226247
end_of_record"
227248

228-
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))"
249+
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
229250
}
230251

231252
function setup_external_cc_target() {
@@ -306,42 +327,17 @@ EOF
306327
}
307328

308329
function test_external_cc_target_can_collect_coverage() {
309-
local -r clang="/usr/bin/clang"
310-
if [[ ! -x ${clang} ]]; then
311-
return
312-
fi
313-
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
314-
if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then
315-
# No lcov produced with <9.0, no branch coverage with 10.0 and 11.0.
316-
echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return
317-
fi
318-
319-
local -r llvm_profdata="/usr/bin/llvm-profdata"
320-
if [[ ! -x ${llvm_profdata} ]]; then
321-
return
322-
fi
323-
324-
local -r llvm_cov="/usr/bin/llvm-cov"
325-
if [[ ! -x ${llvm_cov} ]]; then
326-
return
327-
fi
328-
330+
setup_llvm_coverage_tools_for_lcov || return 0
329331
setup_external_cc_target
330332

331-
BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \
332-
BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \
333-
--combined_report=lcov --test_output=all \
334-
@other_repo//:t --instrumentation_filter=// &>$TEST_log || fail "Coverage for @other_repo//:t failed"
333+
bazel coverage --combined_report=lcov --test_output=all \
334+
@other_repo//:t --instrumentation_filter=// &>$TEST_log || fail "Coverage for @other_repo//:t failed"
335335

336336
local expected_result='SF:b.cc
337337
FN:1,_Z1bb
338338
FNDA:1,_Z1bb
339339
FNF:1
340340
FNH:1
341-
BRDA:2,0,0,1
342-
BRDA:2,0,1,0
343-
BRF:2
344-
BRH:1
345341
DA:1,1
346342
DA:2,1
347343
DA:3,1
@@ -357,10 +353,6 @@ FN:4,_Z1ab
357353
FNDA:1,_Z1ab
358354
FNF:1
359355
FNH:1
360-
BRDA:5,0,0,1
361-
BRDA:5,0,1,0
362-
BRF:2
363-
BRH:1
364356
DA:4,1
365357
DA:5,1
366358
DA:6,1
@@ -372,47 +364,22 @@ LH:5
372364
LF:7
373365
end_of_record'
374366

375-
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))"
376-
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat)"
367+
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
368+
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')"
377369
}
378370

379371
function test_external_cc_target_coverage_not_collected_by_default() {
380-
local -r clang="/usr/bin/clang"
381-
if [[ ! -x ${clang} ]]; then
382-
return
383-
fi
384-
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
385-
if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then
386-
# No lcov produced with <9.0, no branch coverage with 10.0 and 11.0.
387-
echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return
388-
fi
389-
390-
local -r llvm_profdata="/usr/bin/llvm-profdata"
391-
if [[ ! -x ${llvm_profdata} ]]; then
392-
return
393-
fi
394-
395-
local -r llvm_cov="/usr/bin/llvm-cov"
396-
if [[ ! -x ${llvm_cov} ]]; then
397-
return
398-
fi
399-
372+
setup_llvm_coverage_tools_for_lcov || return 0
400373
setup_external_cc_target
401374

402-
BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \
403-
BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \
404-
--combined_report=lcov --test_output=all \
405-
@other_repo//:t &>$TEST_log || fail "Coverage for @other_repo//:t failed"
375+
bazel coverage --combined_report=lcov --test_output=all \
376+
@other_repo//:t &>$TEST_log || fail "Coverage for @other_repo//:t failed"
406377

407378
local expected_result='SF:b.cc
408379
FN:1,_Z1bb
409380
FNDA:1,_Z1bb
410381
FNF:1
411382
FNH:1
412-
BRDA:2,0,0,1
413-
BRDA:2,0,1,0
414-
BRF:2
415-
BRH:1
416383
DA:1,1
417384
DA:2,1
418385
DA:3,1
@@ -424,30 +391,12 @@ LH:5
424391
LF:7
425392
end_of_record'
426393

427-
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))"
428-
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat)"
394+
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
395+
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')"
429396
}
430397

431398
function test_coverage_with_tmp_in_path() {
432-
local -r clang="/usr/bin/clang"
433-
if [[ ! -x ${clang} ]]; then
434-
return
435-
fi
436-
local -r clang_version=$(clang --version | grep -o "clang version [0-9]*" | cut -d " " -f 3)
437-
if [ "$clang_version" -lt 9 ] || [ "$clang_version" -eq 10 ] || [ "$clang_version" -eq 11 ]; then
438-
# No lcov produced with <9.0, no branch coverage with 10.0 and 11.0.
439-
echo "clang versions <9.0 as well as 10.0 and 11.0 are not supported." && return
440-
fi
441-
442-
local -r llvm_profdata="/usr/bin/llvm-profdata"
443-
if [[ ! -x ${llvm_profdata} ]]; then
444-
return
445-
fi
446-
447-
local -r llvm_cov="/usr/bin/llvm-cov"
448-
if [[ ! -x ${llvm_cov} ]]; then
449-
return
450-
fi
399+
setup_llvm_coverage_tools_for_lcov || return 0
451400

452401
mkdir -p foo/tmp
453402
cat > foo/tmp/BUILD <<'EOF'
@@ -490,20 +439,14 @@ int main(void) {
490439
}
491440
EOF
492441

493-
BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=$llvm_profdata CC=$clang \
494-
BAZEL_LLVM_COV=$llvm_cov bazel coverage --experimental_generate_llvm_lcov \
495-
--combined_report=lcov --test_output=all \
496-
//foo/tmp:t --instrumentation_filter=// &>$TEST_log || fail "Coverage failed"
442+
bazel coverage --combined_report=lcov --test_output=all \
443+
//foo/tmp:t --instrumentation_filter=// &>$TEST_log || fail "Coverage failed"
497444

498445
local expected_result='SF:foo/tmp/a.cc
499446
FN:3,_Z1ab
500447
FNDA:1,_Z1ab
501448
FNF:1
502449
FNH:1
503-
BRDA:4,0,0,1
504-
BRDA:4,0,1,0
505-
BRF:2
506-
BRH:1
507450
DA:3,1
508451
DA:4,1
509452
DA:5,1
@@ -515,8 +458,8 @@ LH:5
515458
LF:7
516459
end_of_record'
517460

518-
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log))"
519-
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat)"
461+
assert_equals "$expected_result" "$(cat $(get_coverage_file_path_from_test_log) | grep -v '^BR')"
462+
assert_equals "$expected_result" "$(cat bazel-out/_coverage/_coverage_report.dat | grep -v '^BR')"
520463
}
521464

522465
run_suite "test tests"

0 commit comments

Comments
 (0)