1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
# needs-profiler-support
# ignore-windows-gnu
# FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
# properly. Since we only have GCC on the CI ignore the test for now.
# ISSUE(76038): When targeting MSVC, Rust binaries built with both `-Z instrument-coverage` and
# `-C link-dead-code` typically crash (with a seg-fault) or at best generate an empty `*.profraw`.
# See ../coverage/coverage_tools.mk for more information.
-include ../coverage/coverage_tools.mk
BASEDIR=../coverage-reports-base
SOURCEDIR=../coverage
ifeq ($(UNAME),Darwin)
# FIXME(richkadel): It appears that --debug is not available on MacOS even when not running
# under CI.
NO_LLVM_ASSERTIONS=1
endif
# The `llvm-cov show` flag `--debug`, used to generate the `counters` output files, is only enabled
# if LLVM assertions are enabled. Some CI builds disable debug assertions.
ifndef NO_LLVM_ASSERTIONS
DEBUG_FLAG=--debug
endif
# When generating `expected_*` results (using `x.py test --bless`), the `--debug` flag is forced.
# If assertions are disabled, the command will fail with an error, rather than attempt to generate
# only partial results.
ifdef RUSTC_BLESS_TEST
DEBUG_FLAG=--debug
endif
all: $(patsubst $(SOURCEDIR)/%.rs,%,$(wildcard $(SOURCEDIR)/*.rs))
# Ensure there are no `expected` results for tests that may have been removed or renamed
.PHONY: clear_expected_if_blessed
clear_expected_if_blessed:
ifdef RUSTC_BLESS_TEST
rm -f expected_export_coverage.*.json
rm -f expected_show_coverage.*.txt
rm -f expected_show_coverage_counters.*.txt
endif
-include clear_expected_if_blessed
%: $(SOURCEDIR)/%.rs
# Compile the test program with coverage instrumentation and generate relevant MIR.
$(RUSTC) $(SOURCEDIR)/$@.rs \
-Zinstrument-coverage \
-Clink-dead-code=$(LINK_DEAD_CODE)
# Run it in order to generate some profiling data,
# with `LLVM_PROFILE_FILE=<profdata_file>` environment variable set to
# output the coverage stats for this run.
LLVM_PROFILE_FILE="$(TMPDIR)"/$@.profraw \
$(call RUN,$@) || \
( \
status=$$?; \
grep -q "^\/\/ expect-exit-status-$$status" $(SOURCEDIR)/$@.rs || \
( >&2 echo "program exited with an unexpected exit status: $$status"; \
false \
) \
)
# Postprocess the profiling data so it can be used by the llvm-cov tool
"$(LLVM_BIN_DIR)"/llvm-profdata merge --sparse \
"$(TMPDIR)"/$@.profraw \
-o "$(TMPDIR)"/$@.profdata
# Generate a coverage report using `llvm-cov show`.
"$(LLVM_BIN_DIR)"/llvm-cov show \
$(DEBUG_FLAG) \
--Xdemangler="$(RUST_DEMANGLER)" \
--show-line-counts-or-regions \
--instr-profile="$(TMPDIR)"/$@.profdata \
$(call BIN,"$(TMPDIR)"/$@) \
> "$(TMPDIR)"/actual_show_coverage.$@.txt \
2> "$(TMPDIR)"/show_coverage_stderr.$@.txt || \
( status=$$? ; \
>&2 cat "$(TMPDIR)"/show_coverage_stderr.$@.txt ; \
exit $$status \
)
ifdef DEBUG_FLAG
# The first line (beginning with "Args:" contains hard-coded, build-specific
# file paths. Strip that line and keep the remaining lines with counter debug
# data.
tail -n +2 "$(TMPDIR)"/show_coverage_stderr.$@.txt \
> "$(TMPDIR)"/actual_show_coverage_counters.$@.txt
endif
ifdef RUSTC_BLESS_TEST
cp "$(TMPDIR)"/actual_show_coverage.$@.txt \
expected_show_coverage.$@.txt
cp "$(TMPDIR)"/actual_show_coverage_counters.$@.txt \
expected_show_coverage_counters.$@.txt
else
# Compare the show coverage output (`--bless` refreshes `typical` files)
# Note `llvm-cov show` output for some programs can vary, but can be ignored
# by inserting `// ignore-llvm-cov-show-diffs` at the top of the source file.
$(DIFF) expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \
( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
>&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
) || \
( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
false \
)
ifdef DEBUG_FLAG
$(DIFF) expected_show_coverage_counters.$@.txt "$(TMPDIR)"/actual_show_coverage_counters.$@.txt || \
( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
>&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
) || \
( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
>&2 echo '(Ignore anyway until mangled function names in "counters" files are demangled.)' \
)
# FIXME(richkadel): Apply the demangler to the `*_show_coverage_counters.*.txt` files,
# so the crate disambiguator differences will be stripped away. At that point, these files
# will be less likely to vary, and the last `echo` above (starting with "Ignore anyway")
# can be replaced with `false` to fail the test.
endif
endif
# Generate a coverage report in JSON, using `llvm-cov export`, and fail if
# there are differences from the expected output.
"$(LLVM_BIN_DIR)"/llvm-cov export \
--summary-only \
--instr-profile="$(TMPDIR)"/$@.profdata \
$(call BIN,"$(TMPDIR)"/$@) \
| "$(PYTHON)" $(BASEDIR)/prettify_json.py \
> "$(TMPDIR)"/actual_export_coverage.$@.json
ifdef RUSTC_BLESS_TEST
cp "$(TMPDIR)"/actual_export_coverage.$@.json expected_export_coverage.$@.json
else
# Check that exported JSON coverage data matches what we expect (`--bless` refreshes `expected`)
$(DIFF) expected_export_coverage.$@.json "$(TMPDIR)"/actual_export_coverage.$@.json
endif
|
