1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
|
# needs-profiler-support
# ignore-windows-gnu
# min-llvm-version: 11.0
# FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
# properly. Since we only have GCC on the CI ignore the test for now.
-include ../coverage/coverage_tools.mk
BASEDIR=../coverage-reports
SOURCEDIR=../coverage
# The `llvm-cov show` flag `--debug`, used to generate the `counters` output files, is only
# enabled if LLVM assertions are enabled. This requires Rust config `llvm/optimize` and not
# `llvm/release_debuginfo`. Note that some CI builds disable debug assertions (by setting
# `NO_LLVM_ASSERTIONS=1`), so the tests must still pass even if the `--debug` flag is
# not supported. (Note that `counters` files are only produced in the `$(TMPDIR)`
# directory, for inspection and debugging support. They are *not* copied to `expected_*`
# files when `--bless`ed.)
LLVM_COV_DEBUG := $(shell \
"$(LLVM_BIN_DIR)"/llvm-cov show --debug 2>&1 | \
grep -q "Unknown command line argument '--debug'"; \
echo $$?)
ifeq ($(LLVM_COV_DEBUG), 1)
DEBUG_FLAG=--debug
endif
# FIXME(richkadel): I'm adding `--ignore-filename-regex=` line(s) for specific test(s) that produce
# `llvm-cov` results for multiple files (for example `uses_crate.rs` and `used_crate/mod.rs`) as a
# workaround for two problems causing tests to fail on Windows:
#
# 1. When multiple files appear in the `llvm-cov show` results, each file's coverage results can
# appear in different a different order. Whether this is random or, somehow, platform-specific,
# the Windows output flips the order of the files, compared to Linux. In the `uses_crate.rs`
# test, the only test-unique (interesting) results we care about are the results for only one
# of the two files, `mod/uses_crate.rs`, so the workaround is to ignore all but this one file.
# In the future, we may want a more sophisticated solution that splits apart `llvm-cov show`
# results into separate results files for each result (taking care not to create new file
# paths that might be too long for Windows MAX_PATH limits when creating these new sub-results,
# as well).
# 2. When multiple files appear in the `llvm-cov show` results, the results for each file are
# prefixed with their filename, including platform-specific path separators (`\` for Windows,
# and `/` everywhere else). This could be filtered or normalized of course, but by ignoring
# coverage results for all but one of the file, the filenames are no longer included anyway.
# If this changes (if/when we decide to support `llvm-cov show` results for multiple files),
# the file path separator differences may need to be addressed.
#
# Since this is only a workaround, I decided to implement the override by adding an option for
# each file to be ignored, using a `--ignore-filename-regex=` entry for each one, rather than
# implement some more sophisticated solution with a new custom test directive in the test file
# itself (similar to `expect-exit-status`) because that would add a lot of complexity and still
# be a workaround, with the same result, with no benefit.
#
# Yes these `--ignore-filename-regex=` options are included in all invocations of `llvm-cov show`
# for now, but it is effectively ignored for all tests that don't include this file anyway.
#
# (Note that it's also possible the `_counters.<test>.txt` and `<test>.json` files (if generated)
# may order results from multiple files inconsistently, which might also have to be accomodated
# if and when we allow `llvm-cov` to produce results for multiple files. Note, the path separators
# appear to be normalized to `/` in those files, thankfully.)
LLVM_COV_IGNORE_FILES=\
--ignore-filename-regex=uses_crate.rs
# When generating `expected_*` results (using `x.py test --bless`), the `--debug` flag is forced.
# If assertions are disabled, the command will fail with an error, rather than attempt to generate
# only partial results.
ifdef RUSTC_BLESS_TEST
DEBUG_FLAG=--debug
endif
all: $(patsubst $(SOURCEDIR)/lib/%.rs,%,$(wildcard $(SOURCEDIR)/lib/*.rs)) $(patsubst $(SOURCEDIR)/%.rs,%,$(wildcard $(SOURCEDIR)/*.rs))
# Ensure there are no `expected` results for tests that may have been removed or renamed
.PHONY: clear_expected_if_blessed
clear_expected_if_blessed:
ifdef RUSTC_BLESS_TEST
rm -f expected_*
endif
-include clear_expected_if_blessed
%: $(SOURCEDIR)/lib/%.rs
# Compile the test library with coverage instrumentation
$(RUSTC) $(SOURCEDIR)/lib/$@.rs \
$$( sed -nE 's#^// compile-flags:(.*)#\1# p' $(SOURCEDIR)/lib/$@.rs) \
--crate-type rlib -Zinstrument-coverage
%: $(SOURCEDIR)/%.rs
# Compile the test program with coverage instrumentation
$(RUSTC) $(SOURCEDIR)/$@.rs \
$$( sed -nE 's#^// compile-flags:(.*)#\1# p' $(SOURCEDIR)/$@.rs) \
-L "$(TMPDIR)" -Zinstrument-coverage
# Run it in order to generate some profiling data,
# with `LLVM_PROFILE_FILE=<profdata_file>` environment variable set to
# output the coverage stats for this run.
LLVM_PROFILE_FILE="$(TMPDIR)"/$@-%p.profraw \
$(call RUN,$@) || \
( \
status=$$?; \
grep -q "^\/\/ expect-exit-status-$$status" $(SOURCEDIR)/$@.rs || \
( >&2 echo "program exited with an unexpected exit status: $$status"; \
false \
) \
)
# Run it through rustdoc as well to cover doctests
LLVM_PROFILE_FILE="$(TMPDIR)"/$@-%p.profraw \
$(RUSTDOC) --crate-name workaround_for_79771 --test $(SOURCEDIR)/$@.rs \
$$( sed -nE 's#^// compile-flags:(.*)#\1# p' $(SOURCEDIR)/$@.rs) \
-L "$(TMPDIR)" -Zinstrument-coverage \
-Z unstable-options --persist-doctests=$(TMPDIR)/rustdoc-$@
# Postprocess the profiling data so it can be used by the llvm-cov tool
"$(LLVM_BIN_DIR)"/llvm-profdata merge --sparse \
"$(TMPDIR)"/$@-*.profraw \
-o "$(TMPDIR)"/$@.profdata
# Generate a coverage report using `llvm-cov show`.
"$(LLVM_BIN_DIR)"/llvm-cov show \
$(DEBUG_FLAG) \
$(LLVM_COV_IGNORE_FILES) \
--Xdemangler="$(RUST_DEMANGLER)" \
--show-line-counts-or-regions \
--instr-profile="$(TMPDIR)"/$@.profdata \
$(call BIN,"$(TMPDIR)"/$@) \
$$( \
for file in $(TMPDIR)/rustdoc-$@/*/rust_out; \
do \
[ -x "$$file" ] && printf "%s %s " -object $$file; \
done \
) \
2> "$(TMPDIR)"/show_coverage_stderr.$@.txt \
| "$(PYTHON)" $(BASEDIR)/normalize_paths.py \
> "$(TMPDIR)"/actual_show_coverage.$@.txt || \
( status=$$? ; \
>&2 cat "$(TMPDIR)"/show_coverage_stderr.$@.txt ; \
exit $$status \
)
ifdef DEBUG_FLAG
# The first line (beginning with "Args:" contains hard-coded, build-specific
# file paths. Strip that line and keep the remaining lines with counter debug
# data.
tail -n +2 "$(TMPDIR)"/show_coverage_stderr.$@.txt \
> "$(TMPDIR)"/actual_show_coverage_counters.$@.txt
endif
ifdef RUSTC_BLESS_TEST
cp "$(TMPDIR)"/actual_show_coverage.$@.txt \
expected_show_coverage.$@.txt
else
# Compare the show coverage output (`--bless` refreshes `typical` files).
#
# FIXME(richkadel): None of the Rust test source samples have the
# `// ignore-llvm-cov-show-diffs` anymore. This directive exists to work around a limitation
# with `llvm-cov show`. When reporting coverage for multiple instantiations of a generic function,
# with different type substitutions, `llvm-cov show` prints these in a non-deterministic order,
# breaking the `diff` comparision.
#
# A partial workaround is implemented below, with `diff --ignore-matching-lines=RE`
# to ignore each line prefixing each generic instantiation coverage code region.
#
# This workaround only works if the coverage counts are identical across all reported
# instantiations. If there is no way to ensure this, you may need to apply the
# `// ignore-llvm-cov-show-diffs` directive, and check for differences using the
# `.json` files to validate that results have not changed. (Until then, the JSON
# files are redundant, so there is no need to generate `expected_*.json` files or
# compare actual JSON results.)
$(DIFF) --ignore-matching-lines='^ | .*::<.*>.*:$$' --ignore-matching-lines='^ | <.*>::.*:$$' \
expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \
( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
>&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
) || \
( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
false \
)
endif
####################################################################################################
# The following Makefile content was used to copy the generated `counters` files
# to `expected_` files (when `--bless`ed) and to compare them via `diff`; but for
# multiple reasons, these files cannot easily be used for test validation:
#
# * Output lines can be produced in non-deterministic order (depending on the
# target platform, and sometimes on unrelated codegen changes).
# * Some lines include demangled function names, making them more challenging
# to interpret and compare.
#
# The files are still generated (in `$(TMPDIR)`) to support developers wanting
# to inspect the counters, for debugging purposes.
#
# ifdef RUSTC_BLESS_TEST
# cp "$(TMPDIR)"/actual_show_coverage_counters.$@.txt \
# expected_show_coverage_counters.$@.txt
# else
#
# ifdef DEBUG_FLAG
# $(DIFF) expected_show_coverage_counters.$@.txt "$(TMPDIR)"/actual_show_coverage_counters.$@.txt || \
# ( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
# >&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
# ) || \
# ( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
# >&2 echo '(Ignore anyway until mangled function names in "counters" files are demangled.)' \
# )
# endif
#
# endif
####################################################################################################
# The following Makefile content, and short JSON script, were used to generate
# coverage reports in JSON when the `llvm-cov show` reports were less reliable for
# testing. At the present time, however, the `llvm-cov show` results, and methods
# for comparing them, are working for all tests, making the JSON reports redundant.
#
# If this changes in the future, the scripts are left here, commented out, but can
# be resurrected if desired. This could be used to compare *only* the JSON files;
# and in that case, the `llvm-cov show` reports can be ignored by inserting
# `// ignore-llvm-cov-show-diffs` at the top of the source file.
#
# # Generate a coverage report in JSON, using `llvm-cov export`, and fail if
# # there are differences from the expected output.
# "$(LLVM_BIN_DIR)"/llvm-cov export \
# $(LLVM_COV_IGNORE_FILES) \
# --summary-only \
# --instr-profile="$(TMPDIR)"/$@.profdata \
# $(call BIN,"$(TMPDIR)"/$@) \
# | "$(PYTHON)" $(BASEDIR)/prettify_json.py \
# > "$(TMPDIR)"/actual_export_coverage.$@.json
#
# ifdef RUSTC_BLESS_TEST
# cp "$(TMPDIR)"/actual_export_coverage.$@.json expected_export_coverage.$@.json
# else
# # Check that exported JSON coverage data matches what we expect (`--bless` refreshes `expected`)
# $(DIFF) expected_export_coverage.$@.json "$(TMPDIR)"/actual_export_coverage.$@.json
# endif
#
# # # If generating coverage reports in JSON, this Makefile is accompanied by
# # # a Python script, `prettify_json.py`, which is defined:
# #
# # #!/usr/bin/env python
# #
# # import sys
# # import json
# #
# # # Try to decode line in order to ensure it is a valid JSON document
# # for line in sys.stdin:
# # parsed = json.loads(line)
# # print (json.dumps(parsed, indent=2, separators=(',', ': '), sort_keys=True))
|
