1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
#!/usr/bin/env python3
'''
Test whether cargo-miri works properly.
Assumes the `MIRI_SYSROOT` env var to be set appropriately,
and the working directory to contain the cargo-miri-test project.
'''
import difflib
import os
import re
import subprocess
import sys
CGREEN = '\33[32m'
CBOLD = '\33[1m'
CEND = '\33[0m'
CARGO_EXTRA_FLAGS = os.environ.get("CARGO_EXTRA_FLAGS", "").split()
def fail(msg):
print("\nTEST FAIL: {}".format(msg))
sys.exit(1)
def cargo_miri(cmd, quiet = True):
args = ["cargo", "miri", cmd] + CARGO_EXTRA_FLAGS
if quiet:
args += ["-q"]
if 'MIRI_TEST_TARGET' in os.environ:
args += ["--target", os.environ['MIRI_TEST_TARGET']]
return args
def normalize_stdout(str):
str = str.replace("src\\", "src/") # normalize paths across platforms
str = re.sub("finished in \d+\.\d\ds", "finished in $TIME", str) # the time keeps changing, obviously
return str
def normalize_stderr(str):
str = re.sub("Preparing a sysroot for Miri \(target: [a-z0-9_-]+\)\.\.\. done\n", "", str) # remove leading cargo-miri setup output
return str
def check_output(actual, path, name):
if os.environ.get("RUSTC_BLESS", "0") != "0":
# Write the output only if bless is set
open(path, mode='w').write(actual)
return True
expected = open(path).read()
if expected == actual:
return True
print(f"{name} output did not match reference in {path}!")
print(f"--- BEGIN diff {name} ---")
for text in difflib.unified_diff(expected.split("\n"), actual.split("\n")):
print(text)
print(f"--- END diff {name} ---")
return False
def test(name, cmd, stdout_ref, stderr_ref, stdin=b'', env=None):
if env is None:
env = {}
print("Testing {}...".format(name))
## Call `cargo miri`, capture all output
p_env = os.environ.copy()
p_env.update(env)
p = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=p_env,
)
(stdout, stderr) = p.communicate(input=stdin)
stdout = normalize_stdout(stdout.decode("UTF-8"))
stderr = normalize_stderr(stderr.decode("UTF-8"))
stdout_matches = check_output(stdout, stdout_ref, "stdout")
stderr_matches = check_output(stderr, stderr_ref, "stderr")
if p.returncode == 0 and stdout_matches and stderr_matches:
# All good!
return
fail("exit code was {}".format(p.returncode))
def test_no_rebuild(name, cmd, env=None):
if env is None:
env = {}
print("Testing {}...".format(name))
p_env = os.environ.copy()
p_env.update(env)
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=p_env,
)
(stdout, stderr) = p.communicate()
stdout = stdout.decode("UTF-8")
stderr = stderr.decode("UTF-8")
if p.returncode != 0:
fail("rebuild failed")
# Also check for 'Running' as a sanity check.
if stderr.count(" Compiling ") > 0 or stderr.count(" Running ") == 0:
print("--- BEGIN stderr ---")
print(stderr, end="")
print("--- END stderr ---")
fail("Something was being rebuilt when it should not be (or we got no output)")
def test_cargo_miri_run():
test("`cargo miri run` (no isolation)",
cargo_miri("run"),
"run.default.stdout.ref", "run.default.stderr.ref",
stdin=b'12\n21\n',
env={
'MIRIFLAGS': "-Zmiri-disable-isolation",
'MIRITESTVAR': "wrongval", # make sure the build.rs value takes precedence
},
)
# Special test: run it again *without* `-q` to make sure nothing is being rebuilt (Miri issue #1722)
test_no_rebuild("`cargo miri run` (no rebuild)",
cargo_miri("run", quiet=False) + ["--", ""],
env={'MIRITESTVAR': "wrongval"}, # changing the env var causes a rebuild (re-runs build.rs),
# so keep it set
)
# This also covers passing arguments without `--`: Cargo will forward unused positional arguments to the program.
test("`cargo miri run` (with arguments and target)",
cargo_miri("run") + ["--bin", "cargo-miri-test", "hello world", '"hello world"', r'he\\llo\"world'],
"run.args.stdout.ref", "run.args.stderr.ref",
)
test("`cargo miri r` (subcrate, no isolation)",
cargo_miri("r") + ["-p", "subcrate"],
"run.subcrate.stdout.ref", "run.subcrate.stderr.ref",
env={'MIRIFLAGS': "-Zmiri-disable-isolation"},
)
test("`cargo miri run` (custom target dir)",
# Attempt to confuse the argument parser.
cargo_miri("run") + ["--target-dir=custom-run", "--", "--target-dir=target/custom-run"],
"run.args.stdout.ref", "run.custom-target-dir.stderr.ref",
)
def test_cargo_miri_test():
# rustdoc is not run on foreign targets
is_foreign = 'MIRI_TEST_TARGET' in os.environ
default_ref = "test.cross-target.stdout.ref" if is_foreign else "test.default.stdout.ref"
filter_ref = "test.filter.cross-target.stdout.ref" if is_foreign else "test.filter.stdout.ref"
# macOS needs permissive provenance inside getrandom_1.
test("`cargo miri test`",
cargo_miri("test"),
default_ref, "test.stderr-empty.ref",
env={'MIRIFLAGS': "-Zmiri-permissive-provenance -Zmiri-seed=4242"},
)
test("`cargo miri test` (no isolation, no doctests)",
cargo_miri("test") + ["--bins", "--tests"], # no `--lib`, we disabled that in `Cargo.toml`
"test.cross-target.stdout.ref", "test.stderr-empty.ref",
env={'MIRIFLAGS': "-Zmiri-permissive-provenance -Zmiri-disable-isolation"},
)
test("`cargo miri test` (with filter)",
cargo_miri("test") + ["--", "--format=pretty", "pl"],
filter_ref, "test.stderr-empty.ref",
)
test("`cargo miri test` (test target)",
cargo_miri("test") + ["--test", "test", "--", "--format=pretty"],
"test.test-target.stdout.ref", "test.stderr-empty.ref",
env={'MIRIFLAGS': "-Zmiri-permissive-provenance"},
)
test("`cargo miri test` (bin target)",
cargo_miri("test") + ["--bin", "cargo-miri-test", "--", "--format=pretty"],
"test.bin-target.stdout.ref", "test.stderr-empty.ref",
)
test("`cargo miri t` (subcrate, no isolation)",
cargo_miri("t") + ["-p", "subcrate"],
"test.subcrate.stdout.ref", "test.stderr-proc-macro.ref",
env={'MIRIFLAGS': "-Zmiri-disable-isolation"},
)
test("`cargo miri test` (subcrate, doctests)",
cargo_miri("test") + ["-p", "subcrate", "--doc"],
"test.stdout-empty.ref", "test.stderr-proc-macro-doctest.ref",
)
test("`cargo miri test` (custom target dir)",
cargo_miri("test") + ["--target-dir=custom-test"],
default_ref, "test.stderr-empty.ref",
env={'MIRIFLAGS': "-Zmiri-permissive-provenance"},
)
del os.environ["CARGO_TARGET_DIR"] # this overrides `build.target-dir` passed by `--config`, so unset it
test("`cargo miri test` (config-cli)",
cargo_miri("test") + ["--config=build.target-dir=\"config-cli\""],
default_ref, "test.stderr-empty.ref",
env={'MIRIFLAGS': "-Zmiri-permissive-provenance"},
)
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.environ["CARGO_TARGET_DIR"] = "target" # this affects the location of the target directory that we need to check
os.environ["RUST_TEST_NOCAPTURE"] = "0" # this affects test output, so make sure it is not set
os.environ["RUST_TEST_THREADS"] = "1" # avoid non-deterministic output due to concurrent test runs
target_str = " for target {}".format(os.environ['MIRI_TEST_TARGET']) if 'MIRI_TEST_TARGET' in os.environ else ""
print(CGREEN + CBOLD + "## Running `cargo miri` tests{}".format(target_str) + CEND)
test_cargo_miri_run()
test_cargo_miri_test()
# Ensure we did not create anything outside the expected target dir.
for target_dir in ["target", "custom-run", "custom-test", "config-cli"]:
if os.listdir(target_dir) != ["miri"]:
fail(f"`{target_dir}` contains unexpected files")
# Ensure something exists inside that target dir.
os.access(os.path.join(target_dir, "miri", "debug", "deps"), os.F_OK)
print("\nTEST SUCCESSFUL!")
sys.exit(0)
|
