diff options
Diffstat (limited to 'compiler')
71 files changed, 1360 insertions, 1556 deletions
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index d396f18d59c..ef0c763ac20 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -49,14 +49,6 @@ bitflags! { } } -/// Which niches (beyond the `null` niche) are available on references. -#[derive(Default, Copy, Clone, Hash, Debug, Eq, PartialEq)] -#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_Generic))] -pub struct ReferenceNichePolicy { - pub size: bool, - pub align: bool, -} - #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_Generic))] pub enum IntegerType { @@ -355,33 +347,6 @@ impl TargetDataLayout { } #[inline] - pub fn target_usize_max(&self) -> u64 { - self.pointer_size.unsigned_int_max().try_into().unwrap() - } - - #[inline] - pub fn target_isize_min(&self) -> i64 { - self.pointer_size.signed_int_min().try_into().unwrap() - } - - #[inline] - pub fn target_isize_max(&self) -> i64 { - self.pointer_size.signed_int_max().try_into().unwrap() - } - - /// Returns the (inclusive) range of possible addresses for an allocation with - /// the given size and alignment. - /// - /// Note that this doesn't take into account target-specific limitations. - #[inline] - pub fn address_range_for(&self, size: Size, align: Align) -> (u64, u64) { - let end = Size::from_bytes(self.target_usize_max()); - let min = align.bytes(); - let max = (end - size).align_down_to(align).bytes(); - (min, max) - } - - #[inline] pub fn vector_align(&self, vec_size: Size) -> AbiAndPrefAlign { for &(size, align) in &self.vector_align { if size == vec_size { @@ -509,12 +474,6 @@ impl Size { } #[inline] - pub fn align_down_to(self, align: Align) -> Size { - let mask = align.bytes() - 1; - Size::from_bytes(self.bytes() & !mask) - } - - #[inline] pub fn is_aligned(self, align: Align) -> bool { let mask = align.bytes() - 1; self.bytes() & mask == 0 @@ -1008,43 +967,6 @@ impl WrappingRange { } } - /// Returns `true` if `range` is contained in `self`. - #[inline(always)] - pub fn contains_range<I: Into<u128> + Ord>(&self, range: RangeInclusive<I>) -> bool { - if range.is_empty() { - return true; - } - - let (vmin, vmax) = range.into_inner(); - let (vmin, vmax) = (vmin.into(), vmax.into()); - - if self.start <= self.end { - self.start <= vmin && vmax <= self.end - } else { - // The last check is needed to cover the following case: - // `vmin ... start, end ... vmax`. In this special case there is no gap - // between `start` and `end` so we must return true. - self.start <= vmin || vmax <= self.end || self.start == self.end + 1 - } - } - - /// Returns `true` if `range` has an overlap with `self`. - #[inline(always)] - pub fn overlaps_range<I: Into<u128> + Ord>(&self, range: RangeInclusive<I>) -> bool { - if range.is_empty() { - return false; - } - - let (vmin, vmax) = range.into_inner(); - let (vmin, vmax) = (vmin.into(), vmax.into()); - - if self.start <= self.end { - self.start <= vmax && vmin <= self.end - } else { - self.start <= vmax || vmin <= self.end - } - } - /// Returns `self` with replaced `start` #[inline(always)] pub fn with_start(mut self, start: u128) -> Self { @@ -1062,15 +984,9 @@ impl WrappingRange { /// Returns `true` if `size` completely fills the range. #[inline] pub fn is_full_for(&self, size: Size) -> bool { - debug_assert!(self.is_in_range_for(size)); - self.start == (self.end.wrapping_add(1) & size.unsigned_int_max()) - } - - /// Returns `true` if the range is valid for `size`. - #[inline(always)] - pub fn is_in_range_for(&self, size: Size) -> bool { let max_value = size.unsigned_int_max(); - self.start <= max_value && self.end <= max_value + debug_assert!(self.start <= max_value && self.end <= max_value); + self.start == (self.end.wrapping_add(1) & max_value) } } @@ -1511,21 +1427,16 @@ impl Niche { pub fn reserve<C: HasDataLayout>(&self, cx: &C, count: u128) -> Option<(u128, Scalar)> { assert!(count > 0); - if count > self.available(cx) { - return None; - } let Self { value, valid_range: v, .. } = *self; - let max_value = value.size(cx).unsigned_int_max(); - let distance_end_zero = max_value - v.end; + let size = value.size(cx); + assert!(size.bits() <= 128); + let max_value = size.unsigned_int_max(); - // Null-pointer optimization. This is guaranteed by Rust (at least for `Option<_>`), - // and offers better codegen opportunities. - if count == 1 && matches!(value, Pointer(_)) && !v.contains(0) { - // Select which bound to move to minimize the number of lost niches. - let valid_range = - if v.start - 1 > distance_end_zero { v.with_end(0) } else { v.with_start(0) }; - return Some((0, Scalar::Initialized { value, valid_range })); + let niche = v.end.wrapping_add(1)..v.start; + let available = niche.end.wrapping_sub(niche.start) & max_value; + if count > available { + return None; } // Extend the range of valid values being reserved by moving either `v.start` or `v.end` bound. @@ -1548,6 +1459,7 @@ impl Niche { let end = v.end.wrapping_add(count) & max_value; Some((start, Scalar::Initialized { value, valid_range: v.with_end(end) })) }; + let distance_end_zero = max_value - v.end; if v.start > v.end { // zero is unavailable because wrapping occurs move_end(v) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index a7198fbf887..17b73468a31 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -2353,7 +2353,12 @@ impl Param { /// Builds a `Param` object from `ExplicitSelf`. pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param { let span = eself.span.to(eself_ident.span); - let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None }); + let infer_ty = P(Ty { + id: DUMMY_NODE_ID, + kind: TyKind::ImplicitSelf, + span: eself_ident.span, + tokens: None, + }); let (mutbl, ty) = match eself.node { SelfKind::Explicit(ty, mutbl) => (mutbl, ty), SelfKind::Value(mutbl) => (mutbl, infer_ty), diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml index 8e6c1e8ade0..652d6eca3f6 100644 --- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml +++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml @@ -12,9 +12,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install rustfmt + - name: Avoid installing rustc-dev run: | - rustup component add rustfmt + sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain + echo 'profile = "minimal"' >> rust-toolchain + rustfmt -v - name: Rustfmt run: | @@ -127,7 +129,7 @@ jobs: - uses: actions/checkout@v3 - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Disable JIT tests run: | @@ -136,7 +138,7 @@ jobs: - name: Test env: TARGET_TRIPLE: x86_64-unknown-linux-gnu - run: ./y.rs test --use-backend llvm + run: ./y.sh test --use-backend llvm bench: runs-on: ubuntu-latest diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock index 904233d4242..af8e43da4ea 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -51,18 +51,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6160c0a96253993b79fb7e0983534a4515ecf666120ddf8f92068114997ebc" +checksum = "ec27af72e56235eb326b5bf2de4e70ab7c5ac1fb683a1829595badaf821607fd" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b38da5f63562e42f3c929d7c76871098e5ad12c8ab44b0659ffc529f22a5b3a" +checksum = "2231e12925e6c5f4bc9c95b62a798eea6ed669a95bc3e00f8b2adb3b7b9b7a80" dependencies = [ "bumpalo", "cranelift-bforest", @@ -81,39 +81,39 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011371e213e163b55dd9e8404b3f2d9fa52cd14dc2f3dc5b83e61ffceff126db" +checksum = "413b00b8dfb3aab85674a534677e7ca08854b503f164a70ec0634fce80996e2c" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf97dde7f5ad571161cdd203a2c9c88682ef669830aea3c14ea5d164ef8bb43" +checksum = "cd0feb9ecc8193ef5cb04f494c5bd835e5bfec4bde726e7ac0444fc9dd76229e" [[package]] name = "cranelift-control" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9a9254aee733b0f2b68e5eaaf0337ad53cb23252a056c10a35370551be8d40" +checksum = "72eedd2afcf5fee1e042eaaf18d3750e48ad0eca364a9f5971ecfdd5ef85bf71" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf39a33ee39479d1337cd9333f3c09786c5a0ca1ec509edcaf9d1346d5de0e5" +checksum = "7af19157be42671073cf8c2a52d6a4ae1e7b11f1dcb4131fede356d9f91c29dd" [[package]] name = "cranelift-frontend" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e260b92a193a0a2dccc3938f133d9532e7dcfe8d03e36bf8b7d3518c1c1793" +checksum = "c2dc7636c5fad156be7d9ae691cd1aaecd97326caf2ab534ba168056d56aa76c" dependencies = [ "cranelift-codegen", "log", @@ -123,15 +123,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9446c8e1aadfcdacee1a49592bc2c25d1d9bf5484782c163e7f5485c92cd3c1c" +checksum = "c1111aea4fb6fade5779903f184249a3fc685a799fe4ec59126f9af59c7c2a74" [[package]] name = "cranelift-jit" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "689a6df165d0f860c1e1a3d53c28944e2743c3e9ee4c678cf190fe60ad7a6ef5" +checksum = "dadf88076317f6286ec77ebbe65978734fb43b6befdc96f52ff4c4c511841644" dependencies = [ "anyhow", "cranelift-codegen", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1402d6ff1695b429536b2eaa126db560fc94c375ed0e9cfb15051fc07427f7" +checksum = "c6bae8a82dbf82241b1083e57e06870d2c2bdc9852727be99d58477513816953" dependencies = [ "anyhow", "cranelift-codegen", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac916f3c5aff4b817e42fc2e682292b931495b3fe2603d5e3c3cf602d74e344" +checksum = "1ecfc01a634448468a698beac433d98040033046678a0eed3ca39a3a9f63ae86" dependencies = [ "cranelift-codegen", "libc", @@ -171,9 +171,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23860f4cd064017f2108e6bc5d25660a77cd6eea77f1ac0756870a00abb12e93" +checksum = "0ee14a7276999f0dcaae2de84043e2c2de50820fb89b3db56fab586a4ad26734" dependencies = [ "anyhow", "cranelift-codegen", @@ -194,6 +194,12 @@ dependencies = [ ] [[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + +[[package]] name = "fallible-iterator" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -206,7 +212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 1.9.3", "stable_deref_trait", ] @@ -226,6 +232,12 @@ dependencies = [ ] [[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] name = "indexmap" version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -236,6 +248,16 @@ dependencies = [ ] [[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] name = "libc" version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -283,7 +305,7 @@ checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" dependencies = [ "crc32fast", "hashbrown 0.13.2", - "indexmap", + "indexmap 1.9.3", "memchr", ] @@ -295,9 +317,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "regalloc2" -version = "0.8.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a52e724646c6c0800fc456ec43b4165d2f91fba88ceaca06d9e0b400023478" +checksum = "5b4dcbd3a2ae7fb94b5813fa0e957c6ab51bf5d0a8ee1b69e0c2d0f1e6eb8485" dependencies = [ "hashbrown 0.13.2", "log", @@ -335,7 +357,7 @@ dependencies = [ "cranelift-native", "cranelift-object", "gimli", - "indexmap", + "indexmap 2.0.0", "libloading", "object", "smallvec", @@ -374,9 +396,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasmtime-jit-icache-coherence" -version = "9.0.1" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d90933b781e1cef7656baed671c7a90bdba0c1c694e04fdd4124419308f5cbb" +checksum = "e34eb67f0829a5614ec54716c8e0c9fe68fab7b9df3686c85f719c9d247f7169" dependencies = [ "cfg-if", "libc", diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml index 1c1f2d8577b..8ded81d7399 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.toml +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -8,17 +8,17 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.96.1", features = ["unwind", "all-arch"] } -cranelift-frontend = { version = "0.96.1" } -cranelift-module = { version = "0.96.1" } -cranelift-native = { version = "0.96.1" } -cranelift-jit = { version = "0.96.1", optional = true } -cranelift-object = { version = "0.96.1" } +cranelift-codegen = { version = "0.98", features = ["unwind", "all-arch"] } +cranelift-frontend = { version = "0.98" } +cranelift-module = { version = "0.98" } +cranelift-native = { version = "0.98" } +cranelift-jit = { version = "0.98", optional = true } +cranelift-object = { version = "0.98" } target-lexicon = "0.12.0" gimli = { version = "0.27.2", default-features = false, features = ["write"]} object = { version = "0.30.3", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } -indexmap = "1.9.3" +indexmap = "2.0.0" libloading = { version = "0.7.3", optional = true } smallvec = "1.8.1" diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md index 9469feea0cb..62eaef359af 100644 --- a/compiler/rustc_codegen_cranelift/Readme.md +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -65,12 +65,12 @@ to `./build/host/stage2/bin/`. Note that you would need to do this every time yo 5. Copy cargo from another toolchain: `cp $(rustup which cargo) .build/<your hostname triple>/stage2/bin/cargo` * Another option is to build it at step 3 and copy with other executables at step 4. 6. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`. -7. (Windows only) compile y.rs: `rustc +stage2 -O y.rs`. -8. You need to prefix every `./y.rs` (or `y` if you built `y.rs`) command by `rustup run stage2` to make cg_clif use your local changes in rustc. +7. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`. +8. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc. - * `rustup run stage2 ./y.rs prepare` - * `rustup run stage2 ./y.rs build` - * (Optional) run tests: `rustup run stage2 ./y.rs test` + * `rustup run stage2 ./y.sh prepare` + * `rustup run stage2 ./y.sh build` + * (Optional) run tests: `rustup run stage2 ./y.sh test` 9. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`. ## Configuration diff --git a/compiler/rustc_codegen_cranelift/build_system/bench.rs b/compiler/rustc_codegen_cranelift/build_system/bench.rs index 2bb11800034..cec608ea042 100644 --- a/compiler/rustc_codegen_cranelift/build_system/bench.rs +++ b/compiler/rustc_codegen_cranelift/build_system/bench.rs @@ -1,4 +1,5 @@ use std::env; +use std::io::Write; use std::path::Path; use super::path::{Dirs, RelPath}; @@ -30,6 +31,12 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); + let mut gha_step_summary = if let Ok(file) = std::env::var("GITHUB_STEP_SUMMARY") { + Some(std::fs::OpenOptions::new().append(true).open(file).unwrap()) + } else { + None + }; + eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); let cargo_clif = RelPath::DIST .to_path(dirs) @@ -60,36 +67,64 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { target_dir = target_dir.display(), ); + let bench_compile_markdown = RelPath::DIST.to_path(dirs).join("bench_compile.md"); + let bench_compile = hyperfine_command( 1, bench_runs, Some(&clean_cmd), - &[&llvm_build_cmd, &clif_build_cmd, &clif_build_opt_cmd], + &[ + ("cargo build", &llvm_build_cmd), + ("cargo-clif build", &clif_build_cmd), + ("cargo-clif build --release", &clif_build_opt_cmd), + ], + &bench_compile_markdown, ); spawn_and_wait(bench_compile); + if let Some(gha_step_summary) = gha_step_summary.as_mut() { + gha_step_summary.write_all(b"## Compile ebobby/simple-raytracer\n\n").unwrap(); + gha_step_summary.write_all(&std::fs::read(bench_compile_markdown).unwrap()).unwrap(); + gha_step_summary.write_all(b"\n").unwrap(); + } + eprintln!("[BENCH RUN] ebobby/simple-raytracer"); + let bench_run_markdown = RelPath::DIST.to_path(dirs).join("bench_run.md"); + + let raytracer_cg_llvm = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_llvm", + "bin", + )); + let raytracer_cg_clif = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_clif", + "bin", + )); + let raytracer_cg_clif_opt = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_clif_opt", + "bin", + )); let mut bench_run = hyperfine_command( 0, bench_runs, None, &[ - Path::new(".") - .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_llvm", "bin")) - .to_str() - .unwrap(), - Path::new(".") - .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif", "bin")) - .to_str() - .unwrap(), - Path::new(".") - .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif_opt", "bin")) - .to_str() - .unwrap(), + ("", raytracer_cg_llvm.to_str().unwrap()), + ("", raytracer_cg_clif.to_str().unwrap()), + ("", raytracer_cg_clif_opt.to_str().unwrap()), ], + &bench_run_markdown, ); bench_run.current_dir(RelPath::BUILD.to_path(dirs)); spawn_and_wait(bench_run); + + if let Some(gha_step_summary) = gha_step_summary.as_mut() { + gha_step_summary.write_all(b"## Run ebobby/simple-raytracer\n\n").unwrap(); + gha_step_summary.write_all(&std::fs::read(bench_run_markdown).unwrap()).unwrap(); + gha_step_summary.write_all(b"\n").unwrap(); + } } diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs index 6855c1a7fc5..1c5db23299d 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; -use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler}; +use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler, LogGroup}; pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); @@ -13,6 +13,8 @@ pub(crate) fn build_backend( bootstrap_host_compiler: &Compiler, use_unstable_features: bool, ) -> PathBuf { + let _group = LogGroup::guard("Build backend"); + let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); maybe_incremental(&mut cmd); diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs index 74bba9ed5eb..04097936d03 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs @@ -6,6 +6,7 @@ use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; use super::utils::{ maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, + LogGroup, }; use super::{CodegenBackend, SysrootKind}; @@ -22,6 +23,8 @@ pub(crate) fn build_sysroot( rustup_toolchain_name: Option<&str>, target_triple: String, ) -> Compiler { + let _guard = LogGroup::guard("Build sysroot"); + eprintln!("[BUILD] sysroot {:?}", sysroot_kind); DIST_DIR.ensure_fresh(dirs); @@ -251,7 +254,10 @@ fn build_clif_sysroot_for_triple( rustflags .push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap())); if channel == "release" { - rustflags.push_str(" -Zmir-opt-level=3"); + // Incremental compilation by default disables mir inlining. This leads to both a decent + // compile perf and a significant runtime perf regression. As such forcefully enable mir + // inlining. + rustflags.push_str(" -Zinline-mir"); } compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs index e31e39a483f..3ee2e8f4a4e 100644 --- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs +++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs @@ -27,6 +27,7 @@ pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { STDLIB_SRC.to_path(dirs).join("Cargo.toml"), r#" [workspace] +resolver = "1" members = ["./library/sysroot"] [patch.crates-io] diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs index 08d8f708c7d..0254d18cf7c 100644 --- a/compiler/rustc_codegen_cranelift/build_system/tests.rs +++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs @@ -3,7 +3,7 @@ use super::config; use super::path::{Dirs, RelPath}; use super::prepare::{apply_patches, GitRepo}; use super::rustc_info::get_default_sysroot; -use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; +use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, LogGroup}; use super::{CodegenBackend, SysrootKind}; use std::env; use std::ffi::OsStr; @@ -21,6 +21,7 @@ struct TestCase { enum TestCaseCmd { Custom { func: &'static dyn Fn(&TestRunner<'_>) }, BuildLib { source: &'static str, crate_types: &'static str }, + BuildBin { source: &'static str }, BuildBinAndRun { source: &'static str, args: &'static [&'static str] }, JitBin { source: &'static str, args: &'static str }, } @@ -39,6 +40,10 @@ impl TestCase { Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } } } + const fn build_bin(config: &'static str, source: &'static str) -> Self { + Self { config, cmd: TestCaseCmd::BuildBin { source } } + } + const fn build_bin_and_run( config: &'static str, source: &'static str, @@ -92,6 +97,7 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]), TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]), TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]), + TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"), ]; // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is @@ -119,8 +125,8 @@ pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", - "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", - "800548f8000e31bd", + "7c7dbe0c505ccbc02ff30c1e37381ab1d47bf46f", + "5bcc9c544f6fa7bd", "portable-simd", ); @@ -380,15 +386,17 @@ impl<'a> TestRunner<'a> { let tag = tag.to_uppercase(); let is_jit_test = tag == "JIT"; - if !config::get_bool(config) + let _guard = if !config::get_bool(config) || (is_jit_test && !self.jit_supported) || self.skip_tests.contains(&config) { eprintln!("[{tag}] {testname} (skipped)"); continue; } else { + let guard = LogGroup::guard(&format!("[{tag}] {testname}")); eprintln!("[{tag}] {testname}"); - } + guard + }; match *cmd { TestCaseCmd::Custom { func } => func(self), @@ -405,6 +413,13 @@ impl<'a> TestRunner<'a> { ]); } } + TestCaseCmd::BuildBin { source } => { + if self.use_unstable_features { + self.run_rustc([source]); + } else { + self.run_rustc([source, "--cfg", "no_unstable_features"]); + } + } TestCaseCmd::BuildBinAndRun { source, args } => { if self.use_unstable_features { self.run_rustc([source]); diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs index 41fc366e290..97c82d501c5 100644 --- a/compiler/rustc_codegen_cranelift/build_system/utils.rs +++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs @@ -3,6 +3,7 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{self, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; use super::path::{Dirs, RelPath}; @@ -136,10 +137,13 @@ pub(crate) fn hyperfine_command( warmup: u64, runs: u64, prepare: Option<&str>, - cmds: &[&str], + cmds: &[(&str, &str)], + markdown_export: &Path, ) -> Command { let mut bench = Command::new("hyperfine"); + bench.arg("--export-markdown").arg(markdown_export); + if warmup != 0 { bench.arg("--warmup").arg(warmup.to_string()); } @@ -152,7 +156,12 @@ pub(crate) fn hyperfine_command( bench.arg("--prepare").arg(prepare); } - bench.args(cmds); + for &(name, cmd) in cmds { + if name != "" { + bench.arg("-n").arg(name); + } + bench.arg(cmd); + } bench } @@ -167,6 +176,8 @@ pub(crate) fn git_command<'a>(repo_dir: impl Into<Option<&'a Path>>, cmd: &str) .arg("user.email=dummy@example.com") .arg("-c") .arg("core.autocrlf=false") + .arg("-c") + .arg("commit.gpgSign=false") .arg(cmd); if let Some(repo_dir) = repo_dir.into() { git_cmd.current_dir(repo_dir); @@ -259,6 +270,33 @@ pub(crate) fn is_ci_opt() -> bool { env::var("CI_OPT").is_ok() } +static IN_GROUP: AtomicBool = AtomicBool::new(false); +pub(crate) struct LogGroup { + is_gha: bool, +} + +impl LogGroup { + pub(crate) fn guard(name: &str) -> LogGroup { + let is_gha = env::var("GITHUB_ACTIONS").is_ok(); + + assert!(!IN_GROUP.swap(true, Ordering::SeqCst)); + if is_gha { + eprintln!("::group::{name}"); + } + + LogGroup { is_gha } + } +} + +impl Drop for LogGroup { + fn drop(&mut self) { + if self.is_gha { + eprintln!("::endgroup::"); + } + IN_GROUP.store(false, Ordering::SeqCst); + } +} + pub(crate) fn maybe_incremental(cmd: &mut Command) { if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") { // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt index d6e3924a24d..fa1c9f4259c 100644 --- a/compiler/rustc_codegen_cranelift/config.txt +++ b/compiler/rustc_codegen_cranelift/config.txt @@ -41,6 +41,7 @@ aot.track-caller-attribute aot.float-minmax-pass aot.mod_bench aot.issue-72793 +aot.issue-59326 testsuite.extended_sysroot test.rust-random/rand diff --git a/compiler/rustc_codegen_cranelift/example/issue-59326.rs b/compiler/rustc_codegen_cranelift/example/issue-59326.rs new file mode 100644 index 00000000000..70b7c94e15c --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/issue-59326.rs @@ -0,0 +1,27 @@ +// Based on https://github.com/rust-lang/rust/blob/689511047a75a30825e367d4fd45c74604d0b15e/tests/ui/issues/issue-59326.rs#L1 +// check-pass +trait Service { + type S; +} + +trait Framing { + type F; +} + +impl Framing for () { + type F = (); +} + +trait HttpService<F: Framing>: Service<S = F::F> {} + +type BoxService = Box<dyn HttpService<(), S = ()>>; + +fn build_server<F: FnOnce() -> BoxService>(_: F) {} + +fn make_server<F: Framing>() -> Box<dyn HttpService<F, S = F::F>> { + unimplemented!() +} + +fn main() { + build_server(|| make_server()) +} diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs index 79ca4c03985..9ecc4c5dd5b 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs @@ -547,7 +547,9 @@ impl<T> Box<T> { impl<T: ?Sized, A> Drop for Box<T, A> { fn drop(&mut self) { // inner value is dropped by compiler - libc::free(self.0.pointer.0 as *mut u8); + unsafe { + libc::free(self.0.pointer.0 as *mut u8); + } } } diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index 1bf0ff64c92..490cc2404f6 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -1,4 +1,12 @@ -#![feature(core_intrinsics, generators, generator_trait, is_sorted, repr_simd)] +#![feature( + core_intrinsics, + generators, + generator_trait, + is_sorted, + repr_simd, + tuple_trait, + unboxed_closures +)] #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; @@ -155,12 +163,34 @@ fn main() { } foo(I64X2(0, 0)); + + transmute_fat_pointer(); + + rust_call_abi(); } fn panic(_: u128) { panic!(); } +use std::mem::transmute; + +#[cfg(target_pointer_width = "32")] +type TwoPtrs = i64; +#[cfg(target_pointer_width = "64")] +type TwoPtrs = i128; + +fn transmute_fat_pointer() -> TwoPtrs { + unsafe { transmute::<_, TwoPtrs>("true !") } +} + +extern "rust-call" fn rust_call_abi_callee<T: std::marker::Tuple>(_: T) {} + +fn rust_call_abi() { + rust_call_abi_callee(()); + rust_call_abi_callee((1, 2)); +} + #[repr(simd)] struct I64X2(i64, i64); diff --git a/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch index 1d5479bedde..a650e10110b 100644 --- a/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch +++ b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch @@ -10,6 +10,18 @@ Cranelift doesn't support them yet library/core/tests/atomic.rs | 4 --- 4 files changed, 4 insertions(+), 50 deletions(-) +diff --git a/lib.rs b/lib.rs +index 897a5e9..331f66f 100644 +--- a/lib.rs ++++ b/lib.rs +@@ -93,7 +93,6 @@ + #![feature(const_option)] + #![feature(const_option_ext)] + #![feature(const_result)] +-#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))] + #![feature(int_roundings)] + #![feature(slice_group_by)] + #![feature(split_array)] diff --git a/atomic.rs b/atomic.rs index b735957..ea728b6 100644 --- a/atomic.rs diff --git a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch index 45f73f36b93..646928893e9 100644 --- a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch +++ b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch @@ -38,9 +38,9 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index d9de37e..8293fce 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs -@@ -2234,46 +2234,6 @@ atomic_int! { - "AtomicU64::new(0)", - u64 AtomicU64 ATOMIC_U64_INIT +@@ -2996,42 +2996,6 @@ atomic_int! { + 8, + u64 AtomicU64 } -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { @@ -53,14 +53,12 @@ index d9de37e..8293fce 100644 - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), -- unstable(feature = "integer_atomics", issue = "99069"), - cfg_attr(not(test), rustc_diagnostic_item = "AtomicI128"), - "i128", - "#![feature(integer_atomics)]\n\n", - atomic_min, atomic_max, - 16, -- "AtomicI128::new(0)", -- i128 AtomicI128 ATOMIC_I128_INIT +- i128 AtomicI128 -} -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { @@ -73,16 +71,15 @@ index d9de37e..8293fce 100644 - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), -- unstable(feature = "integer_atomics", issue = "99069"), - cfg_attr(not(test), rustc_diagnostic_item = "AtomicU128"), - "u128", - "#![feature(integer_atomics)]\n\n", - atomic_umin, atomic_umax, - 16, -- "AtomicU128::new(0)", -- u128 AtomicU128 ATOMIC_U128_INIT +- u128 AtomicU128 -} + #[cfg(target_has_atomic_load_store = "ptr")] macro_rules! atomic_int_ptr_sized { ( $($target_pointer_width:literal $align:literal)* ) => { $( -- diff --git a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml index 1dde9e54d7e..aea47bdfba2 100644 --- a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml +++ b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ "compiler_builtins", "gimli", @@ -35,6 +35,12 @@ dependencies = [ ] [[package]] +name = "allocator-api2" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" + +[[package]] name = "auxv" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -68,9 +74,9 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.93" +version = "0.1.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76630810d973ecea3dbf611e1b7aecfb1012751ef1ff8de3998f89014a166781" +checksum = "6866e0f3638013234db3c89ead7a14d278354338e7237257407500009012b23f" dependencies = [ "cc", "rustc-std-workspace-core", @@ -145,10 +151,11 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ + "allocator-api2", "compiler_builtins", "rustc-std-workspace-alloc", "rustc-std-workspace-core", @@ -156,9 +163,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -186,9 +193,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", "compiler_builtins", @@ -198,9 +205,9 @@ dependencies = [ [[package]] name = "object" -version = "0.30.4" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ "compiler_builtins", "memchr", diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain index fa3a10b9adc..34514658359 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-06-15" +channel = "nightly-2023-07-22" components = ["rust-src", "rustc-dev", "llvm-tools"] diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs index 99b97be24e6..f73b2012684 100644 --- a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs +++ b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs @@ -40,14 +40,22 @@ fn main() { "cargo" }; - let args: Vec<_> = match env::args().nth(1).as_deref() { + let mut args = env::args().skip(1).collect::<Vec<_>>(); + if args.get(0).map(|arg| &**arg) == Some("clif") { + // Avoid infinite recursion when invoking `cargo-clif` as cargo subcommand using + // `cargo clif`. + args.remove(0); + } + + let args: Vec<_> = match args.get(0).map(|arg| &**arg) { Some("jit") => { env::set_var( "RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic", ); + args.remove(0); IntoIterator::into_iter(["rustc".to_string()]) - .chain(env::args().skip(2)) + .chain(args) .chain([ "--".to_string(), "-Zunstable-options".to_string(), @@ -60,8 +68,9 @@ fn main() { "RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic", ); + args.remove(0); IntoIterator::into_iter(["rustc".to_string()]) - .chain(env::args().skip(2)) + .chain(args) .chain([ "--".to_string(), "-Zunstable-options".to_string(), @@ -69,7 +78,7 @@ fn main() { ]) .collect() } - _ => env::args().skip(1).collect(), + _ => args, }; #[cfg(unix)] diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh index 15b16b42be5..e6bbac647e5 100644 --- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh +++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh @@ -10,7 +10,8 @@ git fetch git checkout -- . git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')" -git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-stdlib-*.patch +git -c user.name=Dummy -c user.email=dummy@example.com -c commit.gpgSign=false \ + am ../patches/*-stdlib-*.patch git apply - <<EOF diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml @@ -51,7 +52,7 @@ popd # FIXME remove once inline asm is fully supported export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc" -export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)" +export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build/stdlib; pwd)" # Allow the testsuite to use llvm tools host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index a7920cc54ea..83cbe0db633 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -32,6 +32,8 @@ rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR # missing features # ================ +rm -r tests/run-make/comment-section # cg_clif doesn't yet write the .comment section + # requires stack unwinding # FIXME add needs-unwind to this test rm -r tests/run-make/libtest-junit @@ -98,8 +100,11 @@ rm -r tests/run-make/sepcomp-inlining # same rm -r tests/run-make/sepcomp-separate # same rm -r tests/run-make/sepcomp-cci-copies # same rm -r tests/run-make/volatile-intrinsics # same +rm -r tests/run-make/llvm-ident # same +rm -r tests/run-make/no-builtins-attribute # same rm tests/ui/abi/stack-protector.rs # requires stack protector support rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes +rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific # giving different but possibly correct results # ============================================= @@ -118,6 +123,7 @@ rm tests/ui/suggestions/derive-trait-for-method-call.rs # same rm tests/ui/typeck/issue-46112.rs # same rm tests/ui/consts/const_cmp_type_id.rs # same rm tests/ui/consts/issue-73976-monomorphic.rs # same +rm tests/ui/rfcs/rfc-3348-c-string-literals/non-ascii.rs # same # rustdoc-clif passes extra args, suppressing the help message when no args are passed rm -r tests/run-make/issue-88756-default-output @@ -143,6 +149,8 @@ rm -r tests/run-make/used # same rm -r tests/run-make/no-alloc-shim rm -r tests/run-make/emit-to-stdout +rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported + # bugs in the test suite # ====================== rm tests/ui/backtrace.rs # TODO warning @@ -150,6 +158,8 @@ rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd +rm tests/ui/panic-handler/weak-lang-item-2.rs # Will be fixed by #113568 + cp ../dist/bin/rustdoc-clif ../dist/bin/rustdoc # some tests expect bin/rustdoc to exist # prevent $(RUSTDOC) from picking up the sysroot built by x.py. It conflicts with the one used by diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index eddb479073c..2c038f22ca9 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -445,9 +445,14 @@ pub(crate) fn codegen_terminator_call<'tcx>( // Unpack arguments tuple for closures let mut args = if fn_sig.abi() == Abi::RustCall { - assert_eq!(args.len(), 2, "rust-call abi requires two arguments"); - let self_arg = codegen_call_argument_operand(fx, &args[0]); - let pack_arg = codegen_call_argument_operand(fx, &args[1]); + let (self_arg, pack_arg) = match args { + [pack_arg] => (None, codegen_call_argument_operand(fx, pack_arg)), + [self_arg, pack_arg] => ( + Some(codegen_call_argument_operand(fx, self_arg)), + codegen_call_argument_operand(fx, pack_arg), + ), + _ => panic!("rust-call abi requires one or two arguments"), + }; let tupled_arguments = match pack_arg.value.layout().ty.kind() { ty::Tuple(ref tupled_arguments) => tupled_arguments, @@ -455,7 +460,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( }; let mut args = Vec::with_capacity(1 + tupled_arguments.len()); - args.push(self_arg); + args.extend(self_arg); for i in 0..tupled_arguments.len() { args.push(CallArgument { value: pack_arg.value.value_field(fx, FieldIdx::new(i)), diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index e05f2146f0c..522dd7189fe 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -7,6 +7,8 @@ use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; use cranelift_codegen::ir::UserFuncName; +use cranelift_codegen::CodegenError; +use cranelift_module::ModuleError; use crate::constant::ConstantCx; use crate::debuginfo::FunctionDebugContext; @@ -172,7 +174,21 @@ pub(crate) fn compile_fn( // Define function cx.profiler.generic_activity("define function").run(|| { context.want_disasm = cx.should_write_ir; - module.define_function(codegened_func.func_id, context).unwrap(); + match module.define_function(codegened_func.func_id, context) { + Ok(()) => {} + Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => { + let handler = rustc_session::EarlyErrorHandler::new( + rustc_session::config::ErrorOutputType::default(), + ); + handler.early_error(format!( + "backend implementation limit exceeded while compiling {name}", + name = codegened_func.symbol_name + )); + } + Err(err) => { + panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name); + } + } }); if cx.should_write_ir { @@ -356,7 +372,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { codegen_panic_inner( fx, - rustc_hir::LangItem::PanicBoundsCheck, + rustc_hir::LangItem::PanicMisalignedPointerDereference, &[required, found, location], source_info.span, ); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs index 1b454b6667c..50bc7a127af 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -165,7 +165,7 @@ impl FunctionDebugContext { for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() { debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start); if !loc.is_default() { - let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap(); + let source_loc = self.source_loc_set[loc.bits() as usize]; create_row_for_span(debug_context, source_loc); } else { create_row_for_span(debug_context, self.function_source_loc); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs index 3a7421d8b30..8a4b1cccf14 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -38,7 +38,7 @@ pub(crate) struct DebugContext { pub(crate) struct FunctionDebugContext { entry_id: UnitEntryId, function_source_loc: (FileId, u64, u64), - source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>, + source_loc_set: IndexSet<(FileId, u64, u64)>, } impl DebugContext { diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 41e24acefbe..3ea38842148 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -114,9 +114,9 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .iter() .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) .flatten() - .collect::<FxHashMap<_, (_, _)>>() + .collect::<FxHashMap<_, _>>() .into_iter() - .collect::<Vec<(_, (_, _))>>(); + .collect::<Vec<(_, _)>>(); tcx.sess.time("codegen mono items", || { super::predefine_mono_items(tcx, &mut jit_module, &mono_items); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index 18162fb5ab2..fdd27a454e0 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -18,6 +18,20 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( // Spin loop hint } + // Used by is_x86_feature_detected!(); + "llvm.x86.xgetbv" => { + // FIXME use the actual xgetbv instruction + intrinsic_args!(fx, args => (v); intrinsic); + + let v = v.load_scalar(fx); + + // As of writing on XCR0 exists + fx.bcx.ins().trapnz(v, TrapCode::UnreachableCodeReached); + + let res = fx.bcx.ins().iconst(types::I64, 1 /* bit 0 must be set */); + ret.write_cvalue(fx, CValue::by_val(res, fx.layout_of(fx.tcx.types.i64))); + } + // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" @@ -53,7 +67,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); } - "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { + "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { let (x, y, kind) = match args { [x, y, kind] => (x, y, kind), _ => bug!("wrong number of args for intrinsic {intrinsic}"), @@ -66,18 +80,95 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let flt_cc = match kind .try_to_bits(Size::from_bytes(1)) .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) + .try_into() + .unwrap() { - 0 => FloatCC::Equal, - 1 => FloatCC::LessThan, - 2 => FloatCC::LessThanOrEqual, - 7 => FloatCC::Ordered, - 3 => FloatCC::Unordered, - 4 => FloatCC::NotEqual, - 5 => FloatCC::UnorderedOrGreaterThanOrEqual, - 6 => FloatCC::UnorderedOrGreaterThan, + _CMP_EQ_OQ | _CMP_EQ_OS => FloatCC::Equal, + _CMP_LT_OS | _CMP_LT_OQ => FloatCC::LessThan, + _CMP_LE_OS | _CMP_LE_OQ => FloatCC::LessThanOrEqual, + _CMP_UNORD_Q | _CMP_UNORD_S => FloatCC::Unordered, + _CMP_NEQ_UQ | _CMP_NEQ_US => FloatCC::NotEqual, + _CMP_NLT_US | _CMP_NLT_UQ => FloatCC::UnorderedOrGreaterThanOrEqual, + _CMP_NLE_US | _CMP_NLE_UQ => FloatCC::UnorderedOrGreaterThan, + _CMP_ORD_Q | _CMP_ORD_S => FloatCC::Ordered, + _CMP_EQ_UQ | _CMP_EQ_US => FloatCC::UnorderedOrEqual, + _CMP_NGE_US | _CMP_NGE_UQ => FloatCC::UnorderedOrLessThan, + _CMP_NGT_US | _CMP_NGT_UQ => FloatCC::UnorderedOrLessThanOrEqual, + _CMP_FALSE_OQ | _CMP_FALSE_OS => todo!(), + _CMP_NEQ_OQ | _CMP_NEQ_OS => FloatCC::OrderedNotEqual, + _CMP_GE_OS | _CMP_GE_OQ => FloatCC::GreaterThanOrEqual, + _CMP_GT_OS | _CMP_GT_OQ => FloatCC::GreaterThan, + _CMP_TRUE_UQ | _CMP_TRUE_US => todo!(), + kind => unreachable!("kind {:?}", kind), }; + // Copied from stdarch + /// Equal (ordered, non-signaling) + const _CMP_EQ_OQ: i32 = 0x00; + /// Less-than (ordered, signaling) + const _CMP_LT_OS: i32 = 0x01; + /// Less-than-or-equal (ordered, signaling) + const _CMP_LE_OS: i32 = 0x02; + /// Unordered (non-signaling) + const _CMP_UNORD_Q: i32 = 0x03; + /// Not-equal (unordered, non-signaling) + const _CMP_NEQ_UQ: i32 = 0x04; + /// Not-less-than (unordered, signaling) + const _CMP_NLT_US: i32 = 0x05; + /// Not-less-than-or-equal (unordered, signaling) + const _CMP_NLE_US: i32 = 0x06; + /// Ordered (non-signaling) + const _CMP_ORD_Q: i32 = 0x07; + /// Equal (unordered, non-signaling) + const _CMP_EQ_UQ: i32 = 0x08; + /// Not-greater-than-or-equal (unordered, signaling) + const _CMP_NGE_US: i32 = 0x09; + /// Not-greater-than (unordered, signaling) + const _CMP_NGT_US: i32 = 0x0a; + /// False (ordered, non-signaling) + const _CMP_FALSE_OQ: i32 = 0x0b; + /// Not-equal (ordered, non-signaling) + const _CMP_NEQ_OQ: i32 = 0x0c; + /// Greater-than-or-equal (ordered, signaling) + const _CMP_GE_OS: i32 = 0x0d; + /// Greater-than (ordered, signaling) + const _CMP_GT_OS: i32 = 0x0e; + /// True (unordered, non-signaling) + const _CMP_TRUE_UQ: i32 = 0x0f; + /// Equal (ordered, signaling) + const _CMP_EQ_OS: i32 = 0x10; + /// Less-than (ordered, non-signaling) + const _CMP_LT_OQ: i32 = 0x11; + /// Less-than-or-equal (ordered, non-signaling) + const _CMP_LE_OQ: i32 = 0x12; + /// Unordered (signaling) + const _CMP_UNORD_S: i32 = 0x13; + /// Not-equal (unordered, signaling) + const _CMP_NEQ_US: i32 = 0x14; + /// Not-less-than (unordered, non-signaling) + const _CMP_NLT_UQ: i32 = 0x15; + /// Not-less-than-or-equal (unordered, non-signaling) + const _CMP_NLE_UQ: i32 = 0x16; + /// Ordered (signaling) + const _CMP_ORD_S: i32 = 0x17; + /// Equal (unordered, signaling) + const _CMP_EQ_US: i32 = 0x18; + /// Not-greater-than-or-equal (unordered, non-signaling) + const _CMP_NGE_UQ: i32 = 0x19; + /// Not-greater-than (unordered, non-signaling) + const _CMP_NGT_UQ: i32 = 0x1a; + /// False (ordered, signaling) + const _CMP_FALSE_OS: i32 = 0x1b; + /// Not-equal (ordered, signaling) + const _CMP_NEQ_OS: i32 = 0x1c; + /// Greater-than-or-equal (ordered, non-signaling) + const _CMP_GE_OQ: i32 = 0x1d; + /// Greater-than (ordered, non-signaling) + const _CMP_GT_OQ: i32 = 0x1e; + /// True (unordered, signaling) + const _CMP_TRUE_US: i32 = 0x1f; + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| { let res_lane = match lane_ty.kind() { ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane), @@ -103,6 +194,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.sse2.psrai.d" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.sse2.psrai.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.sse2.pslli.d" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -137,6 +245,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.sse2.psrai.w" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.sse2.psrai.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.sse2.pslli.w" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -171,6 +296,57 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.avx.psrai.d" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.psrai.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } + "llvm.x86.sse2.psrli.q" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.psrli.q imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 64 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } + "llvm.x86.sse2.pslli.q" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.pslli.q imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 64 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.avx.pslli.d" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -205,6 +381,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.avx2.psrai.w" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.psrai.w imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.avx2.pslli.w" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -313,7 +506,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.place_lane(fx, 2).to_ptr().store(fx, res_2, MemFlags::trusted()); ret.place_lane(fx, 3).to_ptr().store(fx, res_3, MemFlags::trusted()); } - "llvm.x86.sse2.storeu.dq" => { + "llvm.x86.sse2.storeu.dq" | "llvm.x86.sse2.storeu.pd" => { intrinsic_args!(fx, args => (mem_addr, a); intrinsic); let mem_addr = mem_addr.load_scalar(fx); @@ -321,17 +514,45 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); } - "llvm.x86.addcarry.64" => { + "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { + let a = match args { + [a] => a, + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().iabs(lane) + }); + } + "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => { intrinsic_args!(fx, args => (c_in, a, b); intrinsic); let c_in = c_in.load_scalar(fx); - llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b); + let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); + + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); + let val = CValue::by_val_pair(cb_out, c, layout); + ret.write_cvalue(fx, val); } - "llvm.x86.subborrow.64" => { + "llvm.x86.addcarryx.u32" | "llvm.x86.addcarryx.u64" => { + intrinsic_args!(fx, args => (c_in, a, b, out); intrinsic); + let c_in = c_in.load_scalar(fx); + + let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); + + Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted()); + ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8))); + } + "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => { intrinsic_args!(fx, args => (b_in, a, b); intrinsic); let b_in = b_in.load_scalar(fx); - llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b); + let (cb_out, c) = llvm_add_sub(fx, BinOp::Sub, b_in, a, b); + + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); + let val = CValue::by_val_pair(cb_out, c, layout); + ret.write_cvalue(fx, val); } _ => { fx.tcx @@ -356,21 +577,11 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( fn llvm_add_sub<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, - ret: CPlace<'tcx>, cb_in: Value, a: CValue<'tcx>, b: CValue<'tcx>, -) { - assert_eq!( - a.layout().ty, - fx.tcx.types.u64, - "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64" - ); - assert_eq!( - b.layout().ty, - fx.tcx.types.u64, - "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64" - ); +) -> (Value, Value) { + assert_eq!(a.layout().ty, b.layout().ty); // c + carry -> c + first intermediate carry or borrow respectively let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b); @@ -378,15 +589,14 @@ fn llvm_add_sub<'tcx>( let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx); // c + carry -> c + second intermediate carry or borrow respectively - let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in); - let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64)); - let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64); + let clif_ty = fx.clif_type(a.layout().ty).unwrap(); + let cb_in_as_int = fx.bcx.ins().uextend(clif_ty, cb_in); + let cb_in_as_int = CValue::by_val(cb_in_as_int, fx.layout_of(a.layout().ty)); + let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_int); let (c, cb1) = int1.load_scalar_pair(fx); // carry0 | carry1 -> carry or borrow respectively let cb_out = fx.bcx.ins().bor(cb0, cb1); - let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, fx.tcx.types.u64])); - let val = CValue::by_val_pair(cb_out, c, layout); - ret.write_cvalue(fx, val); + (cb_out, c) } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index da8ab361331..e3006b253b7 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -647,12 +647,13 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); ret.write_cvalue(fx, val); } - sym::volatile_store | sym::unaligned_volatile_store => { + sym::volatile_store | sym::unaligned_volatile_store | sym::nontemporal_store => { intrinsic_args!(fx, args => (ptr, val); intrinsic); let ptr = ptr.load_scalar(fx); // Cranelift treats stores as volatile by default // FIXME correctly handle unaligned_volatile_store + // FIXME actually do nontemporal stores if requested let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); } diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 0de2dccda71..ebd153cb71d 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -268,8 +268,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar }; flags_builder.set("tls_model", tls_model).unwrap(); - flags_builder.set("enable_simd", "true").unwrap(); - flags_builder.set("enable_llvm_abi_extensions", "true").unwrap(); use rustc_session::config::OptLevel; diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index ec0b61a7ce5..ff95141ce90 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -2,6 +2,8 @@ use crate::prelude::*; +use rustc_middle::ty::FnSig; + use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::immediates::Offset32; @@ -160,6 +162,7 @@ impl<'tcx> CValue<'tcx> { } /// Load a value with layout.abi of scalar + #[track_caller] pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value { let layout = self.1; match self.0 { @@ -182,6 +185,7 @@ impl<'tcx> CValue<'tcx> { } /// Load a value pair with layout.abi of scalar pair + #[track_caller] pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) { let layout = self.1; match self.0 { @@ -583,17 +587,25 @@ impl<'tcx> CPlace<'tcx> { let dst_layout = self.layout(); match self.inner { CPlaceInner::Var(_local, var) => { - let data = CValue(from.0, dst_layout).load_scalar(fx); + let data = match from.1.abi { + Abi::Scalar(_) => CValue(from.0, dst_layout).load_scalar(fx), + _ => { + let (ptr, meta) = from.force_stack(fx); + assert!(meta.is_none()); + CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar(fx) + } + }; let dst_ty = fx.clif_type(self.layout().ty).unwrap(); transmute_scalar(fx, var, data, dst_ty); } CPlaceInner::VarPair(_local, var1, var2) => { - let (data1, data2) = if from.layout().ty == dst_layout.ty { - CValue(from.0, dst_layout).load_scalar_pair(fx) - } else { - let (ptr, meta) = from.force_stack(fx); - assert!(meta.is_none()); - CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx) + let (data1, data2) = match from.1.abi { + Abi::ScalarPair(_, _) => CValue(from.0, dst_layout).load_scalar_pair(fx), + _ => { + let (ptr, meta) = from.force_stack(fx); + assert!(meta.is_none()); + CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx) + } }; let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap(); transmute_scalar(fx, var1, data1, dst_ty1); @@ -607,30 +619,38 @@ impl<'tcx> CPlace<'tcx> { let mut flags = MemFlags::new(); flags.set_notrap(); - match from.layout().abi { - Abi::Scalar(_) => { - let val = from.load_scalar(fx); - to_ptr.store(fx, val, flags); - return; - } - Abi::ScalarPair(a_scalar, b_scalar) => { - let (value, extra) = from.load_scalar_pair(fx); - let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); - to_ptr.store(fx, value, flags); - to_ptr.offset(fx, b_offset).store(fx, extra, flags); - return; - } - _ => {} - } match from.0 { CValueInner::ByVal(val) => { to_ptr.store(fx, val, flags); } - CValueInner::ByValPair(_, _) => { - bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi); - } + CValueInner::ByValPair(val1, val2) => match from.layout().abi { + Abi::ScalarPair(a_scalar, b_scalar) => { + let b_offset = + scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + to_ptr.store(fx, val1, flags); + to_ptr.offset(fx, b_offset).store(fx, val2, flags); + } + _ => bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi), + }, CValueInner::ByRef(from_ptr, None) => { + match from.layout().abi { + Abi::Scalar(_) => { + let val = from.load_scalar(fx); + to_ptr.store(fx, val, flags); + return; + } + Abi::ScalarPair(a_scalar, b_scalar) => { + let b_offset = + scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let (val1, val2) = from.load_scalar_pair(fx); + to_ptr.store(fx, val1, flags); + to_ptr.offset(fx, b_offset).store(fx, val2, flags); + return; + } + _ => {} + } + let from_addr = from_ptr.get_addr(fx); let to_addr = to_ptr.get_addr(fx); let src_layout = from.1; @@ -815,11 +835,42 @@ pub(crate) fn assert_assignable<'tcx>( ParamEnv::reveal_all(), from_ty.fn_sig(fx.tcx), ); + let FnSig { + inputs_and_output: types_from, + c_variadic: c_variadic_from, + unsafety: unsafety_from, + abi: abi_from, + } = from_sig; let to_sig = fx .tcx .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx)); + let FnSig { + inputs_and_output: types_to, + c_variadic: c_variadic_to, + unsafety: unsafety_to, + abi: abi_to, + } = to_sig; + let mut types_from = types_from.iter(); + let mut types_to = types_to.iter(); + loop { + match (types_from.next(), types_to.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => break, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + assert_eq!( + c_variadic_from, c_variadic_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + assert_eq!( + unsafety_from, unsafety_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); assert_eq!( - from_sig, to_sig, + abi_from, abi_to, "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", from_sig, to_sig, fx, ); diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs index a30bce0a313..84d57838512 100644 --- a/compiler/rustc_codegen_gcc/src/type_of.rs +++ b/compiler/rustc_codegen_gcc/src/type_of.rs @@ -339,8 +339,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> { return pointee; } - let assume_valid_ptr = true; - let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset, assume_valid_ptr); + let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset); cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); result diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 29dd53ff763..2dbd467cc84 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -411,8 +411,8 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) { return pointee; } - let assume_valid_ptr = true; - let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset, assume_valid_ptr); + + let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset); cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); result diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 0c7b8a79612..92792ab6477 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -501,7 +501,22 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { }); // #73631: closures inherit `#[target_feature]` annotations - if tcx.features().target_feature_11 && tcx.is_closure(did.to_def_id()) { + // + // If this closure is marked `#[inline(always)]`, simply skip adding `#[target_feature]`. + // + // At this point, `unsafe` has already been checked and `#[target_feature]` only affects codegen. + // Emitting both `#[inline(always)]` and `#[target_feature]` can potentially result in an + // ICE, because LLVM errors when the function fails to be inlined due to a target feature + // mismatch. + // + // Using `#[inline(always)]` implies that this closure will most likely be inlined into + // its parent function, which effectively inherits the features anyway. Boxing this closure + // would result in this closure being compiled without the inherited target features, but this + // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute. + if tcx.features().target_feature_11 + && tcx.is_closure(did.to_def_id()) + && codegen_fn_attrs.inline != InlineAttr::Always + { let owner_id = tcx.parent(did.to_def_id()); if tcx.def_kind(owner_id).has_codegen_attrs() { codegen_fn_attrs diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl index 8833f55831c..d8eade5bd2a 100644 --- a/compiler/rustc_const_eval/messages.ftl +++ b/compiler/rustc_const_eval/messages.ftl @@ -244,6 +244,7 @@ const_eval_not_enough_caller_args = const_eval_null_box = {$front_matter}: encountered a null box const_eval_null_fn_ptr = {$front_matter}: encountered a null function pointer const_eval_null_ref = {$front_matter}: encountered a null reference +const_eval_nullable_ptr_out_of_range = {$front_matter}: encountered a potentially null pointer, but expected something that cannot possibly fail to be {$in_range} const_eval_nullary_intrinsic_fail = could not evaluate nullary intrinsic diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 0a9a47b2837..267795a6cb4 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -1,6 +1,7 @@ use rustc_hir::def::DefKind; use rustc_hir::{LangItem, CRATE_HIR_ID}; use rustc_middle::mir; +use rustc_middle::mir::interpret::PointerArithmetic; use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::lint::builtin::INVALID_ALIGNMENT; @@ -16,7 +17,7 @@ use rustc_ast::Mutability; use rustc_hir::def_id::DefId; use rustc_middle::mir::AssertMessage; use rustc_span::symbol::{sym, Symbol}; -use rustc_target::abi::{Align, HasDataLayout as _, Size}; +use rustc_target::abi::{Align, Size}; use rustc_target::spec::abi::Abi as CallAbi; use crate::errors::{LongRunning, LongRunningWarn}; @@ -303,8 +304,8 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { Ok(ControlFlow::Break(())) } else { // Not alignable in const, return `usize::MAX`. - let usize_max = self.data_layout().target_usize_max(); - self.write_scalar(Scalar::from_target_usize(usize_max, self), dest)?; + let usize_max = Scalar::from_target_usize(self.target_usize_max(), self); + self.write_scalar(usize_max, dest)?; self.return_to_block(ret)?; Ok(ControlFlow::Break(())) } @@ -332,7 +333,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { // Inequality with integers other than null can never be known for sure. (Scalar::Int(int), ptr @ Scalar::Ptr(..)) | (ptr @ Scalar::Ptr(..), Scalar::Int(int)) - if int.is_null() && !self.ptr_scalar_range(ptr)?.contains(&0) => + if int.is_null() && !self.scalar_may_be_null(ptr)? => { 0 } diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs index 61ce695ccd2..ca38cce710e 100644 --- a/compiler/rustc_const_eval/src/errors.rs +++ b/compiler/rustc_const_eval/src/errors.rs @@ -617,6 +617,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { MutableRefInConst => const_eval_mutable_ref_in_const, NullFnPtr => const_eval_null_fn_ptr, NeverVal => const_eval_never_val, + NullablePtrOutOfRange { .. } => const_eval_nullable_ptr_out_of_range, PtrOutOfRange { .. } => const_eval_ptr_out_of_range, OutOfRange { .. } => const_eval_out_of_range, UnsafeCell => const_eval_unsafe_cell, @@ -731,7 +732,9 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { | InvalidFnPtr { value } => { err.set_arg("value", value); } - PtrOutOfRange { range, max_value } => add_range_arg(range, max_value, handler, err), + NullablePtrOutOfRange { range, max_value } | PtrOutOfRange { range, max_value } => { + add_range_arg(range, max_value, handler, err) + } OutOfRange { range, max_value, value } => { err.set_arg("value", value); add_range_arg(range, max_value, handler, err); diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs index 99ea0ab18bc..f23a455c2ca 100644 --- a/compiler/rustc_const_eval/src/interpret/discriminant.rs +++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs @@ -2,7 +2,8 @@ use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt}; use rustc_middle::{mir, ty}; -use rustc_target::abi::{self, TagEncoding, VariantIdx, Variants, WrappingRange}; +use rustc_target::abi::{self, TagEncoding}; +use rustc_target::abi::{VariantIdx, Variants}; use super::{ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Scalar}; @@ -179,24 +180,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // discriminant (encoded in niche/tag) and variant index are the same. let variants_start = niche_variants.start().as_u32(); let variants_end = niche_variants.end().as_u32(); - let variants_len = u128::from(variants_end - variants_start); let variant = match tag_val.try_to_int() { Err(dbg_val) => { // So this is a pointer then, and casting to an int failed. // Can only happen during CTFE. - // The pointer and niches ranges must be disjoint, then we know - // this is the untagged variant (as the value is not in the niche). - // Everything else, we conservatively reject. - let range = self.ptr_scalar_range(tag_val)?; - let niches = WrappingRange { - start: niche_start, - end: niche_start.wrapping_add(variants_len), - }; - if niches.overlaps_range(range) { + // The niche must be just 0, and the ptr not null, then we know this is + // okay. Everything else, we conservatively reject. + let ptr_valid = niche_start == 0 + && variants_start == variants_end + && !self.scalar_may_be_null(tag_val)?; + if !ptr_valid { throw_ub!(InvalidTag(dbg_val)) - } else { - untagged_variant } + untagged_variant } Ok(tag_bits) => { let tag_bits = tag_bits.assert_bits(tag_layout.size); @@ -209,7 +205,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let variant_index_relative = variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size); // Check if this is in the range that indicates an actual discriminant. - if variant_index_relative <= variants_len { + if variant_index_relative <= u128::from(variants_end - variants_start) { let variant_index_relative = u32::try_from(variant_index_relative) .expect("we checked that this fits into a u32"); // Then computing the absolute variant idx should not overflow any more. diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 8ec9a71bf3a..04cae23f852 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -5,7 +5,9 @@ use rustc_hir::def_id::DefId; use rustc_middle::mir::{ self, - interpret::{Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, Scalar}, + interpret::{ + Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar, + }, BinOp, NonDivergingIntrinsic, }; use rustc_middle::ty; @@ -13,7 +15,7 @@ use rustc_middle::ty::layout::{LayoutOf as _, ValidityRequirement}; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_span::symbol::{sym, Symbol}; -use rustc_target::abi::{Abi, Align, HasDataLayout as _, Primitive, Size}; +use rustc_target::abi::{Abi, Align, Primitive, Size}; use super::{ util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy, @@ -359,12 +361,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { )?; // Perform division by size to compute return value. - let dl = self.data_layout(); let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned { - assert!(0 <= dist && dist <= dl.target_isize_max()); + assert!(0 <= dist && dist <= self.target_isize_max()); usize_layout } else { - assert!(dl.target_isize_min() <= dist && dist <= dl.target_isize_max()); + assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max()); isize_layout }; let pointee_layout = self.layout_of(instance_args.type_at(0))?; diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 29fc5ffcfe7..7b44a20ef03 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -10,7 +10,6 @@ use std::assert_matches::assert_matches; use std::borrow::Cow; use std::collections::VecDeque; use std::fmt; -use std::ops::RangeInclusive; use std::ptr; use rustc_ast::Mutability; @@ -1223,34 +1222,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Machine pointer introspection. impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { - /// Turn a pointer-sized scalar into a (non-empty) range of possible values. + /// Test if this value might be null. /// If the machine does not support ptr-to-int casts, this is conservative. - pub fn ptr_scalar_range( - &self, - scalar: Scalar<M::Provenance>, - ) -> InterpResult<'tcx, RangeInclusive<u64>> { - if let Ok(int) = scalar.to_target_usize(self) { - return Ok(int..=int); - } - - let ptr = scalar.to_pointer(self)?; - - // Can only happen during CTFE. - Ok(match self.ptr_try_get_alloc_id(ptr) { - Ok((alloc_id, offset, _)) => { - let offset = offset.bytes(); - let (size, align, _) = self.get_alloc_info(alloc_id); - let dl = self.data_layout(); - if offset > size.bytes() { - // If the pointer is out-of-bounds, we do not have a - // meaningful range to return. - 0..=dl.target_usize_max() - } else { - let (min, max) = dl.address_range_for(size, align); - (min + offset)..=(max + offset) + pub fn scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool> { + Ok(match scalar.try_to_int() { + Ok(int) => int.is_null(), + Err(_) => { + // Can only happen during CTFE. + let ptr = scalar.to_pointer(self)?; + match self.ptr_try_get_alloc_id(ptr) { + Ok((alloc_id, offset, _)) => { + let (size, _align, _kind) = self.get_alloc_info(alloc_id); + // If the pointer is out-of-bounds, it may be null. + // Note that one-past-the-end (offset == size) is still inbounds, and never null. + offset > size + } + Err(_offset) => bug!("a non-int scalar is always a pointer"), } } - Err(_offset) => bug!("a non-int scalar is always a pointer"), }) } diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 108394d224b..21c655988a0 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -19,7 +19,9 @@ use rustc_middle::mir::interpret::{ use rustc_middle::ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_span::symbol::{sym, Symbol}; -use rustc_target::abi::{Abi, FieldIdx, Scalar as ScalarAbi, Size, VariantIdx, Variants}; +use rustc_target::abi::{ + Abi, FieldIdx, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange, +}; use std::hash::Hash; @@ -552,7 +554,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // FIXME: Check if the signature matches } else { // Otherwise (for standalone Miri), we have to still check it to be non-null. - if self.ecx.ptr_scalar_range(value)?.contains(&0) { + if self.ecx.scalar_may_be_null(value)? { throw_validation_failure!(self.path, NullFnPtr); } } @@ -593,36 +595,46 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' ) -> InterpResult<'tcx> { let size = scalar_layout.size(self.ecx); let valid_range = scalar_layout.valid_range(self.ecx); + let WrappingRange { start, end } = valid_range; let max_value = size.unsigned_int_max(); - assert!(valid_range.end <= max_value); - match scalar.try_to_int() { - Ok(int) => { - // We have an explicit int: check it against the valid range. - let bits = int.assert_bits(size); - if valid_range.contains(bits) { - Ok(()) - } else { - throw_validation_failure!( - self.path, - OutOfRange { value: format!("{bits}"), range: valid_range, max_value } - ) - } - } + assert!(end <= max_value); + let bits = match scalar.try_to_int() { + Ok(int) => int.assert_bits(size), Err(_) => { // So this is a pointer then, and casting to an int failed. // Can only happen during CTFE. - // We check if the possible addresses are compatible with the valid range. - let range = self.ecx.ptr_scalar_range(scalar)?; - if valid_range.contains_range(range) { - Ok(()) + // We support 2 kinds of ranges here: full range, and excluding zero. + if start == 1 && end == max_value { + // Only null is the niche. So make sure the ptr is NOT null. + if self.ecx.scalar_may_be_null(scalar)? { + throw_validation_failure!( + self.path, + NullablePtrOutOfRange { range: valid_range, max_value } + ) + } else { + return Ok(()); + } + } else if scalar_layout.is_always_valid(self.ecx) { + // Easy. (This is reachable if `enforce_number_validity` is set.) + return Ok(()); } else { - // Reject conservatively, because the pointer *could* have a bad value. + // Conservatively, we reject, because the pointer *could* have a bad + // value. throw_validation_failure!( self.path, PtrOutOfRange { range: valid_range, max_value } ) } } + }; + // Now compare. + if valid_range.contains(bits) { + Ok(()) + } else { + throw_validation_failure!( + self.path, + OutOfRange { value: format!("{bits}"), range: valid_range, max_value } + ) } } } diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index ea0d90dbd51..7b2bed302db 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -58,11 +58,10 @@ impl<'tcx> MirPass<'tcx> for Validator { .iterate_to_fixpoint() .into_results_cursor(body); - let mut checker = TypeChecker { + let mut cfg_checker = CfgChecker { when: &self.when, body, tcx, - param_env, mir_phase, unwind_edge_count: 0, reachable_blocks: traversal::reachable_as_bitset(body), @@ -70,13 +69,17 @@ impl<'tcx> MirPass<'tcx> for Validator { place_cache: FxHashSet::default(), value_cache: FxHashSet::default(), }; - checker.visit_body(body); - checker.check_cleanup_control_flow(); + cfg_checker.visit_body(body); + cfg_checker.check_cleanup_control_flow(); + + for (location, msg) in validate_types(tcx, self.mir_phase, param_env, body) { + cfg_checker.fail(location, msg); + } if let MirPhase::Runtime(_) = body.phase { if let ty::InstanceDef::Item(_) = body.source.instance { if body.has_free_regions() { - checker.fail( + cfg_checker.fail( Location::START, format!("Free regions in optimized {} MIR", body.phase.name()), ); @@ -86,11 +89,10 @@ impl<'tcx> MirPass<'tcx> for Validator { } } -struct TypeChecker<'a, 'tcx> { +struct CfgChecker<'a, 'tcx> { when: &'a str, body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, mir_phase: MirPhase, unwind_edge_count: usize, reachable_blocks: BitSet<BasicBlock>, @@ -99,7 +101,7 @@ struct TypeChecker<'a, 'tcx> { value_cache: FxHashSet<u128>, } -impl<'a, 'tcx> TypeChecker<'a, 'tcx> { +impl<'a, 'tcx> CfgChecker<'a, 'tcx> { #[track_caller] fn fail(&self, location: Location, msg: impl AsRef<str>) { let span = self.body.source_info(location).span; @@ -248,30 +250,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { UnwindAction::Unreachable | UnwindAction::Terminate => (), } } - - /// Check if src can be assigned into dest. - /// This is not precise, it will accept some incorrect assignments. - fn mir_assign_valid_types(&self, src: Ty<'tcx>, dest: Ty<'tcx>) -> bool { - // Fast path before we normalize. - if src == dest { - // Equal types, all is good. - return true; - } - - // We sometimes have to use `defining_opaque_types` for subtyping - // to succeed here and figuring out how exactly that should work - // is annoying. It is harmless enough to just not validate anything - // in that case. We still check this after analysis as all opaque - // types have been revealed at this point. - if (src, dest).has_opaque_types() { - return true; - } - - crate::util::is_subtype(self.tcx, self.param_env, src, dest) - } } -impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { +impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { if self.body.local_decls.get(local).is_none() { self.fail( @@ -296,6 +277,275 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + match &statement.kind { + StatementKind::Assign(box (dest, rvalue)) => { + // FIXME(JakobDegen): Check this for all rvalues, not just this one. + if let Rvalue::Use(Operand::Copy(src) | Operand::Move(src)) = rvalue { + // The sides of an assignment must not alias. Currently this just checks whether + // the places are identical. + if dest == src { + self.fail( + location, + "encountered `Assign` statement with overlapping memory", + ); + } + } + } + StatementKind::AscribeUserType(..) => { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + self.fail( + location, + "`AscribeUserType` should have been removed after drop lowering phase", + ); + } + } + StatementKind::FakeRead(..) => { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + self.fail( + location, + "`FakeRead` should have been removed after drop lowering phase", + ); + } + } + StatementKind::SetDiscriminant { .. } => { + if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { + self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); + } + } + StatementKind::Deinit(..) => { + if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { + self.fail(location, "`Deinit`is not allowed until deaggregation"); + } + } + StatementKind::Retag(kind, _) => { + // FIXME(JakobDegen) The validator should check that `self.mir_phase < + // DropsLowered`. However, this causes ICEs with generation of drop shims, which + // seem to fail to set their `MirPhase` correctly. + if matches!(kind, RetagKind::Raw | RetagKind::TwoPhase) { + self.fail(location, format!("explicit `{:?}` is forbidden", kind)); + } + } + StatementKind::StorageLive(local) => { + // We check that the local is not live when entering a `StorageLive` for it. + // Technically, violating this restriction is only UB and not actually indicative + // of not well-formed MIR. This means that an optimization which turns MIR that + // already has UB into MIR that fails this check is not necessarily wrong. However, + // we have no such optimizations at the moment, and so we include this check anyway + // to help us catch bugs. If you happen to write an optimization that might cause + // this to incorrectly fire, feel free to remove this check. + if self.reachable_blocks.contains(location.block) { + self.storage_liveness.seek_before_primary_effect(location); + let locals_with_storage = self.storage_liveness.get(); + if locals_with_storage.contains(*local) { + self.fail( + location, + format!("StorageLive({local:?}) which already has storage here"), + ); + } + } + } + StatementKind::StorageDead(_) + | StatementKind::Intrinsic(_) + | StatementKind::Coverage(_) + | StatementKind::ConstEvalCounter + | StatementKind::PlaceMention(..) + | StatementKind::Nop => {} + } + + self.super_statement(statement, location); + } + + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { + match &terminator.kind { + TerminatorKind::Goto { target } => { + self.check_edge(location, *target, EdgeKind::Normal); + } + TerminatorKind::SwitchInt { targets, discr: _ } => { + for (_, target) in targets.iter() { + self.check_edge(location, target, EdgeKind::Normal); + } + self.check_edge(location, targets.otherwise(), EdgeKind::Normal); + + self.value_cache.clear(); + self.value_cache.extend(targets.iter().map(|(value, _)| value)); + let has_duplicates = targets.iter().len() != self.value_cache.len(); + if has_duplicates { + self.fail( + location, + format!( + "duplicated values in `SwitchInt` terminator: {:?}", + terminator.kind, + ), + ); + } + } + TerminatorKind::Drop { target, unwind, .. } => { + self.check_edge(location, *target, EdgeKind::Normal); + self.check_unwind_edge(location, *unwind); + } + TerminatorKind::Call { args, destination, target, unwind, .. } => { + if let Some(target) = target { + self.check_edge(location, *target, EdgeKind::Normal); + } + self.check_unwind_edge(location, *unwind); + + // The call destination place and Operand::Move place used as an argument might be + // passed by a reference to the callee. Consequently they must be non-overlapping. + // Currently this simply checks for duplicate places. + self.place_cache.clear(); + self.place_cache.insert(destination.as_ref()); + let mut has_duplicates = false; + for arg in args { + if let Operand::Move(place) = arg { + has_duplicates |= !self.place_cache.insert(place.as_ref()); + } + } + + if has_duplicates { + self.fail( + location, + format!( + "encountered overlapping memory in `Call` terminator: {:?}", + terminator.kind, + ), + ); + } + } + TerminatorKind::Assert { target, unwind, .. } => { + self.check_edge(location, *target, EdgeKind::Normal); + self.check_unwind_edge(location, *unwind); + } + TerminatorKind::Yield { resume, drop, .. } => { + if self.body.generator.is_none() { + self.fail(location, "`Yield` cannot appear outside generator bodies"); + } + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + self.fail(location, "`Yield` should have been replaced by generator lowering"); + } + self.check_edge(location, *resume, EdgeKind::Normal); + if let Some(drop) = drop { + self.check_edge(location, *drop, EdgeKind::Normal); + } + } + TerminatorKind::FalseEdge { real_target, imaginary_target } => { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + self.fail( + location, + "`FalseEdge` should have been removed after drop elaboration", + ); + } + self.check_edge(location, *real_target, EdgeKind::Normal); + self.check_edge(location, *imaginary_target, EdgeKind::Normal); + } + TerminatorKind::FalseUnwind { real_target, unwind } => { + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + self.fail( + location, + "`FalseUnwind` should have been removed after drop elaboration", + ); + } + self.check_edge(location, *real_target, EdgeKind::Normal); + self.check_unwind_edge(location, *unwind); + } + TerminatorKind::InlineAsm { destination, unwind, .. } => { + if let Some(destination) = destination { + self.check_edge(location, *destination, EdgeKind::Normal); + } + self.check_unwind_edge(location, *unwind); + } + TerminatorKind::GeneratorDrop => { + if self.body.generator.is_none() { + self.fail(location, "`GeneratorDrop` cannot appear outside generator bodies"); + } + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + self.fail( + location, + "`GeneratorDrop` should have been replaced by generator lowering", + ); + } + } + TerminatorKind::Resume | TerminatorKind::Terminate => { + let bb = location.block; + if !self.body.basic_blocks[bb].is_cleanup { + self.fail( + location, + "Cannot `Resume` or `Terminate` from non-cleanup basic block", + ) + } + } + TerminatorKind::Return => { + let bb = location.block; + if self.body.basic_blocks[bb].is_cleanup { + self.fail(location, "Cannot `Return` from cleanup basic block") + } + } + TerminatorKind::Unreachable => {} + } + + self.super_terminator(terminator, location); + } + + fn visit_source_scope(&mut self, scope: SourceScope) { + if self.body.source_scopes.get(scope).is_none() { + self.tcx.sess.diagnostic().delay_span_bug( + self.body.span, + format!( + "broken MIR in {:?} ({}):\ninvalid source scope {:?}", + self.body.source.instance, self.when, scope, + ), + ); + } + } +} + +pub fn validate_types<'tcx>( + tcx: TyCtxt<'tcx>, + mir_phase: MirPhase, + param_env: ty::ParamEnv<'tcx>, + body: &Body<'tcx>, +) -> Vec<(Location, String)> { + let mut type_checker = TypeChecker { body, tcx, param_env, mir_phase, failures: Vec::new() }; + type_checker.visit_body(body); + type_checker.failures +} + +struct TypeChecker<'a, 'tcx> { + body: &'a Body<'tcx>, + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + mir_phase: MirPhase, + failures: Vec<(Location, String)>, +} + +impl<'a, 'tcx> TypeChecker<'a, 'tcx> { + fn fail(&mut self, location: Location, msg: impl Into<String>) { + self.failures.push((location, msg.into())); + } + + /// Check if src can be assigned into dest. + /// This is not precise, it will accept some incorrect assignments. + fn mir_assign_valid_types(&self, src: Ty<'tcx>, dest: Ty<'tcx>) -> bool { + // Fast path before we normalize. + if src == dest { + // Equal types, all is good. + return true; + } + + // We sometimes have to use `defining_opaque_types` for subtyping + // to succeed here and figuring out how exactly that should work + // is annoying. It is harmless enough to just not validate anything + // in that case. We still check this after analysis as all opaque + // types have been revealed at this point. + if (src, dest).has_opaque_types() { + return true; + } + + crate::util::is_subtype(self.tcx, self.param_env, src, dest) + } +} + +impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { // This check is somewhat expensive, so only run it when -Zvalidate-mir is passed. if self.tcx.sess.opts.unstable_opts.validate_mir @@ -342,10 +592,10 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } ProjectionElem::Field(f, ty) => { let parent_ty = place_ref.ty(&self.body.local_decls, self.tcx); - let fail_out_of_bounds = |this: &Self, location| { + let fail_out_of_bounds = |this: &mut Self, location| { this.fail(location, format!("Out of bounds field {:?} for {:?}", f, parent_ty)); }; - let check_equal = |this: &Self, location, f_ty| { + let check_equal = |this: &mut Self, location, f_ty| { if !this.mir_assign_valid_types(ty, f_ty) { this.fail( location, @@ -440,9 +690,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } fn visit_var_debug_info(&mut self, debuginfo: &VarDebugInfo<'tcx>) { - let check_place = |place: Place<'_>| { + let check_place = |this: &mut Self, place: Place<'_>| { if place.projection.iter().any(|p| !p.can_use_in_debuginfo()) { - self.fail( + this.fail( START_BLOCK.start_location(), format!("illegal place {:?} in debuginfo for {:?}", place, debuginfo.name), ); @@ -451,7 +701,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { match debuginfo.value { VarDebugInfoContents::Const(_) => {} VarDebugInfoContents::Place(place) => { - check_place(place); + check_place(self, place); if debuginfo.references != 0 && place.projection.last() == Some(&PlaceElem::Deref) { self.fail( START_BLOCK.start_location(), @@ -461,7 +711,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } VarDebugInfoContents::Composite { ty, ref fragments } => { for f in fragments { - check_place(f.contents); + check_place(self, f.contents); if ty.is_union() || ty.is_enum() { self.fail( START_BLOCK.start_location(), @@ -718,7 +968,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } Rvalue::NullaryOp(NullOp::OffsetOf(fields), container) => { - let fail_out_of_bounds = |this: &Self, location, field, ty| { + let fail_out_of_bounds = |this: &mut Self, location, field, ty| { this.fail(location, format!("Out of bounds field {field:?} for {ty:?}")); }; @@ -894,26 +1144,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.fail(location, format!("explicit `{:?}` is forbidden", kind)); } } - StatementKind::StorageLive(local) => { - // We check that the local is not live when entering a `StorageLive` for it. - // Technically, violating this restriction is only UB and not actually indicative - // of not well-formed MIR. This means that an optimization which turns MIR that - // already has UB into MIR that fails this check is not necessarily wrong. However, - // we have no such optimizations at the moment, and so we include this check anyway - // to help us catch bugs. If you happen to write an optimization that might cause - // this to incorrectly fire, feel free to remove this check. - if self.reachable_blocks.contains(location.block) { - self.storage_liveness.seek_before_primary_effect(location); - let locals_with_storage = self.storage_liveness.get(); - if locals_with_storage.contains(*local) { - self.fail( - location, - format!("StorageLive({local:?}) which already has storage here"), - ); - } - } - } - StatementKind::StorageDead(_) + StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) | StatementKind::Coverage(_) | StatementKind::ConstEvalCounter | StatementKind::PlaceMention(..) @@ -925,9 +1157,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { match &terminator.kind { - TerminatorKind::Goto { target } => { - self.check_edge(location, *target, EdgeKind::Normal); - } TerminatorKind::SwitchInt { targets, discr } => { let switch_ty = discr.ty(&self.body.local_decls, self.tcx); @@ -941,36 +1170,16 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { other => bug!("unhandled type: {:?}", other), }); - for (value, target) in targets.iter() { + for (value, _) in targets.iter() { if Scalar::<()>::try_from_uint(value, size).is_none() { self.fail( location, format!("the value {:#x} is not a proper {:?}", value, switch_ty), ) } - - self.check_edge(location, target, EdgeKind::Normal); - } - self.check_edge(location, targets.otherwise(), EdgeKind::Normal); - - self.value_cache.clear(); - self.value_cache.extend(targets.iter().map(|(value, _)| value)); - let has_duplicates = targets.iter().len() != self.value_cache.len(); - if has_duplicates { - self.fail( - location, - format!( - "duplicated values in `SwitchInt` terminator: {:?}", - terminator.kind, - ), - ); } } - TerminatorKind::Drop { target, unwind, .. } => { - self.check_edge(location, *target, EdgeKind::Normal); - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::Call { func, args, destination, target, unwind, .. } => { + TerminatorKind::Call { func, .. } => { let func_ty = func.ty(&self.body.local_decls, self.tcx); match func_ty.kind() { ty::FnPtr(..) | ty::FnDef(..) => {} @@ -979,34 +1188,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { format!("encountered non-callable type {} in `Call` terminator", func_ty), ), } - if let Some(target) = target { - self.check_edge(location, *target, EdgeKind::Normal); - } - self.check_unwind_edge(location, *unwind); - - // The call destination place and Operand::Move place used as an argument might be - // passed by a reference to the callee. Consequently they must be non-overlapping. - // Currently this simply checks for duplicate places. - self.place_cache.clear(); - self.place_cache.insert(destination.as_ref()); - let mut has_duplicates = false; - for arg in args { - if let Operand::Move(place) = arg { - has_duplicates |= !self.place_cache.insert(place.as_ref()); - } - } - - if has_duplicates { - self.fail( - location, - format!( - "encountered overlapping memory in `Call` terminator: {:?}", - terminator.kind, - ), - ); - } } - TerminatorKind::Assert { cond, target, unwind, .. } => { + TerminatorKind::Assert { cond, .. } => { let cond_ty = cond.ty(&self.body.local_decls, self.tcx); if cond_ty != self.tcx.types.bool { self.fail( @@ -1017,88 +1200,20 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ), ); } - self.check_edge(location, *target, EdgeKind::Normal); - self.check_unwind_edge(location, *unwind); } - TerminatorKind::Yield { resume, drop, .. } => { - if self.body.generator.is_none() { - self.fail(location, "`Yield` cannot appear outside generator bodies"); - } - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail(location, "`Yield` should have been replaced by generator lowering"); - } - self.check_edge(location, *resume, EdgeKind::Normal); - if let Some(drop) = drop { - self.check_edge(location, *drop, EdgeKind::Normal); - } - } - TerminatorKind::FalseEdge { real_target, imaginary_target } => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`FalseEdge` should have been removed after drop elaboration", - ); - } - self.check_edge(location, *real_target, EdgeKind::Normal); - self.check_edge(location, *imaginary_target, EdgeKind::Normal); - } - TerminatorKind::FalseUnwind { real_target, unwind } => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`FalseUnwind` should have been removed after drop elaboration", - ); - } - self.check_edge(location, *real_target, EdgeKind::Normal); - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::InlineAsm { destination, unwind, .. } => { - if let Some(destination) = destination { - self.check_edge(location, *destination, EdgeKind::Normal); - } - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::GeneratorDrop => { - if self.body.generator.is_none() { - self.fail(location, "`GeneratorDrop` cannot appear outside generator bodies"); - } - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`GeneratorDrop` should have been replaced by generator lowering", - ); - } - } - TerminatorKind::Resume | TerminatorKind::Terminate => { - let bb = location.block; - if !self.body.basic_blocks[bb].is_cleanup { - self.fail( - location, - "Cannot `Resume` or `Terminate` from non-cleanup basic block", - ) - } - } - TerminatorKind::Return => { - let bb = location.block; - if self.body.basic_blocks[bb].is_cleanup { - self.fail(location, "Cannot `Return` from cleanup basic block") - } - } - TerminatorKind::Unreachable => {} + TerminatorKind::Goto { .. } + | TerminatorKind::Drop { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } + | TerminatorKind::InlineAsm { .. } + | TerminatorKind::GeneratorDrop + | TerminatorKind::Resume + | TerminatorKind::Terminate + | TerminatorKind::Return + | TerminatorKind::Unreachable => {} } self.super_terminator(terminator, location); } - - fn visit_source_scope(&mut self, scope: SourceScope) { - if self.body.source_scopes.get(scope).is_none() { - self.tcx.sess.diagnostic().delay_span_bug( - self.body.span, - format!( - "broken MIR in {:?} ({}):\ninvalid source scope {:?}", - self.body.source.instance, self.when, scope, - ), - ); - } - } } diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs index 44f116ef2da..5e261f80387 100644 --- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs +++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs @@ -137,12 +137,6 @@ enum Scope<'a> { s: ScopeRef<'a>, }, - /// A scope which either determines unspecified lifetimes or errors - /// on them (e.g., due to ambiguity). - Elision { - s: ScopeRef<'a>, - }, - /// Use a specific lifetime (if `Some`) or leave it unset (to be /// inferred in a function body or potentially error outside one), /// for the default choice of lifetime in a trait object type. @@ -211,7 +205,6 @@ impl<'a> fmt::Debug for TruncatedScopeDebug<'a> { Scope::Body { id, s: _ } => { f.debug_struct("Body").field("id", id).field("s", &"..").finish() } - Scope::Elision { s: _ } => f.debug_struct("Elision").field("s", &"..").finish(), Scope::ObjectLifetimeDefault { lifetime, s: _ } => f .debug_struct("ObjectLifetimeDefault") .field("lifetime", lifetime) @@ -325,9 +318,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { break (vec![], BinderScopeType::Normal); } - Scope::Elision { s, .. } - | Scope::ObjectLifetimeDefault { s, .. } - | Scope::AnonConstBoundary { s } => { + Scope::ObjectLifetimeDefault { s, .. } | Scope::AnonConstBoundary { s } => { scope = s; } @@ -526,16 +517,12 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> { | hir::ItemKind::Macro(..) | hir::ItemKind::Mod(..) | hir::ItemKind::ForeignMod { .. } + | hir::ItemKind::Static(..) + | hir::ItemKind::Const(..) | hir::ItemKind::GlobalAsm(..) => { // These sorts of items have no lifetime parameters at all. intravisit::walk_item(self, item); } - hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => { - // No lifetime parameters, but implied 'static. - self.with(Scope::Elision { s: self.scope }, |this| { - intravisit::walk_item(this, item) - }); - } hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin: hir::OpaqueTyOrigin::TyAlias { .. }, .. @@ -727,12 +714,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> { // Elided lifetimes are not allowed in non-return // position impl Trait let scope = Scope::TraitRefBoundary { s: self.scope }; - self.with(scope, |this| { - let scope = Scope::Elision { s: this.scope }; - this.with(scope, |this| { - intravisit::walk_item(this, opaque_ty); - }) - }); + self.with(scope, |this| intravisit::walk_item(this, opaque_ty)); return; } @@ -1293,8 +1275,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { scope = s; } - Scope::Elision { s, .. } - | Scope::ObjectLifetimeDefault { s, .. } + Scope::ObjectLifetimeDefault { s, .. } | Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } | Scope::AnonConstBoundary { s } => { @@ -1357,7 +1338,6 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { Scope::Root { .. } => break, Scope::Binder { s, .. } | Scope::Body { s, .. } - | Scope::Elision { s, .. } | Scope::ObjectLifetimeDefault { s, .. } | Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } @@ -1409,8 +1389,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { scope = s; } - Scope::Elision { s, .. } - | Scope::ObjectLifetimeDefault { s, .. } + Scope::ObjectLifetimeDefault { s, .. } | Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } => { scope = s; @@ -1483,7 +1462,6 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { Scope::Root { .. } => break, Scope::Binder { s, .. } | Scope::Body { s, .. } - | Scope::Elision { s, .. } | Scope::ObjectLifetimeDefault { s, .. } | Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } @@ -1564,7 +1542,6 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { Scope::Body { .. } => break true, Scope::Binder { s, .. } - | Scope::Elision { s, .. } | Scope::ObjectLifetimeDefault { s, .. } | Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } @@ -1832,14 +1809,20 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { output: Option<&'tcx hir::Ty<'tcx>>, in_closure: bool, ) { - self.with(Scope::Elision { s: self.scope }, |this| { - for input in inputs { - this.visit_ty(input); - } - if !in_closure && let Some(output) = output { - this.visit_ty(output); - } - }); + self.with( + Scope::ObjectLifetimeDefault { + lifetime: Some(ResolvedArg::StaticLifetime), + s: self.scope, + }, + |this| { + for input in inputs { + this.visit_ty(input); + } + if !in_closure && let Some(output) = output { + this.visit_ty(output); + } + }, + ); if in_closure && let Some(output) = output { self.visit_ty(output); } @@ -1859,7 +1842,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> { scope = s; } - Scope::Root { .. } | Scope::Elision { .. } => break ResolvedArg::StaticLifetime, + Scope::Root { .. } => break ResolvedArg::StaticLifetime, Scope::Body { .. } | Scope::ObjectLifetimeDefault { lifetime: None, .. } => return, diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index aedc662b067..12124f14a82 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -28,7 +28,6 @@ use rustc_span::edition::{Edition, DEFAULT_EDITION}; use rustc_span::symbol::sym; use rustc_span::FileName; use rustc_span::SourceFileHashAlgorithm; -use rustc_target::abi::ReferenceNichePolicy; use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel}; use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel}; @@ -821,7 +820,6 @@ fn test_unstable_options_tracking_hash() { tracked!(profile_emit, Some(PathBuf::from("abc"))); tracked!(profile_sample_use, Some(PathBuf::from("abc"))); tracked!(profiler_runtime, "abc".to_string()); - tracked!(reference_niches, Some(ReferenceNichePolicy { size: true, align: false })); tracked!(relax_elf_relocations, Some(true)); tracked!(relro_level, Some(RelroLevel::Full)); tracked!(remap_cwd_prefix, Some(PathBuf::from("abc"))); diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 77c33336dff..a8815ee0908 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -301,7 +301,6 @@ provide! { tcx, def_id, other, cdata, is_profiler_runtime => { cdata.root.profiler_runtime } required_panic_strategy => { cdata.root.required_panic_strategy } panic_in_drop_strategy => { cdata.root.panic_in_drop_strategy } - reference_niches_policy => { cdata.root.reference_niches_policy } extern_crate => { let r = *cdata.extern_crate.lock(); r.map(|c| &*tcx.arena.alloc(c)) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 46571e7796d..ac86110f2bd 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -673,7 +673,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { stable_crate_id: tcx.def_path_hash(LOCAL_CRATE.as_def_id()).stable_crate_id(), required_panic_strategy: tcx.required_panic_strategy(LOCAL_CRATE), panic_in_drop_strategy: tcx.sess.opts.unstable_opts.panic_in_drop, - reference_niches_policy: tcx.reference_niches_policy(LOCAL_CRATE), edition: tcx.sess.edition(), has_global_allocator: tcx.has_global_allocator(LOCAL_CRATE), has_alloc_error_handler: tcx.has_alloc_error_handler(LOCAL_CRATE), diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 8bc2e0aa5a9..0bc16fc64ff 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -32,7 +32,7 @@ use rustc_span::edition::Edition; use rustc_span::hygiene::{ExpnIndex, MacroKind}; use rustc_span::symbol::{Ident, Symbol}; use rustc_span::{self, ExpnData, ExpnHash, ExpnId, Span}; -use rustc_target::abi::{FieldIdx, ReferenceNichePolicy, VariantIdx}; +use rustc_target::abi::{FieldIdx, VariantIdx}; use rustc_target::spec::{PanicStrategy, TargetTriple}; use std::marker::PhantomData; @@ -251,7 +251,6 @@ pub(crate) struct CrateRoot { stable_crate_id: StableCrateId, required_panic_strategy: Option<PanicStrategy>, panic_in_drop_strategy: PanicStrategy, - reference_niches_policy: ReferenceNichePolicy, edition: Edition, has_global_allocator: bool, has_alloc_error_handler: bool, diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index 1bcef17d73b..372452ea29a 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -388,6 +388,7 @@ pub enum ValidationErrorKind<'tcx> { MutableRefInConst, NullFnPtr, NeverVal, + NullablePtrOutOfRange { range: WrappingRange, max_value: u128 }, PtrOutOfRange { range: WrappingRange, max_value: u128 }, OutOfRange { value: String, range: WrappingRange, max_value: u128 }, UnsafeCell, diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs index c8133bcc387..65d04919357 100644 --- a/compiler/rustc_middle/src/mir/interpret/pointer.rs +++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs @@ -19,19 +19,33 @@ pub trait PointerArithmetic: HasDataLayout { #[inline(always)] fn max_size_of_val(&self) -> Size { - Size::from_bytes(self.data_layout().target_isize_max()) + Size::from_bytes(self.target_isize_max()) + } + + #[inline] + fn target_usize_max(&self) -> u64 { + self.pointer_size().unsigned_int_max().try_into().unwrap() + } + + #[inline] + fn target_isize_min(&self) -> i64 { + self.pointer_size().signed_int_min().try_into().unwrap() + } + + #[inline] + fn target_isize_max(&self) -> i64 { + self.pointer_size().signed_int_max().try_into().unwrap() } #[inline] fn target_usize_to_isize(&self, val: u64) -> i64 { - let dl = self.data_layout(); let val = val as i64; // Now wrap-around into the machine_isize range. - if val > dl.target_isize_max() { + if val > self.target_isize_max() { // This can only happen if the ptr size is < 64, so we know max_usize_plus_1 fits into // i64. - debug_assert!(dl.pointer_size.bits() < 64); - let max_usize_plus_1 = 1u128 << dl.pointer_size.bits(); + debug_assert!(self.pointer_size().bits() < 64); + let max_usize_plus_1 = 1u128 << self.pointer_size().bits(); val - i64::try_from(max_usize_plus_1).unwrap() } else { val @@ -44,7 +58,7 @@ pub trait PointerArithmetic: HasDataLayout { #[inline] fn truncate_to_ptr(&self, (val, over): (u64, bool)) -> (u64, bool) { let val = u128::from(val); - let max_ptr_plus_1 = 1u128 << self.data_layout().pointer_size.bits(); + let max_ptr_plus_1 = 1u128 << self.pointer_size().bits(); (u64::try_from(val % max_ptr_plus_1).unwrap(), over || val >= max_ptr_plus_1) } @@ -62,11 +76,11 @@ pub trait PointerArithmetic: HasDataLayout { let n = i.unsigned_abs(); if i >= 0 { let (val, over) = self.overflowing_offset(val, n); - (val, over || i > self.data_layout().target_isize_max()) + (val, over || i > self.target_isize_max()) } else { let res = val.overflowing_sub(n); let (val, over) = self.truncate_to_ptr(res); - (val, over || i < self.data_layout().target_isize_min()) + (val, over || i < self.target_isize_min()) } } diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index 9bf02267005..2c481745d98 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -111,11 +111,6 @@ impl EraseType >()]; } -impl EraseType for Result<ty::layout::TyAndNaiveLayout<'_>, &ty::layout::LayoutError<'_>> { - type Result = - [u8; size_of::<Result<ty::layout::TyAndNaiveLayout<'_>, &ty::layout::LayoutError<'_>>>()]; -} - impl EraseType for Result<ty::Const<'_>, mir::interpret::LitToConstError> { type Result = [u8; size_of::<Result<ty::Const<'static>, mir::interpret::LitToConstError>>()]; } @@ -296,7 +291,6 @@ trivial! { rustc_span::Symbol, rustc_span::symbol::Ident, rustc_target::spec::PanicStrategy, - rustc_target::abi::ReferenceNichePolicy, rustc_type_ir::Variance, u32, usize, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index b5b00b7b640..b36f0df78f1 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1394,18 +1394,6 @@ rustc_queries! { desc { "computing layout of `{}`", key.value } } - /// Computes the naive layout approximation of a type. Note that this implicitly - /// executes in "reveal all" mode, and will normalize the input type. - /// - /// Unlike `layout_of`, this doesn't look past references (beyond the `Pointee::Metadata` - /// projection), and as such can be called on generic types like `Option<&T>`. - query naive_layout_of( - key: ty::ParamEnvAnd<'tcx, Ty<'tcx>> - ) -> Result<ty::layout::TyAndNaiveLayout<'tcx>, &'tcx ty::layout::LayoutError<'tcx>> { - depth_limit - desc { "computing layout (naive) of `{}`", key.value } - } - /// Compute a `FnAbi` suitable for indirect calls, i.e. to `fn` pointers. /// /// NB: this doesn't handle virtual calls - those should use `fn_abi_of_instance` @@ -1481,11 +1469,6 @@ rustc_queries! { desc { "getting a crate's configured panic-in-drop strategy" } separate_provide_extern } - query reference_niches_policy(_: CrateNum) -> abi::ReferenceNichePolicy { - fatal_cycle - desc { "getting a crate's policy for size and alignment niches of references" } - separate_provide_extern - } query is_no_builtins(_: CrateNum) -> bool { fatal_cycle desc { "getting whether a crate has `#![no_builtins]`" } diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 26137e86fa0..62805d1e8b5 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -313,16 +313,7 @@ impl<'tcx> SizeSkeleton<'tcx> { ) -> Result<SizeSkeleton<'tcx>, &'tcx LayoutError<'tcx>> { debug_assert!(!ty.has_non_region_infer()); - // First, try computing an exact naive layout (this covers simple types with generic - // references, where a full static layout would fail). - if let Ok(layout) = tcx.naive_layout_of(param_env.and(ty)) { - if layout.exact { - return Ok(SizeSkeleton::Known(layout.size)); - } - } - - // Second, try computing a full static layout (this covers cases when the naive layout - // wasn't smart enough, but cannot deal with generic references). + // First try computing a static layout. let err = match tcx.layout_of(param_env.and(ty)) { Ok(layout) => { return Ok(SizeSkeleton::Known(layout.size)); @@ -336,7 +327,6 @@ impl<'tcx> SizeSkeleton<'tcx> { ) => return Err(e), }; - // Third, fall back to ad-hoc cases. match *ty.kind() { ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => { let non_zero = !ty.is_unsafe_ptr(); @@ -631,219 +621,6 @@ impl<T, E> MaybeResult<T> for Result<T, E> { pub type TyAndLayout<'tcx> = rustc_target::abi::TyAndLayout<'tcx, Ty<'tcx>>; -#[derive(Copy, Clone, Debug, HashStable)] -pub struct TyAndNaiveLayout<'tcx> { - pub ty: Ty<'tcx>, - pub layout: NaiveLayout, -} - -impl std::ops::Deref for TyAndNaiveLayout<'_> { - type Target = NaiveLayout; - fn deref(&self) -> &Self::Target { - &self.layout - } -} - -impl std::ops::DerefMut for TyAndNaiveLayout<'_> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.layout - } -} - -/// Extremely simplified approximation of a type's layout returned by the -/// `naive_layout_of` query. -#[derive(Copy, Clone, Debug, HashStable)] -pub struct NaiveLayout { - pub abi: NaiveAbi, - /// Niche information, required for tracking non-null enum optimizations. - pub niches: NaiveNiches, - /// An underestimate of the layout's size. - pub size: Size, - /// An underestimate of the layout's required alignment. - pub align: Align, - /// If `true`, `size` and `align` must be exact values. - pub exact: bool, -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq, HashStable)] -pub enum NaiveNiches { - None, - Some, - Maybe, -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq, HashStable)] -pub enum NaiveAbi { - /// A scalar layout, always implies `exact` and a non-zero `size`. - Scalar(Primitive), - /// An uninhabited layout. (needed to properly track `Scalar` and niches) - Uninhabited, - /// An unsized aggregate. (needed to properly track `Scalar` and niches) - Unsized, - /// Any other sized layout. - Sized, -} - -impl NaiveAbi { - #[inline] - pub fn as_aggregate(self) -> Self { - match self { - NaiveAbi::Scalar(_) => NaiveAbi::Sized, - _ => self, - } - } -} - -impl NaiveLayout { - /// The layout of an empty aggregate, e.g. `()`. - pub const EMPTY: Self = Self { - size: Size::ZERO, - align: Align::ONE, - exact: true, - abi: NaiveAbi::Sized, - niches: NaiveNiches::None, - }; - - /// Returns whether `self` is a valid approximation of the given full `layout`. - /// - /// This should always return `true` when both layouts are computed from the same type. - pub fn is_refined_by(&self, layout: Layout<'_>) -> bool { - if self.size > layout.size() || self.align > layout.align().abi { - return false; - } - - if let NaiveAbi::Scalar(prim) = self.abi { - if !self.exact - || self.size == Size::ZERO - || !matches!(layout.abi(), Abi::Scalar(s) if s.primitive() == prim) - { - return false; - } - } - - match (self.niches, layout.largest_niche()) { - (NaiveNiches::None, Some(_)) => return false, - (NaiveNiches::Some, None) => return false, - _ => (), - } - - !self.exact || (self.size, self.align) == (layout.size(), layout.align().abi) - } - - /// Returns if this layout is known to be pointer-like (`None` if uncertain) - /// - /// See the corresponding `Layout::is_pointer_like` method. - pub fn is_pointer_like(&self, dl: &TargetDataLayout) -> Option<bool> { - match self.abi { - NaiveAbi::Scalar(_) => { - assert!(self.exact); - Some(self.size == dl.pointer_size && self.align == dl.pointer_align.abi) - } - NaiveAbi::Uninhabited | NaiveAbi::Unsized => Some(false), - NaiveAbi::Sized if self.exact => Some(false), - NaiveAbi::Sized => None, - } - } - - /// Artificially lowers the alignment of this layout. - #[must_use] - #[inline] - pub fn packed(mut self, align: Align) -> Self { - if self.align > align { - self.align = align; - self.abi = self.abi.as_aggregate(); - } - self - } - - /// Artificially raises the alignment of this layout. - #[must_use] - #[inline] - pub fn align_to(mut self, align: Align) -> Self { - if align > self.align { - self.align = align; - self.abi = self.abi.as_aggregate(); - } - self - } - - /// Artificially makes this layout inexact. - #[must_use] - #[inline] - pub fn inexact(mut self) -> Self { - self.abi = self.abi.as_aggregate(); - self.exact = false; - self - } - - /// Pads this layout so that its size is a multiple of `align`. - #[must_use] - #[inline] - pub fn pad_to_align(mut self, align: Align) -> Self { - let new_size = self.size.align_to(align); - if new_size > self.size { - self.abi = self.abi.as_aggregate(); - self.size = new_size; - } - self - } - - /// Returns the layout of `self` immediately followed by `other`, without any - /// padding between them, as in a packed `struct` or tuple. - #[must_use] - #[inline] - pub fn concat(&self, other: &Self, dl: &TargetDataLayout) -> Option<Self> { - use NaiveAbi::*; - - let size = self.size.checked_add(other.size, dl)?; - let align = cmp::max(self.align, other.align); - let exact = self.exact && other.exact; - let abi = match (self.abi, other.abi) { - // The uninhabited and unsized ABIs override everything. - (Uninhabited, _) | (_, Uninhabited) => Uninhabited, - (Unsized, _) | (_, Unsized) => Unsized, - // A scalar struct must have a single non ZST-field. - (_, s @ Scalar(_)) if exact && self.size == Size::ZERO => s, - (s @ Scalar(_), _) if exact && other.size == Size::ZERO => s, - // Default case. - (_, _) => Sized, - }; - let niches = match (self.niches, other.niches) { - (NaiveNiches::Some, _) | (_, NaiveNiches::Some) => NaiveNiches::Some, - (NaiveNiches::None, NaiveNiches::None) => NaiveNiches::None, - (_, _) => NaiveNiches::Maybe, - }; - Some(Self { abi, size, align, exact, niches }) - } - - /// Returns the layout of `self` superposed with `other`, as in an `enum` - /// or an `union`. - /// - /// Note: This always ignore niche information from `other`. - #[must_use] - #[inline] - pub fn union(&self, other: &Self) -> Self { - use NaiveAbi::*; - - let size = cmp::max(self.size, other.size); - let align = cmp::max(self.align, other.align); - let exact = self.exact && other.exact; - let abi = match (self.abi, other.abi) { - // The unsized ABI overrides everything. - (Unsized, _) | (_, Unsized) => Unsized, - // A scalar union must have a single non ZST-field... - (_, s @ Scalar(_)) if exact && self.size == Size::ZERO => s, - (s @ Scalar(_), _) if exact && other.size == Size::ZERO => s, - // ...or identical scalar fields. - (Scalar(s1), Scalar(s2)) if s1 == s2 => Scalar(s1), - // Default cases. - (Uninhabited, Uninhabited) => Uninhabited, - (_, _) => Sized, - }; - Self { abi, size, align, exact, niches: self.niches } - } -} - /// Trait for contexts that want to be able to compute layouts of types. /// This automatically gives access to `LayoutOf`, through a blanket `impl`. pub trait LayoutOfHelpers<'tcx>: HasDataLayout + HasTyCtxt<'tcx> + HasParamEnv<'tcx> { @@ -896,19 +673,6 @@ pub trait LayoutOf<'tcx>: LayoutOfHelpers<'tcx> { .map_err(|err| self.handle_layout_err(*err, span, ty)), ) } - - /// Computes the naive layout estimate of a type. Note that this implicitly - /// executes in "reveal all" mode, and will normalize the input type. - /// - /// Unlike `layout_of`, this doesn't look past references (beyond the `Pointee::Metadata` - /// projection), and as such can be called on generic types like `Option<&T>`. - #[inline] - fn naive_layout_of( - &self, - ty: Ty<'tcx>, - ) -> Result<TyAndNaiveLayout<'tcx>, &'tcx LayoutError<'tcx>> { - self.tcx().naive_layout_of(self.param_env().and(ty)) - } } impl<'tcx, C: LayoutOfHelpers<'tcx>> LayoutOf<'tcx> for C {} @@ -1205,9 +969,6 @@ where this: TyAndLayout<'tcx>, cx: &C, offset: Size, - // If true, assume that pointers are either null or valid (according to their type), - // enabling extra optimizations. - mut assume_valid_ptr: bool, ) -> Option<PointeeInfo> { let tcx = cx.tcx(); let param_env = cx.param_env(); @@ -1230,19 +991,19 @@ where // Freeze/Unpin queries, and can save time in the codegen backend (noalias // attributes in LLVM have compile-time cost even in unoptimized builds). let optimize = tcx.sess.opts.optimize != OptLevel::No; - let safe = match (assume_valid_ptr, mt) { - (true, hir::Mutability::Not) => Some(PointerKind::SharedRef { + let kind = match mt { + hir::Mutability::Not => PointerKind::SharedRef { frozen: optimize && ty.is_freeze(tcx, cx.param_env()), - }), - (true, hir::Mutability::Mut) => Some(PointerKind::MutableRef { + }, + hir::Mutability::Mut => PointerKind::MutableRef { unpin: optimize && ty.is_unpin(tcx, cx.param_env()), - }), - (false, _) => None, + }, }; + tcx.layout_of(param_env.and(ty)).ok().map(|layout| PointeeInfo { size: layout.size, align: layout.align.abi, - safe, + safe: Some(kind), }) } @@ -1251,21 +1012,19 @@ where // Within the discriminant field, only the niche itself is // always initialized, so we only check for a pointer at its // offset. + // + // If the niche is a pointer, it's either valid (according + // to its type), or null (which the niche field's scalar + // validity range encodes). This allows using + // `dereferenceable_or_null` for e.g., `Option<&T>`, and + // this will continue to work as long as we don't start + // using more niches than just null (e.g., the first page of + // the address space, or unaligned pointers). Variants::Multiple { - tag_encoding: - TagEncoding::Niche { - untagged_variant, - niche_variants: ref variants, - niche_start, - }, + tag_encoding: TagEncoding::Niche { untagged_variant, .. }, tag_field, .. } if this.fields.offset(tag_field) == offset => { - // We can only continue assuming pointer validity if the only possible - // discriminant value is null. The null special-case is permitted by LLVM's - // `dereferenceable_or_null`, and allow types like `Option<&T>` to benefit - // from optimizations. - assume_valid_ptr &= niche_start == 0 && variants.start() == variants.end(); Some(this.for_variant(cx, untagged_variant)) } _ => Some(this), @@ -1291,12 +1050,9 @@ where result = field.to_result().ok().and_then(|field| { if ptr_end <= field_start + field.size { // We found the right field, look inside it. - Self::ty_and_layout_pointee_info_at( - field, - cx, - offset - field_start, - assume_valid_ptr, - ) + let field_info = + field.pointee_info_at(cx, offset - field_start); + field_info } else { None } @@ -1311,7 +1067,7 @@ where // FIXME(eddyb) This should be for `ptr::Unique<T>`, not `Box<T>`. if let Some(ref mut pointee) = result { if let ty::Adt(def, _) = this.ty.kind() { - if assume_valid_ptr && def.is_box() && offset.bytes() == 0 { + if def.is_box() && offset.bytes() == 0 { let optimize = tcx.sess.opts.optimize != OptLevel::No; pointee.safe = Some(PointerKind::Box { unpin: optimize && this.ty.boxed_ty().is_unpin(tcx, cx.param_env()), diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index e08edfe143a..f13c8214af1 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -1,6 +1,7 @@ //! Inlining pass for MIR functions use crate::deref_separator::deref_finder; use rustc_attr::InlineAttr; +use rustc_const_eval::transform::validate::validate_types; use rustc_hir::def_id::DefId; use rustc_index::bit_set::BitSet; use rustc_index::Idx; @@ -10,7 +11,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_session::config::OptLevel; -use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; +use rustc_target::abi::FieldIdx; use rustc_target::spec::abi::Abi; use crate::simplify::{remove_dead_blocks, CfgSimplifier}; @@ -200,6 +201,19 @@ impl<'tcx> Inliner<'tcx> { return Err("failed to normalize callee body"); }; + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + if !validate_types( + self.tcx, + MirPhase::Runtime(RuntimePhase::Optimized), + self.param_env, + &callee_body, + ) + .is_empty() + { + return Err("failed to validate callee body"); + } + // Check call signature compatibility. // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. @@ -437,13 +451,8 @@ impl<'tcx> Inliner<'tcx> { instance: callsite.callee, callee_body, cost: 0, - validation: Ok(()), }; - for var_debug_info in callee_body.var_debug_info.iter() { - checker.visit_var_debug_info(var_debug_info); - } - // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); @@ -480,9 +489,6 @@ impl<'tcx> Inliner<'tcx> { } } - // Abort if type validation found anything fishy. - checker.validation?; - // N.B. We still apply our cost threshold to #[inline(always)] functions. // That attribute is often applied to very large functions that exceed LLVM's (very // generous) inlining threshold. Such functions are very poor MIR inlining candidates. @@ -774,11 +780,10 @@ struct CostChecker<'b, 'tcx> { cost: usize, callee_body: &'b Body<'tcx>, instance: ty::Instance<'tcx>, - validation: Result<(), &'static str>, } impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { - fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) { // Don't count StorageLive/StorageDead in the inlining cost. match statement.kind { StatementKind::StorageLive(_) @@ -787,11 +792,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { | StatementKind::Nop => {} _ => self.cost += INSTR_COST, } - - self.super_statement(statement, location); } - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) { let tcx = self.tcx; match terminator.kind { TerminatorKind::Drop { ref place, unwind, .. } => { @@ -835,109 +838,6 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } _ => self.cost += INSTR_COST, } - - self.super_terminator(terminator, location); - } - - /// This method duplicates code from MIR validation in an attempt to detect type mismatches due - /// to normalization failure. - fn visit_projection_elem( - &mut self, - place_ref: PlaceRef<'tcx>, - elem: PlaceElem<'tcx>, - context: PlaceContext, - location: Location, - ) { - if let ProjectionElem::Field(f, ty) = elem { - let parent_ty = place_ref.ty(&self.callee_body.local_decls, self.tcx); - let check_equal = |this: &mut Self, f_ty| { - // Fast path if there is nothing to substitute. - if ty == f_ty { - return; - } - let ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder::bind(&ty)); - let f_ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder::bind(&f_ty)); - if ty == f_ty { - return; - } - if !util::is_subtype(this.tcx, this.param_env, ty, f_ty) { - trace!(?ty, ?f_ty); - this.validation = Err("failed to normalize projection type"); - return; - } - }; - - let kind = match parent_ty.ty.kind() { - &ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) => { - self.tcx.type_of(def_id).instantiate(self.tcx, args).kind() - } - kind => kind, - }; - - match kind { - ty::Tuple(fields) => { - let Some(f_ty) = fields.get(f.as_usize()) else { - self.validation = Err("malformed MIR"); - return; - }; - check_equal(self, *f_ty); - } - ty::Adt(adt_def, args) => { - let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT); - let Some(field) = adt_def.variant(var).fields.get(f) else { - self.validation = Err("malformed MIR"); - return; - }; - check_equal(self, field.ty(self.tcx, args)); - } - ty::Closure(_, args) => { - let args = args.as_closure(); - let Some(f_ty) = args.upvar_tys().nth(f.as_usize()) else { - self.validation = Err("malformed MIR"); - return; - }; - check_equal(self, f_ty); - } - &ty::Generator(def_id, args, _) => { - let f_ty = if let Some(var) = parent_ty.variant_index { - let gen_body = if def_id == self.callee_body.source.def_id() { - self.callee_body - } else { - self.tcx.optimized_mir(def_id) - }; - - let Some(layout) = gen_body.generator_layout() else { - self.validation = Err("malformed MIR"); - return; - }; - - let Some(&local) = layout.variant_fields[var].get(f) else { - self.validation = Err("malformed MIR"); - return; - }; - - let Some(f_ty) = layout.field_tys.get(local) else { - self.validation = Err("malformed MIR"); - return; - }; - - f_ty.ty - } else { - let Some(f_ty) = args.as_generator().prefix_tys().nth(f.index()) else { - self.validation = Err("malformed MIR"); - return; - }; - - f_ty - }; - - check_equal(self, f_ty); - } - _ => self.validation = Err("malformed MIR"), - } - } - - self.super_projection_elem(place_ref, elem, context, location); } } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 3ecdbc36248..769415b614b 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -3003,7 +3003,8 @@ impl<'a> Parser<'a> { fn is_do_catch_block(&self) -> bool { self.token.is_keyword(kw::Do) && self.is_keyword_ahead(1, &[kw::Catch]) - && self.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace)) + && self + .look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()) && !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL) } @@ -3013,7 +3014,8 @@ impl<'a> Parser<'a> { fn is_try_block(&self) -> bool { self.token.is_keyword(kw::Try) - && self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace)) + && self + .look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()) && self.token.uninterpolated_span().at_least_rust_2018() } @@ -3032,10 +3034,14 @@ impl<'a> Parser<'a> { && (( // `async move {` self.is_keyword_ahead(1, &[kw::Move]) - && self.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace)) + && self.look_ahead(2, |t| { + *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block() + }) ) || ( // `async {` - self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace)) + self.look_ahead(1, |t| { + *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block() + }) )) } diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index a53d1fcc69e..d2140161f1d 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -176,8 +176,7 @@ impl QueryJobId { while let Some(id) = current_id { let info = query_map.get(&id).unwrap(); // FIXME: This string comparison should probably not be done. - let query_name = format!("{:?}", info.query.dep_kind); - if query_name == "layout_of" || query_name == "naive_layout_of" { + if format!("{:?}", info.query.dep_kind) == "layout_of" { depth += 1; last_layout = Some((info.clone(), depth)); } diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 05128a51016..be7ef92d175 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -2569,7 +2569,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> { let res = match kind { RibKind::Item(..) | RibKind::AssocItem => Res::Def(def_kind, def_id.to_def_id()), RibKind::Normal => { - if self.r.tcx.sess.features_untracked().non_lifetime_binders { + if self.r.tcx.features().non_lifetime_binders { Res::Def(def_kind, def_id.to_def_id()) } else { Res::Err diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs index d16b7902f60..266e37e4cef 100644 --- a/compiler/rustc_resolve/src/macros.rs +++ b/compiler/rustc_resolve/src/macros.rs @@ -576,10 +576,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> { } // We are trying to avoid reporting this error if other related errors were reported. - if res != Res::Err - && inner_attr - && !self.tcx.sess.features_untracked().custom_inner_attributes - { + if res != Res::Err && inner_attr && !self.tcx.features().custom_inner_attributes { let msg = match res { Res::Def(..) => "inner macro attributes are unstable", Res::NonMacroAttr(..) => "custom inner attributes are unstable", diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 1766e97b67d..a8147ede970 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3117,7 +3117,6 @@ pub(crate) mod dep_tracking { use rustc_feature::UnstableFeatures; use rustc_span::edition::Edition; use rustc_span::RealFileName; - use rustc_target::abi::ReferenceNichePolicy; use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel}; use rustc_target::spec::{ RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TargetTriple, TlsModel, @@ -3213,7 +3212,6 @@ pub(crate) mod dep_tracking { OomStrategy, LanguageIdentifier, TraitSolver, - ReferenceNichePolicy, ); impl<T1, T2> DepTrackingHash for (T1, T2) diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 0c66121c72f..39efe9abeec 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -6,7 +6,6 @@ use crate::{lint, EarlyErrorHandler}; use rustc_data_structures::profiling::TimePassesFormat; use rustc_errors::ColorConfig; use rustc_errors::{LanguageIdentifier, TerminalUrl}; -use rustc_target::abi::ReferenceNichePolicy; use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, SanitizerSet}; use rustc_target::spec::{ RelocModel, RelroLevel, SplitDebuginfo, StackProtector, TargetTriple, TlsModel, @@ -422,8 +421,6 @@ mod desc { pub const parse_proc_macro_execution_strategy: &str = "one of supported execution strategies (`same-thread`, or `cross-thread`)"; pub const parse_dump_solver_proof_tree: &str = "one of: `always`, `on-request`, `on-error`"; - pub const parse_opt_reference_niches: &str = - "`null`, or a `,` separated combination of `size` or `align`"; } mod parse { @@ -1256,31 +1253,6 @@ mod parse { }; true } - - pub(crate) fn parse_opt_reference_niches( - slot: &mut Option<ReferenceNichePolicy>, - v: Option<&str>, - ) -> bool { - let Some(s) = v else { - return false; - }; - - let slot = slot.get_or_insert_default(); - - if s == "null" { - return true; - } - - for opt in s.split(",") { - match opt { - "size" => slot.size = true, - "align" => slot.align = true, - _ => return false, - } - } - - true - } } options! { @@ -1729,8 +1701,6 @@ options! { "enable queries of the dependency graph for regression testing (default: no)"), randomize_layout: bool = (false, parse_bool, [TRACKED], "randomize the layout of types (default: no)"), - reference_niches: Option<ReferenceNichePolicy> = (None, parse_opt_reference_niches, [TRACKED], - "override the set of discriminant niches that may be exposed by references"), relax_elf_relocations: Option<bool> = (None, parse_opt_bool, [TRACKED], "whether ELF relocations can be relaxed"), relro_level: Option<RelroLevel> = (None, parse_relro_level, [TRACKED], diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs index 11ba551dccc..084c917cc31 100644 --- a/compiler/rustc_target/src/abi/mod.rs +++ b/compiler/rustc_target/src/abi/mod.rs @@ -50,9 +50,6 @@ pub trait TyAbiInterface<'a, C>: Sized { this: TyAndLayout<'a, Self>, cx: &C, offset: Size, - // If true, assume that pointers are either null or valid (according to their type), - // enabling extra optimizations. - assume_valid_ptr: bool, ) -> Option<PointeeInfo>; fn is_adt(this: TyAndLayout<'a, Self>) -> bool; fn is_never(this: TyAndLayout<'a, Self>) -> bool; @@ -79,8 +76,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> { where Ty: TyAbiInterface<'a, C>, { - let assume_valid_ptr = true; - Ty::ty_and_layout_pointee_info_at(self, cx, offset, assume_valid_ptr) + Ty::ty_and_layout_pointee_info_at(self, cx, offset) } pub fn is_single_fp_element<C>(self, cx: &C) -> bool diff --git a/compiler/rustc_trait_selection/messages.ftl b/compiler/rustc_trait_selection/messages.ftl index 217ba71b631..f57f1bad15d 100644 --- a/compiler/rustc_trait_selection/messages.ftl +++ b/compiler/rustc_trait_selection/messages.ftl @@ -1,3 +1,13 @@ +trait_selection_adjust_signature_borrow = consider adjusting the signature so it borrows its {$len -> + [one] argument + *[other] arguments + } + +trait_selection_adjust_signature_remove_borrow = consider adjusting the signature so it does not borrow its {$len -> + [one] argument + *[other] arguments + } + trait_selection_dump_vtable_entries = vtable entries for `{$trait_ref}`: {$entries} trait_selection_empty_on_clause_in_rustc_on_unimplemented = empty `on`-clause in `#[rustc_on_unimplemented]` diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs index 54e22cc3d7f..dde9e9c9ac6 100644 --- a/compiler/rustc_trait_selection/src/errors.rs +++ b/compiler/rustc_trait_selection/src/errors.rs @@ -1,5 +1,8 @@ use crate::fluent_generated as fluent; -use rustc_errors::{ErrorGuaranteed, Handler, IntoDiagnostic}; +use rustc_errors::{ + AddToDiagnostic, Applicability, Diagnostic, ErrorGuaranteed, Handler, IntoDiagnostic, + SubdiagnosticMessage, +}; use rustc_macros::Diagnostic; use rustc_middle::ty::{self, PolyTraitRef, Ty}; use rustc_span::{Span, Symbol}; @@ -97,3 +100,34 @@ pub struct InherentProjectionNormalizationOverflow { pub span: Span, pub ty: String, } + +pub enum AdjustSignatureBorrow { + Borrow { to_borrow: Vec<(Span, String)> }, + RemoveBorrow { remove_borrow: Vec<(Span, String)> }, +} + +impl AddToDiagnostic for AdjustSignatureBorrow { + fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, _: F) + where + F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage, + { + match self { + AdjustSignatureBorrow::Borrow { to_borrow } => { + diag.set_arg("len", to_borrow.len()); + diag.multipart_suggestion_verbose( + fluent::trait_selection_adjust_signature_borrow, + to_borrow, + Applicability::MaybeIncorrect, + ); + } + AdjustSignatureBorrow::RemoveBorrow { remove_borrow } => { + diag.set_arg("len", remove_borrow.len()); + diag.multipart_suggestion_verbose( + fluent::trait_selection_adjust_signature_remove_borrow, + remove_borrow, + Applicability::MaybeIncorrect, + ); + } + } + } +} diff --git a/compiler/rustc_trait_selection/src/solve/alias_relate.rs b/compiler/rustc_trait_selection/src/solve/alias_relate.rs index 73362d82306..3c7f91e6ca9 100644 --- a/compiler/rustc_trait_selection/src/solve/alias_relate.rs +++ b/compiler/rustc_trait_selection/src/solve/alias_relate.rs @@ -66,24 +66,27 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { Invert::Yes, )); // Relate via args - let subst_relate_response = self - .assemble_subst_relate_candidate(param_env, alias_lhs, alias_rhs, direction); - candidates.extend(subst_relate_response); + candidates.extend( + self.assemble_subst_relate_candidate( + param_env, alias_lhs, alias_rhs, direction, + ), + ); debug!(?candidates); if let Some(merged) = self.try_merge_responses(&candidates) { Ok(merged) } else { - // When relating two aliases and we have ambiguity, we prefer - // relating the generic arguments of the aliases over normalizing - // them. This is necessary for inference during typeck. + // When relating two aliases and we have ambiguity, if both + // aliases can be normalized to something, we prefer + // "bidirectionally normalizing" both of them within the same + // candidate. + // + // See <https://github.com/rust-lang/trait-system-refactor-initiative/issues/25>. // // As this is incomplete, we must not do so during coherence. match self.solver_mode() { SolverMode::Normal => { - if let Ok(subst_relate_response) = subst_relate_response { - Ok(subst_relate_response) - } else if let Ok(bidirectional_normalizes_to_response) = self + if let Ok(bidirectional_normalizes_to_response) = self .assemble_bidirectional_normalizes_to_candidate( param_env, lhs, rhs, direction, ) diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs index 761f5327f6d..930e62d6388 100644 --- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs @@ -223,20 +223,9 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> { return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS); } - // First, try computing an exact naive layout in case the type is generic. - let is_pointer_like = if let Ok(layout) = tcx.naive_layout_of(key) { - layout.is_pointer_like(&tcx.data_layout).unwrap_or_else(|| { - // Second, we fall back to full layout computation. - tcx.layout_of(key) - .ok() - .filter(|l| l.layout.is_pointer_like(&tcx.data_layout)) - .is_some() - }) - } else { - false - }; - - if is_pointer_like { + if let Ok(layout) = tcx.layout_of(key) + && layout.layout.is_pointer_like(&tcx.data_layout) + { // FIXME: We could make this faster by making a no-constraints response ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) } else { diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs index 073a2a2b1a0..05d2934d4c5 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs @@ -5,6 +5,7 @@ use super::{ PredicateObligation, }; +use crate::errors; use crate::infer::InferCtxt; use crate::traits::{NormalizeExt, ObligationCtxt}; @@ -4031,6 +4032,10 @@ fn hint_missing_borrow<'tcx>( found_node: Node<'_>, err: &mut Diagnostic, ) { + if matches!(found_node, Node::TraitItem(..)) { + return; + } + let found_args = match found.kind() { ty::FnPtr(f) => infcx.instantiate_binder_with_placeholders(*f).inputs().iter(), kind => { @@ -4102,19 +4107,11 @@ fn hint_missing_borrow<'tcx>( } if !to_borrow.is_empty() { - err.multipart_suggestion_verbose( - "consider borrowing the argument", - to_borrow, - Applicability::MaybeIncorrect, - ); + err.subdiagnostic(errors::AdjustSignatureBorrow::Borrow { to_borrow }); } if !remove_borrow.is_empty() { - err.multipart_suggestion_verbose( - "do not borrow the argument", - remove_borrow, - Applicability::MaybeIncorrect, - ); + err.subdiagnostic(errors::AdjustSignatureBorrow::RemoveBorrow { remove_borrow }); } } diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index f1d870269a6..aa195d70a9f 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -979,20 +979,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { return; } - // First, try computing an exact naive layout in case the type is generic. - let is_pointer_like = if let Ok(layout) = tcx.naive_layout_of(key) { - layout.is_pointer_like(&tcx.data_layout).unwrap_or_else(|| { - // Second, we fall back to full layout computation. - tcx.layout_of(key) - .ok() - .filter(|l| l.layout.is_pointer_like(&tcx.data_layout)) - .is_some() - }) - } else { - false - }; - - if is_pointer_like { + if let Ok(layout) = tcx.layout_of(key) + && layout.layout.is_pointer_like(&tcx.data_layout) + { candidates.vec.push(BuiltinCandidate { has_nested: false }); } } diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index da1eba68d53..b840ff184e0 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -3,7 +3,7 @@ use rustc_hir as hir; use rustc_index::bit_set::BitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::{GeneratorLayout, GeneratorSavedLocal}; -use rustc_middle::query::{LocalCrate, Providers}; +use rustc_middle::query::Providers; use rustc_middle::ty::layout::{ IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, MAX_SIMD_LANES, }; @@ -24,28 +24,32 @@ use crate::errors::{ use crate::layout_sanity_check::sanity_check_layout; pub fn provide(providers: &mut Providers) { - *providers = Providers { layout_of, reference_niches_policy, ..*providers }; + *providers = Providers { layout_of, ..*providers }; } -#[instrument(skip(tcx), level = "debug")] -fn reference_niches_policy<'tcx>(tcx: TyCtxt<'tcx>, _: LocalCrate) -> ReferenceNichePolicy { - tcx.sess.opts.unstable_opts.reference_niches.unwrap_or(DEFAULT_REF_NICHES) -} - -/// The reference niche policy for builtin types, and for types in -/// crates not specifying `-Z reference-niches`. -const DEFAULT_REF_NICHES: ReferenceNichePolicy = ReferenceNichePolicy { size: false, align: false }; - #[instrument(skip(tcx, query), level = "debug")] fn layout_of<'tcx>( tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, ) -> Result<TyAndLayout<'tcx>, &'tcx LayoutError<'tcx>> { - let (param_env, unnormalized_ty) = query.into_parts(); + let (param_env, ty) = query.into_parts(); + debug!(?ty); + let param_env = param_env.with_reveal_all_normalized(tcx); - // `naive_layout_of` takes care of normalizing the type. - let naive = tcx.naive_layout_of(query)?; - let ty = naive.ty; + let unnormalized_ty = ty; + + // FIXME: We might want to have two different versions of `layout_of`: + // One that can be called after typecheck has completed and can use + // `normalize_erasing_regions` here and another one that can be called + // before typecheck has completed and uses `try_normalize_erasing_regions`. + let ty = match tcx.try_normalize_erasing_regions(param_env, ty) { + Ok(t) => t, + Err(normalization_error) => { + return Err(tcx + .arena + .alloc(LayoutError::NormalizationFailure(ty, normalization_error))); + } + }; if ty != unnormalized_ty { // Ensure this layout is also cached for the normalized type. @@ -53,11 +57,13 @@ fn layout_of<'tcx>( } let cx = LayoutCx { tcx, param_env }; - let layout = layout_of_uncached(&cx, ty)?; + let layout = layout_of_uncached(&cx, ty)?; let layout = TyAndLayout { ty, layout }; + record_layout_for_printing(&cx, layout); - sanity_check_layout(&cx, &layout, &naive); + + sanity_check_layout(&cx, &layout); Ok(layout) } @@ -77,10 +83,12 @@ fn univariant_uninterned<'tcx>( kind: StructKind, ) -> Result<LayoutS, &'tcx LayoutError<'tcx>> { let dl = cx.data_layout(); - assert!( - !(repr.pack.is_some() && repr.align.is_some()), - "already rejected by `naive_layout_of`" - ); + let pack = repr.pack; + if pack.is_some() && repr.align.is_some() { + cx.tcx.sess.delay_span_bug(DUMMY_SP, "struct cannot be packed and aligned"); + return Err(cx.tcx.arena.alloc(LayoutError::Unknown(ty))); + } + cx.univariant(dl, fields, repr, kind).ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty))) } @@ -138,35 +146,75 @@ fn layout_of_uncached<'tcx>( ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => { let mut data_ptr = scalar_unit(Pointer(AddressSpace::DATA)); if !ty.is_unsafe_ptr() { - // Calling `layout_of` here would cause a query cycle for recursive types; - // so use a conservative estimate that doesn't look past references. - let naive = cx.naive_layout_of(pointee)?.layout; - - let niches = match *pointee.kind() { - ty::FnDef(def, ..) - | ty::Foreign(def) - | ty::Generator(def, ..) - | ty::Closure(def, ..) => tcx.reference_niches_policy(def.krate), - ty::Adt(def, _) => tcx.reference_niches_policy(def.did().krate), - _ => DEFAULT_REF_NICHES, + data_ptr.valid_range_mut().start = 1; + } + + let pointee = tcx.normalize_erasing_regions(param_env, pointee); + if pointee.is_sized(tcx, param_env) { + return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr))); + } + + let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type() + // Projection eagerly bails out when the pointee references errors, + // fall back to structurally deducing metadata. + && !pointee.references_error() + { + let pointee_metadata = Ty::new_projection(tcx,metadata_def_id, [pointee]); + let metadata_ty = match tcx.try_normalize_erasing_regions( + param_env, + pointee_metadata, + ) { + Ok(metadata_ty) => metadata_ty, + Err(mut err) => { + // Usually `<Ty as Pointee>::Metadata` can't be normalized because + // its struct tail cannot be normalized either, so try to get a + // more descriptive layout error here, which will lead to less confusing + // diagnostics. + match tcx.try_normalize_erasing_regions( + param_env, + tcx.struct_tail_without_normalization(pointee), + ) { + Ok(_) => {}, + Err(better_err) => { + err = better_err; + } + } + return Err(error(cx, LayoutError::NormalizationFailure(pointee, err))); + }, }; - let (min_addr, max_addr) = dl.address_range_for( - if niches.size { naive.size } else { Size::ZERO }, - if niches.align { naive.align } else { Align::ONE }, - ); + let metadata_layout = cx.layout_of(metadata_ty)?; + // If the metadata is a 1-zst, then the pointer is thin. + if metadata_layout.is_zst() && metadata_layout.align.abi.bytes() == 1 { + return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr))); + } - *data_ptr.valid_range_mut() = - WrappingRange { start: min_addr.into(), end: max_addr.into() }; - } + let Abi::Scalar(metadata) = metadata_layout.abi else { + return Err(error(cx, LayoutError::Unknown(pointee))); + }; - if let Some(metadata) = ptr_metadata_scalar(cx, pointee)? { - // Effectively a (ptr, meta) tuple. - tcx.mk_layout(cx.scalar_pair(data_ptr, metadata)) + metadata } else { - // No metadata, this is a thin pointer. - tcx.mk_layout(LayoutS::scalar(cx, data_ptr)) - } + let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env); + + match unsized_part.kind() { + ty::Foreign(..) => { + return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr))); + } + ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)), + ty::Dynamic(..) => { + let mut vtable = scalar_unit(Pointer(AddressSpace::DATA)); + vtable.valid_range_mut().start = 1; + vtable + } + _ => { + return Err(error(cx, LayoutError::Unknown(pointee))); + } + } + }; + + // Effectively a (ptr, meta) tuple. + tcx.mk_layout(cx.scalar_pair(data_ptr, metadata)) } ty::Dynamic(_, _, ty::DynStar) => { @@ -178,8 +226,16 @@ fn layout_of_uncached<'tcx>( } // Arrays and slices. - ty::Array(element, count) => { - let count = compute_array_count(cx, count) + ty::Array(element, mut count) => { + if count.has_projections() { + count = tcx.normalize_erasing_regions(param_env, count); + if count.has_projections() { + return Err(error(cx, LayoutError::Unknown(ty))); + } + } + + let count = count + .try_eval_target_usize(tcx, param_env) .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?; let element = cx.layout_of(element)?; let size = element @@ -502,104 +558,20 @@ fn layout_of_uncached<'tcx>( } // Types with no meaningful known layout. - ty::Alias(..) - | ty::Bound(..) - | ty::GeneratorWitness(..) - | ty::GeneratorWitnessMIR(..) - | ty::Infer(_) - | ty::Placeholder(..) - | ty::Param(_) - | ty::Error(_) => { - unreachable!("already rejected by `naive_layout_of`"); + ty::Alias(..) => { + // NOTE(eddyb) `layout_of` query should've normalized these away, + // if that was possible, so there's no reason to try again here. + return Err(error(cx, LayoutError::Unknown(ty))); } - }) -} -pub(crate) fn compute_array_count<'tcx>( - cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, - mut count: ty::Const<'tcx>, -) -> Option<u64> { - let LayoutCx { tcx, param_env } = *cx; - if count.has_projections() { - count = tcx.normalize_erasing_regions(param_env, count); - if count.has_projections() { - return None; + ty::Bound(..) | ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) | ty::Infer(_) => { + bug!("Layout::compute: unexpected type `{}`", ty) } - } - - count.try_eval_target_usize(tcx, param_env) -} - -pub(crate) fn ptr_metadata_scalar<'tcx>( - cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, - pointee: Ty<'tcx>, -) -> Result<Option<Scalar>, &'tcx LayoutError<'tcx>> { - let dl = cx.data_layout(); - let scalar_unit = |value: Primitive| { - let size = value.size(dl); - assert!(size.bits() <= 128); - Scalar::Initialized { value, valid_range: WrappingRange::full(size) } - }; - let LayoutCx { tcx, param_env } = *cx; - - let pointee = tcx.normalize_erasing_regions(param_env, pointee); - if pointee.is_sized(tcx, param_env) { - return Ok(None); - } - - if let Some(metadata_def_id) = tcx.lang_items().metadata_type() - // Projection eagerly bails out when the pointee references errors, - // fall back to structurally deducing metadata. - && !pointee.references_error() - { - let pointee_metadata = Ty::new_projection(tcx,metadata_def_id, [pointee]); - let metadata_ty = match tcx.try_normalize_erasing_regions( - param_env, - pointee_metadata, - ) { - Ok(metadata_ty) => metadata_ty, - Err(mut err) => { - // Usually `<Ty as Pointee>::Metadata` can't be normalized because - // its struct tail cannot be normalized either, so try to get a - // more descriptive layout error here, which will lead to less confusing - // diagnostics. - match tcx.try_normalize_erasing_regions( - param_env, - tcx.struct_tail_without_normalization(pointee), - ) { - Ok(_) => {}, - Err(better_err) => { - err = better_err; - } - } - return Err(error(cx, LayoutError::NormalizationFailure(pointee, err))); - }, - }; - - let metadata_layout = cx.layout_of(metadata_ty)?; - - if metadata_layout.is_zst() && metadata_layout.align.abi.bytes() == 1 { - Ok(None) // If the metadata is a 1-zst, then the pointer is thin. - } else if let Abi::Scalar(metadata) = metadata_layout.abi { - Ok(Some(metadata)) - } else { - Err(error(cx, LayoutError::Unknown(pointee))) - } - } else { - let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env); - - match unsized_part.kind() { - ty::Foreign(..) => Ok(None), - ty::Slice(_) | ty::Str => Ok(Some(scalar_unit(Int(dl.ptr_sized_integer(), false)))), - ty::Dynamic(..) => { - let mut vtable = scalar_unit(Pointer(AddressSpace::DATA)); - vtable.valid_range_mut().start = 1; - Ok(Some(vtable)) - } - _ => Err(error(cx, LayoutError::Unknown(pointee))), + ty::Placeholder(..) | ty::Param(_) | ty::Error(_) => { + return Err(error(cx, LayoutError::Unknown(ty))); } - } + }) } /// Overlap eligibility and variant assignment for each GeneratorSavedLocal. diff --git a/compiler/rustc_ty_utils/src/layout_naive.rs b/compiler/rustc_ty_utils/src/layout_naive.rs deleted file mode 100644 index 3070ab59d53..00000000000 --- a/compiler/rustc_ty_utils/src/layout_naive.rs +++ /dev/null @@ -1,322 +0,0 @@ -use rustc_middle::query::Providers; -use rustc_middle::ty::layout::{ - IntegerExt, LayoutCx, LayoutError, LayoutOf, NaiveAbi, NaiveLayout, NaiveNiches, - TyAndNaiveLayout, -}; -use rustc_middle::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitableExt}; -use rustc_span::DUMMY_SP; -use rustc_target::abi::*; - -use std::ops::Bound; - -use crate::layout::{compute_array_count, ptr_metadata_scalar}; - -pub fn provide(providers: &mut Providers) { - *providers = Providers { naive_layout_of, ..*providers }; -} - -#[instrument(skip(tcx, query), level = "debug")] -fn naive_layout_of<'tcx>( - tcx: TyCtxt<'tcx>, - query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, -) -> Result<TyAndNaiveLayout<'tcx>, &'tcx LayoutError<'tcx>> { - let (param_env, ty) = query.into_parts(); - debug!(?ty); - - let param_env = param_env.with_reveal_all_normalized(tcx); - let unnormalized_ty = ty; - - // FIXME: We might want to have two different versions of `layout_of`: - // One that can be called after typecheck has completed and can use - // `normalize_erasing_regions` here and another one that can be called - // before typecheck has completed and uses `try_normalize_erasing_regions`. - let ty = match tcx.try_normalize_erasing_regions(param_env, ty) { - Ok(t) => t, - Err(normalization_error) => { - return Err(tcx - .arena - .alloc(LayoutError::NormalizationFailure(ty, normalization_error))); - } - }; - - if ty != unnormalized_ty { - // Ensure this layout is also cached for the normalized type. - return tcx.naive_layout_of(param_env.and(ty)); - } - - let cx = LayoutCx { tcx, param_env }; - let layout = naive_layout_of_uncached(&cx, ty)?; - Ok(TyAndNaiveLayout { ty, layout }) -} - -fn error<'tcx>( - cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, - err: LayoutError<'tcx>, -) -> &'tcx LayoutError<'tcx> { - cx.tcx.arena.alloc(err) -} - -fn naive_layout_of_uncached<'tcx>( - cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, - ty: Ty<'tcx>, -) -> Result<NaiveLayout, &'tcx LayoutError<'tcx>> { - let tcx = cx.tcx; - let dl = cx.data_layout(); - - let scalar = |niched: bool, value: Primitive| NaiveLayout { - abi: NaiveAbi::Scalar(value), - niches: if niched { NaiveNiches::Some } else { NaiveNiches::None }, - size: value.size(dl), - align: value.align(dl).abi, - exact: true, - }; - - let univariant = |fields: &mut dyn Iterator<Item = Ty<'tcx>>, - repr: &ReprOptions| - -> Result<NaiveLayout, &'tcx LayoutError<'tcx>> { - if repr.pack.is_some() && repr.align.is_some() { - cx.tcx.sess.delay_span_bug(DUMMY_SP, "struct cannot be packed and aligned"); - return Err(error(cx, LayoutError::Unknown(ty))); - } - - let linear = repr.inhibit_struct_field_reordering_opt(); - let pack = repr.pack.unwrap_or(Align::MAX); - let mut layout = NaiveLayout::EMPTY; - - for field in fields { - let field = cx.naive_layout_of(field)?.packed(pack); - if linear { - layout = layout.pad_to_align(field.align); - } - layout = layout - .concat(&field, dl) - .ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))?; - } - - if let Some(align) = repr.align { - layout = layout.align_to(align); - } - - if linear { - layout.abi = layout.abi.as_aggregate(); - } - - Ok(layout.pad_to_align(layout.align)) - }; - - debug_assert!(!ty.has_non_region_infer()); - - Ok(match *ty.kind() { - // Basic scalars - ty::Bool => scalar(true, Int(I8, false)), - ty::Char => scalar(true, Int(I32, false)), - ty::Int(ity) => scalar(false, Int(Integer::from_int_ty(dl, ity), true)), - ty::Uint(ity) => scalar(false, Int(Integer::from_uint_ty(dl, ity), false)), - ty::Float(fty) => scalar( - false, - match fty { - ty::FloatTy::F32 => F32, - ty::FloatTy::F64 => F64, - }, - ), - ty::FnPtr(_) => scalar(true, Pointer(dl.instruction_address_space)), - - // The never type. - ty::Never => NaiveLayout { abi: NaiveAbi::Uninhabited, ..NaiveLayout::EMPTY }, - - // Potentially-wide pointers. - ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => { - let data_ptr = scalar(!ty.is_unsafe_ptr(), Pointer(AddressSpace::DATA)); - if let Some(metadata) = ptr_metadata_scalar(cx, pointee)? { - // Effectively a (ptr, meta) tuple. - let meta = scalar(!metadata.is_always_valid(dl), metadata.primitive()); - let l = data_ptr - .concat(&meta, dl) - .ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))?; - l.pad_to_align(l.align) - } else { - // No metadata, this is a thin pointer. - data_ptr - } - } - - ty::Dynamic(_, _, ty::DynStar) => { - let ptr = scalar(false, Pointer(AddressSpace::DATA)); - let vtable = scalar(true, Pointer(AddressSpace::DATA)); - ptr.concat(&vtable, dl).ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))? - } - - // Arrays and slices. - ty::Array(element, count) => { - let count = compute_array_count(cx, count) - .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?; - let element = cx.naive_layout_of(element)?; - NaiveLayout { - abi: element.abi.as_aggregate(), - size: element - .size - .checked_mul(count, cx) - .ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))?, - niches: if count == 0 { NaiveNiches::None } else { element.niches }, - ..*element - } - } - ty::Slice(element) => NaiveLayout { - abi: NaiveAbi::Unsized, - size: Size::ZERO, - niches: NaiveNiches::None, - ..*cx.naive_layout_of(element)? - }, - - ty::FnDef(..) => NaiveLayout::EMPTY, - - // Unsized types. - ty::Str | ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => { - NaiveLayout { abi: NaiveAbi::Unsized, ..NaiveLayout::EMPTY } - } - - // FIXME(reference_niches): try to actually compute a reasonable layout estimate, - // without duplicating too much code from `generator_layout`. - ty::Generator(..) => { - NaiveLayout { exact: false, niches: NaiveNiches::Maybe, ..NaiveLayout::EMPTY } - } - - ty::Closure(_, ref substs) => { - univariant(&mut substs.as_closure().upvar_tys(), &ReprOptions::default())? - } - - ty::Tuple(tys) => univariant(&mut tys.iter(), &ReprOptions::default())?, - - ty::Adt(def, substs) if def.is_union() => { - assert_eq!(def.variants().len(), 1, "union should have a single variant"); - let repr = def.repr(); - let pack = repr.pack.unwrap_or(Align::MAX); - if repr.pack.is_some() && repr.align.is_some() { - cx.tcx.sess.delay_span_bug(DUMMY_SP, "union cannot be packed and aligned"); - return Err(error(cx, LayoutError::Unknown(ty))); - } - - let mut layout = NaiveLayout { - // Unions never have niches. - niches: NaiveNiches::None, - ..NaiveLayout::EMPTY - }; - - for f in &def.variants()[FIRST_VARIANT].fields { - let field = cx.naive_layout_of(f.ty(tcx, substs))?; - layout = layout.union(&field.packed(pack)); - } - - // Unions are always inhabited, and never scalar if `repr(C)`. - if !matches!(layout.abi, NaiveAbi::Scalar(_)) || repr.inhibit_enum_layout_opt() { - layout.abi = NaiveAbi::Sized; - } - - if let Some(align) = repr.align { - layout = layout.align_to(align); - } - layout.pad_to_align(layout.align) - } - - ty::Adt(def, substs) => { - let repr = def.repr(); - let mut layout = NaiveLayout { - // An ADT with no inhabited variants should have an uninhabited ABI. - abi: NaiveAbi::Uninhabited, - ..NaiveLayout::EMPTY - }; - - let mut empty_variants = 0; - for v in def.variants() { - let mut fields = v.fields.iter().map(|f| f.ty(tcx, substs)); - let vlayout = univariant(&mut fields, &repr)?; - - if vlayout.size == Size::ZERO && vlayout.exact { - empty_variants += 1; - } else { - // Remember the niches of the last seen variant. - layout.niches = vlayout.niches; - } - - layout = layout.union(&vlayout); - } - - if def.is_enum() { - let may_need_discr = match def.variants().len() { - 0 | 1 => false, - // Simple Option-like niche optimization. - // Handling this special case allows enums like `Option<&T>` - // to be recognized as `PointerLike` and to be transmutable - // in generic contexts. - 2 if empty_variants == 1 && layout.niches == NaiveNiches::Some => { - layout.niches = NaiveNiches::Maybe; // fill up the niche. - false - } - _ => true, - }; - - if may_need_discr || repr.inhibit_enum_layout_opt() { - // For simplicity, assume that the discriminant always get niched. - // This will be wrong in many cases, which will cause the size (and - // sometimes the alignment) to be underestimated. - // FIXME(reference_niches): Be smarter here. - layout.niches = NaiveNiches::Maybe; - layout = layout.inexact(); - } - } else { - assert_eq!(def.variants().len(), 1, "struct should have a single variant"); - - // We don't compute exact alignment for SIMD structs. - if repr.simd() { - layout = layout.inexact(); - } - - // `UnsafeCell` hides all niches. - if def.is_unsafe_cell() { - layout.niches = NaiveNiches::None; - } - } - - let valid_range = tcx.layout_scalar_valid_range(def.did()); - if valid_range != (Bound::Unbounded, Bound::Unbounded) { - let get = |bound, default| match bound { - Bound::Unbounded => default, - Bound::Included(v) => v, - Bound::Excluded(_) => bug!("exclusive `layout_scalar_valid_range` bound"), - }; - - let valid_range = WrappingRange { - start: get(valid_range.0, 0), - // FIXME: this is wrong for scalar-pair ABIs. Fortunately, the - // only type this could currently affect is`NonNull<T: !Sized>`, - // and the `NaiveNiches` result still ends up correct. - end: get(valid_range.1, layout.size.unsigned_int_max()), - }; - assert!( - valid_range.is_in_range_for(layout.size), - "`layout_scalar_valid_range` values are out of bounds", - ); - if !valid_range.is_full_for(layout.size) { - layout.niches = NaiveNiches::Some; - } - } - - layout.pad_to_align(layout.align) - } - - // Types with no meaningful known layout. - ty::Alias(..) => { - // NOTE(eddyb) `layout_of` query should've normalized these away, - // if that was possible, so there's no reason to try again here. - return Err(error(cx, LayoutError::Unknown(ty))); - } - - ty::Bound(..) | ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) | ty::Infer(_) => { - bug!("Layout::compute: unexpected type `{}`", ty) - } - - ty::Placeholder(..) | ty::Param(_) | ty::Error(_) => { - return Err(error(cx, LayoutError::Unknown(ty))); - } - }) -} diff --git a/compiler/rustc_ty_utils/src/layout_sanity_check.rs b/compiler/rustc_ty_utils/src/layout_sanity_check.rs index 2e3fe4e7fb8..8633334381a 100644 --- a/compiler/rustc_ty_utils/src/layout_sanity_check.rs +++ b/compiler/rustc_ty_utils/src/layout_sanity_check.rs @@ -1,5 +1,5 @@ use rustc_middle::ty::{ - layout::{LayoutCx, NaiveLayout, TyAndLayout}, + layout::{LayoutCx, TyAndLayout}, TyCtxt, }; use rustc_target::abi::*; @@ -10,7 +10,6 @@ use std::assert_matches::assert_matches; pub(super) fn sanity_check_layout<'tcx>( cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: &TyAndLayout<'tcx>, - naive: &NaiveLayout, ) { // Type-level uninhabitedness should always imply ABI uninhabitedness. if layout.ty.is_privately_uninhabited(cx.tcx, cx.param_env) { @@ -21,10 +20,6 @@ pub(super) fn sanity_check_layout<'tcx>( bug!("size is not a multiple of align, in the following layout:\n{layout:#?}"); } - if !naive.is_refined_by(layout.layout) { - bug!("the naive layout isn't refined by the actual layout:\n{:#?}\n{:#?}", naive, layout); - } - if !cfg!(debug_assertions) { // Stop here, the rest is kind of expensive. return; diff --git a/compiler/rustc_ty_utils/src/lib.rs b/compiler/rustc_ty_utils/src/lib.rs index e2db6a6993f..55b8857ed39 100644 --- a/compiler/rustc_ty_utils/src/lib.rs +++ b/compiler/rustc_ty_utils/src/lib.rs @@ -31,7 +31,6 @@ mod errors; mod implied_bounds; pub mod instance; mod layout; -mod layout_naive; mod layout_sanity_check; mod needs_drop; mod opaque_types; @@ -48,7 +47,6 @@ pub fn provide(providers: &mut Providers) { consts::provide(providers); implied_bounds::provide(providers); layout::provide(providers); - layout_naive::provide(providers); needs_drop::provide(providers); opaque_types::provide(providers); representability::provide(providers); |