diff options
Diffstat (limited to 'compiler')
29 files changed, 568 insertions, 350 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index d2828669d43..f918facc86d 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -150,7 +150,10 @@ impl LlvmType for CastTarget { // Simplify to a single unit or an array if there's no prefix. // This produces the same layout, but using a simpler type. if self.prefix.iter().all(|x| x.is_none()) { - if rest_count == 1 { + // We can't do this if is_consecutive is set and the unit would get + // split on the target. Currently, this is only relevant for i128 + // registers. + if rest_count == 1 && (!self.rest.is_consecutive || self.rest.unit != Reg::i128()) { return rest_ll_unit; } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 8ec1f5a99e7..a6894a7e089 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1524,8 +1524,8 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVM } extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { - return wrap(llvm::Intrinsic::getDeclaration(unwrap(M), - (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment)); + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_increment)); } extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index 4a7720b38f8..f8f59fbeab4 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -520,7 +520,10 @@ impl<'tcx> Instance<'tcx> { // Reify `Trait::method` implementations if KCFI is enabled // FIXME(maurer) only reify it if it is a vtable-safe function _ if tcx.sess.is_sanitizer_kcfi_enabled() - && tcx.associated_item(def_id).trait_item_def_id.is_some() => + && tcx + .opt_associated_item(def_id) + .and_then(|assoc| assoc.trait_item_def_id) + .is_some() => { // If this function could also go in a vtable, we need to `ReifyShim` it with // KCFI because it can only attach one type per function. diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index de43f9faff9..320d8fd3977 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -58,16 +58,24 @@ //! borrowing from the outer closure, and we simply peel off a `deref` projection //! from them. This second body is stored alongside the first body, and optimized //! with it in lockstep. When we need to resolve a body for `FnOnce` or `AsyncFnOnce`, -//! we use this "by move" body instead. - -use itertools::Itertools; +//! we use this "by-move" body instead. +//! +//! ## How does this work? +//! +//! This pass essentially remaps the body of the (child) closure of the coroutine-closure +//! to take the set of upvars of the parent closure by value. This at least requires +//! changing a by-ref upvar to be by-value in the case that the outer coroutine-closure +//! captures something by value; however, it may also require renumbering field indices +//! in case precise captures (edition 2021 closure capture rules) caused the inner coroutine +//! to split one field capture into two. -use rustc_data_structures::unord::UnordSet; +use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; +use rustc_middle::hir::place::{PlaceBase, Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, dump_mir, MirPass}; use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt, TypeVisitableExt}; -use rustc_target::abi::FieldIdx; +use rustc_target::abi::{FieldIdx, VariantIdx}; pub struct ByMoveBody; @@ -116,32 +124,116 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { .tuple_fields() .len(); - let mut by_ref_fields = UnordSet::default(); - for (idx, (coroutine_capture, parent_capture)) in tcx + let mut field_remapping = UnordMap::default(); + + // One parent capture may correspond to several child captures if we end up + // refining the set of captures via edition-2021 precise captures. We want to + // match up any number of child captures with one parent capture, so we keep + // peeking off this `Peekable` until the child doesn't match anymore. + let mut parent_captures = + tcx.closure_captures(parent_def_id).iter().copied().enumerate().peekable(); + // Make sure we use every field at least once, b/c why are we capturing something + // if it's not used in the inner coroutine. + let mut field_used_at_least_once = false; + + for (child_field_idx, child_capture) in tcx .closure_captures(coroutine_def_id) .iter() + .copied() // By construction we capture all the args first. .skip(num_args) - .zip_eq(tcx.closure_captures(parent_def_id)) .enumerate() { - // This upvar is captured by-move from the parent closure, but by-ref - // from the inner async block. That means that it's being borrowed from - // the outer closure body -- we need to change the coroutine to take the - // upvar by value. - if coroutine_capture.is_by_ref() && !parent_capture.is_by_ref() { - assert_ne!( - coroutine_kind, - ty::ClosureKind::FnOnce, - "`FnOnce` coroutine-closures return coroutines that capture from \ - their body; it will always result in a borrowck error!" + loop { + let Some(&(parent_field_idx, parent_capture)) = parent_captures.peek() else { + bug!("we ran out of parent captures!") + }; + + let PlaceBase::Upvar(parent_base) = parent_capture.place.base else { + bug!("expected capture to be an upvar"); + }; + let PlaceBase::Upvar(child_base) = child_capture.place.base else { + bug!("expected capture to be an upvar"); + }; + + assert!( + child_capture.place.projections.len() >= parent_capture.place.projections.len() ); - by_ref_fields.insert(FieldIdx::from_usize(num_args + idx)); + // A parent matches a child they share the same prefix of projections. + // The child may have more, if it is capturing sub-fields out of + // something that is captured by-move in the parent closure. + if parent_base.var_path.hir_id != child_base.var_path.hir_id + || !std::iter::zip( + &child_capture.place.projections, + &parent_capture.place.projections, + ) + .all(|(child, parent)| child.kind == parent.kind) + { + // Make sure the field was used at least once. + assert!( + field_used_at_least_once, + "we captured {parent_capture:#?} but it was not used in the child coroutine?" + ); + field_used_at_least_once = false; + // Skip this field. + let _ = parent_captures.next().unwrap(); + continue; + } + + // Store this set of additional projections (fields and derefs). + // We need to re-apply them later. + let child_precise_captures = + &child_capture.place.projections[parent_capture.place.projections.len()..]; + + // If the parent captures by-move, and the child captures by-ref, then we + // need to peel an additional `deref` off of the body of the child. + let needs_deref = child_capture.is_by_ref() && !parent_capture.is_by_ref(); + if needs_deref { + assert_ne!( + coroutine_kind, + ty::ClosureKind::FnOnce, + "`FnOnce` coroutine-closures return coroutines that capture from \ + their body; it will always result in a borrowck error!" + ); + } + + // Finally, store the type of the parent's captured place. We need + // this when building the field projection in the MIR body later on. + let mut parent_capture_ty = parent_capture.place.ty(); + parent_capture_ty = match parent_capture.info.capture_kind { + ty::UpvarCapture::ByValue => parent_capture_ty, + ty::UpvarCapture::ByRef(kind) => Ty::new_ref( + tcx, + tcx.lifetimes.re_erased, + parent_capture_ty, + kind.to_mutbl_lossy(), + ), + }; + + field_remapping.insert( + FieldIdx::from_usize(child_field_idx + num_args), + ( + FieldIdx::from_usize(parent_field_idx + num_args), + parent_capture_ty, + needs_deref, + child_precise_captures, + ), + ); + + field_used_at_least_once = true; + break; } + } + + // Pop the last parent capture + if field_used_at_least_once { + let _ = parent_captures.next().unwrap(); + } + assert_eq!(parent_captures.next(), None, "leftover parent captures?"); - // Make sure we're actually talking about the same capture. - // FIXME(async_closures): We could look at the `hir::Upvar` instead? - assert_eq!(coroutine_capture.place.ty(), parent_capture.place.ty()); + if coroutine_kind == ty::ClosureKind::FnOnce { + assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); + return; } let by_move_coroutine_ty = tcx @@ -157,7 +249,7 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { ); let mut by_move_body = body.clone(); - MakeByMoveBody { tcx, by_ref_fields, by_move_coroutine_ty }.visit_body(&mut by_move_body); + MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(())); by_move_body.source = mir::MirSource::from_instance(InstanceDef::CoroutineKindShim { coroutine_def_id: coroutine_def_id.to_def_id(), @@ -168,7 +260,7 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - by_ref_fields: UnordSet<FieldIdx>, + field_remapping: UnordMap<FieldIdx, (FieldIdx, Ty<'tcx>, bool, &'tcx [Projection<'tcx>])>, by_move_coroutine_ty: Ty<'tcx>, } @@ -183,24 +275,59 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { context: mir::visit::PlaceContext, location: mir::Location, ) { + // Initializing an upvar local always starts with `CAPTURE_STRUCT_LOCAL` and a + // field projection. If this is in `field_remapping`, then it must not be an + // arg from calling the closure, but instead an upvar. if place.local == ty::CAPTURE_STRUCT_LOCAL - && let Some((&mir::ProjectionElem::Field(idx, ty), projection)) = + && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && self.by_ref_fields.contains(&idx) + && let Some(&(remapped_idx, remapped_ty, needs_deref, additional_projections)) = + self.field_remapping.get(&idx) { - let (begin, end) = projection.split_first().unwrap(); - // FIXME(async_closures): I'm actually a bit surprised to see that we always - // initially deref the by-ref upvars. If this is not actually true, then we - // will at least get an ICE that explains why this isn't true :^) - assert_eq!(*begin, mir::ProjectionElem::Deref); - // Peel one ref off of the ty. - let peeled_ty = ty.builtin_deref(true).unwrap().ty; + // As noted before, if the parent closure captures a field by value, and + // the child captures a field by ref, then for the by-move body we're + // generating, we also are taking that field by value. Peel off a deref, + // since a layer of reffing has now become redundant. + let final_deref = if needs_deref { + let Some((mir::ProjectionElem::Deref, projection)) = projection.split_first() + else { + bug!( + "There should be at least a single deref for an upvar local initialization, found {projection:#?}" + ); + }; + // There may be more derefs, since we may also implicitly reborrow + // a captured mut pointer. + projection + } else { + projection + }; + + // The only thing that should be left is a deref, if the parent captured + // an upvar by-ref. + std::assert_matches::assert_matches!(final_deref, [] | [mir::ProjectionElem::Deref]); + + // For all of the additional projections that come out of precise capturing, + // re-apply these projections. + let additional_projections = + additional_projections.iter().map(|elem| match elem.kind { + ProjectionKind::Deref => mir::ProjectionElem::Deref, + ProjectionKind::Field(idx, VariantIdx::ZERO) => { + mir::ProjectionElem::Field(idx, elem.ty) + } + _ => unreachable!("precise captures only through fields and derefs"), + }); + + // We start out with an adjusted field index (and ty), representing the + // upvar that we get from our parent closure. We apply any of the additional + // projections to make sure that to the rest of the body of the closure, the + // place looks the same, and then apply that final deref if necessary. *place = mir::Place { local: place.local, projection: self.tcx.mk_place_elems_from_iter( - [mir::ProjectionElem::Field(idx, peeled_ty)] + [mir::ProjectionElem::Field(remapped_idx, remapped_ty)] .into_iter() - .chain(end.iter().copied()), + .chain(additional_projections) + .chain(final_deref.iter().copied()), ), }; } diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 69b48bf0aff..353c3f41ed8 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -34,8 +34,7 @@ use unescape_error_reporting::{emit_unescape_error, escaped_char}; rustc_data_structures::static_assert_size!(rustc_lexer::Token, 12); #[derive(Clone, Debug)] -pub struct UnmatchedDelim { - pub expected_delim: Delimiter, +pub(crate) struct UnmatchedDelim { pub found_delim: Option<Delimiter>, pub found_span: Span, pub unclosed_span: Option<Span>, diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index a506f98bf3a..b5a5a2a90ee 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -77,7 +77,6 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { for &(_, sp) in &self.diag_info.open_braces { err.span_label(sp, "unclosed delimiter"); self.diag_info.unmatched_delims.push(UnmatchedDelim { - expected_delim: Delimiter::Brace, found_delim: None, found_span: self.token.span, unclosed_span: Some(sp), @@ -163,9 +162,8 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> { candidate = Some(*brace_span); } } - let (tok, _) = self.diag_info.open_braces.pop().unwrap(); + let (_, _) = self.diag_info.open_braces.pop().unwrap(); self.diag_info.unmatched_delims.push(UnmatchedDelim { - expected_delim: tok, found_delim: Some(close_delim), found_span: self.token.span, unclosed_span: unclosed_delimiter, diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs index f99f6a3b721..04020d13f22 100644 --- a/compiler/rustc_target/src/abi/call/aarch64.rs +++ b/compiler/rustc_target/src/abi/call/aarch64.rs @@ -31,7 +31,7 @@ where RegKind::Vector => size.bits() == 64 || size.bits() == 128, }; - valid_unit.then_some(Uniform { unit, total: size }) + valid_unit.then_some(Uniform::consecutive(unit, size)) }) } @@ -60,7 +60,7 @@ where let size = ret.layout.size; let bits = size.bits(); if bits <= 128 { - ret.cast_to(Uniform { unit: Reg::i64(), total: size }); + ret.cast_to(Uniform::new(Reg::i64(), size)); return; } ret.make_indirect(); @@ -100,9 +100,9 @@ where }; if size.bits() <= 128 { if align.bits() == 128 { - arg.cast_to(Uniform { unit: Reg::i128(), total: size }); + arg.cast_to(Uniform::new(Reg::i128(), size)); } else { - arg.cast_to(Uniform { unit: Reg::i64(), total: size }); + arg.cast_to(Uniform::new(Reg::i64(), size)); } return; } diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs index 95f6691d42a..9371e1b3958 100644 --- a/compiler/rustc_target/src/abi/call/arm.rs +++ b/compiler/rustc_target/src/abi/call/arm.rs @@ -21,7 +21,7 @@ where RegKind::Vector => size.bits() == 64 || size.bits() == 128, }; - valid_unit.then_some(Uniform { unit, total: size }) + valid_unit.then_some(Uniform::consecutive(unit, size)) }) } @@ -49,7 +49,7 @@ where let size = ret.layout.size; let bits = size.bits(); if bits <= 32 { - ret.cast_to(Uniform { unit: Reg::i32(), total: size }); + ret.cast_to(Uniform::new(Reg::i32(), size)); return; } ret.make_indirect(); @@ -78,7 +78,7 @@ where let align = arg.layout.align.abi.bytes(); let total = arg.layout.size; - arg.cast_to(Uniform { unit: if align <= 4 { Reg::i32() } else { Reg::i64() }, total }); + arg.cast_to(Uniform::consecutive(if align <= 4 { Reg::i32() } else { Reg::i64() }, total)); } pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) diff --git a/compiler/rustc_target/src/abi/call/csky.rs b/compiler/rustc_target/src/abi/call/csky.rs index 8b4328db52e..7951f28beea 100644 --- a/compiler/rustc_target/src/abi/call/csky.rs +++ b/compiler/rustc_target/src/abi/call/csky.rs @@ -18,7 +18,7 @@ fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) { if total.bits() > 64 { arg.make_indirect(); } else if total.bits() > 32 { - arg.cast_to(Uniform { unit: Reg::i32(), total }); + arg.cast_to(Uniform::new(Reg::i32(), total)); } else { arg.cast_to(Reg::i32()); } @@ -38,7 +38,7 @@ fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) { if arg.layout.is_aggregate() { let total = arg.layout.size; if total.bits() > 32 { - arg.cast_to(Uniform { unit: Reg::i32(), total }); + arg.cast_to(Uniform::new(Reg::i32(), total)); } else { arg.cast_to(Reg::i32()); } diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs index 35d4b331cb4..943b12a9fbf 100644 --- a/compiler/rustc_target/src/abi/call/loongarch.rs +++ b/compiler/rustc_target/src/abi/call/loongarch.rs @@ -195,7 +195,7 @@ where if total.bits() <= xlen { arg.cast_to(xlen_reg); } else { - arg.cast_to(Uniform { unit: xlen_reg, total: Size::from_bits(xlen * 2) }); + arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2))); } return false; } @@ -278,10 +278,10 @@ fn classify_arg<'a, Ty, C>( if total.bits() > xlen { let align_regs = align > xlen; if is_loongarch_aggregate(arg) { - arg.cast_to(Uniform { - unit: if align_regs { double_xlen_reg } else { xlen_reg }, - total: Size::from_bits(xlen * 2), - }); + arg.cast_to(Uniform::new( + if align_regs { double_xlen_reg } else { xlen_reg }, + Size::from_bits(xlen * 2), + )); } if align_regs && is_vararg { *avail_gprs -= *avail_gprs % 2; diff --git a/compiler/rustc_target/src/abi/call/mips.rs b/compiler/rustc_target/src/abi/call/mips.rs index 57ccfe2152b..0e5a7f37a09 100644 --- a/compiler/rustc_target/src/abi/call/mips.rs +++ b/compiler/rustc_target/src/abi/call/mips.rs @@ -27,7 +27,7 @@ where if arg.layout.is_aggregate() { let pad_i32 = !offset.is_aligned(align); - arg.cast_to_and_pad_i32(Uniform { unit: Reg::i32(), total: size }, pad_i32); + arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32); } else { arg.extend_integer_width_to(32); } diff --git a/compiler/rustc_target/src/abi/call/mips64.rs b/compiler/rustc_target/src/abi/call/mips64.rs index 2700f67b209..b2a2c34b980 100644 --- a/compiler/rustc_target/src/abi/call/mips64.rs +++ b/compiler/rustc_target/src/abi/call/mips64.rs @@ -68,7 +68,7 @@ where } // Cast to a uniform int structure - ret.cast_to(Uniform { unit: Reg::i64(), total: size }); + ret.cast_to(Uniform::new(Reg::i64(), size)); } else { ret.make_indirect(); } @@ -139,7 +139,7 @@ where let rest_size = size - Size::from_bytes(8) * prefix_index as u64; arg.cast_to(CastTarget { prefix, - rest: Uniform { unit: Reg::i64(), total: rest_size }, + rest: Uniform::new(Reg::i64(), rest_size), attrs: ArgAttributes { regular: ArgAttribute::default(), arg_ext: ArgExtension::None, diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index cdd3f0afd79..4502df339d1 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -255,11 +255,16 @@ pub struct Uniform { /// for 64-bit integers with a total size of 20 bytes. When the argument is actually passed, /// this size will be rounded up to the nearest multiple of `unit.size`. pub total: Size, + + /// Indicate that the argument is consecutive, in the sense that either all values need to be + /// passed in register, or all on the stack. If they are passed on the stack, there should be + /// no additional padding between elements. + pub is_consecutive: bool, } impl From<Reg> for Uniform { fn from(unit: Reg) -> Uniform { - Uniform { unit, total: unit.size } + Uniform { unit, total: unit.size, is_consecutive: false } } } @@ -267,6 +272,18 @@ impl Uniform { pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { self.unit.align(cx) } + + /// Pass using one or more values of the given type, without requiring them to be consecutive. + /// That is, some values may be passed in register and some on the stack. + pub fn new(unit: Reg, total: Size) -> Self { + Uniform { unit, total, is_consecutive: false } + } + + /// Pass using one or more consecutive values of the given type. Either all values will be + /// passed in registers, or all on the stack. + pub fn consecutive(unit: Reg, total: Size) -> Self { + Uniform { unit, total, is_consecutive: true } + } } /// Describes the type used for `PassMode::Cast`. diff --git a/compiler/rustc_target/src/abi/call/nvptx64.rs b/compiler/rustc_target/src/abi/call/nvptx64.rs index 5c040ce9c3b..f85fa2419f0 100644 --- a/compiler/rustc_target/src/abi/call/nvptx64.rs +++ b/compiler/rustc_target/src/abi/call/nvptx64.rs @@ -35,7 +35,7 @@ where 16 => Reg::i128(), _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"), }; - arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes) }); + arg.cast_to(Uniform::new(unit, Size::from_bytes(2 * align_bytes))); } else { // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271. arg.make_direct_deprecated(); diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs index 2d41f77e50e..11a6cb52bab 100644 --- a/compiler/rustc_target/src/abi/call/powerpc64.rs +++ b/compiler/rustc_target/src/abi/call/powerpc64.rs @@ -2,7 +2,7 @@ // Alignment of 128 bit types is not currently handled, this will // need to be fixed when PowerPC vector support is added. -use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind, Uniform}; +use crate::abi::call::{Align, ArgAbi, FnAbi, Reg, RegKind, Uniform}; use crate::abi::{Endian, HasDataLayout, TyAbiInterface}; use crate::spec::HasTargetSpec; @@ -37,7 +37,7 @@ where RegKind::Vector => arg.layout.size.bits() == 128, }; - valid_unit.then_some(Uniform { unit, total: arg.layout.size }) + valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size)) }) } @@ -81,7 +81,7 @@ where Reg::i64() }; - ret.cast_to(Uniform { unit, total: size }); + ret.cast_to(Uniform::new(unit, size)); return; } @@ -108,18 +108,20 @@ where } let size = arg.layout.size; - let (unit, total) = if size.bits() <= 64 { + if size.bits() <= 64 { // Aggregates smaller than a doubleword should appear in // the least-significant bits of the parameter doubleword. - (Reg { kind: RegKind::Integer, size }, size) + arg.cast_to(Reg { kind: RegKind::Integer, size }) } else { - // Aggregates larger than a doubleword should be padded - // at the tail to fill out a whole number of doublewords. - let reg_i64 = Reg::i64(); - (reg_i64, size.align_to(reg_i64.align(cx))) + // Aggregates larger than i64 should be padded at the tail to fill out a whole number + // of i64s or i128s, depending on the aggregate alignment. Always use an array for + // this, even if there is only a single element. + let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() }; + arg.cast_to(Uniform::consecutive( + reg, + size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()), + )) }; - - arg.cast_to(Uniform { unit, total }); } pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs index 6a38496dc57..5d4b3a9d245 100644 --- a/compiler/rustc_target/src/abi/call/riscv.rs +++ b/compiler/rustc_target/src/abi/call/riscv.rs @@ -201,7 +201,7 @@ where if total.bits() <= xlen { arg.cast_to(xlen_reg); } else { - arg.cast_to(Uniform { unit: xlen_reg, total: Size::from_bits(xlen * 2) }); + arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2))); } return false; } @@ -284,10 +284,10 @@ fn classify_arg<'a, Ty, C>( if total.bits() > xlen { let align_regs = align > xlen; if is_riscv_aggregate(arg) { - arg.cast_to(Uniform { - unit: if align_regs { double_xlen_reg } else { xlen_reg }, - total: Size::from_bits(xlen * 2), - }); + arg.cast_to(Uniform::new( + if align_regs { double_xlen_reg } else { xlen_reg }, + Size::from_bits(xlen * 2), + )); } if align_regs && is_vararg { *avail_gprs -= *avail_gprs % 2; diff --git a/compiler/rustc_target/src/abi/call/sparc.rs b/compiler/rustc_target/src/abi/call/sparc.rs index 57ccfe2152b..0e5a7f37a09 100644 --- a/compiler/rustc_target/src/abi/call/sparc.rs +++ b/compiler/rustc_target/src/abi/call/sparc.rs @@ -27,7 +27,7 @@ where if arg.layout.is_aggregate() { let pad_i32 = !offset.is_aligned(align); - arg.cast_to_and_pad_i32(Uniform { unit: Reg::i32(), total: size }, pad_i32); + arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32); } else { arg.extend_integer_width_to(32); } diff --git a/compiler/rustc_target/src/abi/call/sparc64.rs b/compiler/rustc_target/src/abi/call/sparc64.rs index cbed5b4afc1..acdcd5cc0d4 100644 --- a/compiler/rustc_target/src/abi/call/sparc64.rs +++ b/compiler/rustc_target/src/abi/call/sparc64.rs @@ -192,7 +192,7 @@ where arg.cast_to(CastTarget { prefix: data.prefix, - rest: Uniform { unit: Reg::i64(), total: rest_size }, + rest: Uniform::new(Reg::i64(), rest_size), attrs: ArgAttributes { regular: data.arg_attribute, arg_ext: ArgExtension::None, @@ -205,7 +205,7 @@ where } } - arg.cast_to(Uniform { unit: Reg::i64(), total }); + arg.cast_to(Uniform::new(Reg::i64(), total)); } pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) diff --git a/compiler/rustc_target/src/abi/call/wasm.rs b/compiler/rustc_target/src/abi/call/wasm.rs index a7a2b314a94..a773fb1e814 100644 --- a/compiler/rustc_target/src/abi/call/wasm.rs +++ b/compiler/rustc_target/src/abi/call/wasm.rs @@ -1,4 +1,4 @@ -use crate::abi::call::{ArgAbi, FnAbi, Uniform}; +use crate::abi::call::{ArgAbi, FnAbi}; use crate::abi::{HasDataLayout, TyAbiInterface}; fn unwrap_trivial_aggregate<'a, Ty, C>(cx: &C, val: &mut ArgAbi<'a, Ty>) -> bool @@ -10,7 +10,7 @@ where if let Some(unit) = val.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()) { let size = val.layout.size; if unit.size == size { - val.cast_to(Uniform { unit, total: size }); + val.cast_to(unit); return true; } } diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs index 10c03387a5b..837b784f272 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs @@ -40,7 +40,7 @@ pub struct ImplCandidate<'tcx> { enum GetSafeTransmuteErrorAndReason { Silent, - Error { err_msg: String, safe_transmute_explanation: String }, + Error { err_msg: String, safe_transmute_explanation: Option<String> }, } struct UnsatisfiedConst(pub bool); diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs index afdc2c5a7f7..1b8b09ddda1 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs @@ -558,7 +558,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation, - } => (err_msg, Some(safe_transmute_explanation)), + } => (err_msg, safe_transmute_explanation), } } else { (err_msg, None) @@ -3068,28 +3068,33 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { return GetSafeTransmuteErrorAndReason::Silent; }; + let dst = trait_ref.args.type_at(0); + let src = trait_ref.args.type_at(1); + let err_msg = format!("`{src}` cannot be safely transmuted into `{dst}`"); + match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable( obligation.cause, src_and_dst, assume, ) { Answer::No(reason) => { - let dst = trait_ref.args.type_at(0); - let src = trait_ref.args.type_at(1); - let err_msg = format!("`{src}` cannot be safely transmuted into `{dst}`"); let safe_transmute_explanation = match reason { rustc_transmute::Reason::SrcIsNotYetSupported => { - format!("analyzing the transmutability of `{src}` is not yet supported.") + format!("analyzing the transmutability of `{src}` is not yet supported") } rustc_transmute::Reason::DstIsNotYetSupported => { - format!("analyzing the transmutability of `{dst}` is not yet supported.") + format!("analyzing the transmutability of `{dst}` is not yet supported") } rustc_transmute::Reason::DstIsBitIncompatible => { format!("at least one value of `{src}` isn't a bit-valid value of `{dst}`") } + rustc_transmute::Reason::DstUninhabited => { + format!("`{dst}` is uninhabited") + } + rustc_transmute::Reason::DstMayHaveSafetyInvariants => { format!("`{dst}` may carry safety invariants") } @@ -3135,14 +3140,23 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { format!("`{dst}` has an unknown layout") } }; - GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation } + GetSafeTransmuteErrorAndReason::Error { + err_msg, + safe_transmute_explanation: Some(safe_transmute_explanation), + } } // Should never get a Yes at this point! We already ran it before, and did not get a Yes. Answer::Yes => span_bug!( span, "Inconsistent rustc_transmute::is_transmutable(...) result, got Yes", ), - other => span_bug!(span, "Unsupported rustc_transmute::Answer variant: `{other:?}`"), + // Reached when a different obligation (namely `Freeze`) causes the + // transmutability analysis to fail. In this case, silence the + // transmutability error message in favor of that more specific + // error. + Answer::If(_) => { + GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation: None } + } } } diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs index 2c8116b779b..98d5b466cd0 100644 --- a/compiler/rustc_trait_selection/src/traits/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/mod.rs @@ -119,7 +119,9 @@ pub fn predicates_for_generics<'tcx>( /// Determines whether the type `ty` is known to meet `bound` and /// returns true if so. Returns false if `ty` either does not meet -/// `bound` or is not known to meet bound. +/// `bound` or is not known to meet bound (note that this is +/// conservative towards *no impl*, which is the opposite of the +/// `evaluate` methods). pub fn type_known_to_meet_bound_modulo_regions<'tcx>( infcx: &InferCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>, @@ -127,8 +129,50 @@ pub fn type_known_to_meet_bound_modulo_regions<'tcx>( def_id: DefId, ) -> bool { let trait_ref = ty::TraitRef::new(infcx.tcx, def_id, [ty]); - let obligation = Obligation::new(infcx.tcx, ObligationCause::dummy(), param_env, trait_ref); - infcx.predicate_must_hold_modulo_regions(&obligation) + pred_known_to_hold_modulo_regions(infcx, param_env, trait_ref) +} + +/// FIXME(@lcnr): this function doesn't seem right and shouldn't exist? +/// +/// Ping me on zulip if you want to use this method and need help with finding +/// an appropriate replacement. +#[instrument(level = "debug", skip(infcx, param_env, pred), ret)] +fn pred_known_to_hold_modulo_regions<'tcx>( + infcx: &InferCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + pred: impl ToPredicate<'tcx>, +) -> bool { + let obligation = Obligation::new(infcx.tcx, ObligationCause::dummy(), param_env, pred); + + let result = infcx.evaluate_obligation_no_overflow(&obligation); + debug!(?result); + + if result.must_apply_modulo_regions() { + true + } else if result.may_apply() { + // Sometimes obligations are ambiguous because the recursive evaluator + // is not smart enough, so we fall back to fulfillment when we're not certain + // that an obligation holds or not. Even still, we must make sure that + // the we do no inference in the process of checking this obligation. + let goal = infcx.resolve_vars_if_possible((obligation.predicate, obligation.param_env)); + infcx.probe(|_| { + let ocx = ObligationCtxt::new(infcx); + ocx.register_obligation(obligation); + + let errors = ocx.select_all_or_error(); + match errors.as_slice() { + // Only known to hold if we did no inference. + [] => infcx.shallow_resolve(goal) == goal, + + errors => { + debug!(?errors); + false + } + } + }) + } else { + false + } } #[instrument(level = "debug", skip(tcx, elaborated_env))] diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index f98a1714a3f..25ba985397e 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -314,12 +314,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { .flat_map(|cond| flatten_answer_tree(tcx, obligation, predicate, cond)) .collect(), Condition::IfTransmutable { src, dst } => { - let trait_def_id = obligation.predicate.def_id(); + let transmute_trait = obligation.predicate.def_id(); let assume_const = predicate.trait_ref.args.const_at(2); - let make_obl = |from_ty, to_ty| { - let trait_ref1 = ty::TraitRef::new( + let make_transmute_obl = |from_ty, to_ty| { + let trait_ref = ty::TraitRef::new( tcx, - trait_def_id, + transmute_trait, [ ty::GenericArg::from(to_ty), ty::GenericArg::from(from_ty), @@ -331,17 +331,45 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { obligation.cause.clone(), obligation.recursion_depth + 1, obligation.param_env, - trait_ref1, + trait_ref, ) }; + let make_freeze_obl = |ty| { + let trait_ref = ty::TraitRef::new( + tcx, + tcx.lang_items().freeze_trait().unwrap(), + [ty::GenericArg::from(ty)], + ); + Obligation::with_depth( + tcx, + obligation.cause.clone(), + obligation.recursion_depth + 1, + obligation.param_env, + trait_ref, + ) + }; + + let mut obls = vec![]; + + // If the source is a shared reference, it must be `Freeze`; + // otherwise, transmuting could lead to data races. + if src.mutability == Mutability::Not { + obls.extend([make_freeze_obl(src.ty), make_freeze_obl(dst.ty)]) + } + // If Dst is mutable, check bidirectionally. // For example, transmuting bool -> u8 is OK as long as you can't update that u8 // to be > 1, because you could later transmute the u8 back to a bool and get UB. match dst.mutability { - Mutability::Not => vec![make_obl(src.ty, dst.ty)], - Mutability::Mut => vec![make_obl(src.ty, dst.ty), make_obl(dst.ty, src.ty)], + Mutability::Not => obls.push(make_transmute_obl(src.ty, dst.ty)), + Mutability::Mut => obls.extend([ + make_transmute_obl(src.ty, dst.ty), + make_transmute_obl(dst.ty, src.ty), + ]), } + + obls } } } diff --git a/compiler/rustc_transmute/src/layout/dfa.rs b/compiler/rustc_transmute/src/layout/dfa.rs index b8922696e30..77d5a48f158 100644 --- a/compiler/rustc_transmute/src/layout/dfa.rs +++ b/compiler/rustc_transmute/src/layout/dfa.rs @@ -35,6 +35,7 @@ impl<R> Transitions<R> where R: Ref, { + #[allow(dead_code)] fn insert(&mut self, transition: Transition<R>, state: State) { match transition { Transition::Byte(b) => { @@ -82,6 +83,7 @@ impl<R> Dfa<R> where R: Ref, { + #[allow(dead_code)] pub(crate) fn unit() -> Self { let transitions: Map<State, Transitions<R>> = Map::default(); let start = State::new(); diff --git a/compiler/rustc_transmute/src/layout/nfa.rs b/compiler/rustc_transmute/src/layout/nfa.rs index 78fcceb5f2c..3c5963202c6 100644 --- a/compiler/rustc_transmute/src/layout/nfa.rs +++ b/compiler/rustc_transmute/src/layout/nfa.rs @@ -160,6 +160,7 @@ where Self { transitions, start, accepting } } + #[allow(dead_code)] pub(crate) fn edges_from(&self, start: State) -> Option<&Map<Transition<R>, Set<State>>> { self.transitions.get(&start) } diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs index f6bc224c7e7..12c984f1603 100644 --- a/compiler/rustc_transmute/src/layout/tree.rs +++ b/compiler/rustc_transmute/src/layout/tree.rs @@ -173,16 +173,20 @@ pub(crate) mod rustc { use super::Tree; use crate::layout::rustc::{Def, Ref}; + use rustc_middle::ty::layout::HasTyCtxt; + use rustc_middle::ty::layout::LayoutCx; use rustc_middle::ty::layout::LayoutError; + use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::AdtDef; - use rustc_middle::ty::GenericArgsRef; - use rustc_middle::ty::ParamEnv; + use rustc_middle::ty::AdtKind; + use rustc_middle::ty::List; use rustc_middle::ty::ScalarInt; - use rustc_middle::ty::VariantDef; use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt}; use rustc_span::ErrorGuaranteed; - use rustc_target::abi::Align; - use std::alloc; + use rustc_target::abi::FieldsShape; + use rustc_target::abi::Size; + use rustc_target::abi::TyAndLayout; + use rustc_target::abi::Variants; #[derive(Debug, Copy, Clone)] pub(crate) enum Err { @@ -206,176 +210,64 @@ pub(crate) mod rustc { } } - trait LayoutExt { - fn clamp_align(&self, min_align: Align, max_align: Align) -> Self; - } - - impl LayoutExt for alloc::Layout { - fn clamp_align(&self, min_align: Align, max_align: Align) -> Self { - let min_align = min_align.bytes().try_into().unwrap(); - let max_align = max_align.bytes().try_into().unwrap(); - Self::from_size_align(self.size(), self.align().clamp(min_align, max_align)).unwrap() - } - } - - struct LayoutSummary { - total_align: Align, - total_size: usize, - discriminant_size: usize, - discriminant_align: Align, - } - - impl LayoutSummary { - fn from_ty<'tcx>(ty: Ty<'tcx>, ctx: TyCtxt<'tcx>) -> Result<Self, &'tcx LayoutError<'tcx>> { - use rustc_middle::ty::ParamEnvAnd; - use rustc_target::abi::{TyAndLayout, Variants}; - - let param_env = ParamEnv::reveal_all(); - let param_env_and_type = ParamEnvAnd { param_env, value: ty }; - let TyAndLayout { layout, .. } = ctx.layout_of(param_env_and_type)?; - - let total_size: usize = layout.size().bytes_usize(); - let total_align: Align = layout.align().abi; - let discriminant_align: Align; - let discriminant_size: usize; - - if let Variants::Multiple { tag, .. } = layout.variants() { - discriminant_align = tag.align(&ctx).abi; - discriminant_size = tag.size(&ctx).bytes_usize(); - } else { - discriminant_align = Align::ONE; - discriminant_size = 0; - }; - - Ok(Self { total_align, total_size, discriminant_align, discriminant_size }) - } - - fn into(&self) -> alloc::Layout { - alloc::Layout::from_size_align( - self.total_size, - self.total_align.bytes().try_into().unwrap(), - ) - .unwrap() - } - } - impl<'tcx> Tree<Def<'tcx>, Ref<'tcx>> { - pub fn from_ty(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Result<Self, Err> { - use rustc_middle::ty::FloatTy::*; - use rustc_middle::ty::IntTy::*; - use rustc_middle::ty::UintTy::*; + pub fn from_ty( + ty_and_layout: TyAndLayout<'tcx, Ty<'tcx>>, + cx: LayoutCx<'tcx, TyCtxt<'tcx>>, + ) -> Result<Self, Err> { use rustc_target::abi::HasDataLayout; - if let Err(e) = ty.error_reported() { + if let Err(e) = ty_and_layout.ty.error_reported() { return Err(Err::TypeError(e)); } - let target = tcx.data_layout(); + let target = cx.tcx.data_layout(); + let pointer_size = target.pointer_size; - match ty.kind() { + match ty_and_layout.ty.kind() { ty::Bool => Ok(Self::bool()), - ty::Int(I8) | ty::Uint(U8) => Ok(Self::u8()), - ty::Int(I16) | ty::Uint(U16) => Ok(Self::number(2)), - ty::Int(I32) | ty::Uint(U32) | ty::Float(F32) => Ok(Self::number(4)), - ty::Int(I64) | ty::Uint(U64) | ty::Float(F64) => Ok(Self::number(8)), - ty::Int(I128) | ty::Uint(U128) => Ok(Self::number(16)), - ty::Int(Isize) | ty::Uint(Usize) => { - Ok(Self::number(target.pointer_size.bytes_usize())) + ty::Float(nty) => { + let width = nty.bit_width() / 8; + Ok(Self::number(width as _)) } - ty::Tuple(members) => { - if members.len() == 0 { - Ok(Tree::unit()) - } else { - Err(Err::NotYetSupported) - } + ty::Int(nty) => { + let width = nty.normalize(pointer_size.bits() as _).bit_width().unwrap() / 8; + Ok(Self::number(width as _)) } - ty::Array(ty, len) => { - let len = len - .try_eval_target_usize(tcx, ParamEnv::reveal_all()) - .ok_or(Err::NotYetSupported)?; - let elt = Tree::from_ty(*ty, tcx)?; - Ok(std::iter::repeat(elt) - .take(len as usize) - .fold(Tree::unit(), |tree, elt| tree.then(elt))) + ty::Uint(nty) => { + let width = nty.normalize(pointer_size.bits() as _).bit_width().unwrap() / 8; + Ok(Self::number(width as _)) } - ty::Adt(adt_def, args_ref) => { - use rustc_middle::ty::AdtKind; + ty::Tuple(members) => Self::from_tuple(ty_and_layout, members, cx), - // If the layout is ill-specified, halt. - if !(adt_def.repr().c() || adt_def.repr().int.is_some()) { + ty::Array(inner_ty, len) => { + let FieldsShape::Array { stride, count } = &ty_and_layout.fields else { return Err(Err::NotYetSupported); - } + }; + let inner_ty_and_layout = cx.layout_of(*inner_ty)?; + assert_eq!(*stride, inner_ty_and_layout.size); + let elt = Tree::from_ty(inner_ty_and_layout, cx)?; + Ok(std::iter::repeat(elt) + .take(*count as usize) + .fold(Tree::unit(), |tree, elt| tree.then(elt))) + } - // Compute a summary of the type's layout. - let layout_summary = LayoutSummary::from_ty(ty, tcx)?; - - // The layout begins with this adt's visibility. - let vis = Self::def(Def::Adt(*adt_def)); - - // And is followed the layout(s) of its variants - Ok(vis.then(match adt_def.adt_kind() { - AdtKind::Struct => Self::from_repr_c_variant( - ty, - *adt_def, - args_ref, - &layout_summary, - None, - adt_def.non_enum_variant(), - tcx, - )?, - AdtKind::Enum => { - trace!(?adt_def, "treeifying enum"); - let mut tree = Tree::uninhabited(); - - for (idx, variant) in adt_def.variants().iter_enumerated() { - let tag = tcx.tag_for_variant((ty, idx)); - tree = tree.or(Self::from_repr_c_variant( - ty, - *adt_def, - args_ref, - &layout_summary, - tag, - variant, - tcx, - )?); - } - - tree - } - AdtKind::Union => { - // is the layout well-defined? - if !adt_def.repr().c() { - return Err(Err::NotYetSupported); - } - - let ty_layout = layout_of(tcx, ty)?; - - let mut tree = Tree::uninhabited(); - - for field in adt_def.all_fields() { - let variant_ty = field.ty(tcx, args_ref); - let variant_layout = layout_of(tcx, variant_ty)?; - let padding_needed = ty_layout.size() - variant_layout.size(); - let variant = Self::def(Def::Field(field)) - .then(Self::from_ty(variant_ty, tcx)?) - .then(Self::padding(padding_needed)); - - tree = tree.or(variant); - } - - tree - } - })) + ty::Adt(adt_def, _args_ref) if !ty_and_layout.ty.is_box() => { + match adt_def.adt_kind() { + AdtKind::Struct => Self::from_struct(ty_and_layout, *adt_def, cx), + AdtKind::Enum => Self::from_enum(ty_and_layout, *adt_def, cx), + AdtKind::Union => Self::from_union(ty_and_layout, *adt_def, cx), + } } ty::Ref(lifetime, ty, mutability) => { - let layout = layout_of(tcx, *ty)?; - let align = layout.align(); - let size = layout.size(); + let ty_and_layout = cx.layout_of(*ty)?; + let align = ty_and_layout.align.abi.bytes() as usize; + let size = ty_and_layout.size.bytes_usize(); Ok(Tree::Ref(Ref { lifetime: *lifetime, ty: *ty, @@ -389,80 +281,143 @@ pub(crate) mod rustc { } } - fn from_repr_c_variant( - ty: Ty<'tcx>, - adt_def: AdtDef<'tcx>, - args_ref: GenericArgsRef<'tcx>, - layout_summary: &LayoutSummary, - tag: Option<ScalarInt>, - variant_def: &'tcx VariantDef, - tcx: TyCtxt<'tcx>, + /// Constructs a `Tree` from a tuple. + fn from_tuple( + ty_and_layout: TyAndLayout<'tcx, Ty<'tcx>>, + members: &'tcx List<Ty<'tcx>>, + cx: LayoutCx<'tcx, TyCtxt<'tcx>>, ) -> Result<Self, Err> { - let mut tree = Tree::unit(); - - let repr = adt_def.repr(); - let min_align = repr.align.unwrap_or(Align::ONE); - let max_align = repr.pack.unwrap_or(Align::MAX); - - let variant_span = trace_span!( - "treeifying variant", - min_align = ?min_align, - max_align = ?max_align, - ) - .entered(); - - let mut variant_layout = alloc::Layout::from_size_align( - 0, - layout_summary.total_align.bytes().try_into().unwrap(), - ) - .unwrap(); - - // The layout of the variant is prefixed by the tag, if any. - if let Some(tag) = tag { - let tag_layout = - alloc::Layout::from_size_align(tag.size().bytes_usize(), 1).unwrap(); - tree = tree.then(Self::from_tag(tag, tcx)); - variant_layout = variant_layout.extend(tag_layout).unwrap().0; + match &ty_and_layout.fields { + FieldsShape::Primitive => { + assert_eq!(members.len(), 1); + let inner_ty = members[0]; + let inner_ty_and_layout = cx.layout_of(inner_ty)?; + assert_eq!(ty_and_layout.layout, inner_ty_and_layout.layout); + Self::from_ty(inner_ty_and_layout, cx) + } + FieldsShape::Arbitrary { offsets, .. } => { + assert_eq!(offsets.len(), members.len()); + Self::from_variant(Def::Primitive, None, ty_and_layout, ty_and_layout.size, cx) + } + FieldsShape::Array { .. } | FieldsShape::Union(_) => Err(Err::NotYetSupported), + } + } + + /// Constructs a `Tree` from a struct. + /// + /// # Panics + /// + /// Panics if `def` is not a struct definition. + fn from_struct( + ty_and_layout: TyAndLayout<'tcx, Ty<'tcx>>, + def: AdtDef<'tcx>, + cx: LayoutCx<'tcx, TyCtxt<'tcx>>, + ) -> Result<Self, Err> { + assert!(def.is_struct()); + let def = Def::Adt(def); + Self::from_variant(def, None, ty_and_layout, ty_and_layout.size, cx) + } + + /// Constructs a `Tree` from an enum. + /// + /// # Panics + /// + /// Panics if `def` is not an enum definition. + fn from_enum( + ty_and_layout: TyAndLayout<'tcx, Ty<'tcx>>, + def: AdtDef<'tcx>, + cx: LayoutCx<'tcx, TyCtxt<'tcx>>, + ) -> Result<Self, Err> { + assert!(def.is_enum()); + let layout = ty_and_layout.layout; + + if let Variants::Multiple { tag_field, .. } = layout.variants() { + // For enums (but not coroutines), the tag field is + // currently always the first field of the layout. + assert_eq!(*tag_field, 0); } - // Next come fields. - let fields_span = trace_span!("treeifying fields").entered(); - for field_def in variant_def.fields.iter() { - let field_ty = field_def.ty(tcx, args_ref); - let _span = trace_span!("treeifying field", field = ?field_ty).entered(); + let variants = def.discriminants(cx.tcx()).try_fold( + Self::uninhabited(), + |variants, (idx, ref discriminant)| { + let tag = cx.tcx.tag_for_variant((ty_and_layout.ty, idx)); + let variant_def = Def::Variant(def.variant(idx)); + let variant_ty_and_layout = ty_and_layout.for_variant(&cx, idx); + let variant = Self::from_variant( + variant_def, + tag, + variant_ty_and_layout, + layout.size, + cx, + )?; + Result::<Self, Err>::Ok(variants.or(variant)) + }, + )?; - // begin with the field's visibility - tree = tree.then(Self::def(Def::Field(field_def))); + return Ok(Self::def(Def::Adt(def)).then(variants)); + } - // compute the field's layout characteristics - let field_layout = layout_of(tcx, field_ty)?.clamp_align(min_align, max_align); + /// Constructs a `Tree` from a 'variant-like' layout. + /// + /// A 'variant-like' layout includes those of structs and, of course, + /// enum variants. Pragmatically speaking, this method supports anything + /// with `FieldsShape::Arbitrary`. + /// + /// Note: This routine assumes that the optional `tag` is the first + /// field, and enum callers should check that `tag_field` is, in fact, + /// `0`. + fn from_variant( + def: Def<'tcx>, + tag: Option<ScalarInt>, + ty_and_layout: TyAndLayout<'tcx, Ty<'tcx>>, + total_size: Size, + cx: LayoutCx<'tcx, TyCtxt<'tcx>>, + ) -> Result<Self, Err> { + // This constructor does not support non-`FieldsShape::Arbitrary` + // layouts. + let FieldsShape::Arbitrary { offsets, memory_index } = ty_and_layout.layout.fields() + else { + return Err(Err::NotYetSupported); + }; - // next comes the field's padding - let padding_needed = variant_layout.padding_needed_for(field_layout.align()); - if padding_needed > 0 { - tree = tree.then(Self::padding(padding_needed)); - } + // When this function is invoked with enum variants, + // `ty_and_layout.size` does not encompass the entire size of the + // enum. We rely on `total_size` for this. + assert!(ty_and_layout.size <= total_size); - // finally, the field's layout - tree = tree.then(Self::from_ty(field_ty, tcx)?); + let mut size = Size::ZERO; + let mut struct_tree = Self::def(def); - // extend the variant layout with the field layout - variant_layout = variant_layout.extend(field_layout).unwrap().0; + // If a `tag` is provided, place it at the start of the layout. + if let Some(tag) = tag { + size += tag.size(); + struct_tree = struct_tree.then(Self::from_tag(tag, cx.tcx)); } - drop(fields_span); - // finally: padding - let padding_span = trace_span!("adding trailing padding").entered(); - if layout_summary.total_size > variant_layout.size() { - let padding_needed = layout_summary.total_size - variant_layout.size(); - tree = tree.then(Self::padding(padding_needed)); - }; - drop(padding_span); - drop(variant_span); - Ok(tree) + // Append the fields, in memory order, to the layout. + let inverse_memory_index = memory_index.invert_bijective_mapping(); + for (memory_idx, field_idx) in inverse_memory_index.iter_enumerated() { + // Add interfield padding. + let padding_needed = offsets[*field_idx] - size; + let padding = Self::padding(padding_needed.bytes_usize()); + + let field_ty_and_layout = ty_and_layout.field(&cx, field_idx.as_usize()); + let field_tree = Self::from_ty(field_ty_and_layout, cx)?; + + struct_tree = struct_tree.then(padding).then(field_tree); + + size += padding_needed + field_ty_and_layout.size; + } + + // Add trailing padding. + let padding_needed = total_size - size; + let trailing_padding = Self::padding(padding_needed.bytes_usize()); + + Ok(struct_tree.then(trailing_padding)) } - pub fn from_tag(tag: ScalarInt, tcx: TyCtxt<'tcx>) -> Self { + /// Constructs a `Tree` representing the value of a enum tag. + fn from_tag(tag: ScalarInt, tcx: TyCtxt<'tcx>) -> Self { use rustc_target::abi::Endian; let size = tag.size(); let bits = tag.to_bits(size).unwrap(); @@ -479,24 +434,42 @@ pub(crate) mod rustc { }; Self::Seq(bytes.iter().map(|&b| Self::from_bits(b)).collect()) } - } - fn layout_of<'tcx>( - ctx: TyCtxt<'tcx>, - ty: Ty<'tcx>, - ) -> Result<alloc::Layout, &'tcx LayoutError<'tcx>> { - use rustc_middle::ty::ParamEnvAnd; - use rustc_target::abi::TyAndLayout; - - let param_env = ParamEnv::reveal_all(); - let param_env_and_type = ParamEnvAnd { param_env, value: ty }; - let TyAndLayout { layout, .. } = ctx.layout_of(param_env_and_type)?; - let layout = alloc::Layout::from_size_align( - layout.size().bytes_usize(), - layout.align().abi.bytes().try_into().unwrap(), - ) - .unwrap(); - trace!(?ty, ?layout, "computed layout for type"); - Ok(layout) + /// Constructs a `Tree` from a union. + /// + /// # Panics + /// + /// Panics if `def` is not a union definition. + fn from_union( + ty_and_layout: TyAndLayout<'tcx, Ty<'tcx>>, + def: AdtDef<'tcx>, + cx: LayoutCx<'tcx, TyCtxt<'tcx>>, + ) -> Result<Self, Err> { + assert!(def.is_union()); + + let union_layout = ty_and_layout.layout; + + // This constructor does not support non-`FieldsShape::Union` + // layouts. Fields of this shape are all placed at offset 0. + let FieldsShape::Union(fields) = union_layout.fields() else { + return Err(Err::NotYetSupported); + }; + + let fields = &def.non_enum_variant().fields; + let fields = fields.iter_enumerated().try_fold( + Self::uninhabited(), + |fields, (idx, ref field_def)| { + let field_def = Def::Field(field_def); + let field_ty_and_layout = ty_and_layout.field(&cx, idx.as_usize()); + let field = Self::from_ty(field_ty_and_layout, cx)?; + let trailing_padding_needed = union_layout.size - field_ty_and_layout.size; + let trailing_padding = Self::padding(trailing_padding_needed.bytes_usize()); + let field_and_padding = field.then(trailing_padding); + Result::<Self, Err>::Ok(fields.or(field_and_padding)) + }, + )?; + + Ok(Self::def(Def::Adt(def)).then(fields)) + } } } diff --git a/compiler/rustc_transmute/src/lib.rs b/compiler/rustc_transmute/src/lib.rs index b6f937ebe47..12312271646 100644 --- a/compiler/rustc_transmute/src/lib.rs +++ b/compiler/rustc_transmute/src/lib.rs @@ -1,6 +1,6 @@ #![feature(alloc_layout_extra)] #![feature(never_type)] -#![allow(dead_code, unused_variables)] +#![allow(unused_variables)] #[macro_use] extern crate tracing; @@ -49,6 +49,8 @@ pub enum Reason<T> { DstIsNotYetSupported, /// The layout of the destination type is bit-incompatible with the source type. DstIsBitIncompatible, + /// The destination type is uninhabited. + DstUninhabited, /// The destination type may carry safety invariants. DstMayHaveSafetyInvariants, /// `Dst` is larger than `Src`, and the excess bytes were not exclusively uninitialized. diff --git a/compiler/rustc_transmute/src/maybe_transmutable/mod.rs b/compiler/rustc_transmute/src/maybe_transmutable/mod.rs index 16d15580a05..2789fe8f6b1 100644 --- a/compiler/rustc_transmute/src/maybe_transmutable/mod.rs +++ b/compiler/rustc_transmute/src/maybe_transmutable/mod.rs @@ -33,6 +33,9 @@ mod rustc { use super::*; use crate::layout::tree::rustc::Err; + use rustc_middle::ty::layout::LayoutCx; + use rustc_middle::ty::layout::LayoutOf; + use rustc_middle::ty::ParamEnv; use rustc_middle::ty::Ty; use rustc_middle::ty::TyCtxt; @@ -43,12 +46,20 @@ mod rustc { pub fn answer(self) -> Answer<<TyCtxt<'tcx> as QueryContext>::Ref> { let Self { src, dst, assume, context } = self; + let layout_cx = LayoutCx { tcx: context, param_env: ParamEnv::reveal_all() }; + let layout_of = |ty| { + layout_cx + .layout_of(ty) + .map_err(|_| Err::NotYetSupported) + .and_then(|tl| Tree::from_ty(tl, layout_cx)) + }; + // Convert `src` and `dst` from their rustc representations, to `Tree`-based // representations. If these conversions fail, conclude that the transmutation is // unacceptable; the layouts of both the source and destination types must be // well-defined. - let src = Tree::from_ty(src, context); - let dst = Tree::from_ty(dst, context); + let src = layout_of(src); + let dst = layout_of(dst); match (src, dst) { (Err(Err::TypeError(_)), _) | (_, Err(Err::TypeError(_))) => { @@ -86,6 +97,10 @@ where // references. let src = src.prune(&|def| false); + if src.is_inhabited() && !dst.is_inhabited() { + return Answer::No(Reason::DstUninhabited); + } + trace!(?src, "pruned src"); // Remove all `Def` nodes from `dst`, additionally... diff --git a/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs b/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs index 54ed03d44e6..1ccb6f36c8e 100644 --- a/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs +++ b/compiler/rustc_transmute/src/maybe_transmutable/query_context.rs @@ -5,8 +5,6 @@ pub(crate) trait QueryContext { type Def: layout::Def; type Ref: layout::Ref; type Scope: Copy; - - fn min_align(&self, reference: Self::Ref) -> usize; } #[cfg(test)] @@ -31,10 +29,6 @@ pub(crate) mod test { type Def = Def; type Ref = !; type Scope = (); - - fn min_align(&self, reference: !) -> usize { - unimplemented!() - } } } @@ -48,9 +42,5 @@ mod rustc { type Ref = layout::rustc::Ref<'tcx>; type Scope = Ty<'tcx>; - - fn min_align(&self, reference: Self::Ref) -> usize { - unimplemented!() - } } } |