diff options
Diffstat (limited to 'compiler')
29 files changed, 390 insertions, 140 deletions
diff --git a/compiler/rustc_codegen_gcc/src/type_.rs b/compiler/rustc_codegen_gcc/src/type_.rs index bdf7318ce48..89a415cdb36 100644 --- a/compiler/rustc_codegen_gcc/src/type_.rs +++ b/compiler/rustc_codegen_gcc/src/type_.rs @@ -300,4 +300,8 @@ impl<'gcc, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'gcc, 'tcx> { // Unsupported. self.context.new_rvalue_from_int(self.int_type, 0) } + + fn set_kcfi_type_metadata(&self, _function: RValue<'gcc>, _kcfi_typeid: u32) { + // Unsupported. + } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index fed56cdd438..668d9292705 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -88,7 +88,8 @@ pub(crate) unsafe fn codegen( callee, args.as_ptr(), args.len() as c_uint, - None, + [].as_ptr(), + 0 as c_uint, ); llvm::LLVMSetTailCall(ret, True); if output.is_some() { @@ -132,8 +133,15 @@ pub(crate) unsafe fn codegen( .enumerate() .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::<Vec<_>>(); - let ret = - llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); llvm::LLVMSetTailCall(ret, True); llvm::LLVMBuildRetVoid(llbuilder); llvm::LLVMDisposeBuilder(llbuilder); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 77dd15ef4d8..83bffb20e0c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -20,6 +20,7 @@ use rustc_middle::ty::layout::{ }; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; +use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, Target}; use std::borrow::Cow; @@ -225,9 +226,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("invoke {:?} with args ({:?})", llfn, args); let args = self.check_call("invoke", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles = vec![funclet_bundle]; + + // Set KCFI operand bundle + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + let kcfi_bundle = + if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap()); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + if kcfi_bundle.is_some() { + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); + } + bundles.retain(|bundle| bundle.is_some()); let invoke = unsafe { llvm::LLVMRustBuildInvoke( self.llbuilder, @@ -237,7 +254,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { args.len() as c_uint, then, catch, - bundle, + bundles.as_ptr(), + bundles.len() as c_uint, UNNAMED, ) }; @@ -1143,7 +1161,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, - None, + [].as_ptr(), + 0 as c_uint, ); } } @@ -1159,9 +1178,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("call {:?} with args ({:?})", llfn, args); let args = self.check_call("call", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles = vec![funclet_bundle]; + + // Set KCFI operand bundle + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + let kcfi_bundle = + if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap()); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + if kcfi_bundle.is_some() { + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); + } + bundles.retain(|bundle| bundle.is_some()); let call = unsafe { llvm::LLVMRustBuildCall( self.llbuilder, @@ -1169,7 +1204,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, - bundle, + bundles.as_ptr(), + bundles.len() as c_uint, ) }; if let Some(fn_abi) = fn_abi { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4dcc7cd5447..aa1735f38ac 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -250,6 +250,11 @@ pub unsafe fn create_module<'ll>( ); } + if sess.is_sanitizer_kcfi_enabled() { + let kcfi = "kcfi\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); + } + // Control Flow Guard is currently only supported by the MSVC linker on Windows. if sess.target.is_like_msvc { match sess.opts.cg.control_flow_guard { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index d87117dffdc..a9e3dcf4cb3 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -27,9 +27,7 @@ use rustc_codegen_ssa::traits::*; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; -use rustc_middle::mir::{self, GeneratorLayout}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ @@ -1026,33 +1024,6 @@ fn build_struct_type_di_node<'ll, 'tcx>( // Tuples //=----------------------------------------------------------------------------- -/// Returns names of captured upvars for closures and generators. -/// -/// Here are some examples: -/// - `name__field1__field2` when the upvar is captured by value. -/// - `_ref__name__field` when the upvar is captured by reference. -/// -/// For generators this only contains upvars that are shared by all states. -fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec<String> { - let body = tcx.optimized_mir(def_id); - - body.var_debug_info - .iter() - .filter_map(|var| { - let is_ref = match var.value { - mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => { - // The projection is either `[.., Field, Deref]` or `[.., Field]`. It - // implies whether the variable is captured by value or by reference. - matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) - } - _ => return None, - }; - let prefix = if is_ref { "_ref__" } else { "" }; - Some(prefix.to_owned() + var.name.as_str()) - }) - .collect() -} - /// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. /// For a generator, this will handle upvars shared by all states. fn build_upvar_field_di_nodes<'ll, 'tcx>( @@ -1083,7 +1054,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) ); - let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); + let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); let layout = cx.layout_of(closure_or_generator_ty); up_var_tys @@ -1229,43 +1200,6 @@ fn build_union_type_di_node<'ll, 'tcx>( ) } -// FIXME(eddyb) maybe precompute this? Right now it's computed once -// per generator monomorphization, but it doesn't depend on substs. -fn generator_layout_and_saved_local_names<'tcx>( - tcx: TyCtxt<'tcx>, - def_id: DefId, -) -> (&'tcx GeneratorLayout<'tcx>, IndexVec<mir::GeneratorSavedLocal, Option<Symbol>>) { - let body = tcx.optimized_mir(def_id); - let generator_layout = body.generator_layout().unwrap(); - let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); - - let state_arg = mir::Local::new(1); - for var in &body.var_debug_info { - let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; - if place.local != state_arg { - continue; - } - match place.projection[..] { - [ - // Deref of the `Pin<&mut Self>` state argument. - mir::ProjectionElem::Field(..), - mir::ProjectionElem::Deref, - // Field of a variant of the state. - mir::ProjectionElem::Downcast(_, variant), - mir::ProjectionElem::Field(field, _), - ] => { - let name = &mut generator_saved_local_names - [generator_layout.variant_fields[variant][field]]; - if name.is_none() { - name.replace(var.name); - } - } - _ => {} - } - } - (generator_layout, generator_saved_local_names) -} - /// Computes the type parameters for a type, if any, for the given metadata. fn build_generic_type_param_di_nodes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 53e8a291d1e..69443b9b828 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -22,9 +22,9 @@ use crate::{ common::CodegenCx, debuginfo::{ metadata::{ - build_field_di_node, closure_saved_names_of_captured_variables, + build_field_di_node, enums::{tag_base_type, DiscrResult}, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER, @@ -677,9 +677,9 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( }; let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + cx.tcx.generator_layout_and_saved_local_names(generator_def_id); - let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index becbccc434d..93419d27a62 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -4,9 +4,8 @@ use crate::{ common::CodegenCx, debuginfo::{ metadata::{ - closure_saved_names_of_captured_variables, enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, StubInfo, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, UNKNOWN_LINE_NUMBER, @@ -157,7 +156,7 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( ), |cx, generator_type_di_node| { let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + cx.tcx.generator_layout_and_saved_local_names(generator_def_id); let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { bug!( @@ -167,7 +166,7 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( }; let common_upvar_names = - closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index dc21a02cec4..6a575095f7e 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -20,7 +20,7 @@ use crate::type_::Type; use crate::value::Value; use rustc_codegen_ssa::traits::TypeMembershipMethods; use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::typeid_for_fnabi; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi}; use smallvec::SmallVec; /// Declare a function. @@ -136,6 +136,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { self.set_type_metadata(llfn, typeid); } + if self.tcx.sess.is_sanitizer_kcfi_enabled() { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } + llfn } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 8a9392255b8..e61dbe8b8fc 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -427,6 +427,7 @@ pub enum MetadataType { MD_type = 19, MD_vcall_visibility = 28, MD_noundef = 29, + MD_kcfi_type = 36, } /// LLVMRustAsmDialect @@ -1063,6 +1064,7 @@ extern "C" { pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>; // Operations on constants of any type pub fn LLVMConstNull(Ty: &Type) -> &Value; @@ -1273,7 +1275,8 @@ extern "C" { NumArgs: c_uint, Then: &'a BasicBlock, Catch: &'a BasicBlock, - Bundle: Option<&OperandBundleDef<'a>>, + OpBundles: *const Option<&OperandBundleDef<'a>>, + NumOpBundles: c_uint, Name: *const c_char, ) -> &'a Value; pub fn LLVMBuildLandingPad<'a>( @@ -1643,7 +1646,8 @@ extern "C" { Fn: &'a Value, Args: *const &'a Value, NumArgs: c_uint, - Bundle: Option<&OperandBundleDef<'a>>, + OpBundles: *const Option<&OperandBundleDef<'a>>, + NumOpBundles: c_uint, ) -> &'a Value; pub fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 5772b7e1d81..ff111d96f84 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -316,4 +316,19 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { ) } } + + fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) + } + } } diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 86481d5d758..109161ccc83 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -122,6 +122,7 @@ pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { pub trait TypeMembershipMethods<'tcx>: Backend<'tcx> { fn set_type_metadata(&self, function: Self::Function, typeid: String); fn typeid_metadata(&self, typeid: String) -> Self::Value; + fn set_kcfi_type_metadata(&self, function: Self::Function, typeid: u32); } pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> { diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 4b6e068db43..6d8e78a0f18 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -394,7 +394,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ ungated!(instruction_set, Normal, template!(List: "set"), ErrorPreceding), gated!( no_sanitize, Normal, - template!(List: "address, memory, thread"), DuplicatesOk, + template!(List: "address, kcfi, memory, thread"), DuplicatesOk, experimental!(no_sanitize) ), gated!(no_coverage, Normal, template!(Word), WarnFollowing, experimental!(no_coverage)), diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs index 18f08ff6677..72288e5bc76 100644 --- a/compiler/rustc_hir_analysis/src/collect.rs +++ b/compiler/rustc_hir_analysis/src/collect.rs @@ -1846,6 +1846,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs { codegen_fn_attrs.no_sanitize |= SanitizerSet::ADDRESS; } else if item.has_name(sym::cfi) { codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI; + } else if item.has_name(sym::kcfi) { + codegen_fn_attrs.no_sanitize |= SanitizerSet::KCFI; } else if item.has_name(sym::memory) { codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMORY; } else if item.has_name(sym::memtag) { @@ -1859,7 +1861,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs { } else { tcx.sess .struct_span_err(item.span(), "invalid argument for `no_sanitize`") - .note("expected one of: `address`, `cfi`, `hwaddress`, `memory`, `memtag`, `shadow-call-stack`, or `thread`") + .note("expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread`") .emit(); } } diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs index b5db94f8c06..777f7ca5c9b 100644 --- a/compiler/rustc_lint/src/unused.rs +++ b/compiler/rustc_lint/src/unused.rs @@ -617,7 +617,10 @@ trait UnusedDelimLint { lhs_needs_parens || (followed_by_block && match &inner.kind { - ExprKind::Ret(_) | ExprKind::Break(..) | ExprKind::Yield(..) => true, + ExprKind::Ret(_) + | ExprKind::Break(..) + | ExprKind::Yield(..) + | ExprKind::Yeet(..) => true, ExprKind::Range(_lhs, Some(rhs), _limits) => { matches!(rhs.kind, ExprKind::Block(..)) } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 3a748f38995..5831d4693f1 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1476,13 +1476,13 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, - OperandBundleDef *Bundle) { + OperandBundleDef **OpBundles, + unsigned NumOpBundles) { Value *Callee = unwrap(Fn); FunctionType *FTy = unwrap<FunctionType>(Ty); - unsigned Len = Bundle ? 1 : 0; - ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len); return wrap(unwrap(B)->CreateCall( - FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles)); + FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), + makeArrayRef(*OpBundles, NumOpBundles))); } extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { @@ -1522,14 +1522,14 @@ extern "C" LLVMValueRef LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, - OperandBundleDef *Bundle, const char *Name) { + OperandBundleDef **OpBundles, unsigned NumOpBundles, + const char *Name) { Value *Callee = unwrap(Fn); FunctionType *FTy = unwrap<FunctionType>(Ty); - unsigned Len = Bundle ? 1 : 0; - ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len); return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), makeArrayRef(unwrap(Args), NumArgs), - Bundles, Name)); + makeArrayRef(*OpBundles, NumOpBundles), + Name)); } extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 47c1ce80756..9ea8dc6e69f 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -1,6 +1,7 @@ //! Miscellaneous type-system utilities that are too small to deserve their own modules. use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use crate::mir; use crate::ty::layout::IntegerExt; use crate::ty::{ self, DefIdTree, FallibleTypeFolder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, @@ -15,6 +16,7 @@ use rustc_hir as hir; use rustc_hir::def::{CtorOf, DefKind, Res}; use rustc_hir::def_id::DefId; use rustc_index::bit_set::GrowableBitSet; +use rustc_index::vec::{Idx, IndexVec}; use rustc_macros::HashStable; use rustc_span::{sym, DUMMY_SP}; use rustc_target::abi::{Integer, IntegerType, Size, TargetDataLayout}; @@ -692,6 +694,80 @@ impl<'tcx> TyCtxt<'tcx> { pub fn bound_impl_subject(self, def_id: DefId) -> ty::EarlyBinder<ty::ImplSubject<'tcx>> { ty::EarlyBinder(self.impl_subject(def_id)) } + + /// Returns names of captured upvars for closures and generators. + /// + /// Here are some examples: + /// - `name__field1__field2` when the upvar is captured by value. + /// - `_ref__name__field` when the upvar is captured by reference. + /// + /// For generators this only contains upvars that are shared by all states. + pub fn closure_saved_names_of_captured_variables( + self, + def_id: DefId, + ) -> SmallVec<[String; 16]> { + let body = self.optimized_mir(def_id); + + body.var_debug_info + .iter() + .filter_map(|var| { + let is_ref = match var.value { + mir::VarDebugInfoContents::Place(place) + if place.local == mir::Local::new(1) => + { + // The projection is either `[.., Field, Deref]` or `[.., Field]`. It + // implies whether the variable is captured by value or by reference. + matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) + } + _ => return None, + }; + let prefix = if is_ref { "_ref__" } else { "" }; + Some(prefix.to_owned() + var.name.as_str()) + }) + .collect() + } + + // FIXME(eddyb) maybe precompute this? Right now it's computed once + // per generator monomorphization, but it doesn't depend on substs. + pub fn generator_layout_and_saved_local_names( + self, + def_id: DefId, + ) -> ( + &'tcx ty::GeneratorLayout<'tcx>, + IndexVec<mir::GeneratorSavedLocal, Option<rustc_span::Symbol>>, + ) { + let tcx = self; + let body = tcx.optimized_mir(def_id); + let generator_layout = body.generator_layout().unwrap(); + let mut generator_saved_local_names = + IndexVec::from_elem(None, &generator_layout.field_tys); + + let state_arg = mir::Local::new(1); + for var in &body.var_debug_info { + let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; + if place.local != state_arg { + continue; + } + match place.projection[..] { + [ + // Deref of the `Pin<&mut Self>` state argument. + mir::ProjectionElem::Field(..), + mir::ProjectionElem::Deref, + // Field of a variant of the state. + mir::ProjectionElem::Downcast(_, variant), + mir::ProjectionElem::Field(field, _), + ] => { + let name = &mut generator_saved_local_names + [generator_layout.variant_fields[variant][field]]; + if name.is_none() { + name.replace(var.name); + } + } + _ => {} + } + } + (generator_layout, generator_saved_local_names) + } } struct OpaqueTypeExpander<'tcx> { diff --git a/compiler/rustc_session/src/code_stats.rs b/compiler/rustc_session/src/code_stats.rs index eede4d16ea3..1085bce4475 100644 --- a/compiler/rustc_session/src/code_stats.rs +++ b/compiler/rustc_session/src/code_stats.rs @@ -19,7 +19,7 @@ pub enum SizeKind { Min, } -#[derive(Clone, PartialEq, Eq, Hash, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub struct FieldInfo { pub name: Symbol, pub offset: u64, @@ -33,6 +33,7 @@ pub enum DataTypeKind { Union, Enum, Closure, + Generator, } #[derive(PartialEq, Eq, Hash, Debug)] @@ -114,7 +115,7 @@ impl CodeStats { let struct_like = match kind { DataTypeKind::Struct | DataTypeKind::Closure => true, - DataTypeKind::Enum | DataTypeKind::Union => false, + DataTypeKind::Enum | DataTypeKind::Union | DataTypeKind::Generator => false, }; for (i, variant_info) in variants.into_iter().enumerate() { let VariantInfo { ref name, kind: _, align: _, size, ref fields } = *variant_info; diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 8e9198b79df..dab9c736d14 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -368,7 +368,7 @@ mod desc { pub const parse_opt_panic_strategy: &str = parse_panic_strategy; pub const parse_oom_strategy: &str = "either `panic` or `abort`"; pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; - pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`"; + pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `kcfi`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`"; pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2"; pub const parse_cfguard: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`"; @@ -675,6 +675,7 @@ mod parse { *slot |= match s { "address" => SanitizerSet::ADDRESS, "cfi" => SanitizerSet::CFI, + "kcfi" => SanitizerSet::KCFI, "leak" => SanitizerSet::LEAK, "memory" => SanitizerSet::MEMORY, "memtag" => SanitizerSet::MEMTAG, diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 4c049a8d628..8859b76d289 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -686,6 +686,10 @@ impl Session { self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI) } + pub fn is_sanitizer_kcfi_enabled(&self) -> bool { + self.opts.unstable_opts.sanitizer.contains(SanitizerSet::KCFI) + } + /// Check whether this compile session and crate type use static crt. pub fn crt_static(&self, crate_type: Option<CrateType>) -> bool { if !self.target.crt_static_respected { @@ -1544,6 +1548,14 @@ fn validate_commandline_args_with_session_available(sess: &Session) { } } + // LLVM CFI and KCFI are mutually exclusive + if sess.is_sanitizer_cfi_enabled() && sess.is_sanitizer_kcfi_enabled() { + sess.emit_err(CannotMixAndMatchSanitizers { + first: "cfi".to_string(), + second: "kcfi".to_string(), + }); + } + if sess.opts.unstable_opts.stack_protector != StackProtector::None { if !sess.target.options.supports_stack_protector { sess.emit_warning(StackProtectorNotSupportedForTarget { diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index cef4c6f79ce..335bfc3302f 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -491,6 +491,10 @@ impl SpanData { pub fn is_dummy(self) -> bool { self.lo.0 == 0 && self.hi.0 == 0 } + #[inline] + pub fn is_visible(self, sm: &SourceMap) -> bool { + !self.is_dummy() && sm.is_span_accessible(self.span()) + } /// Returns `true` if `self` fully encloses `other`. pub fn contains(self, other: Self) -> bool { self.lo <= other.lo && other.hi <= self.hi @@ -556,6 +560,11 @@ impl Span { self.data_untracked().is_dummy() } + #[inline] + pub fn is_visible(self, sm: &SourceMap) -> bool { + self.data_untracked().is_visible(sm) + } + /// Returns `true` if this span comes from any kind of macro, desugaring or inlining. #[inline] pub fn from_expansion(self) -> bool { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 61253845497..1fcf8c7a8bf 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -828,6 +828,7 @@ symbols! { item_like_imports, iter, iter_repeat, + kcfi, keyword, kind, kreg, diff --git a/compiler/rustc_symbol_mangling/Cargo.toml b/compiler/rustc_symbol_mangling/Cargo.toml index 2a29ad6a9e5..4e447eab02e 100644 --- a/compiler/rustc_symbol_mangling/Cargo.toml +++ b/compiler/rustc_symbol_mangling/Cargo.toml @@ -10,6 +10,7 @@ bitflags = "1.2.1" tracing = "0.1" punycode = "0.4.0" rustc-demangle = "0.1.21" +twox-hash = "1.6.3" rustc_span = { path = "../rustc_span" } rustc_middle = { path = "../rustc_middle" } diff --git a/compiler/rustc_symbol_mangling/src/typeid.rs b/compiler/rustc_symbol_mangling/src/typeid.rs index 9228bea43f9..53983bed718 100644 --- a/compiler/rustc_symbol_mangling/src/typeid.rs +++ b/compiler/rustc_symbol_mangling/src/typeid.rs @@ -3,6 +3,8 @@ use rustc_middle::ty::{FnSig, Ty, TyCtxt}; use rustc_target::abi::call::FnAbi; +use std::hash::Hasher; +use twox_hash::XxHash64; mod typeid_itanium_cxx_abi; use typeid_itanium_cxx_abi::TypeIdOptions; @@ -16,3 +18,25 @@ pub fn typeid_for_fnabi<'tcx>(tcx: TyCtxt<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) pub fn typeid_for_fnsig<'tcx>(tcx: TyCtxt<'tcx>, fn_sig: &FnSig<'tcx>) -> String { typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, TypeIdOptions::NO_OPTIONS) } + +/// Returns an LLVM KCFI type metadata identifier for the specified FnAbi. +pub fn kcfi_typeid_for_fnabi<'tcx>(tcx: TyCtxt<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> u32 { + // An LLVM KCFI type metadata identifier is a 32-bit constant produced by taking the lower half + // of the xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.) + let mut hash: XxHash64 = Default::default(); + hash.write( + typeid_itanium_cxx_abi::typeid_for_fnabi(tcx, fn_abi, TypeIdOptions::NO_OPTIONS).as_bytes(), + ); + hash.finish() as u32 +} + +/// Returns an LLVM KCFI type metadata identifier for the specified FnSig. +pub fn kcfi_typeid_for_fnsig<'tcx>(tcx: TyCtxt<'tcx>, fn_sig: &FnSig<'tcx>) -> u32 { + // An LLVM KCFI type metadata identifier is a 32-bit constant produced by taking the lower half + // of the xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.) + let mut hash: XxHash64 = Default::default(); + hash.write( + typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, TypeIdOptions::NO_OPTIONS).as_bytes(), + ); + hash.finish() as u32 +} diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_none.rs b/compiler/rustc_target/src/spec/aarch64_unknown_none.rs index 4ae6d4120c9..aca52e1478e 100644 --- a/compiler/rustc_target/src/spec/aarch64_unknown_none.rs +++ b/compiler/rustc_target/src/spec/aarch64_unknown_none.rs @@ -6,13 +6,16 @@ // // For example, `-C target-cpu=cortex-a53`. -use super::{Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, Target, TargetOptions}; +use super::{ + Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, SanitizerSet, Target, TargetOptions, +}; pub fn target() -> Target { let opts = TargetOptions { linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes), linker: Some("rust-lld".into()), features: "+strict-align,+neon,+fp-armv8".into(), + supported_sanitizers: SanitizerSet::KCFI, relocation_model: RelocModel::Static, disable_redzone: true, max_atomic_width: Some(128), diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index d05b8aa4200..1db1d7e85ad 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -804,7 +804,7 @@ impl ToJson for StackProbeType { bitflags::bitflags! { #[derive(Default, Encodable, Decodable)] - pub struct SanitizerSet: u8 { + pub struct SanitizerSet: u16 { const ADDRESS = 1 << 0; const LEAK = 1 << 1; const MEMORY = 1 << 2; @@ -813,6 +813,7 @@ bitflags::bitflags! { const CFI = 1 << 5; const MEMTAG = 1 << 6; const SHADOWCALLSTACK = 1 << 7; + const KCFI = 1 << 8; } } @@ -824,6 +825,7 @@ impl SanitizerSet { Some(match self { SanitizerSet::ADDRESS => "address", SanitizerSet::CFI => "cfi", + SanitizerSet::KCFI => "kcfi", SanitizerSet::LEAK => "leak", SanitizerSet::MEMORY => "memory", SanitizerSet::MEMTAG => "memtag", @@ -859,6 +861,7 @@ impl IntoIterator for SanitizerSet { [ SanitizerSet::ADDRESS, SanitizerSet::CFI, + SanitizerSet::KCFI, SanitizerSet::LEAK, SanitizerSet::MEMORY, SanitizerSet::MEMTAG, @@ -2327,6 +2330,7 @@ impl Target { base.$key_name |= match s.as_str() { Some("address") => SanitizerSet::ADDRESS, Some("cfi") => SanitizerSet::CFI, + Some("kcfi") => SanitizerSet::KCFI, Some("leak") => SanitizerSet::LEAK, Some("memory") => SanitizerSet::MEMORY, Some("memtag") => SanitizerSet::MEMTAG, diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_none.rs b/compiler/rustc_target/src/spec/x86_64_unknown_none.rs index e4d33c2b8c6..32060c35c11 100644 --- a/compiler/rustc_target/src/spec/x86_64_unknown_none.rs +++ b/compiler/rustc_target/src/spec/x86_64_unknown_none.rs @@ -5,7 +5,7 @@ // features. use super::{Cc, CodeModel, LinkerFlavor, Lld, PanicStrategy}; -use super::{RelroLevel, StackProbeType, Target, TargetOptions}; +use super::{RelroLevel, SanitizerSet, StackProbeType, Target, TargetOptions}; pub fn target() -> Target { let opts = TargetOptions { @@ -20,6 +20,7 @@ pub fn target() -> Target { features: "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float" .into(), + supported_sanitizers: SanitizerSet::KCFI, disable_redzone: true, panic_strategy: PanicStrategy::Abort, code_model: Some(CodeModel::Kernel), diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs index 6ea54b625bb..443d57aaf3d 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs @@ -2413,19 +2413,19 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> { | ObligationCauseCode::ExprBindingObligation(item_def_id, span, ..) => { let item_name = tcx.def_path_str(item_def_id); let mut multispan = MultiSpan::from(span); + let sm = tcx.sess.source_map(); if let Some(ident) = tcx.opt_item_ident(item_def_id) { - let sm = tcx.sess.source_map(); let same_line = match (sm.lookup_line(ident.span.hi()), sm.lookup_line(span.lo())) { (Ok(l), Ok(r)) => l.line == r.line, _ => true, }; - if !ident.span.is_dummy() && !ident.span.overlaps(span) && !same_line { + if ident.span.is_visible(sm) && !ident.span.overlaps(span) && !same_line { multispan.push_span_label(ident.span, "required by a bound in this"); } } let descr = format!("required by a bound in `{}`", item_name); - if !span.is_dummy() { + if span.is_visible(sm) { let msg = format!("required by this bound in `{}`", item_name); multispan.push_span_label(span, msg); err.span_note(multispan, &descr); diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 73c7eb6992f..d644cbccea1 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -85,7 +85,7 @@ fn fn_sig_for_fn_abi<'tcx>( bound_vars, ) } - ty::Generator(_, substs, _) => { + ty::Generator(did, substs, _) => { let sig = substs.as_generator().poly_sig(); let bound_vars = tcx.mk_bound_variable_kinds( @@ -104,10 +104,22 @@ fn fn_sig_for_fn_abi<'tcx>( let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs); let sig = sig.skip_binder(); - let state_did = tcx.require_lang_item(LangItem::GeneratorState, None); - let state_adt_ref = tcx.adt_def(state_did); - let state_substs = tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]); - let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); + // The `FnSig` and the `ret_ty` here is for a generators main + // `Generator::resume(...) -> GeneratorState` function in case we + // have an ordinary generator, or the `Future::poll(...) -> Poll` + // function in case this is a special generator backing an async construct. + let ret_ty = if tcx.generator_is_async(did) { + let state_did = tcx.require_lang_item(LangItem::Poll, None); + let state_adt_ref = tcx.adt_def(state_did); + let state_substs = tcx.intern_substs(&[sig.return_ty.into()]); + tcx.mk_adt(state_adt_ref, state_substs) + } else { + let state_did = tcx.require_lang_item(LangItem::GeneratorState, None); + let state_adt_ref = tcx.adt_def(state_did); + let state_substs = tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]); + tcx.mk_adt(state_adt_ref, state_substs) + }; + ty::Binder::bind_with_vars( tcx.mk_fn_sig( [env_ty, sig.resume_ty].iter(), diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index fbc055b5d23..f4672a70072 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -1,3 +1,4 @@ +use hir::def_id::DefId; use rustc_hir as hir; use rustc_index::bit_set::BitSet; use rustc_index::vec::{Idx, IndexVec}; @@ -6,7 +7,7 @@ use rustc_middle::ty::layout::{ IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, MAX_SIMD_LANES, }; use rustc_middle::ty::{ - self, subst::SubstsRef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitable, + self, subst::SubstsRef, AdtDef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitable, }; use rustc_session::{DataTypeKind, FieldInfo, SizeKind, VariantInfo}; use rustc_span::symbol::Symbol; @@ -814,27 +815,39 @@ fn record_layout_for_printing_outlined<'tcx>( ); }; - let adt_def = match *layout.ty.kind() { - ty::Adt(ref adt_def, _) => { + match *layout.ty.kind() { + ty::Adt(adt_def, _) => { debug!("print-type-size t: `{:?}` process adt", layout.ty); - adt_def + let adt_kind = adt_def.adt_kind(); + let adt_packed = adt_def.repr().pack.is_some(); + let (variant_infos, opt_discr_size) = variant_info_for_adt(cx, layout, adt_def); + record(adt_kind.into(), adt_packed, opt_discr_size, variant_infos); + } + + ty::Generator(def_id, substs, _) => { + debug!("print-type-size t: `{:?}` record generator", layout.ty); + // Generators always have a begin/poisoned/end state with additional suspend points + let (variant_infos, opt_discr_size) = + variant_info_for_generator(cx, layout, def_id, substs); + record(DataTypeKind::Generator, false, opt_discr_size, variant_infos); } ty::Closure(..) => { debug!("print-type-size t: `{:?}` record closure", layout.ty); record(DataTypeKind::Closure, false, None, vec![]); - return; } _ => { debug!("print-type-size t: `{:?}` skip non-nominal", layout.ty); - return; } }; +} - let adt_kind = adt_def.adt_kind(); - let adt_packed = adt_def.repr().pack.is_some(); - +fn variant_info_for_adt<'tcx>( + cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, + layout: TyAndLayout<'tcx>, + adt_def: AdtDef<'tcx>, +) -> (Vec<VariantInfo>, Option<Size>) { let build_variant_info = |n: Option<Symbol>, flds: &[Symbol], layout: TyAndLayout<'tcx>| { let mut min_size = Size::ZERO; let field_info: Vec<_> = flds @@ -843,10 +856,7 @@ fn record_layout_for_printing_outlined<'tcx>( .map(|(i, &name)| { let field_layout = layout.field(cx, i); let offset = layout.fields.offset(i); - let field_end = offset + field_layout.size; - if min_size < field_end { - min_size = field_end; - } + min_size = min_size.max(offset + field_layout.size); FieldInfo { name, offset: offset.bytes(), @@ -871,16 +881,9 @@ fn record_layout_for_printing_outlined<'tcx>( debug!("print-type-size `{:#?}` variant {}", layout, adt_def.variant(index).name); let variant_def = &adt_def.variant(index); let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect(); - record( - adt_kind.into(), - adt_packed, - None, - vec![build_variant_info(Some(variant_def.name), &fields, layout)], - ); + (vec![build_variant_info(Some(variant_def.name), &fields, layout)], None) } else { - // (This case arises for *empty* enums; so give it - // zero variants.) - record(adt_kind.into(), adt_packed, None, vec![]); + (vec![], None) } } @@ -898,15 +901,101 @@ fn record_layout_for_printing_outlined<'tcx>( build_variant_info(Some(variant_def.name), &fields, layout.for_variant(cx, i)) }) .collect(); - record( - adt_kind.into(), - adt_packed, + + ( + variant_infos, match tag_encoding { TagEncoding::Direct => Some(tag.size(cx)), _ => None, }, - variant_infos, - ); + ) } } } + +fn variant_info_for_generator<'tcx>( + cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, + layout: TyAndLayout<'tcx>, + def_id: DefId, + substs: ty::SubstsRef<'tcx>, +) -> (Vec<VariantInfo>, Option<Size>) { + let Variants::Multiple { tag, ref tag_encoding, .. } = layout.variants else { + return (vec![], None); + }; + + let (generator, state_specific_names) = cx.tcx.generator_layout_and_saved_local_names(def_id); + let upvar_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); + + let mut upvars_size = Size::ZERO; + let upvar_fields: Vec<_> = substs + .as_generator() + .upvar_tys() + .zip(upvar_names) + .enumerate() + .map(|(field_idx, (_, name))| { + let field_layout = layout.field(cx, field_idx); + let offset = layout.fields.offset(field_idx); + upvars_size = upvars_size.max(offset + field_layout.size); + FieldInfo { + name: Symbol::intern(&name), + offset: offset.bytes(), + size: field_layout.size.bytes(), + align: field_layout.align.abi.bytes(), + } + }) + .collect(); + + let variant_infos: Vec<_> = generator + .variant_fields + .iter_enumerated() + .map(|(variant_idx, variant_def)| { + let variant_layout = layout.for_variant(cx, variant_idx); + let mut variant_size = Size::ZERO; + let fields = variant_def + .iter() + .enumerate() + .map(|(field_idx, local)| { + let field_layout = variant_layout.field(cx, field_idx); + let offset = variant_layout.fields.offset(field_idx); + // The struct is as large as the last field's end + variant_size = variant_size.max(offset + field_layout.size); + FieldInfo { + name: state_specific_names.get(*local).copied().flatten().unwrap_or( + Symbol::intern(&format!(".generator_field{}", local.as_usize())), + ), + offset: offset.bytes(), + size: field_layout.size.bytes(), + align: field_layout.align.abi.bytes(), + } + }) + .chain(upvar_fields.iter().copied()) + .collect(); + + // If the variant has no state-specific fields, then it's the size of the upvars. + if variant_size == Size::ZERO { + variant_size = upvars_size; + } + // We need to add the discriminant size back into min_size, since it is subtracted + // later during printing. + variant_size += match tag_encoding { + TagEncoding::Direct => tag.size(cx), + _ => Size::ZERO, + }; + + VariantInfo { + name: Some(Symbol::intern(&ty::GeneratorSubsts::variant_name(variant_idx))), + kind: SizeKind::Exact, + size: variant_size.bytes(), + align: variant_layout.align.abi.bytes(), + fields, + } + }) + .collect(); + ( + variant_infos, + match tag_encoding { + TagEncoding::Direct => Some(tag.size(cx)), + _ => None, + }, + ) +} |