diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
28 files changed, 3014 insertions, 2271 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b14a4f28c75..cc8b3a1a4e4 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -393,6 +393,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_cconv(&self) -> llvm::CallConv { match self.conv { Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, @@ -510,9 +511,9 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected // by the LLVM verifier. - if let Int(..) = scalar.value { + if let Int(..) = scalar.primitive() { if !scalar.is_bool() && !scalar.is_always_valid(bx) { - bx.range_metadata(callsite, scalar.valid_range); + bx.range_metadata(callsite, scalar.valid_range(bx)); } } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index eb19e427217..f935acb1a7e 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -3,7 +3,7 @@ use libc::c_uint; use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; -use rustc_session::config::DebugInfo; +use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_span::symbol::sym; use crate::debuginfo; @@ -139,9 +139,19 @@ pub(crate) unsafe fn codegen( llvm::LLVMBuildRetVoid(llbuilder); llvm::LLVMDisposeBuilder(llbuilder); + // __rust_alloc_error_handler_should_panic + let name = OomStrategy::SYMBOL; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let val = tcx.sess.opts.debugging_opts.oom.should_panic(); + let llval = llvm::LLVMConstInt(i8, val as u64, False); + llvm::LLVMSetInitializer(ll_g, llval); + if tcx.sess.opts.debuginfo != DebugInfo::None { - let dbg_cx = debuginfo::CrateDebugContext::new(llmod); - debuginfo::metadata::compile_unit_metadata(tcx, module_name, &dbg_cx); + let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); dbg_cx.finalize(tcx.sess); } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 96c7d884b7b..a53946995ee 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -290,6 +290,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); + // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` + if let Some((dest, _, _)) = dest_catch_funclet { + self.switch_to_block(dest); + } + // Write results to outputs for (idx, op) in operands.iter().enumerate() { if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } @@ -307,11 +312,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -impl AsmMethods for CodegenCx<'_, '_> { +impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { fn codegen_global_asm( &self, template: &[InlineAsmTemplatePiece], - operands: &[GlobalAsmOperandRef], + operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span], ) { @@ -337,6 +342,29 @@ impl AsmMethods for CodegenCx<'_, '_> { // here unlike normal inline assembly. template_str.push_str(string); } + GlobalAsmOperandRef::SymFn { instance } => { + let llval = self.get_fn(instance); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + GlobalAsmOperandRef::SymStatic { def_id } => { + let llval = self + .renamed_statics + .borrow() + .get(&def_id) + .copied() + .unwrap_or_else(|| self.get_static(def_id)); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } } } } @@ -574,7 +602,10 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", InlineAsmRegClass::X86( - X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg, + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, ) => unreachable!("clobber-only"), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", @@ -659,7 +690,12 @@ fn modifier_to_llvm( _ => unreachable!(), }, InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, - InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg) => { + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => { unreachable!("clobber-only") } InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, @@ -729,7 +765,12 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), - InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg) => { + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => { unreachable!("clobber-only") } InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), @@ -753,7 +794,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' /// Helper function to get the LLVM type for a Scalar. Pointers are returned as /// the equivalent integer type. fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { - match scalar.value { + match scalar.primitive() { Primitive::Int(Integer::I8, _) => cx.type_i8(), Primitive::Int(Integer::I16, _) => cx.type_i16(), Primitive::Int(Integer::I32, _) => cx.type_i32(), @@ -774,7 +815,7 @@ fn llvm_fixup_input<'ll, 'tcx>( ) -> &'ll Value { match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { - if let Primitive::Int(Integer::I8, _) = s.value { + if let Primitive::Int(Integer::I8, _) = s.primitive() { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } else { @@ -785,7 +826,7 @@ fn llvm_fixup_input<'ll, 'tcx>( let elem_ty = llvm_asm_scalar_type(bx.cx, s); let count = 16 / layout.size.bytes(); let vec_ty = bx.cx.type_vector(elem_ty, count); - if let Primitive::Pointer = s.value { + if let Primitive::Pointer = s.primitive() { value = bx.ptrtoint(value, bx.cx.type_isize()); } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) @@ -800,7 +841,7 @@ fn llvm_fixup_input<'ll, 'tcx>( bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.value == Primitive::F64 => + if s.primitive() == Primitive::F64 => { bx.bitcast(value, bx.cx.type_i64()) } @@ -812,7 +853,7 @@ fn llvm_fixup_input<'ll, 'tcx>( InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s), ) => { - if let Primitive::Int(Integer::I32, _) = s.value { + if let Primitive::Int(Integer::I32, _) = s.primitive() { bx.bitcast(value, bx.cx.type_f32()) } else { value @@ -826,19 +867,21 @@ fn llvm_fixup_input<'ll, 'tcx>( ), Abi::Scalar(s), ) => { - if let Primitive::Int(Integer::I64, _) = s.value { + if let Primitive::Int(Integer::I64, _) = s.primitive() { bx.bitcast(value, bx.cx.type_f64()) } else { value } } - (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => match s.value { - // MIPS only supports register-length arithmetics. - Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), - Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), - Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), - _ => value, - }, + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), + Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), + Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), + _ => value, + } + } _ => value, } } @@ -852,7 +895,7 @@ fn llvm_fixup_output<'ll, 'tcx>( ) -> &'ll Value { match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { - if let Primitive::Int(Integer::I8, _) = s.value { + if let Primitive::Int(Integer::I8, _) = s.primitive() { bx.extract_element(value, bx.const_i32(0)) } else { value @@ -860,7 +903,7 @@ fn llvm_fixup_output<'ll, 'tcx>( } (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { value = bx.extract_element(value, bx.const_i32(0)); - if let Primitive::Pointer = s.value { + if let Primitive::Pointer = s.primitive() { value = bx.inttoptr(value, layout.llvm_type(bx.cx)); } value @@ -875,7 +918,7 @@ fn llvm_fixup_output<'ll, 'tcx>( bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.value == Primitive::F64 => + if s.primitive() == Primitive::F64 => { bx.bitcast(value, bx.cx.type_f64()) } @@ -887,7 +930,7 @@ fn llvm_fixup_output<'ll, 'tcx>( InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s), ) => { - if let Primitive::Int(Integer::I32, _) = s.value { + if let Primitive::Int(Integer::I32, _) = s.primitive() { bx.bitcast(value, bx.cx.type_i32()) } else { value @@ -901,20 +944,22 @@ fn llvm_fixup_output<'ll, 'tcx>( ), Abi::Scalar(s), ) => { - if let Primitive::Int(Integer::I64, _) = s.value { + if let Primitive::Int(Integer::I64, _) = s.primitive() { bx.bitcast(value, bx.cx.type_i64()) } else { value } } - (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => match s.value { - // MIPS only supports register-length arithmetics. - Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), - Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), - Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), - Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), - _ => value, - }, + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), + Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), + Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), + Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), + _ => value, + } + } _ => value, } } @@ -927,7 +972,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( ) -> &'ll Type { match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { - if let Primitive::Int(Integer::I8, _) = s.value { + if let Primitive::Int(Integer::I8, _) = s.primitive() { cx.type_vector(cx.type_i8(), 8) } else { layout.llvm_type(cx) @@ -946,7 +991,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( cx.type_vector(elem_ty, count * 2) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.value == Primitive::F64 => + if s.primitive() == Primitive::F64 => { cx.type_i64() } @@ -958,7 +1003,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s), ) => { - if let Primitive::Int(Integer::I32, _) = s.value { + if let Primitive::Int(Integer::I32, _) = s.primitive() { cx.type_f32() } else { layout.llvm_type(cx) @@ -972,19 +1017,21 @@ fn llvm_fixup_output_type<'ll, 'tcx>( ), Abi::Scalar(s), ) => { - if let Primitive::Int(Integer::I64, _) = s.value { + if let Primitive::Int(Integer::I64, _) = s.primitive() { cx.type_f64() } else { layout.llvm_type(cx) } } - (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => match s.value { - // MIPS only supports register-length arithmetics. - Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), - Primitive::F32 => cx.type_i32(), - Primitive::F64 => cx.type_i64(), - _ => layout.llvm_type(cx), - }, + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), + Primitive::F32 => cx.type_i32(), + Primitive::F64 => cx.type_i64(), + _ => layout.llvm_type(cx), + } + } _ => layout.llvm_type(cx), } } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 101da0012cb..9394d60134f 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -6,6 +6,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::OptLevel; +use rustc_span::symbol::sym; use rustc_target::spec::abi::Abi; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; @@ -116,7 +117,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribu // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = cx.sess().target.mcount.as_str(); + let mcount_name = cx.sess().target.mcount.as_ref(); Some(llvm::CreateAttrStringValue( cx.llcx, @@ -329,9 +330,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( ) { let span = cx .tcx - .get_attrs(instance.def_id()) - .iter() - .find(|a| a.has_name(rustc_span::sym::target_feature)) + .get_attr(instance.def_id(), sym::target_feature) .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); let msg = format!( "the target features {} must all be either enabled or disabled together", diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index f814cc93043..da9d8b5fb33 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -15,19 +15,12 @@ use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_session::cstore::{DllCallingConvention, DllImport}; use rustc_session::Session; -struct ArchiveConfig<'a> { - pub sess: &'a Session, - pub dst: PathBuf, - pub src: Option<PathBuf>, -} - /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] pub struct LlvmArchiveBuilder<'a> { - config: ArchiveConfig<'a>, - removals: Vec<String>, + sess: &'a Session, + dst: PathBuf, additions: Vec<Addition>, - src_archive: Option<Option<ArchiveRO>>, } enum Addition { @@ -50,10 +43,6 @@ fn is_relevant_child(c: &Child<'_>) -> bool { } } -fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) -> ArchiveConfig<'a> { - ArchiveConfig { sess, dst: output.to_path_buf(), src: input.map(|p| p.to_path_buf()) } -} - /// Map machine type strings to values of LLVM's MachineTypes enum. fn llvm_machine_type(cpu: &str) -> LLVMMachineType { match cpu { @@ -68,37 +57,8 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Creates a new static archive, ready for modifying the archive specified /// by `config`. - fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> LlvmArchiveBuilder<'a> { - let config = archive_config(sess, output, input); - LlvmArchiveBuilder { - config, - removals: Vec::new(), - additions: Vec::new(), - src_archive: None, - } - } - - /// Removes a file from this archive - fn remove_file(&mut self, file: &str) { - self.removals.push(file.to_string()); - } - - /// Lists all files in an archive - fn src_files(&mut self) -> Vec<String> { - if self.src_archive().is_none() { - return Vec::new(); - } - - let archive = self.src_archive.as_ref().unwrap().as_ref().unwrap(); - - archive - .iter() - .filter_map(|child| child.ok()) - .filter(is_relevant_child) - .filter_map(|child| child.name()) - .filter(|name| !self.removals.iter().any(|x| x == name)) - .map(|name| name.to_owned()) - .collect() + fn new(sess: &'a Session, output: &Path) -> LlvmArchiveBuilder<'a> { + LlvmArchiveBuilder { sess, dst: output.to_path_buf(), additions: Vec::new() } } fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> @@ -129,13 +89,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. - fn build(mut self) { - let kind = self.llvm_archive_kind().unwrap_or_else(|kind| { - self.config.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) - }); - - if let Err(e) = self.build_with_llvm(kind) { - self.config.sess.fatal(&format!("failed to build archive: {}", e)); + fn build(mut self) -> bool { + match self.build_with_llvm() { + Ok(any_members) => any_members, + Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), } } @@ -151,14 +108,16 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path.with_extension("lib") }; - let target = &self.config.sess.target; - let mingw_gnu_toolchain = - target.vendor == "pc" && target.os == "windows" && target.env == "gnu"; + let target = &self.sess.target; + let mingw_gnu_toolchain = target.vendor == "pc" + && target.os == "windows" + && target.env == "gnu" + && target.abi.is_empty(); let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { - if self.config.sess.target.arch == "x86" { + if self.sess.target.arch == "x86" { ( LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), import.ordinal, @@ -195,11 +154,11 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - self.config.sess.fatal(&format!("Error writing .DEF file: {}", e)); + self.sess.fatal(&format!("Error writing .DEF file: {}", e)); } }; - let dlltool = find_binutils_dlltool(self.config.sess); + let dlltool = find_binutils_dlltool(self.sess); let result = std::process::Command::new(dlltool) .args([ "-d", @@ -213,9 +172,9 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match result { Err(e) => { - self.config.sess.fatal(&format!("Error calling dlltool: {}", e)); + self.sess.fatal(&format!("Error calling dlltool: {}", e)); } - Ok(output) if !output.status.success() => self.config.sess.fatal(&format!( + Ok(output) if !output.status.success() => self.sess.fatal(&format!( "Dlltool could not create import library: {}\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) @@ -261,13 +220,13 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path_z.as_ptr(), ffi_exports.as_ptr(), ffi_exports.len(), - llvm_machine_type(&self.config.sess.target.arch) as u16, - !self.config.sess.target.is_like_msvc, + llvm_machine_type(&self.sess.target.arch) as u16, + !self.sess.target.is_like_msvc, ) }; if result == crate::llvm::LLVMRustResult::Failure { - self.config.sess.fatal(&format!( + self.sess.fatal(&format!( "Error creating import library for {}: {}", lib_name, llvm::last_error().unwrap_or("unknown LLVM error".to_string()) @@ -276,7 +235,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { }; self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { - self.config.sess.fatal(&format!( + self.sess.fatal(&format!( "failed to add native library {}: {}", output_path.display(), e @@ -286,46 +245,19 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { } impl<'a> LlvmArchiveBuilder<'a> { - fn src_archive(&mut self) -> Option<&ArchiveRO> { - if let Some(ref a) = self.src_archive { - return a.as_ref(); - } - let src = self.config.src.as_ref()?; - self.src_archive = Some(ArchiveRO::open(src).ok()); - self.src_archive.as_ref().unwrap().as_ref() - } - - fn llvm_archive_kind(&self) -> Result<ArchiveKind, &str> { - let kind = &*self.config.sess.target.archive_format; - kind.parse().map_err(|_| kind) - } + fn build_with_llvm(&mut self) -> io::Result<bool> { + let kind = &*self.sess.target.archive_format; + let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| { + self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) + }); - fn build_with_llvm(&mut self, kind: ArchiveKind) -> io::Result<()> { - let removals = mem::take(&mut self.removals); let mut additions = mem::take(&mut self.additions); let mut strings = Vec::new(); let mut members = Vec::new(); - let dst = CString::new(self.config.dst.to_str().unwrap())?; + let dst = CString::new(self.dst.to_str().unwrap())?; unsafe { - if let Some(archive) = self.src_archive() { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - let Some(child_name) = child.name() else { continue }; - if removals.iter().any(|r| r == child_name) { - continue; - } - - let name = CString::new(child_name)?; - members.push(llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - )); - strings.push(name); - } - } for addition in &mut additions { match addition { Addition::File { path, name_in_archive } => { @@ -387,7 +319,7 @@ impl<'a> LlvmArchiveBuilder<'a> { }; Err(io::Error::new(io::ErrorKind::Other, msg)) } else { - Ok(()) + Ok(!members.is_empty()) }; for member in members { llvm::LLVMRustArchiveMemberFree(member); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 0f5b1c08ec2..38402e04313 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -6,9 +6,7 @@ use crate::llvm::{self, build_string, False, True}; use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; -use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLTOInput, ModuleConfig, TargetMachineFactoryConfig, -}; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; @@ -16,7 +14,7 @@ use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; -use rustc_middle::middle::exported_symbols::SymbolExportLevel; +use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; use tracing::{debug, info}; @@ -55,8 +53,8 @@ fn prepare_lto( Lto::No => panic!("didn't request LTO but we're doing LTO"), }; - let symbol_filter = &|&(ref name, level): &(String, SymbolExportLevel)| { - if level.is_below_threshold(export_threshold) { + let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { + if info.level.is_below_threshold(export_threshold) || info.used { Some(CString::new(name.as_str()).unwrap()) } else { None @@ -313,7 +311,9 @@ fn fat_lto( for (bc_decoded, name) in serialized_modules { let _timer = cgcx .prof - .generic_activity_with_arg("LLVM_fat_lto_link_module", format!("{:?}", name)); + .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| { + recorder.record_arg(format!("{:?}", name)) + }); info!("linking {:?}", name); let data = bc_decoded.data(); linker.add(data).map_err(|()| { @@ -351,17 +351,17 @@ fn fat_lto( } } - Ok(LtoModuleCodegen::Fat { module: Some(module), _serialized_bitcode: serialized_bitcode }) + Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) } -crate struct Linker<'a>(&'a mut llvm::Linker<'a>); +pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); impl<'a> Linker<'a> { - crate fn new(llmod: &'a llvm::Module) -> Self { + pub(crate) fn new(llmod: &'a llvm::Module) -> Self { unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } } - crate fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { + pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { unsafe { if llvm::LLVMRustLinkerAdd( self.0, @@ -394,15 +394,15 @@ impl Drop for Linker<'_> { /// /// At a high level Thin LTO looks like: /// -/// 1. Prepare a "summary" of each LLVM module in question which describes -/// the values inside, cost of the values, etc. -/// 2. Merge the summaries of all modules in question into one "index" -/// 3. Perform some global analysis on this index -/// 4. For each module, use the index and analysis calculated previously to -/// perform local transformations on the module, for example inlining -/// small functions from other modules. -/// 5. Run thin-specific optimization passes over each module, and then code -/// generate everything at the end. +/// 1. Prepare a "summary" of each LLVM module in question which describes +/// the values inside, cost of the values, etc. +/// 2. Merge the summaries of all modules in question into one "index" +/// 3. Perform some global analysis on this index +/// 4. For each module, use the index and analysis calculated previously to +/// perform local transformations on the module, for example inlining +/// small functions from other modules. +/// 5. Run thin-specific optimization passes over each module, and then code +/// generate everything at the end. /// /// The summary for each module is intended to be quite cheap, and the global /// index is relatively quite cheap to create as well. As a result, the goal of @@ -576,19 +576,31 @@ fn thin_lto( pub(crate) fn run_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, - module: &ModuleCodegen<ModuleLlvm>, - config: &ModuleConfig, + module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, ) -> Result<(), FatalError> { let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); + let config = cgcx.config(module.kind); // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. // // This code is based off the code found in llvm's LTO code generator: - // tools/lto/LTOCodeGenerator.cpp + // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { + if !llvm::LLVMRustHasModuleFlag( + module.module_llvm.llmod(), + "LTOPostLink".as_ptr().cast(), + 11, + ) { + llvm::LLVMRustAddModuleFlag( + module.module_llvm.llmod(), + llvm::LLVMModFlagBehavior::Error, + "LTOPostLink\0".as_ptr().cast(), + 1, + ); + } if llvm_util::should_use_new_llvm_pass_manager( &config.new_llvm_pass_manager, &cgcx.target_arch, @@ -623,7 +635,7 @@ pub(crate) fn run_pass_manager( if thin { llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); } else { - llvm::LLVMPassManagerBuilderPopulateLTOPassManager( + llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( b, pm, /* Internalize = */ False, /* RunInliner = */ True, ); } @@ -724,7 +736,7 @@ impl Drop for ThinBuffer { } pub unsafe fn optimize_thin_module( - thin_module: &mut ThinModule<LlvmCodegenBackend>, + thin_module: ThinModule<LlvmCodegenBackend>, cgcx: &CodegenContext<LlvmCodegenBackend>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { let diag_handler = cgcx.create_diag_handler(); @@ -741,7 +753,7 @@ pub unsafe fn optimize_thin_module( // that LLVM Context and Module. let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; - let module = ModuleCodegen { + let mut module = ModuleCodegen { module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, name: thin_module.name().to_string(), kind: ModuleKind::Regular, @@ -857,8 +869,7 @@ pub unsafe fn optimize_thin_module( // little differently. { info!("running thin lto passes over {}", module.name); - let config = cgcx.config(module.kind); - run_pass_manager(cgcx, &diag_handler, &module, config, true)?; + run_pass_manager(cgcx, &diag_handler, &mut module, true)?; save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c18719d4ad7..ab8874d796d 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -56,28 +56,24 @@ pub fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { + debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { let output_c = path_to_c_string(output); - let result = if let Some(dwo_output) = dwo_output { - let dwo_output_c = path_to_c_string(dwo_output); - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - dwo_output_c.as_ptr(), - file_type, - ) + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() } else { - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - std::ptr::null(), - file_type, - ) + std::ptr::null() }; + let result = llvm::LLVMRustWriteOutputFile( + target, + pm, + m, + output_c.as_ptr(), + dwo_output_ptr, + file_type, + ); // Record artifact sizes for self-profiling if result == llvm::LLVMRustResult::Success { @@ -340,7 +336,7 @@ fn report_inline_asm( } let level = match level { llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning, + llvm::DiagnosticLevel::Warning => Level::Warning(None), llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); @@ -467,7 +463,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); - let extra_passes = config.passes.join(","); + let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; let llvm_plugins = config.llvm_plugins.join(","); @@ -523,6 +519,12 @@ pub(crate) unsafe fn optimize( let module_name = module.name.clone(); let module_name = Some(&module_name[..]); + if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { + diag_handler.warn( + "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", + ); + } + if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); let out = path_to_c_string(&out); @@ -628,8 +630,8 @@ pub(crate) unsafe fn optimize( extra_passes.as_ptr(), extra_passes.len() as size_t, ); - llvm::LLVMPassManagerBuilderPopulateFunctionPassManager(b, fpm); - llvm::LLVMPassManagerBuilderPopulateModulePassManager(b, mpm); + llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); + llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); }); have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; @@ -721,8 +723,7 @@ pub(crate) fn link( let mut linker = Linker::new(first.module_llvm.llmod()); for module in elements { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_link_module", format!("{:?}", module.name)); + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); linker.add(buffer.data()).map_err(|()| { let msg = format!("failed to serialize module {:?}", module.name); @@ -1035,7 +1036,8 @@ unsafe fn embed_bitcode( // reason (see issue #90326 for historical background). let is_apple = cgcx.opts.target_triple.triple().contains("-ios") || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos"); + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos"); if is_apple || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") @@ -1086,7 +1088,7 @@ pub unsafe fn with_llvm_pmb( // Create the PassManagerBuilder for LLVM. We configure it with // reasonable defaults and prepare it to actually populate the pass // manager. - let builder = llvm::LLVMPassManagerBuilderCreate(); + let builder = llvm::LLVMRustPassManagerBuilderCreate(); let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); let inline_threshold = config.inline_threshold; let pgo_gen_path = get_pgo_gen_path(config); @@ -1103,14 +1105,9 @@ pub unsafe fn with_llvm_pmb( pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + opt_size as c_int, ); - llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32); - - if opt_size != llvm::CodeGenOptSizeNone { - llvm::LLVMPassManagerBuilderSetDisableUnrollLoops(builder, 1); - } - llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); // Here we match what clang does (kinda). For O0 we only inline @@ -1119,16 +1116,16 @@ pub unsafe fn with_llvm_pmb( // thresholds copied from clang. match (opt_level, opt_size, inline_threshold) { (.., Some(t)) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); } (llvm::CodeGenOptLevel::Aggressive, ..) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); } (_, llvm::CodeGenOptSizeDefault, _) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 75); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); } (_, llvm::CodeGenOptSizeAggressive, _) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 25); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); } (llvm::CodeGenOptLevel::None, ..) => { llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); @@ -1137,12 +1134,12 @@ pub unsafe fn with_llvm_pmb( llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); } (llvm::CodeGenOptLevel::Default, ..) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 225); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); } } f(builder); - llvm::LLVMPassManagerBuilderDispose(builder); + llvm::LLVMRustPassManagerBuilderDispose(builder); } // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index dd3ada44389..86f92dc0239 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -74,10 +74,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen<ModuleLlvm> { let cgu = tcx.codegen_unit(cgu_name); - let _prof_timer = tcx.prof.generic_activity_with_args( - "codegen_module", - &[cgu_name.to_string(), cgu.size_estimate().to_string()], - ); + let _prof_timer = + tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { + recorder.record_arg(cgu_name.to_string()); + recorder.record_arg(cgu.size_estimate().to_string()); + }); // Instantiate monomorphizations without filling out definitions yet... let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); { @@ -99,15 +100,6 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } - // Run replace-all-uses-with for statics that need it - for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { - unsafe { - let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); - llvm::LLVMReplaceAllUsesWith(old_g, bitcast); - llvm::LLVMDeleteGlobal(old_g); - } - } - // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.compiler.used` variable, created next. if cx.sess().instrument_coverage() { @@ -122,6 +114,16 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen cx.create_compiler_used_variable() } + // Run replace-all-uses-with for statics that need it. This must + // happen after the llvm.used variables are created. + for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } + } + // Finalize debuginfo if cx.sess().opts.debuginfo != DebugInfo::None { cx.debuginfo_finalize(); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index c4eb593d297..8c1e865762c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -484,14 +484,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { bx.noundef_metadata(load); } - match scalar.value { + match scalar.primitive() { abi::Int(..) => { if !scalar.is_always_valid(bx) { - bx.range_metadata(load, scalar.valid_range); + bx.range_metadata(load, scalar.valid_range(bx)); } } abi::Pointer => { - if !scalar.valid_range.contains(0) { + if !scalar.valid_range(bx).contains(0) { bx.nonnull_metadata(load); } @@ -509,15 +509,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Ref(place.llval, Some(llextra), place.align) } else if place.layout.is_llvm_immediate() { let mut const_llval = None; + let llty = place.layout.llvm_type(self); unsafe { if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { if llvm::LLVMIsGlobalConstant(global) == llvm::True { - const_llval = llvm::LLVMGetInitializer(global); + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); + } + } } } } let llval = const_llval.unwrap_or_else(|| { - let load = self.load(place.layout.llvm_type(self), place.llval, place.align); + let load = self.load(llty, place.llval, place.align); if let abi::Abi::Scalar(scalar) = place.layout.abi { scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); } @@ -525,7 +530,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { }); OperandValue::Immediate(self.to_immediate(llval, place.layout)) } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { - let b_offset = a.value.size(self).align_to(b.value.align(self).abi); + let b_offset = a.size(self).align_to(b.align(self).abi); let pair_ty = place.layout.llvm_type(self); let mut load = |i, scalar: abi::Scalar, layout, align, offset| { @@ -1059,11 +1064,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst: &'ll Value, cmp: &'ll Value, src: &'ll Value, - order: rustc_codegen_ssa::common::AtomicOrdering, + mut order: rustc_codegen_ssa::common::AtomicOrdering, failure_order: rustc_codegen_ssa::common::AtomicOrdering, weak: bool, ) -> &'ll Value { let weak = if weak { llvm::True } else { llvm::False }; + if llvm_util::get_version() < (13, 0, 0) { + use rustc_codegen_ssa::common::AtomicOrdering::*; + // Older llvm has the pre-C++17 restriction on + // success and failure memory ordering, + // requiring the former to be at least as strong as the latter. + // So, for llvm 12, we upgrade the success ordering to a stronger + // one if necessary. + match (order, failure_order) { + (Relaxed, Acquire) => order = Acquire, + (Release, Acquire) => order = AcquireRelease, + (_, SequentiallyConsistent) => order = SequentiallyConsistent, + _ => {} + } + } unsafe { llvm::LLVMRustBuildAtomicCmpXchg( self.llbuilder, @@ -1412,7 +1431,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } } - crate fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); self.call(ty, f, args, None) } @@ -1452,7 +1471,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } fn fptoint_sat_broken_in_llvm(&self) -> bool { - match self.tcx.sess.target.arch.as_str() { + match self.tcx.sess.target.arch.as_ref() { // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 "riscv64" => llvm_util::get_version() < (13, 0, 0), _ => false, diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index b10e74625da..d37aadeb523 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -109,8 +109,7 @@ impl<'ll> CodegenCx<'ll, '_> { pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { assert_eq!(idx as c_uint as u64, idx); - let us = &[idx as c_uint]; - let r = llvm::LLVMConstExtractValue(v, us.as_ptr(), us.len() as c_uint); + let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); @@ -147,6 +146,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.const_uint(self.type_i1(), val as u64) } + fn const_i16(&self, i: i16) -> &'ll Value { + self.const_int(self.type_i16(), i as i64) + } + fn const_i32(&self, i: i32) -> &'ll Value { self.const_int(self.type_i32(), i as i64) } @@ -217,16 +220,16 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { - let bitsize = if layout.is_bool() { 1 } else { layout.value.size(self).bits() }; + let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { Scalar::Int(ScalarInt::ZST) => { - assert_eq!(0, layout.value.size(self).bytes()); + assert_eq!(0, layout.size(self).bytes()); self.const_undef(self.type_ix(0)) } Scalar::Int(int) => { - let data = int.assert_bits(layout.value.size(self)); + let data = int.assert_bits(layout.size(self)); let llval = self.const_uint_big(self.type_ix(bitsize), data); - if layout.value == Pointer { + if layout.primitive() == Pointer { unsafe { llvm::LLVMConstIntToPtr(llval, llty) } } else { self.const_bitcast(llval, llty) @@ -265,7 +268,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { 1, ) }; - if layout.value != Pointer { + if layout.primitive() != Pointer { unsafe { llvm::LLVMConstPtrToInt(llval, llty) } } else { self.const_bitcast(llval, llty) diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 7d3fe43eeab..5bbbfe9a4ab 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -109,7 +109,10 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), &cx.tcx, ), - Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } }, + Scalar::Initialized { + value: Primitive::Pointer, + valid_range: WrappingRange::full(dl.pointer_size), + }, cx.type_i8p_ext(address_space), )); next_offset = offset + pointer_size; @@ -209,11 +212,11 @@ pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { } impl<'ll> CodegenCx<'ll, '_> { - crate fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMConstBitCast(val, ty) } } - crate fn static_addr_of_mut( + pub(crate) fn static_addr_of_mut( &self, cv: &'ll Value, align: Align, @@ -238,7 +241,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn get_static(&self, def_id: DefId) -> &'ll Value { + pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); if let Some(&g) = self.instances.borrow().get(&instance) { return g; @@ -409,6 +412,13 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { llvm::LLVMRustSetLinkage(new_g, linkage); llvm::LLVMRustSetVisibility(new_g, visibility); + // The old global has had its name removed but is returned by + // get_static since it is in the instance cache. Provide an + // alternative lookup that points to the new global so that + // global_asm! can compute the correct mangled symbol name + // for the global. + self.renamed_statics.borrow_mut().insert(def_id, new_g); + // To avoid breaking any invariants, we leave around the old // global for the moment; we'll replace all references to it // with the new global later. (See base::codegen_backend.) @@ -428,7 +438,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { llvm::LLVMSetGlobalConstant(g, llvm::True); } - debuginfo::create_global_var_metadata(self, def_id, g); + debuginfo::build_global_var_di_node(self, def_id, g); if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { llvm::set_thread_local_mode(g, self.tls_model); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 52e03e0ad3d..c007728095f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -14,6 +14,7 @@ use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, @@ -95,7 +96,7 @@ pub struct CodegenCx<'ll, 'tcx> { pub isize_ty: &'ll Type, pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, - pub dbg_cx: Option<debuginfo::CrateDebugContext<'ll, 'tcx>>, + pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>, eh_personality: Cell<Option<&'ll Value>>, eh_catch_typeinfo: Cell<Option<&'ll Value>>, @@ -105,6 +106,12 @@ pub struct CodegenCx<'ll, 'tcx> { /// A counter that is used for generating local symbol names local_gen_sym_counter: Cell<usize>, + + /// `codegen_static` will sometimes create a second global variable with a + /// different type and clear the symbol name of the original global. + /// `global_asm!` needs to be able to find this new global so that it can + /// compute the correct mangled symbol name to insert into the asm. + pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, } pub struct TypeLowering<'ll> { @@ -134,7 +141,7 @@ pub unsafe fn create_module<'ll>( let mod_name = SmallCStr::new(mod_name); let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); - let mut target_data_layout = sess.target.data_layout.clone(); + let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); if llvm_version < (13, 0, 0) { if sess.target.arch == "powerpc64" { @@ -319,11 +326,20 @@ pub unsafe fn create_module<'ll>( ) } + if sess.opts.debugging_opts.virtual_function_elimination { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "Virtual Function Elim\0".as_ptr().cast(), + 1, + ); + } + llmod } impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { - crate fn new( + pub(crate) fn new( tcx: TyCtxt<'tcx>, codegen_unit: &'tcx CodegenUnit<'tcx>, llvm_module: &'ll crate::ModuleLlvm, @@ -360,7 +376,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { // feasible. The compiler may be able to get around this, but it may // involve some invasive changes to deal with this. // - // The flipside of this situation is that whenever you link to a dll and + // The flip side of this situation is that whenever you link to a dll and // you import a function from it, the import should be tagged with // `dllimport`. At this time, however, the compiler does not emit // `dllimport` for any declarations other than constants (where it is @@ -396,8 +412,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { }; let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { - let dctx = debuginfo::CrateDebugContext::new(llmod); - debuginfo::metadata::compile_unit_metadata(tcx, codegen_unit.name().as_str(), &dctx); + let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node( + tcx, + codegen_unit.name().as_str(), + &dctx, + ); Some(dctx) } else { None @@ -432,10 +452,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { rust_try_fn: Cell::new(None), intrinsics: Default::default(), local_gen_sym_counter: Cell::new(0), + renamed_statics: Default::default(), } } - crate fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> { + pub(crate) fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> { &self.statics_to_rauw } @@ -587,7 +608,7 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll> CodegenCx<'ll, '_> { - crate fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { + pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { if let Some(v) = self.intrinsics.borrow().get(key).cloned() { return v; } @@ -644,6 +665,7 @@ impl<'ll> CodegenCx<'ll, '_> { let t_isize = self.type_isize(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_metadata = self.type_metadata(); ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); @@ -855,7 +877,10 @@ impl<'ll> CodegenCx<'ll, '_> { // This isn't an "LLVM intrinsic", but LLVM's optimization passes // recognize it like one and we assume it exists in `core::slice::cmp` - ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32); + match self.sess().target.arch.as_ref() { + "avr" | "msp430" => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i16), + _ => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32), + } // variadic intrinsics ifn!("llvm.va_start", fn(i8p) -> void); @@ -866,21 +891,22 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); } - ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1); + ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void); - ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void); + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } None } - crate fn eh_catch_typeinfo(&self) -> &'ll Value { + pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() { return eh_catch_typeinfo; } let tcx = self.tcx; - assert!(self.sess().target.is_like_emscripten); + assert!(self.sess().target.os == "emscripten"); let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { Some(def_id) => self.get_static(def_id), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 39f53235e2c..99e4ded62f1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -20,64 +20,66 @@ pub fn compute_mir_scopes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - fn_dbg_scope: &'ll DIScope, debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, ) { - // Find all the scopes with variables defined in them. - let mut has_variables = BitSet::new_empty(mir.source_scopes.len()); - - // Only consider variables when they're going to be emitted. - // FIXME(eddyb) don't even allocate `has_variables` otherwise. - if cx.sess().opts.debuginfo == DebugInfo::Full { + // Find all scopes with variables defined in them. + let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { + let mut vars = BitSet::new_empty(mir.source_scopes.len()); // FIXME(eddyb) take into account that arguments always have debuginfo, // irrespective of their name (assuming full debuginfo is enabled). // NOTE(eddyb) actually, on second thought, those are always in the // function scope, which always exists. for var_debug_info in &mir.var_debug_info { - has_variables.insert(var_debug_info.source_info.scope); + vars.insert(var_debug_info.source_info.scope); } - } - + Some(vars) + } else { + // Nothing to emit, of course. + None + }; + let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { let scope = SourceScope::new(idx); - make_mir_scope(cx, instance, mir, fn_dbg_scope, &has_variables, debug_context, scope); + make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); } + assert!(instantiated.count() == mir.source_scopes.len()); } fn make_mir_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - fn_dbg_scope: &'ll DIScope, - has_variables: &BitSet<SourceScope>, + variables: &Option<BitSet<SourceScope>>, debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + instantiated: &mut BitSet<SourceScope>, scope: SourceScope, ) { - if debug_context.scopes[scope].dbg_scope.is_some() { + if instantiated.contains(scope) { return; } let scope_data = &mir.source_scopes[scope]; let parent_scope = if let Some(parent) = scope_data.parent_scope { - make_mir_scope(cx, instance, mir, fn_dbg_scope, has_variables, debug_context, parent); + make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); debug_context.scopes[parent] } else { // The root is the function itself. let loc = cx.lookup_debug_loc(mir.span.lo()); debug_context.scopes[scope] = DebugScope { - dbg_scope: Some(fn_dbg_scope), - inlined_at: None, file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_pos, + ..debug_context.scopes[scope] }; + instantiated.insert(scope); return; }; - if !has_variables.contains(scope) && scope_data.inlined.is_none() { + if let Some(vars) = variables && !vars.contains(scope) && scope_data.inlined.is_none() { // Do not create a DIScope if there are no variables defined in this // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. debug_context.scopes[scope] = parent_scope; + instantiated.insert(scope); return; } @@ -99,7 +101,7 @@ fn make_mir_scope<'ll, 'tcx>( None => unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlock( DIB(cx), - parent_scope.dbg_scope.unwrap(), + parent_scope.dbg_scope, file_metadata, loc.line, loc.col, @@ -115,9 +117,10 @@ fn make_mir_scope<'ll, 'tcx>( }); debug_context.scopes[scope] = DebugScope { - dbg_scope: Some(dbg_scope), + dbg_scope, inlined_at: inlined_at.or(parent_scope.inlined_at), file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_pos, }; + instantiated.insert(scope); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md index f983d092039..aaec4e68c17 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md +++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md @@ -27,14 +27,14 @@ The module is thus driven from an outside client with functions like Internally the module will try to reuse already created metadata by utilizing a cache. The way to get a shared metadata node when needed is thus to just call the corresponding function in this module: - - let file_metadata = file_metadata(cx, file); - +```ignore (illustrative) +let file_metadata = file_metadata(cx, file); +``` The function will take care of probing the cache for an existing node for that exact file path. All private state used by the module is stored within either the -CrateDebugContext struct (owned by the CodegenCx) or the +CodegenUnitDebugContext struct (owned by the CodegenCx) or the FunctionDebugContext (owned by the FunctionCx). This file consists of three conceptual sections: @@ -63,7 +63,7 @@ struct List { will generate the following callstack with a naive DFS algorithm: -``` +```ignore (illustrative) describe(t = List) describe(t = i32) describe(t = Option<Box<List>>) @@ -72,7 +72,7 @@ describe(t = List) ... ``` -To break cycles like these, we use "forward declarations". That is, when +To break cycles like these, we use "stubs". That is, when the algorithm encounters a possibly recursive type (any struct or enum), it immediately creates a type description node and inserts it into the cache *before* describing the members of the type. This type description is just @@ -80,13 +80,8 @@ a stub (as type members are not described and added to it yet) but it allows the algorithm to already refer to the type. After the stub is inserted into the cache, the algorithm continues as before. If it now encounters a recursive reference, it will hit the cache and does not try to -describe the type anew. - -This behavior is encapsulated in the 'RecursiveTypeDescription' enum, -which represents a kind of continuation, storing all state needed to -continue traversal at the type members after the type has been registered -with the cache. (This implementation approach might be a tad over- -engineered and may change in the future) +describe the type anew. This behavior is encapsulated in the +`type_map::build_type_with_children()` function. ## Source Locations and Line Information @@ -134,47 +129,3 @@ detection. The `create_argument_metadata()` and related functions take care of linking the `llvm.dbg.declare` instructions to the correct source locations even while source location emission is still disabled, so there is no need to do anything special with source location handling here. - -## Unique Type Identification - -In order for link-time optimization to work properly, LLVM needs a unique -type identifier that tells it across compilation units which types are the -same as others. This type identifier is created by -`TypeMap::get_unique_type_id_of_type()` using the following algorithm: - -1. Primitive types have their name as ID - -2. Structs, enums and traits have a multipart identifier - - 1. The first part is the SVH (strict version hash) of the crate they - were originally defined in - - 2. The second part is the ast::NodeId of the definition in their - original crate - - 3. The final part is a concatenation of the type IDs of their concrete - type arguments if they are generic types. - -3. Tuple-, pointer-, and function types are structurally identified, which - means that they are equivalent if their component types are equivalent - (i.e., `(i32, i32)` is the same regardless in which crate it is used). - -This algorithm also provides a stable ID for types that are defined in one -crate but instantiated from metadata within another crate. We just have to -take care to always map crate and `NodeId`s back to the original crate -context. - -As a side-effect these unique type IDs also help to solve a problem arising -from lifetime parameters. Since lifetime parameters are completely omitted -in debuginfo, more than one `Ty` instance may map to the same debuginfo -type metadata, that is, some struct `Struct<'a>` may have N instantiations -with different concrete substitutions for `'a`, and thus there will be N -`Ty` instances for the type `Struct<'a>` even though it is not generic -otherwise. Unfortunately this means that we cannot use `ty::type_id()` as -cheap identifier for type metadata -- we have done this in the past, but it -led to unnecessary metadata duplication in the best case and LLVM -assertions in the worst. However, the unique type ID as described above -*can* be used as identifier. Since it is comparatively expensive to -construct, though, `ty::type_id()` is still used additionally as an -optimization for cases where the exact same type has been seen before -(which is most of the time). diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 31a09242c5a..5186aee57fb 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -5,11 +5,14 @@ use crate::llvm; use crate::builder::Builder; use crate::common::CodegenCx; use crate::value::Value; +use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_session::config::DebugInfo; +use rustc_session::config::{CrateType, DebugInfo}; use rustc_span::symbol::sym; +use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -37,9 +40,33 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' section_var.unwrap_or_else(|| { let section_name = b".debug_gdb_scripts\0"; - let section_contents = b"\x01gdb_load_rust_pretty_printers.py\0"; + let mut section_contents = Vec::new(); + + // Add the pretty printers for the standard library first. + section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0"); + + // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute. + let visualizers = collect_debugger_visualizers_transitive( + cx.tcx, + DebuggerVisualizerType::GdbPrettyPrinter, + ); + let crate_name = cx.tcx.crate_name(LOCAL_CRATE); + for (index, visualizer) in visualizers.iter().enumerate() { + // The initial byte `4` instructs GDB that the following pretty printer + // is defined inline as opposed to in a standalone file. + section_contents.extend_from_slice(b"\x04"); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name.as_str(), index); + section_contents.extend_from_slice(vis_name.as_bytes()); + section_contents.extend_from_slice(&visualizer.src); + + // The final byte `0` tells GDB that the pretty printer has been + // fully defined and can continue searching for additional + // pretty printers. + section_contents.extend_from_slice(b"\0"); + } unsafe { + let section_contents = section_contents.as_slice(); let llvm_type = cx.type_array(cx.type_i8(), section_contents.len() as u64); let section_var = cx @@ -62,7 +89,32 @@ pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create + // ODR violations at link time, this section will not be emitted for rlibs since + // each rlib could produce a different set of visualizers that would be embedded + // in the `.debug_gdb_scripts` section. For that reason, we make sure that the + // section is only emitted for leaf crates. + let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { + CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { + // These are crate types for which we will embed pretty printers since they + // are treated as leaf crates. + true + } + CrateType::ProcMacro => { + // We could embed pretty printers for proc macro crates too but it does not + // seem like a good default, since this is a rare use case and we don't + // want to slow down the common case. + false + } + CrateType::Rlib => { + // As per the above description, embedding pretty printers for rlibs could + // lead to ODR violations so we skip this crate type as well. + false + } + }); + !omit_gdb_pretty_printer_section && cx.sess().opts.debuginfo != DebugInfo::None && cx.sess().target.emit_debug_gdb_scripts + && embed_visualizers } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 89fc8980037..d5f39a45670 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1,21 +1,22 @@ -use self::MemberDescriptionFactory::*; -use self::RecursiveTypeDescription::*; +use self::type_map::DINodeCreationResult; +use self::type_map::Stub; +use self::type_map::UniqueTypeId; use super::namespace::mangled_name_of_instance; use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; use super::utils::{ create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, DIB, }; -use super::CrateDebugContext; +use super::CodegenUnitDebugContext; use crate::abi; use crate::common::CodegenCx; +use crate::debuginfo::metadata::type_map::build_type_with_children; use crate::debuginfo::utils::fat_pointer_kind; use crate::debuginfo::utils::FatPtrKind; use crate::llvm; use crate::llvm::debuginfo::{ - DIArray, DICompositeType, DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, - DebugEmissionKind, + DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, }; use crate::value::Value; @@ -23,36 +24,34 @@ use cstr::cstr; use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fx::FxHashMap; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{self, GeneratorLayout}; -use rustc_middle::ty::layout::{self, IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ - self, AdtKind, GeneratorSubsts, Instance, ParamEnv, Ty, TyCtxt, COMMON_VTABLE_ENTRIES, + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; -use rustc_session::config::{self, DebugInfo}; +use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; -use rustc_span::FileNameDisplayPreference; -use rustc_span::{self, SourceFile, SourceFileHash}; -use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, TagEncoding}; -use rustc_target::abi::{Int, Pointer, F32, F64}; -use rustc_target::abi::{Primitive, Size, VariantIdx, Variants}; -use smallvec::SmallVec; +use rustc_span::FileName; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; +use rustc_target::abi::{Align, Size}; +use smallvec::smallvec; use tracing::debug; -use libc::{c_longlong, c_uint}; -use std::cell::RefCell; -use std::collections::hash_map::Entry; +use libc::{c_char, c_longlong, c_uint}; +use std::borrow::Cow; use std::fmt::{self, Write}; use std::hash::{Hash, Hasher}; use std::iter; use std::path::{Path, PathBuf}; use std::ptr; +use tracing::instrument; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -88,250 +87,51 @@ const DW_ATE_unsigned: c_uint = 0x07; #[allow(non_upper_case_globals)] const DW_ATE_UTF: c_uint = 0x10; -pub const UNKNOWN_LINE_NUMBER: c_uint = 0; -pub const UNKNOWN_COLUMN_NUMBER: c_uint = 0; +pub(super) const UNKNOWN_LINE_NUMBER: c_uint = 0; +pub(super) const UNKNOWN_COLUMN_NUMBER: c_uint = 0; -pub const NO_SCOPE_METADATA: Option<&DIScope> = None; +const NO_SCOPE_METADATA: Option<&DIScope> = None; +/// A function that returns an empty list of generic parameter debuginfo nodes. +const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = |_| SmallVec::new(); -mod unique_type_id { - use rustc_data_structures::{ - fingerprint::Fingerprint, - stable_hasher::{HashStable, NodeIdHashingMode, StableHasher}, - }; - use rustc_middle::ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}; - use rustc_target::abi::VariantIdx; - - // This type cannot be constructed outside of this module because - // it has a private field. We make use of this in order to prevent - // `UniqueTypeId` from being constructed directly, without asserting - // the preconditions. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] - pub struct HiddenZst { - _inaccessible: (), - } - - /// A unique identifier for anything that we create a debuginfo node for. - /// The types it contains are expected to already be normalized (which - /// is debug_asserted in the constructors). - /// - /// Note that there are some things that only show up in debuginfo, like - /// the separate type descriptions for each enum variant. These get an ID - /// too because they have their own debuginfo node in LLVM IR. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] - pub(super) enum UniqueTypeId<'tcx> { - /// The ID of a regular type as it shows up at the language level. - Ty(Ty<'tcx>, HiddenZst), - /// The ID for the artificial struct type describing a single enum variant. - Variant(Ty<'tcx>, VariantIdx, HiddenZst), - /// The ID for the single DW_TAG_variant_part nested inside the top-level - /// DW_TAG_structure_type that describes enums and generators. - VariantPart(Ty<'tcx>, HiddenZst), - /// The ID of the artificial type we create for VTables. - VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, HiddenZst), - } - - impl<'tcx> UniqueTypeId<'tcx> { - pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { - debug_assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); - UniqueTypeId::Ty(t, HiddenZst { _inaccessible: () }) - } - - pub fn for_enum_variant( - tcx: TyCtxt<'tcx>, - enum_ty: Ty<'tcx>, - variant_idx: VariantIdx, - ) -> Self { - debug_assert_eq!( - enum_ty, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty) - ); - UniqueTypeId::Variant(enum_ty, variant_idx, HiddenZst { _inaccessible: () }) - } - - pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { - debug_assert_eq!( - enum_ty, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty) - ); - UniqueTypeId::VariantPart(enum_ty, HiddenZst { _inaccessible: () }) - } - - pub fn for_vtable_ty( - tcx: TyCtxt<'tcx>, - self_type: Ty<'tcx>, - implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, - ) -> Self { - debug_assert_eq!( - self_type, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type) - ); - debug_assert_eq!( - implemented_trait, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) - ); - UniqueTypeId::VTableTy(self_type, implemented_trait, HiddenZst { _inaccessible: () }) - } - - /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` - /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. - /// - /// Right now this takes the form of a hex-encoded opaque hash value. - pub fn generate_unique_id_string(&self, tcx: TyCtxt<'tcx>) -> String { - let mut hasher = StableHasher::new(); - let mut hcx = tcx.create_stable_hashing_context(); - hcx.while_hashing_spans(false, |hcx| { - hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| { - self.hash_stable(hcx, &mut hasher); - }); - }); - hasher.finish::<Fingerprint>().to_hex() - } - } -} -use unique_type_id::*; - -/// The `TypeMap` is where the debug context holds the type metadata nodes -/// created so far. The metadata nodes are indexed by `UniqueTypeId`. -#[derive(Default)] -pub struct TypeMap<'ll, 'tcx> { - unique_id_to_metadata: RefCell<FxHashMap<UniqueTypeId<'tcx>, &'ll DIType>>, -} - -impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { - /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will - /// fail if the mapping already exists. - fn register_unique_id_with_metadata( - &self, - unique_type_id: UniqueTypeId<'tcx>, - metadata: &'ll DIType, - ) { - if self.unique_id_to_metadata.borrow_mut().insert(unique_type_id, metadata).is_some() { - bug!("type metadata for unique ID '{:?}' is already in the `TypeMap`!", unique_type_id); - } - } - - fn find_metadata_for_unique_id( - &self, - unique_type_id: UniqueTypeId<'tcx>, - ) -> Option<&'ll DIType> { - self.unique_id_to_metadata.borrow().get(&unique_type_id).cloned() - } -} - -/// A description of some recursive type. It can either be already finished (as -/// with `FinalMetadata`) or it is not yet finished, but contains all information -/// needed to generate the missing parts of the description. See the -/// documentation section on Recursive Types at the top of this file for more -/// information. -enum RecursiveTypeDescription<'ll, 'tcx> { - UnfinishedMetadata { - unfinished_type: Ty<'tcx>, - unique_type_id: UniqueTypeId<'tcx>, - metadata_stub: &'ll DICompositeType, - member_holding_stub: &'ll DICompositeType, - member_description_factory: MemberDescriptionFactory<'ll, 'tcx>, - }, - FinalMetadata(&'ll DICompositeType), -} - -fn create_and_register_recursive_type_forward_declaration<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unfinished_type: Ty<'tcx>, - unique_type_id: UniqueTypeId<'tcx>, - metadata_stub: &'ll DICompositeType, - member_holding_stub: &'ll DICompositeType, - member_description_factory: MemberDescriptionFactory<'ll, 'tcx>, -) -> RecursiveTypeDescription<'ll, 'tcx> { - // Insert the stub into the `TypeMap` in order to allow for recursive references. - debug_context(cx).type_map.register_unique_id_with_metadata(unique_type_id, metadata_stub); - - UnfinishedMetadata { - unfinished_type, - unique_type_id, - metadata_stub, - member_holding_stub, - member_description_factory, - } -} +// SmallVec is used quite a bit in this module, so create a shorthand. +// The actual number of elements is not so important. +pub type SmallVec<T> = smallvec::SmallVec<[T; 16]>; -impl<'ll, 'tcx> RecursiveTypeDescription<'ll, 'tcx> { - /// Finishes up the description of the type in question (mostly by providing - /// descriptions of the fields of the given type) and returns the final type - /// metadata. - fn finalize(&self, cx: &CodegenCx<'ll, 'tcx>) -> MetadataCreationResult<'ll> { - match *self { - FinalMetadata(metadata) => MetadataCreationResult::new(metadata, false), - UnfinishedMetadata { - unfinished_type, - unique_type_id, - metadata_stub, - member_holding_stub, - ref member_description_factory, - } => { - // Make sure that we have a forward declaration of the type in - // the TypeMap so that recursive references are possible. This - // will always be the case if the RecursiveTypeDescription has - // been properly created through the - // `create_and_register_recursive_type_forward_declaration()` - // function. - { - if debug_context(cx) - .type_map - .find_metadata_for_unique_id(unique_type_id) - .is_none() - { - bug!( - "Forward declaration of potentially recursive type \ - '{:?}' was not found in TypeMap!", - unfinished_type - ); - } - } +mod enums; +mod type_map; - // ... then create the member descriptions ... - let member_descriptions = member_description_factory.create_member_descriptions(cx); - let type_params = compute_type_parameters(cx, unfinished_type); +pub(crate) use type_map::TypeMap; - // ... and attach them to the stub to complete it. - set_members_of_composite_type( - cx, - member_holding_stub, - member_descriptions, - None, - type_params, - ); - MetadataCreationResult::new(metadata_stub, true) - } - } - } -} - -/// Returns from the enclosing function if the type metadata with the given +/// Returns from the enclosing function if the type debuginfo node with the given /// unique ID can be found in the type map. -macro_rules! return_if_metadata_created_in_meantime { +macro_rules! return_if_di_node_created_in_meantime { ($cx: expr, $unique_type_id: expr) => { - if let Some(metadata) = - debug_context($cx).type_map.find_metadata_for_unique_id($unique_type_id) - { - return MetadataCreationResult::new(metadata, true); + if let Some(di_node) = debug_context($cx).type_map.di_node_for_unique_id($unique_type_id) { + return DINodeCreationResult::new(di_node, true); } }; } +/// Extract size and alignment from a TyAndLayout. +fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { + (ty_and_layout.size, ty_and_layout.align.abi) +} + /// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`). -/// For slices (that is, "arrays" of unknown size) use [slice_type_metadata]. -fn fixed_size_array_metadata<'ll, 'tcx>( +/// For slices (that is, "arrays" of unknown size) use [build_slice_type_di_node]. +fn build_fixed_size_array_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, array_type: Ty<'tcx>, -) -> MetadataCreationResult<'ll> { +) -> DINodeCreationResult<'ll> { let ty::Array(element_type, len) = array_type.kind() else { - bug!("fixed_size_array_metadata() called with non-ty::Array type `{:?}`", array_type) + bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) }; - let element_type_metadata = type_metadata(cx, *element_type); + let element_type_di_node = type_di_node(cx, *element_type); - return_if_metadata_created_in_meantime!(cx, unique_type_id); + return_if_di_node_created_in_meantime!(cx, unique_type_id); let (size, align) = cx.size_and_align_of(array_type); @@ -341,17 +141,17 @@ fn fixed_size_array_metadata<'ll, 'tcx>( unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; let subscripts = create_DIArray(DIB(cx), &[subrange]); - let metadata = unsafe { + let di_node = unsafe { llvm::LLVMRustDIBuilderCreateArrayType( DIB(cx), size.bits(), align.bits() as u32, - element_type_metadata, + element_type_di_node, subscripts, ) }; - MetadataCreationResult::new(metadata, false) + DINodeCreationResult::new(di_node, false) } /// Creates debuginfo for built-in pointer-like things: @@ -362,21 +162,28 @@ fn fixed_size_array_metadata<'ll, 'tcx>( /// /// At some point we might want to remove the special handling of Box /// and treat it the same as other smart pointers (like Rc, Arc, ...). -fn pointer_or_reference_metadata<'ll, 'tcx>( +fn build_pointer_or_reference_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ptr_type: Ty<'tcx>, pointee_type: Ty<'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> MetadataCreationResult<'ll> { - let pointee_type_metadata = type_metadata(cx, pointee_type); +) -> DINodeCreationResult<'ll> { + // The debuginfo generated by this function is only valid if `ptr_type` is really just + // a (fat) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. + debug_assert_eq!( + cx.size_and_align_of(ptr_type), + cx.size_and_align_of(cx.tcx.mk_mut_ptr(pointee_type)) + ); - return_if_metadata_created_in_meantime!(cx, unique_type_id); + let pointee_type_di_node = type_di_node(cx, pointee_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); let (thin_pointer_size, thin_pointer_align) = cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); - let pointer_type_metadata = match fat_pointer_kind(cx, pointee_type) { + match fat_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. debug_assert_eq!( @@ -387,87 +194,100 @@ fn pointer_or_reference_metadata<'ll, 'tcx>( pointee_type, ); - unsafe { + let di_node = unsafe { llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), - pointee_type_metadata, + pointee_type_di_node, thin_pointer_size.bits(), thin_pointer_align.bits() as u32, 0, // Ignore DWARF address space. ptr_type_debuginfo_name.as_ptr().cast(), ptr_type_debuginfo_name.len(), ) - } + }; + + DINodeCreationResult { di_node, already_stored_in_typemap: false } } Some(fat_pointer_kind) => { - let layout = cx.layout_of(ptr_type); - - let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); - let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); - - let (addr_field_name, extra_field_name) = match fat_pointer_kind { - FatPtrKind::Dyn => ("pointer", "vtable"), - FatPtrKind::Slice => ("data_ptr", "length"), - }; + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &ptr_type_debuginfo_name, + cx.size_and_align_of(ptr_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, owner| { + // FIXME: If this fat pointer is a `Box` then we don't want to use its + // type layout and instead use the layout of the raw pointer inside + // of it. + // The proper way to handle this is to not treat Box as a pointer + // at all and instead emit regular struct debuginfo for it. We just + // need to make sure that we don't break existing debuginfo consumers + // by doing that (at least not without a warning period). + let layout_type = + if ptr_type.is_box() { cx.tcx.mk_mut_ptr(pointee_type) } else { ptr_type }; + + let layout = cx.layout_of(layout_type); + let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); + let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); + + let (addr_field_name, extra_field_name) = match fat_pointer_kind { + FatPtrKind::Dyn => ("pointer", "vtable"), + FatPtrKind::Slice => ("data_ptr", "length"), + }; - debug_assert_eq!(abi::FAT_PTR_ADDR, 0); - debug_assert_eq!(abi::FAT_PTR_EXTRA, 1); + debug_assert_eq!(abi::FAT_PTR_ADDR, 0); + debug_assert_eq!(abi::FAT_PTR_EXTRA, 1); - // The data pointer type is a regular, thin pointer, regardless of whether this is a slice - // or a trait object. - let data_ptr_type_metadata = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_metadata, - addr_field.size.bits(), - addr_field.align.abi.bits() as u32, - 0, // Ignore DWARF address space. - std::ptr::null(), - 0, - ) - }; + // The data pointer type is a regular, thin pointer, regardless of whether this + // is a slice or a trait object. + let data_ptr_type_di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + addr_field.size.bits(), + addr_field.align.abi.bits() as u32, + 0, // Ignore DWARF address space. + std::ptr::null(), + 0, + ) + }; - let member_descriptions = vec![ - MemberDescription { - name: addr_field_name.into(), - type_metadata: data_ptr_type_metadata, - offset: layout.fields.offset(abi::FAT_PTR_ADDR), - size: addr_field.size, - align: addr_field.align.abi, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - }, - MemberDescription { - name: extra_field_name.into(), - type_metadata: type_metadata(cx, extra_field.ty), - offset: layout.fields.offset(abi::FAT_PTR_EXTRA), - size: extra_field.size, - align: extra_field.align.abi, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, + smallvec![ + build_field_di_node( + cx, + owner, + addr_field_name, + (addr_field.size, addr_field.align.abi), + layout.fields.offset(abi::FAT_PTR_ADDR), + DIFlags::FlagZero, + data_ptr_type_di_node, + ), + build_field_di_node( + cx, + owner, + extra_field_name, + (extra_field.size, extra_field.align.abi), + layout.fields.offset(abi::FAT_PTR_EXTRA), + DIFlags::FlagZero, + type_di_node(cx, extra_field.ty), + ), + ] }, - ]; - - composite_type_metadata( - cx, - ptr_type, - &ptr_type_debuginfo_name, - unique_type_id, - member_descriptions, - NO_SCOPE_METADATA, + NO_GENERICS, ) } - }; - - MetadataCreationResult { metadata: pointer_type_metadata, already_stored_in_typemap: false } + } } -fn subroutine_type_metadata<'ll, 'tcx>( +fn build_subroutine_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> MetadataCreationResult<'ll> { +) -> DINodeCreationResult<'ll> { // It's possible to create a self-referential // type in Rust by using 'impl trait': // @@ -483,49 +303,46 @@ fn subroutine_type_metadata<'ll, 'tcx>( // Once that is created, we replace the marker in the typemap with the actual type. debug_context(cx) .type_map - .unique_id_to_metadata + .unique_id_to_di_node .borrow_mut() - .insert(unique_type_id, recursion_marker_type(cx)); - - let UniqueTypeId::Ty(fn_ty, _) = unique_type_id else { - bug!("subroutine_type_metadata() called with unexpected input type: {:?}", unique_type_id) - }; + .insert(unique_type_id, recursion_marker_type_di_node(cx)); + let fn_ty = unique_type_id.expect_ty(); let signature = cx .tcx .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), fn_ty.fn_sig(cx.tcx)); - let signature_metadata: SmallVec<[_; 32]> = iter::once( + let signature_di_nodes: SmallVec<_> = iter::once( // return type match signature.output().kind() { ty::Tuple(tys) if tys.is_empty() => { // this is a "void" function None } - _ => Some(type_metadata(cx, signature.output())), + _ => Some(type_di_node(cx, signature.output())), }, ) .chain( // regular arguments - signature.inputs().iter().map(|&argument_type| Some(type_metadata(cx, argument_type))), + signature.inputs().iter().map(|&argument_type| Some(type_di_node(cx, argument_type))), ) .collect(); - debug_context(cx).type_map.unique_id_to_metadata.borrow_mut().remove(&unique_type_id); + debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); - let fn_metadata = unsafe { + let fn_di_node = unsafe { llvm::LLVMRustDIBuilderCreateSubroutineType( DIB(cx), - create_DIArray(DIB(cx), &signature_metadata[..]), + create_DIArray(DIB(cx), &signature_di_nodes[..]), ) }; // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); - let metadata = unsafe { + let di_node = unsafe { llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), - fn_metadata, + fn_di_node, cx.tcx.data_layout.pointer_size.bits(), cx.tcx.data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. @@ -534,21 +351,37 @@ fn subroutine_type_metadata<'ll, 'tcx>( ) }; - MetadataCreationResult::new(metadata, false) + DINodeCreationResult::new(di_node, false) } /// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs /// we with the correct type name (e.g. "dyn SomeTrait<Foo, Item=u32> + Sync"). -fn dyn_type_metadata<'ll, 'tcx>( +fn build_dyn_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, dyn_type: Ty<'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> &'ll DIType { +) -> DINodeCreationResult<'ll> { if let ty::Dynamic(..) = dyn_type.kind() { let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true); - composite_type_metadata(cx, dyn_type, &type_name, unique_type_id, vec![], NO_SCOPE_METADATA) + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + cx.size_and_align_of(dyn_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) } else { - bug!("Only ty::Dynamic is valid for dyn_type_metadata(). Found {:?} instead.", dyn_type) + bug!( + "Only ty::Dynamic is valid for build_dyn_type_di_node(). Found {:?} instead.", + dyn_type + ) } } @@ -569,96 +402,81 @@ fn dyn_type_metadata<'ll, 'tcx>( /// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the /// slice is zero, then accessing `unsized_field` in the debugger would /// result in an out-of-bounds access. -fn slice_type_metadata<'ll, 'tcx>( +fn build_slice_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, slice_type: Ty<'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> MetadataCreationResult<'ll> { +) -> DINodeCreationResult<'ll> { let element_type = match slice_type.kind() { ty::Slice(element_type) => *element_type, ty::Str => cx.tcx.types.u8, _ => { bug!( - "Only ty::Slice is valid for slice_type_metadata(). Found {:?} instead.", + "Only ty::Slice is valid for build_slice_type_di_node(). Found {:?} instead.", slice_type ) } }; - let element_type_metadata = type_metadata(cx, element_type); - return_if_metadata_created_in_meantime!(cx, unique_type_id); - MetadataCreationResult { metadata: element_type_metadata, already_stored_in_typemap: false } + let element_type_di_node = type_di_node(cx, element_type); + return_if_di_node_created_in_meantime!(cx, unique_type_id); + DINodeCreationResult { di_node: element_type_di_node, already_stored_in_typemap: false } } -pub fn type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { +/// Get the debuginfo node for the given type. +/// +/// This function will look up the debuginfo node in the TypeMap. If it can't find it, it +/// will create the node by dispatching to the corresponding `build_*_di_node()` function. +pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); - if let Some(metadata) = debug_context(cx).type_map.find_metadata_for_unique_id(unique_type_id) { - return metadata; + if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) + { + return existing_di_node; } - debug!("type_metadata: {:?}", t); + debug!("type_di_node: {:?}", t); - let MetadataCreationResult { metadata, already_stored_in_typemap } = match *t.kind() { + let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { - MetadataCreationResult::new(basic_type_metadata(cx, t), false) - } - ty::Tuple(elements) if elements.is_empty() => { - MetadataCreationResult::new(basic_type_metadata(cx, t), false) - } - ty::Array(..) => fixed_size_array_metadata(cx, unique_type_id, t), - ty::Slice(_) | ty::Str => slice_type_metadata(cx, t, unique_type_id), - ty::Dynamic(..) => { - MetadataCreationResult::new(dyn_type_metadata(cx, t, unique_type_id), false) - } - ty::Foreign(..) => { - MetadataCreationResult::new(foreign_type_metadata(cx, t, unique_type_id), false) + build_basic_type_di_node(cx, t) } + ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), + ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), + ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { - pointer_or_reference_metadata(cx, t, pointee_type, unique_type_id) + build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) } - ty::Adt(def, _) if def.is_box() => { - pointer_or_reference_metadata(cx, t, t.boxed_ty(), unique_type_id) - } - ty::FnDef(..) | ty::FnPtr(_) => subroutine_type_metadata(cx, unique_type_id), - ty::Closure(def_id, substs) => { - let upvar_tys: Vec<_> = substs.as_closure().upvar_tys().collect(); - let containing_scope = get_namespace_for_item(cx, def_id); - prepare_tuple_metadata(cx, t, &upvar_tys, unique_type_id, Some(containing_scope)) - .finalize(cx) - } - ty::Generator(def_id, substs, _) => { - let upvar_tys: Vec<_> = substs - .as_generator() - .prefix_tys() - .map(|t| cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) - .collect(); - prepare_enum_metadata(cx, t, def_id, unique_type_id, upvar_tys).finalize(cx) + // Box<T, A> may have a non-ZST allocator A. In that case, we + // cannot treat Box<T, A> as just an owned alias of `*mut T`. + ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } + ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), + ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::Generator(..) => enums::build_generator_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => prepare_struct_metadata(cx, t, unique_type_id).finalize(cx), - AdtKind::Union => prepare_union_metadata(cx, t, unique_type_id).finalize(cx), - AdtKind::Enum => { - prepare_enum_metadata(cx, t, def.did, unique_type_id, vec![]).finalize(cx) - } + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), }, - ty::Tuple(tys) => { - prepare_tuple_metadata(cx, t, tys, unique_type_id, NO_SCOPE_METADATA).finalize(cx) - } + ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), // Type parameters from polymorphized functions. - ty::Param(_) => MetadataCreationResult::new(param_type_metadata(cx, t), false), - _ => bug!("debuginfo: unexpected type in type_metadata: {:?}", t), + ty::Param(_) => build_param_type_di_node(cx, t), + _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), }; { if already_stored_in_typemap { // Make sure that we really do have a `TypeMap` entry for the unique type ID. - let metadata_for_uid = - match debug_context(cx).type_map.find_metadata_for_unique_id(unique_type_id) { - Some(metadata) => metadata, + let di_node_for_uid = + match debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) { + Some(di_node) => di_node, None => { bug!( - "expected type metadata for unique \ + "expected type debuginfo node for unique \ type ID '{:?}' to already be in \ the `debuginfo::TypeMap` but it \ was not.", @@ -667,16 +485,17 @@ pub fn type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll } }; - debug_assert_eq!(metadata_for_uid as *const _, metadata as *const _); + debug_assert_eq!(di_node_for_uid as *const _, di_node as *const _); } else { - debug_context(cx).type_map.register_unique_id_with_metadata(unique_type_id, metadata); + debug_context(cx).type_map.insert(unique_type_id, di_node); } } - metadata + di_node } -fn recursion_marker_type<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType { +// FIXME(mw): Cache this via a regular UniqueTypeId instead of an extra field in the debug context. +fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType { *debug_context(cx).recursion_marker_type.get_or_init(move || { unsafe { // The choice of type here is pretty arbitrary - @@ -710,78 +529,105 @@ fn hex_encode(data: &[u8]) -> String { } pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { - debug!("file_metadata: file_name: {:?}", source_file.name); - - let hash = Some(&source_file.src_hash); - let file_name = Some(source_file.name.prefer_remapped().to_string()); - let directory = if source_file.is_real_file() && !source_file.is_imported() { - Some( - cx.sess() - .opts - .working_dir - .to_string_lossy(FileNameDisplayPreference::Remapped) - .to_string(), - ) - } else { - // If the path comes from an upstream crate we assume it has been made - // independent of the compiler's working directory one way or another. - None - }; - file_metadata_raw(cx, file_name, directory, hash) -} - -pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { - file_metadata_raw(cx, None, None, None) -} + let cache_key = Some((source_file.name_hash, source_file.src_hash)); + return debug_context(cx) + .created_files + .borrow_mut() + .entry(cache_key) + .or_insert_with(|| alloc_new_file_metadata(cx, source_file)); -fn file_metadata_raw<'ll>( - cx: &CodegenCx<'ll, '_>, - file_name: Option<String>, - directory: Option<String>, - hash: Option<&SourceFileHash>, -) -> &'ll DIFile { - let key = (file_name, directory); - - match debug_context(cx).created_files.borrow_mut().entry(key) { - Entry::Occupied(o) => o.get(), - Entry::Vacant(v) => { - let (file_name, directory) = v.key(); - debug!("file_metadata: file_name: {:?}, directory: {:?}", file_name, directory); - - let file_name = file_name.as_deref().unwrap_or("<unknown>"); - let directory = directory.as_deref().unwrap_or(""); - - let (hash_kind, hash_value) = match hash { - Some(hash) => { - let kind = match hash.kind { - rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, - rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, - rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, - }; - (kind, hex_encode(hash.hash_bytes())) + #[instrument(skip(cx, source_file), level = "debug")] + fn alloc_new_file_metadata<'ll>( + cx: &CodegenCx<'ll, '_>, + source_file: &SourceFile, + ) -> &'ll DIFile { + debug!(?source_file.name); + + let (directory, file_name) = match &source_file.name { + FileName::Real(filename) => { + let working_directory = &cx.sess().opts.working_dir; + debug!(?working_directory); + + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. + // Because of path remapping we sometimes see strange things here: `abs_path` + // might actually look like a relative path + // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without + // taking the working directory into account, downstream tooling will + // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`, + // which makes no sense. Usually in such cases the working directory will also + // be remapped to `<crate-name-and-version>` or some other prefix of the path + // we are remapping, so we end up with + // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) } - None => (llvm::ChecksumKind::None, String::new()), - }; + } + other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), + }; - let file_metadata = unsafe { - llvm::LLVMRustDIBuilderCreateFile( - DIB(cx), - file_name.as_ptr().cast(), - file_name.len(), - directory.as_ptr().cast(), - directory.len(), - hash_kind, - hash_value.as_ptr().cast(), - hash_value.len(), - ) - }; + let hash_kind = match source_file.src_hash.kind { + rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, + rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, + rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, + }; + let hash_value = hex_encode(source_file.src_hash.hash_bytes()); - v.insert(file_metadata); - file_metadata + unsafe { + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + hash_kind, + hash_value.as_ptr().cast(), + hash_value.len(), + ) } } } +pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { + debug_context(cx).created_files.borrow_mut().entry(None).or_insert_with(|| unsafe { + let file_name = "<unknown>"; + let directory = ""; + let hash_value = ""; + + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + llvm::ChecksumKind::None, + hash_value.as_ptr().cast(), + hash_value.len(), + ) + }) +} + trait MsvcBasicName { fn msvc_basic_name(self) -> &'static str; } @@ -821,8 +667,11 @@ impl MsvcBasicName for ty::FloatTy { } } -fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { - debug!("basic_type_metadata: {:?}", t); +fn build_basic_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_basic_type_di_node: {:?}", t); // When targeting MSVC, emit MSVC style type names for compatibility with // .natvis visualizers (and perhaps other existing native debuggers?) @@ -830,7 +679,13 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l let (name, encoding) = match t.kind() { ty::Never => ("!", DW_ATE_unsigned), - ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned), + ty::Tuple(elements) if elements.is_empty() => { + if cpp_like_debuginfo { + return build_tuple_type_di_node(cx, UniqueTypeId::for_ty(cx.tcx, t)); + } else { + ("()", DW_ATE_unsigned) + } + } ty::Bool => ("bool", DW_ATE_boolean), ty::Char => ("char", DW_ATE_UTF), ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), @@ -839,10 +694,10 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), - _ => bug!("debuginfo::basic_type_metadata - `t` is invalid type"), + _ => bug!("debuginfo::build_basic_type_di_node - `t` is invalid type"), }; - let ty_metadata = unsafe { + let ty_di_node = unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), name.as_ptr().cast(), @@ -853,20 +708,20 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l }; if !cpp_like_debuginfo { - return ty_metadata; + return DINodeCreationResult::new(ty_di_node, false); } let typedef_name = match t.kind() { ty::Int(int_ty) => int_ty.name_str(), ty::Uint(uint_ty) => uint_ty.name_str(), ty::Float(float_ty) => float_ty.name_str(), - _ => return ty_metadata, + _ => return DINodeCreationResult::new(ty_di_node, false), }; - let typedef_metadata = unsafe { + let typedef_di_node = unsafe { llvm::LLVMRustDIBuilderCreateTypedef( DIB(cx), - ty_metadata, + ty_di_node, typedef_name.as_ptr().cast(), typedef_name.len(), unknown_file_metadata(cx), @@ -875,48 +730,60 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l ) }; - typedef_metadata + DINodeCreationResult::new(typedef_di_node, false) } -fn foreign_type_metadata<'ll, 'tcx>( +fn build_foreign_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> &'ll DIType { - debug!("foreign_type_metadata: {:?}", t); +) -> DINodeCreationResult<'ll> { + debug!("build_foreign_type_di_node: {:?}", t); - let name = compute_debuginfo_type_name(cx.tcx, t, false); - let (size, align) = cx.size_and_align_of(t); - create_struct_stub( + let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { + bug!("build_foreign_type_di_node() called with unexpected type: {:?}", unique_type_id.expect_ty()); + }; + + build_type_with_children( cx, - size, - align, - &name, - unique_type_id, - NO_SCOPE_METADATA, - DIFlags::FlagZero, - None, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, t, false), + cx.size_and_align_of(t), + Some(get_namespace_for_item(cx, def_id)), + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, ) } -fn param_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { - debug!("param_type_metadata: {:?}", t); +fn build_param_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_param_type_di_node: {:?}", t); let name = format!("{:?}", t); - unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_ptr().cast(), - name.len(), - Size::ZERO.bits(), - DW_ATE_unsigned, - ) + DINodeCreationResult { + di_node: unsafe { + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + Size::ZERO.bits(), + DW_ATE_unsigned, + ) + }, + already_stored_in_typemap: false, } } -pub fn compile_unit_metadata<'ll, 'tcx>( +pub fn build_compile_unit_di_node<'ll, 'tcx>( tcx: TyCtxt<'tcx>, codegen_unit_name: &str, - debug_context: &CrateDebugContext<'ll, 'tcx>, + debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, ) -> &'ll DIDescriptor { let mut name_in_debuginfo = match tcx.sess.local_crate_source_file { Some(ref path) => path.clone(), @@ -943,7 +810,7 @@ pub fn compile_unit_metadata<'ll, 'tcx>( name_in_debuginfo.push("@"); name_in_debuginfo.push(codegen_unit_name); - debug!("compile_unit_metadata: {:?}", name_in_debuginfo); + debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); let rustc_producer = format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. @@ -1073,165 +940,86 @@ pub fn compile_unit_metadata<'ll, 'tcx>( } } -struct MetadataCreationResult<'ll> { - metadata: &'ll DIType, - already_stored_in_typemap: bool, -} - -impl<'ll> MetadataCreationResult<'ll> { - fn new(metadata: &'ll DIType, already_stored_in_typemap: bool) -> Self { - MetadataCreationResult { metadata, already_stored_in_typemap } - } -} - -#[derive(Debug)] -struct SourceInfo<'ll> { - file: &'ll DIFile, - line: u32, -} - -/// Description of a type member, which can either be a regular field (as in -/// structs or tuples) or an enum variant. -#[derive(Debug)] -struct MemberDescription<'ll> { - name: String, - type_metadata: &'ll DIType, +/// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. +fn build_field_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + owner: &'ll DIScope, + name: &str, + size_and_align: (Size, Align), offset: Size, - size: Size, - align: Align, flags: DIFlags, - discriminant: Option<u64>, - source_info: Option<SourceInfo<'ll>>, -} - -impl<'ll> MemberDescription<'ll> { - fn into_metadata( - self, - cx: &CodegenCx<'ll, '_>, - composite_type_metadata: &'ll DIScope, - ) -> &'ll DIType { - let (file, line) = self - .source_info - .map(|info| (info.file, info.line)) - .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); - unsafe { - llvm::LLVMRustDIBuilderCreateVariantMemberType( - DIB(cx), - composite_type_metadata, - self.name.as_ptr().cast(), - self.name.len(), - file, - line, - self.size.bits(), - self.align.bits() as u32, - self.offset.bits(), - self.discriminant.map(|v| cx.const_u64(v)), - self.flags, - self.type_metadata, - ) - } - } -} - -/// A factory for `MemberDescription`s. It produces a list of member descriptions -/// for some record-like type. `MemberDescriptionFactory`s are used to defer the -/// creation of type member descriptions in order to break cycles arising from -/// recursive type definitions. -enum MemberDescriptionFactory<'ll, 'tcx> { - StructMDF(StructMemberDescriptionFactory<'tcx>), - TupleMDF(TupleMemberDescriptionFactory<'tcx>), - EnumMDF(EnumMemberDescriptionFactory<'ll, 'tcx>), - UnionMDF(UnionMemberDescriptionFactory<'tcx>), - VariantMDF(VariantMemberDescriptionFactory<'tcx>), -} - -impl<'ll, 'tcx> MemberDescriptionFactory<'ll, 'tcx> { - fn create_member_descriptions(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<MemberDescription<'ll>> { - match *self { - StructMDF(ref this) => this.create_member_descriptions(cx), - TupleMDF(ref this) => this.create_member_descriptions(cx), - EnumMDF(ref this) => this.create_member_descriptions(cx), - UnionMDF(ref this) => this.create_member_descriptions(cx), - VariantMDF(ref this) => this.create_member_descriptions(cx), - } - } -} - -//=----------------------------------------------------------------------------- -// Structs -//=----------------------------------------------------------------------------- - -/// Creates `MemberDescription`s for the fields of a struct. -struct StructMemberDescriptionFactory<'tcx> { - ty: Ty<'tcx>, - variant: &'tcx ty::VariantDef, -} - -impl<'tcx> StructMemberDescriptionFactory<'tcx> { - fn create_member_descriptions<'ll>( - &self, - cx: &CodegenCx<'ll, 'tcx>, - ) -> Vec<MemberDescription<'ll>> { - let layout = cx.layout_of(self.ty); - self.variant - .fields - .iter() - .enumerate() - .map(|(i, f)| { - let name = if self.variant.ctor_kind == CtorKind::Fn { - format!("__{}", i) - } else { - f.name.to_string() - }; - let field = layout.field(cx, i); - MemberDescription { - name, - type_metadata: type_metadata(cx, field.ty), - offset: layout.fields.offset(i), - size: field.size, - align: field.align.abi, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - } - }) - .collect() + type_di_node: &'ll DIType, +) -> &'ll DIType { + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + owner, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size_and_align.0.bits(), + size_and_align.1.bits() as u32, + offset.bits(), + flags, + type_di_node, + ) } } -fn prepare_struct_metadata<'ll, 'tcx>( +/// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct. +fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - struct_type: Ty<'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> RecursiveTypeDescription<'ll, 'tcx> { - let struct_name = compute_debuginfo_type_name(cx.tcx, struct_type, false); - - let (struct_def_id, variant) = match struct_type.kind() { - ty::Adt(def, _) => (def.did, def.non_enum_variant()), - _ => bug!("prepare_struct_metadata on a non-ADT"), +) -> DINodeCreationResult<'ll> { + let struct_type = unique_type_id.expect_ty(); + let ty::Adt(adt_def, _) = struct_type.kind() else { + bug!("build_struct_type_di_node() called with non-struct-type: {:?}", struct_type); }; + debug_assert!(adt_def.is_struct()); + let containing_scope = get_namespace_for_item(cx, adt_def.did()); + let struct_type_and_layout = cx.layout_of(struct_type); + let variant_def = adt_def.non_enum_variant(); - let containing_scope = get_namespace_for_item(cx, struct_def_id); - let (size, align) = cx.size_and_align_of(struct_type); - - let struct_metadata_stub = create_struct_stub( + type_map::build_type_with_children( cx, - size, - align, - &struct_name, - unique_type_id, - Some(containing_scope), - DIFlags::FlagZero, - None, - ); - - create_and_register_recursive_type_forward_declaration( - cx, - struct_type, - unique_type_id, - struct_metadata_stub, - struct_metadata_stub, - StructMDF(StructMemberDescriptionFactory { ty: struct_type, variant }), + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, struct_type, false), + size_and_align_of(struct_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_name = if variant_def.ctor_kind == CtorKind::Fn { + // This is a tuple struct + tuple_field_name(i) + } else { + // This is struct with named fields + Cow::Borrowed(f.name.as_str()) + }; + let field_layout = struct_type_and_layout.field(cx, i); + build_field_di_node( + cx, + owner, + &field_name[..], + (field_layout.size, field_layout.align.abi), + struct_type_and_layout.fields.offset(i), + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, struct_type), ) } @@ -1244,7 +1032,9 @@ fn prepare_struct_metadata<'ll, 'tcx>( /// Here are some examples: /// - `name__field1__field2` when the upvar is captured by value. /// - `_ref__name__field` when the upvar is captured by reference. -fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> Vec<String> { +/// +/// For generators this only contains upvars that are shared by all states. +fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec<String> { let body = tcx.optimized_mir(def_id); body.var_debug_info @@ -1261,152 +1051,185 @@ fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> let prefix = if is_ref { "_ref__" } else { "" }; Some(prefix.to_owned() + var.name.as_str()) }) - .collect::<Vec<_>>() + .collect() } -/// Creates `MemberDescription`s for the fields of a tuple. -struct TupleMemberDescriptionFactory<'tcx> { - ty: Ty<'tcx>, - component_types: Vec<Ty<'tcx>>, -} +/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. +/// For a generator, this will handle upvars shared by all states. +fn build_upvar_field_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + closure_or_generator_ty: Ty<'tcx>, + closure_or_generator_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { + ty::Generator(def_id, substs, _) => { + let upvar_tys: SmallVec<_> = substs.as_generator().prefix_tys().collect(); + (def_id, upvar_tys) + } + ty::Closure(def_id, substs) => { + let upvar_tys: SmallVec<_> = substs.as_closure().upvar_tys().collect(); + (def_id, upvar_tys) + } + _ => { + bug!( + "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", + closure_or_generator_ty + ) + } + }; -impl<'tcx> TupleMemberDescriptionFactory<'tcx> { - fn create_member_descriptions<'ll>( - &self, - cx: &CodegenCx<'ll, 'tcx>, - ) -> Vec<MemberDescription<'ll>> { - let mut capture_names = match *self.ty.kind() { - ty::Generator(def_id, ..) | ty::Closure(def_id, ..) => { - Some(closure_saved_names_of_captured_variables(cx.tcx, def_id).into_iter()) - } - _ => None, - }; - let layout = cx.layout_of(self.ty); - self.component_types + debug_assert!( + up_var_tys .iter() - .enumerate() - .map(|(i, &component_type)| { - let (size, align) = cx.size_and_align_of(component_type); - let name = if let Some(names) = capture_names.as_mut() { - names.next().unwrap() - } else { - format!("__{}", i) - }; - MemberDescription { - name, - type_metadata: type_metadata(cx, component_type), - offset: layout.fields.offset(i), - size, - align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - } - }) - .collect() - } + .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) + ); + + let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); + let layout = cx.layout_of(closure_or_generator_ty); + + up_var_tys + .into_iter() + .zip(capture_names.iter()) + .enumerate() + .map(|(index, (up_var_ty, capture_name))| { + build_field_di_node( + cx, + closure_or_generator_di_node, + capture_name, + cx.size_and_align_of(up_var_ty), + layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, up_var_ty), + ) + }) + .collect() } -fn prepare_tuple_metadata<'ll, 'tcx>( +/// Builds the DW_TAG_structure_type debuginfo node for a Rust tuple type. +fn build_tuple_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - tuple_type: Ty<'tcx>, - component_types: &[Ty<'tcx>], unique_type_id: UniqueTypeId<'tcx>, - containing_scope: Option<&'ll DIScope>, -) -> RecursiveTypeDescription<'ll, 'tcx> { - let (size, align) = cx.size_and_align_of(tuple_type); - let tuple_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); +) -> DINodeCreationResult<'ll> { + let tuple_type = unique_type_id.expect_ty(); + let &ty::Tuple(component_types) = tuple_type.kind() else { + bug!("build_tuple_type_di_node() called with non-tuple-type: {:?}", tuple_type) + }; - let struct_stub = create_struct_stub( - cx, - size, - align, - &tuple_name[..], - unique_type_id, - containing_scope, - DIFlags::FlagZero, - None, - ); + let tuple_type_and_layout = cx.layout_of(tuple_type); + let type_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); - create_and_register_recursive_type_forward_declaration( + type_map::build_type_with_children( cx, - tuple_type, - unique_type_id, - struct_stub, - struct_stub, - TupleMDF(TupleMemberDescriptionFactory { - ty: tuple_type, - component_types: component_types.to_vec(), - }), + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + size_and_align_of(tuple_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + // Fields: + |cx, tuple_di_node| { + component_types + .into_iter() + .enumerate() + .map(|(index, component_type)| { + build_field_di_node( + cx, + tuple_di_node, + &tuple_field_name(index), + cx.size_and_align_of(component_type), + tuple_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, component_type), + ) + }) + .collect() + }, + NO_GENERICS, ) } -//=----------------------------------------------------------------------------- -// Unions -//=----------------------------------------------------------------------------- - -struct UnionMemberDescriptionFactory<'tcx> { - layout: TyAndLayout<'tcx>, - variant: &'tcx ty::VariantDef, -} +/// Builds the debuginfo node for a closure environment. +fn build_closure_env_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let closure_env_type = unique_type_id.expect_ty(); + let &ty::Closure(def_id, _substs) = closure_env_type.kind() else { + bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) + }; + let containing_scope = get_namespace_for_item(cx, def_id); + let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); -impl<'tcx> UnionMemberDescriptionFactory<'tcx> { - fn create_member_descriptions<'ll>( - &self, - cx: &CodegenCx<'ll, 'tcx>, - ) -> Vec<MemberDescription<'ll>> { - self.variant - .fields - .iter() - .enumerate() - .map(|(i, f)| { - let field = self.layout.field(cx, i); - MemberDescription { - name: f.name.to_string(), - type_metadata: type_metadata(cx, field.ty), - offset: Size::ZERO, - size: field.size, - align: field.align.abi, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - } - }) - .collect() - } + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + cx.size_and_align_of(closure_env_type), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| build_upvar_field_di_nodes(cx, closure_env_type, owner), + NO_GENERICS, + ) } -fn prepare_union_metadata<'ll, 'tcx>( +/// Build the debuginfo node for a Rust `union` type. +fn build_union_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - union_type: Ty<'tcx>, unique_type_id: UniqueTypeId<'tcx>, -) -> RecursiveTypeDescription<'ll, 'tcx> { - let union_name = compute_debuginfo_type_name(cx.tcx, union_type, false); - - let (union_def_id, variant) = match union_type.kind() { - ty::Adt(def, _) => (def.did, def.non_enum_variant()), - _ => bug!("prepare_union_metadata on a non-ADT"), +) -> DINodeCreationResult<'ll> { + let union_type = unique_type_id.expect_ty(); + let (union_def_id, variant_def) = match union_type.kind() { + ty::Adt(def, _) => (def.did(), def.non_enum_variant()), + _ => bug!("build_union_type_di_node on a non-ADT"), }; - let containing_scope = get_namespace_for_item(cx, union_def_id); + let union_ty_and_layout = cx.layout_of(union_type); + let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); - let union_metadata_stub = - create_union_stub(cx, union_type, &union_name, unique_type_id, containing_scope); - - create_and_register_recursive_type_forward_declaration( + type_map::build_type_with_children( cx, - union_type, - unique_type_id, - union_metadata_stub, - union_metadata_stub, - UnionMDF(UnionMemberDescriptionFactory { layout: cx.layout_of(union_type), variant }), + type_map::stub( + cx, + Stub::Union, + unique_type_id, + &type_name, + size_and_align_of(union_ty_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_layout = union_ty_and_layout.field(cx, i); + build_field_di_node( + cx, + owner, + f.name.as_str(), + size_and_align_of(field_layout), + Size::ZERO, + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + // Generics: + |cx| build_generic_type_param_di_nodes(cx, union_type), ) } -//=----------------------------------------------------------------------------- -// Enums -//=----------------------------------------------------------------------------- - // FIXME(eddyb) maybe precompute this? Right now it's computed once // per generator monomorphization, but it doesn't depend on substs. fn generator_layout_and_saved_local_names<'tcx>( @@ -1444,890 +1267,30 @@ fn generator_layout_and_saved_local_names<'tcx>( (generator_layout, generator_saved_local_names) } -/// Describes the members of an enum value; an enum is described as a union of -/// structs in DWARF. This `MemberDescriptionFactory` provides the description for -/// the members of this union; so for every variant of the given enum, this -/// factory will produce one `MemberDescription` (all with no name and a fixed -/// offset of zero bytes). -struct EnumMemberDescriptionFactory<'ll, 'tcx> { - enum_type: Ty<'tcx>, - layout: TyAndLayout<'tcx>, - tag_type_metadata: Option<&'ll DIType>, - common_members: Vec<Option<&'ll DIType>>, -} - -impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { - fn create_member_descriptions(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<MemberDescription<'ll>> { - let generator_variant_info_data = match *self.enum_type.kind() { - ty::Generator(def_id, ..) => { - Some(generator_layout_and_saved_local_names(cx.tcx, def_id)) - } - _ => None, - }; - - let variant_info_for = |index: VariantIdx| match *self.enum_type.kind() { - ty::Adt(adt, _) => VariantInfo::Adt(&adt.variants[index], index), - ty::Generator(def_id, _, _) => { - let (generator_layout, generator_saved_local_names) = - generator_variant_info_data.as_ref().unwrap(); - VariantInfo::Generator { - def_id, - generator_layout: *generator_layout, - generator_saved_local_names, - variant_index: index, - } - } - _ => bug!(), - }; - - // While LLVM supports generating debuginfo for variant types (enums), it doesn't support - // lowering that debuginfo to CodeView records for msvc targets. So if we are targeting - // msvc, then we need to use a different, fallback encoding of the debuginfo. - let fallback = cpp_like_debuginfo(cx.tcx); - // This will always find the metadata in the type map. - let self_metadata = type_metadata(cx, self.enum_type); - - match self.layout.variants { - Variants::Single { index } => { - if let ty::Adt(adt, _) = self.enum_type.kind() { - if adt.variants.is_empty() { - return vec![]; - } - } - - let variant_info = variant_info_for(index); - let (variant_type_metadata, member_description_factory) = - describe_enum_variant(cx, self.layout, variant_info, self_metadata); - - let member_descriptions = member_description_factory.create_member_descriptions(cx); - let type_params = compute_type_parameters(cx, self.enum_type); - - set_members_of_composite_type( - cx, - variant_type_metadata, - member_descriptions, - Some(&self.common_members), - type_params, - ); - vec![MemberDescription { - name: variant_info.variant_name(), - type_metadata: variant_type_metadata, - offset: Size::ZERO, - size: self.layout.size, - align: self.layout.align.abi, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: variant_info.source_info(cx), - }] - } - Variants::Multiple { - tag_encoding: TagEncoding::Direct, - tag_field, - ref variants, - .. - } => { - let fallback_discr_variant = if fallback { - // For MSVC, we generate a union of structs for each variant and an - // explicit discriminant field roughly equivalent to the following C: - // ```c - // union enum$<{name}> { - // struct {variant 0 name} { - // <variant 0 fields> - // } variant0; - // <other variant structs> - // {name} discriminant; - // } - // ``` - // The natvis in `intrinsic.natvis` then matches on `this.discriminant` to - // determine which variant is active and then displays it. - let enum_layout = self.layout; - let offset = enum_layout.fields.offset(tag_field); - let discr_ty = enum_layout.field(cx, tag_field).ty; - let (size, align) = cx.size_and_align_of(discr_ty); - Some(MemberDescription { - name: "discriminant".into(), - type_metadata: self.tag_type_metadata.unwrap(), - offset, - size, - align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - }) - } else { - None - }; - - variants - .iter_enumerated() - .map(|(i, _)| { - let variant = self.layout.for_variant(cx, i); - let variant_info = variant_info_for(i); - let (variant_type_metadata, member_desc_factory) = - describe_enum_variant(cx, variant, variant_info, self_metadata); - - let member_descriptions = - member_desc_factory.create_member_descriptions(cx); - let type_params = compute_type_parameters(cx, self.enum_type); - - set_members_of_composite_type( - cx, - variant_type_metadata, - member_descriptions, - Some(&self.common_members), - type_params, - ); - - MemberDescription { - name: if fallback { - format!("variant{}", i.as_u32()) - } else { - variant_info.variant_name() - }, - type_metadata: variant_type_metadata, - offset: Size::ZERO, - size: self.layout.size, - align: self.layout.align.abi, - flags: DIFlags::FlagZero, - discriminant: Some( - self.layout.ty.discriminant_for_variant(cx.tcx, i).unwrap().val - as u64, - ), - source_info: variant_info.source_info(cx), - } - }) - .chain(fallback_discr_variant.into_iter()) - .collect() - } - Variants::Multiple { - tag_encoding: - TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, - tag, - ref variants, - tag_field, - } => { - let calculate_niche_value = |i: VariantIdx| { - if i == dataful_variant { - None - } else { - let value = (i.as_u32() as u128) - .wrapping_sub(niche_variants.start().as_u32() as u128) - .wrapping_add(niche_start); - let value = tag.value.size(cx).truncate(value); - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - Some(value as u64) - } - }; - - // For MSVC, we will generate a union of two fields, one for the dataful variant - // and one that just points to the discriminant. We also create an enum that - // contains tag values for the non-dataful variants and make the discriminant field - // that type. We then use natvis to render the enum type correctly in Windbg/VS. - // This will generate debuginfo roughly equivalent to the following C: - // ```c - // union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { - // struct <dataful variant name> { - // <fields in dataful variant> - // } dataful_variant; - // enum Discriminant$ { - // <non-dataful variants> - // } discriminant; - // } - // ``` - // The natvis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` - // and evaluates `this.discriminant`. If the value is between the min niche and max - // niche, then the enum is in the dataful variant and `this.dataful_variant` is - // rendered. Otherwise, the enum is in one of the non-dataful variants. In that - // case, we just need to render the name of the `this.discriminant` enum. - if fallback { - let dataful_variant_layout = self.layout.for_variant(cx, dataful_variant); - - let mut discr_enum_ty = tag.value.to_ty(cx.tcx); - // If the niche is the NULL value of a reference, then `discr_enum_ty` will be a RawPtr. - // CodeView doesn't know what to do with enums whose base type is a pointer so we fix this up - // to just be `usize`. - if let ty::RawPtr(_) = discr_enum_ty.kind() { - discr_enum_ty = cx.tcx.types.usize; - } - - let tags: Vec<_> = variants - .iter_enumerated() - .filter_map(|(variant_idx, _)| { - calculate_niche_value(variant_idx).map(|tag| { - let variant = variant_info_for(variant_idx); - let name = variant.variant_name(); - - Some(unsafe { - llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - name.as_ptr().cast(), - name.len(), - tag as i64, - !discr_enum_ty.is_signed(), - ) - }) - }) - }) - .collect(); - - let discr_enum = unsafe { - llvm::LLVMRustDIBuilderCreateEnumerationType( - DIB(cx), - self_metadata, - "Discriminant$".as_ptr().cast(), - "Discriminant$".len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - tag.value.size(cx).bits(), - tag.value.align(cx).abi.bits() as u32, - create_DIArray(DIB(cx), &tags), - type_metadata(cx, discr_enum_ty), - true, - ) - }; - - let variant_info = variant_info_for(dataful_variant); - let (variant_type_metadata, member_desc_factory) = describe_enum_variant( - cx, - dataful_variant_layout, - variant_info, - self_metadata, - ); - - let member_descriptions = member_desc_factory.create_member_descriptions(cx); - let type_params = compute_type_parameters(cx, self.enum_type); - - set_members_of_composite_type( - cx, - variant_type_metadata, - member_descriptions, - Some(&self.common_members), - type_params, - ); - - let (size, align) = - cx.size_and_align_of(dataful_variant_layout.field(cx, tag_field).ty); - - vec![ - MemberDescription { - // Name the dataful variant so that we can identify it for natvis - name: "dataful_variant".to_string(), - type_metadata: variant_type_metadata, - offset: Size::ZERO, - size: self.layout.size, - align: self.layout.align.abi, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: variant_info.source_info(cx), - }, - MemberDescription { - name: "discriminant".into(), - type_metadata: discr_enum, - offset: dataful_variant_layout.fields.offset(tag_field), - size, - align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - }, - ] - } else { - variants - .iter_enumerated() - .map(|(i, _)| { - let variant = self.layout.for_variant(cx, i); - let variant_info = variant_info_for(i); - let (variant_type_metadata, member_desc_factory) = - describe_enum_variant(cx, variant, variant_info, self_metadata); - - let member_descriptions = - member_desc_factory.create_member_descriptions(cx); - let type_params = compute_type_parameters(cx, self.enum_type); - - set_members_of_composite_type( - cx, - variant_type_metadata, - member_descriptions, - Some(&self.common_members), - type_params, - ); - - let niche_value = calculate_niche_value(i); - - MemberDescription { - name: variant_info.variant_name(), - type_metadata: variant_type_metadata, - offset: Size::ZERO, - size: self.layout.size, - align: self.layout.align.abi, - flags: DIFlags::FlagZero, - discriminant: niche_value, - source_info: variant_info.source_info(cx), - } - }) - .collect() - } - } - } - } -} - -// Creates `MemberDescription`s for the fields of a single enum variant. -struct VariantMemberDescriptionFactory<'tcx> { - /// Cloned from the `layout::Struct` describing the variant. - offsets: Vec<Size>, - args: Vec<(String, Ty<'tcx>)>, -} - -impl<'tcx> VariantMemberDescriptionFactory<'tcx> { - fn create_member_descriptions<'ll>( - &self, - cx: &CodegenCx<'ll, 'tcx>, - ) -> Vec<MemberDescription<'ll>> { - self.args - .iter() - .enumerate() - .map(|(i, &(ref name, ty))| { - let (size, align) = cx.size_and_align_of(ty); - MemberDescription { - name: name.to_string(), - type_metadata: type_metadata(cx, ty), - offset: self.offsets[i], - size, - align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - } - }) - .collect() - } -} - -#[derive(Copy, Clone)] -enum VariantInfo<'a, 'tcx> { - Adt(&'tcx ty::VariantDef, VariantIdx), - Generator { - def_id: DefId, - generator_layout: &'tcx GeneratorLayout<'tcx>, - generator_saved_local_names: &'a IndexVec<mir::GeneratorSavedLocal, Option<Symbol>>, - variant_index: VariantIdx, - }, -} - -impl<'tcx> VariantInfo<'_, 'tcx> { - fn variant_idx(&self) -> VariantIdx { - match self { - VariantInfo::Adt(_, variant_index) | VariantInfo::Generator { variant_index, .. } => { - *variant_index - } - } - } - - fn map_struct_name<R>(&self, f: impl FnOnce(&str) -> R) -> R { - match self { - VariantInfo::Adt(variant, _) => f(variant.name.as_str()), - VariantInfo::Generator { variant_index, .. } => { - f(&GeneratorSubsts::variant_name(*variant_index)) - } - } - } - - fn variant_name(&self) -> String { - match self { - VariantInfo::Adt(variant, _) => variant.name.to_string(), - VariantInfo::Generator { variant_index, .. } => { - // Since GDB currently prints out the raw discriminant along - // with every variant, make each variant name be just the value - // of the discriminant. The struct name for the variant includes - // the actual variant description. - format!("{}", variant_index.as_usize()) - } - } - } - - fn field_name(&self, i: usize) -> String { - let field_name = match *self { - VariantInfo::Adt(variant, _) if variant.ctor_kind != CtorKind::Fn => { - Some(variant.fields[i].name) - } - VariantInfo::Generator { - generator_layout, - generator_saved_local_names, - variant_index, - .. - } => { - generator_saved_local_names - [generator_layout.variant_fields[variant_index][i.into()]] - } - _ => None, - }; - field_name.map(|name| name.to_string()).unwrap_or_else(|| format!("__{}", i)) - } - - fn source_info<'ll>(&self, cx: &CodegenCx<'ll, 'tcx>) -> Option<SourceInfo<'ll>> { - if let VariantInfo::Generator { def_id, variant_index, .. } = self { - let span = - cx.tcx.generator_layout(*def_id).unwrap().variant_source_info[*variant_index].span; - if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - return Some(SourceInfo { file: file_metadata(cx, &loc.file), line: loc.line }); - } - } - None - } -} - -/// Returns a tuple of (1) `type_metadata_stub` of the variant, (2) a -/// `MemberDescriptionFactory` for producing the descriptions of the -/// fields of the variant. This is a rudimentary version of a full -/// `RecursiveTypeDescription`. -fn describe_enum_variant<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - layout: layout::TyAndLayout<'tcx>, - variant: VariantInfo<'_, 'tcx>, - containing_scope: &'ll DIScope, -) -> (&'ll DICompositeType, MemberDescriptionFactory<'ll, 'tcx>) { - let metadata_stub = variant.map_struct_name(|variant_name| { - let unique_type_id = - UniqueTypeId::for_enum_variant(cx.tcx, layout.ty, variant.variant_idx()); - - let (size, align) = cx.size_and_align_of(layout.ty); - - create_struct_stub( - cx, - size, - align, - variant_name, - unique_type_id, - Some(containing_scope), - DIFlags::FlagZero, - None, - ) - }); - - let offsets = (0..layout.fields.count()).map(|i| layout.fields.offset(i)).collect(); - let args = (0..layout.fields.count()) - .map(|i| (variant.field_name(i), layout.field(cx, i).ty)) - .collect(); - - let member_description_factory = VariantMDF(VariantMemberDescriptionFactory { offsets, args }); - - (metadata_stub, member_description_factory) -} - -fn prepare_enum_metadata<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type: Ty<'tcx>, - enum_def_id: DefId, - unique_type_id: UniqueTypeId<'tcx>, - outer_field_tys: Vec<Ty<'tcx>>, -) -> RecursiveTypeDescription<'ll, 'tcx> { - let tcx = cx.tcx; - let enum_name = compute_debuginfo_type_name(tcx, enum_type, false); - - let containing_scope = get_namespace_for_item(cx, enum_def_id); - // FIXME: This should emit actual file metadata for the enum, but we - // currently can't get the necessary information when it comes to types - // imported from other crates. Formerly we violated the ODR when performing - // LTO because we emitted debuginfo for the same type with varying file - // metadata, so as a workaround we pretend that the type comes from - // <unknown> - let file_metadata = unknown_file_metadata(cx); - - let discriminant_type_metadata = |discr: Primitive| { - let enumerators_metadata: Vec<_> = match enum_type.kind() { - ty::Adt(def, _) => iter::zip(def.discriminants(tcx), &def.variants) - .map(|((_, discr), v)| { - let name = v.name.as_str(); - let is_unsigned = match discr.ty.kind() { - ty::Int(_) => false, - ty::Uint(_) => true, - _ => bug!("non integer discriminant"), - }; - unsafe { - Some(llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - name.as_ptr().cast(), - name.len(), - // FIXME: what if enumeration has i128 discriminant? - discr.val as i64, - is_unsigned, - )) - } - }) - .collect(), - ty::Generator(_, substs, _) => substs - .as_generator() - .variant_range(enum_def_id, tcx) - .map(|variant_index| { - debug_assert_eq!(tcx.types.u32, substs.as_generator().discr_ty(tcx)); - let name = GeneratorSubsts::variant_name(variant_index); - unsafe { - Some(llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - name.as_ptr().cast(), - name.len(), - // Generators use u32 as discriminant type, verified above. - variant_index.as_u32().into(), - true, // IsUnsigned - )) - } - }) - .collect(), - _ => bug!(), - }; - - let disr_type_key = (enum_def_id, discr); - let cached_discriminant_type_metadata = - debug_context(cx).created_enum_disr_types.borrow().get(&disr_type_key).cloned(); - match cached_discriminant_type_metadata { - Some(discriminant_type_metadata) => discriminant_type_metadata, - None => { - let (discriminant_size, discriminant_align) = (discr.size(cx), discr.align(cx)); - let discriminant_base_type_metadata = type_metadata(cx, discr.to_ty(tcx)); - - let item_name; - let discriminant_name = match enum_type.kind() { - ty::Adt(..) => { - item_name = tcx.item_name(enum_def_id); - item_name.as_str() - } - ty::Generator(..) => enum_name.as_str(), - _ => bug!(), - }; - - let discriminant_type_metadata = unsafe { - llvm::LLVMRustDIBuilderCreateEnumerationType( - DIB(cx), - containing_scope, - discriminant_name.as_ptr().cast(), - discriminant_name.len(), - file_metadata, - UNKNOWN_LINE_NUMBER, - discriminant_size.bits(), - discriminant_align.abi.bits() as u32, - create_DIArray(DIB(cx), &enumerators_metadata), - discriminant_base_type_metadata, - true, - ) - }; - - debug_context(cx) - .created_enum_disr_types - .borrow_mut() - .insert(disr_type_key, discriminant_type_metadata); - - discriminant_type_metadata - } - } - }; - - let layout = cx.layout_of(enum_type); - - if let (Abi::Scalar(_), Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. }) = - (layout.abi, &layout.variants) - { - return FinalMetadata(discriminant_type_metadata(tag.value)); - } - - // While LLVM supports generating debuginfo for variant types (enums), it doesn't support - // lowering that debuginfo to CodeView records for msvc targets. So if we are targeting - // msvc, then we need to use a different encoding of the debuginfo. - if cpp_like_debuginfo(tcx) { - let discriminant_type_metadata = match layout.variants { - Variants::Single { .. } => None, - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } - | Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { - Some(discriminant_type_metadata(tag.value)) - } - }; - - let enum_metadata = { - let unique_type_id_str = unique_type_id.generate_unique_id_string(tcx); - - unsafe { - llvm::LLVMRustDIBuilderCreateUnionType( - DIB(cx), - None, - enum_name.as_ptr().cast(), - enum_name.len(), - file_metadata, - UNKNOWN_LINE_NUMBER, - layout.size.bits(), - layout.align.abi.bits() as u32, - DIFlags::FlagZero, - None, - 0, // RuntimeLang - unique_type_id_str.as_ptr().cast(), - unique_type_id_str.len(), - ) - } - }; - - return create_and_register_recursive_type_forward_declaration( - cx, - enum_type, - unique_type_id, - enum_metadata, - enum_metadata, - EnumMDF(EnumMemberDescriptionFactory { - enum_type, - layout, - tag_type_metadata: discriminant_type_metadata, - common_members: vec![], - }), - ); - } - - let discriminator_name = match enum_type.kind() { - ty::Generator(..) => "__state", - _ => "", - }; - let discriminator_metadata = match layout.variants { - // A single-variant enum has no discriminant. - Variants::Single { .. } => None, - - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, tag_field, .. } => { - // Find the integer type of the correct size. - let size = tag.value.size(cx); - let align = tag.value.align(cx); - - let tag_type = match tag.value { - Int(t, _) => t, - F32 => Integer::I32, - F64 => Integer::I64, - Pointer => cx.data_layout().ptr_sized_integer(), - } - .to_ty(cx.tcx, false); - - let tag_metadata = basic_type_metadata(cx, tag_type); - unsafe { - Some(llvm::LLVMRustDIBuilderCreateMemberType( - DIB(cx), - containing_scope, - discriminator_name.as_ptr().cast(), - discriminator_name.len(), - file_metadata, - UNKNOWN_LINE_NUMBER, - size.bits(), - align.abi.bits() as u32, - layout.fields.offset(tag_field).bits(), - DIFlags::FlagArtificial, - tag_metadata, - )) - } - } - - Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, tag_field, .. } => { - let discr_type = tag.value.to_ty(cx.tcx); - let (size, align) = cx.size_and_align_of(discr_type); - - let discr_metadata = basic_type_metadata(cx, discr_type); - unsafe { - Some(llvm::LLVMRustDIBuilderCreateMemberType( - DIB(cx), - containing_scope, - discriminator_name.as_ptr().cast(), - discriminator_name.len(), - file_metadata, - UNKNOWN_LINE_NUMBER, - size.bits(), - align.bits() as u32, - layout.fields.offset(tag_field).bits(), - DIFlags::FlagArtificial, - discr_metadata, - )) - } - } - }; - - let outer_fields = match layout.variants { - Variants::Single { .. } => vec![], - Variants::Multiple { .. } => { - let tuple_mdf = - TupleMemberDescriptionFactory { ty: enum_type, component_types: outer_field_tys }; - tuple_mdf - .create_member_descriptions(cx) - .into_iter() - .map(|desc| Some(desc.into_metadata(cx, containing_scope))) - .collect() - } - }; - - let variant_part_unique_type_id_str = - UniqueTypeId::for_enum_variant_part(tcx, enum_type).generate_unique_id_string(tcx); - - let empty_array = create_DIArray(DIB(cx), &[]); - let name = ""; - let variant_part = unsafe { - llvm::LLVMRustDIBuilderCreateVariantPart( - DIB(cx), - containing_scope, - name.as_ptr().cast(), - name.len(), - file_metadata, - UNKNOWN_LINE_NUMBER, - layout.size.bits(), - layout.align.abi.bits() as u32, - DIFlags::FlagZero, - discriminator_metadata, - empty_array, - variant_part_unique_type_id_str.as_ptr().cast(), - variant_part_unique_type_id_str.len(), - ) - }; - - let struct_wrapper = { - // The variant part must be wrapped in a struct according to DWARF. - // All fields except the discriminant (including `outer_fields`) - // should be put into structures inside the variant part, which gives - // an equivalent layout but offers us much better integration with - // debuggers. - let type_array = create_DIArray(DIB(cx), &[Some(variant_part)]); - let unique_type_id_str = unique_type_id.generate_unique_id_string(tcx); - - unsafe { - llvm::LLVMRustDIBuilderCreateStructType( - DIB(cx), - Some(containing_scope), - enum_name.as_ptr().cast(), - enum_name.len(), - file_metadata, - UNKNOWN_LINE_NUMBER, - layout.size.bits(), - layout.align.abi.bits() as u32, - DIFlags::FlagZero, - None, - type_array, - 0, - None, - unique_type_id_str.as_ptr().cast(), - unique_type_id_str.len(), - ) - } - }; - - create_and_register_recursive_type_forward_declaration( - cx, - enum_type, - unique_type_id, - struct_wrapper, - variant_part, - EnumMDF(EnumMemberDescriptionFactory { - enum_type, - layout, - tag_type_metadata: None, - common_members: outer_fields, - }), - ) -} - -/// Creates debug information for a composite type, that is, anything that -/// results in a LLVM struct. -/// -/// Examples of Rust types to use this are: structs, tuples, boxes, vecs, and enums. -fn composite_type_metadata<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - composite_type: Ty<'tcx>, - composite_type_name: &str, - composite_type_unique_id: UniqueTypeId<'tcx>, - member_descriptions: Vec<MemberDescription<'ll>>, - containing_scope: Option<&'ll DIScope>, -) -> &'ll DICompositeType { - let (size, align) = cx.size_and_align_of(composite_type); - - // Create the (empty) struct metadata node ... - let composite_type_metadata = create_struct_stub( - cx, - size, - align, - composite_type_name, - composite_type_unique_id, - containing_scope, - DIFlags::FlagZero, - None, - ); - - // ... and immediately create and add the member descriptions. - set_members_of_composite_type( - cx, - composite_type_metadata, - member_descriptions, - None, - compute_type_parameters(cx, composite_type), - ); - - composite_type_metadata -} - -fn set_members_of_composite_type<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - composite_type_metadata: &'ll DICompositeType, - member_descriptions: Vec<MemberDescription<'ll>>, - common_members: Option<&Vec<Option<&'ll DIType>>>, - type_params: &'ll DIArray, -) { - // In some rare cases LLVM metadata uniquing would lead to an existing type - // description being used instead of a new one created in - // create_struct_stub. This would cause a hard to trace assertion in - // DICompositeType::SetTypeArray(). The following check makes sure that we - // get a better error message if this should happen again due to some - // regression. - { - let mut composite_types_completed = - debug_context(cx).composite_types_completed.borrow_mut(); - if !composite_types_completed.insert(composite_type_metadata) { - bug!( - "debuginfo::set_members_of_composite_type() - \ - Already completed forward declaration re-encountered." - ); - } - } - - let mut member_metadata: Vec<_> = member_descriptions - .into_iter() - .map(|desc| Some(desc.into_metadata(cx, composite_type_metadata))) - .collect(); - if let Some(other_members) = common_members { - member_metadata.extend(other_members.iter()); - } - - unsafe { - let field_array = create_DIArray(DIB(cx), &member_metadata); - llvm::LLVMRustDICompositeTypeReplaceArrays( - DIB(cx), - composite_type_metadata, - Some(field_array), - Some(type_params), - ); - } -} - /// Computes the type parameters for a type, if any, for the given metadata. -fn compute_type_parameters<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) -> &'ll DIArray { +fn build_generic_type_param_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, +) -> SmallVec<&'ll DIType> { if let ty::Adt(def, substs) = *ty.kind() { if substs.types().next().is_some() { - let generics = cx.tcx.generics_of(def.did); + let generics = cx.tcx.generics_of(def.did()); let names = get_parameter_names(cx, generics); - let template_params: Vec<_> = iter::zip(substs, names) + let template_params: SmallVec<_> = iter::zip(substs, names) .filter_map(|(kind, name)| { if let GenericArgKind::Type(ty) = kind.unpack() { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); - let actual_type_metadata = type_metadata(cx, actual_type); + let actual_type_di_node = type_di_node(cx, actual_type); let name = name.as_str(); Some(unsafe { - Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, name.as_ptr().cast(), name.len(), - actual_type_metadata, - )) + actual_type_di_node, + ) }) } else { None @@ -2335,10 +1298,11 @@ fn compute_type_parameters<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) - }) .collect(); - return create_DIArray(DIB(cx), &template_params); + return template_params; } } - return create_DIArray(DIB(cx), &[]); + + return smallvec![]; fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec<Symbol> { let mut names = generics @@ -2349,89 +1313,10 @@ fn compute_type_parameters<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) - } } -/// A convenience wrapper around `LLVMRustDIBuilderCreateStructType()`. Does not do -/// any caching, does not add any fields to the struct. This can be done later -/// with `set_members_of_composite_type()`. -fn create_struct_stub<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - size: Size, - align: Align, - type_name: &str, - unique_type_id: UniqueTypeId<'tcx>, - containing_scope: Option<&'ll DIScope>, - flags: DIFlags, - vtable_holder: Option<&'ll DIType>, -) -> &'ll DICompositeType { - let unique_type_id = unique_type_id.generate_unique_id_string(cx.tcx); - - let metadata_stub = unsafe { - // `LLVMRustDIBuilderCreateStructType()` wants an empty array. A null - // pointer will lead to hard to trace and debug LLVM assertions - // later on in `llvm/lib/IR/Value.cpp`. - let empty_array = create_DIArray(DIB(cx), &[]); - - llvm::LLVMRustDIBuilderCreateStructType( - DIB(cx), - containing_scope, - type_name.as_ptr().cast(), - type_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - size.bits(), - align.bits() as u32, - flags, - None, - empty_array, - 0, - vtable_holder, - unique_type_id.as_ptr().cast(), - unique_type_id.len(), - ) - }; - - metadata_stub -} - -fn create_union_stub<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - union_type: Ty<'tcx>, - union_type_name: &str, - unique_type_id: UniqueTypeId<'tcx>, - containing_scope: &'ll DIScope, -) -> &'ll DICompositeType { - let (union_size, union_align) = cx.size_and_align_of(union_type); - let unique_type_id = unique_type_id.generate_unique_id_string(cx.tcx); - - let metadata_stub = unsafe { - // `LLVMRustDIBuilderCreateUnionType()` wants an empty array. A null - // pointer will lead to hard to trace and debug LLVM assertions - // later on in `llvm/lib/IR/Value.cpp`. - let empty_array = create_DIArray(DIB(cx), &[]); - - llvm::LLVMRustDIBuilderCreateUnionType( - DIB(cx), - Some(containing_scope), - union_type_name.as_ptr().cast(), - union_type_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - union_size.bits(), - union_align.bits() as u32, - DIFlags::FlagZero, - Some(empty_array), - 0, // RuntimeLang - unique_type_id.as_ptr().cast(), - unique_type_id.len(), - ) - }; - - metadata_stub -} - /// Creates debug information for the given global variable. /// -/// Adds the created metadata nodes directly to the crate's IR. -pub fn create_global_var_metadata<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { +/// Adds the created debuginfo nodes directly to the crate's IR. +pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { if cx.dbg_cx.is_none() { return; } @@ -2457,7 +1342,7 @@ pub fn create_global_var_metadata<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, g let is_local_to_unit = is_node_local_to_unit(cx, def_id); let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); - let type_metadata = type_metadata(cx, variable_type); + let type_di_node = type_di_node(cx, variable_type); let var_name = tcx.item_name(def_id); let var_name = var_name.as_str(); let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name; @@ -2477,11 +1362,11 @@ pub fn create_global_var_metadata<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, g linkage_name.len(), file_metadata, line_number, - type_metadata, + type_di_node, is_local_to_unit, global, None, - global_align.bytes() as u32, + global_align.bits() as u32, ); } } @@ -2495,7 +1380,7 @@ pub fn create_global_var_metadata<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, g /// the name of the method they implement. This can be implemented in the future once there /// is a proper disambiguation scheme for dealing with methods from different traits that have /// the same name. -fn vtable_type_metadata<'ll, 'tcx>( +fn build_vtable_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, @@ -2508,14 +1393,14 @@ fn vtable_type_metadata<'ll, 'tcx>( tcx.vtable_entries(trait_ref) } else { - COMMON_VTABLE_ENTRIES + TyCtxt::COMMON_VTABLE_ENTRIES }; // All function pointers are described as opaque pointers. This could be improved in the future // by describing them as actual function pointers. let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); - let void_pointer_type_debuginfo = type_metadata(cx, void_pointer_ty); - let usize_debuginfo = type_metadata(cx, tcx.types.usize); + let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); + let usize_di_node = type_di_node(cx, tcx.types.usize); let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); // If `usize` is not pointer-sized and -aligned then the size and alignment computations // for the vtable as a whole would be wrong. Let's make sure this holds even on weird @@ -2529,72 +1414,155 @@ fn vtable_type_metadata<'ll, 'tcx>( // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate // the vtable to the type it is for. - let vtable_holder = type_metadata(cx, ty); + let vtable_holder = type_di_node(cx, ty); - let vtable_type_metadata = create_struct_stub( + build_type_with_children( cx, - size, - pointer_align, - &vtable_type_name, - unique_type_id, - NO_SCOPE_METADATA, - DIFlags::FlagArtificial, - Some(vtable_holder), - ); + type_map::stub( + cx, + Stub::VtableTy { vtable_holder }, + unique_type_id, + &vtable_type_name, + (size, pointer_align), + NO_SCOPE_METADATA, + DIFlags::FlagArtificial, + ), + |cx, vtable_type_di_node| { + vtable_entries + .iter() + .enumerate() + .filter_map(|(index, vtable_entry)| { + let (field_name, field_type_di_node) = match vtable_entry { + ty::VtblEntry::MetadataDropInPlace => { + ("drop_in_place".to_string(), void_pointer_type_di_node) + } + ty::VtblEntry::Method(_) => { + // Note: This code does not try to give a proper name to each method + // because their might be multiple methods with the same name + // (coming from different traits). + (format!("__method{}", index), void_pointer_type_di_node) + } + ty::VtblEntry::TraitVPtr(_) => { + (format!("__super_trait_ptr{}", index), void_pointer_type_di_node) + } + ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node), + ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node), + ty::VtblEntry::Vacant => return None, + }; - // Create a field for each entry in the vtable. - let fields: Vec<_> = vtable_entries - .iter() - .enumerate() - .filter_map(|(index, vtable_entry)| { - let (field_name, field_type) = match vtable_entry { - ty::VtblEntry::MetadataDropInPlace => { - ("drop_in_place".to_string(), void_pointer_type_debuginfo) - } - ty::VtblEntry::Method(_) => { - // Note: This code does not try to give a proper name to each method - // because there might be multiple methods with the same name - // (coming from different traits). - (format!("__method{}", index), void_pointer_type_debuginfo) - } - ty::VtblEntry::TraitVPtr(_) => { - // Note: In the future we could try to set the type of this pointer - // to the type that we generate for the corresponding vtable. - (format!("__super_trait_ptr{}", index), void_pointer_type_debuginfo) - } - ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_debuginfo), - ty::VtblEntry::MetadataSize => ("size".to_string(), usize_debuginfo), - ty::VtblEntry::Vacant => return None, - }; + let field_offset = pointer_size * index as u64; - Some(MemberDescription { - name: field_name, - type_metadata: field_type, - offset: pointer_size * index as u64, - size: pointer_size, - align: pointer_align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - }) - }) - .collect(); + Some(build_field_di_node( + cx, + vtable_type_di_node, + &field_name, + (pointer_size, pointer_align), + field_offset, + DIFlags::FlagZero, + field_type_di_node, + )) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); - let type_params = create_DIArray(DIB(cx), &[]); - set_members_of_composite_type(cx, vtable_type_metadata, fields, None, type_params); - vtable_type_metadata + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { + VCallVisibility::Public + } + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | ( + Lto::ThinLocal | Lto::Thin | Lto::Fat, + Visibility::Restricted(_) | Visibility::Invisible, + false, + ) => VCallVisibility::LinkageUnit, + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { + VCallVisibility::TranslationUnit + } + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } } /// Creates debug information for the given vtable, which is for the /// given type. /// /// Adds the created metadata nodes directly to the crate's IR. -pub fn create_vtable_metadata<'ll, 'tcx>( +pub fn create_vtable_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.debugging_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + if cx.dbg_cx.is_none() { return; } @@ -2606,7 +1574,7 @@ pub fn create_vtable_metadata<'ll, 'tcx>( let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); - let vtable_type = vtable_type_metadata(cx, ty, poly_trait_ref); + let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); let linkage_name = ""; unsafe { @@ -2619,7 +1587,7 @@ pub fn create_vtable_metadata<'ll, 'tcx>( linkage_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, - vtable_type, + vtable_type_di_node, true, vtable, None, @@ -2637,3 +1605,14 @@ pub fn extend_scope_to_file<'ll>( let file_metadata = file_metadata(cx, file); unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } } + +pub fn tuple_field_name(field_index: usize) -> Cow<'static, str> { + const TUPLE_FIELD_NAMES: [&'static str; 16] = [ + "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11", + "__12", "__13", "__14", "__15", + ]; + TUPLE_FIELD_NAMES + .get(field_index) + .map(|s| Cow::from(*s)) + .unwrap_or_else(|| Cow::from(format!("__{}", field_index))) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs new file mode 100644 index 00000000000..d6e2c8ccdf4 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -0,0 +1,514 @@ +use std::borrow::Cow; + +use libc::c_uint; +use rustc_codegen_ssa::debuginfo::{ + type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +}; +use rustc_middle::{ + bug, + ty::{ + self, + layout::{LayoutOf, TyAndLayout}, + util::Discr, + AdtDef, GeneratorSubsts, + }, +}; +use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use smallvec::smallvec; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + build_field_di_node, closure_saved_names_of_captured_variables, + enums::tag_base_type, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, + type_map::{self, UniqueTypeId}, + unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, + UNKNOWN_LINE_NUMBER, + }, + utils::DIB, + }, + llvm::{ + self, + debuginfo::{DIFile, DIFlags, DIType}, + }, +}; + +/// In CPP-like mode, we generate a union of structs for each variant and an +/// explicit discriminant field roughly equivalent to the following C/C++ code: +/// +/// ```c +/// union enum$<{fully-qualified-name}> { +/// struct {variant 0 name} { +/// <variant 0 fields> +/// } variant0; +/// <other variant structs> +/// {name} discriminant; +/// } +/// ``` +/// +/// As you can see, the type name is wrapped `enum$`. This way we can have a +/// single NatVis rule for handling all enums. +/// +/// At the LLVM IR level this looks like +/// +/// ```txt +/// DW_TAG_union_type (top-level type for enum) +/// DW_TAG_member (member for variant 1) +/// DW_TAG_member (member for variant 2) +/// DW_TAG_member (member for variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// DW_TAG_enumeration_type (type of tag) +/// ``` +/// +/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things +/// differently in order to allow a NatVis visualizer to extract all the information needed: +/// We generate a union of two fields, one for the dataful variant +/// and one that just points to the discriminant (which is some field within the dataful variant). +/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful +/// variants and make the discriminant field that type. We then use NatVis to render the enum type +/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: +/// +/// ```c +/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { +/// struct <dataful variant name> { +/// <fields in dataful variant> +/// } dataful_variant; +/// enum Discriminant$ { +/// <non-dataful variants> +/// } discriminant; +/// } +/// ``` +/// +/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` +/// and evaluates `this.discriminant`. If the value is between the min niche and max +/// niche, then the enum is in the dataful variant and `this.dataful_variant` is +/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that +/// case, we just need to render the name of the `this.discriminant` enum. +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + type_map::Stub::Union, + unique_type_id, + &enum_type_name, + cx.size_and_align_of(enum_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, enum_type_di_node| { + match enum_type_and_layout.variants { + Variants::Single { index: variant_index } => { + if enum_adt_def.variants().is_empty() { + // Uninhabited enums have Variants::Single. We don't generate + // any members for them. + return smallvec![]; + } + + build_single_variant_union_fields( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + variant_index, + ) + } + Variants::Multiple { + tag_encoding: TagEncoding::Direct, + ref variants, + tag_field, + .. + } => build_union_fields_for_direct_tag_enum( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + &mut variants.indices(), + tag_field, + ), + Variants::Multiple { + tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + ref variants, + tag_field, + .. + } => build_union_fields_for_niche_tag_enum( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + dataful_variant, + &mut variants.indices(), + tag_field, + ), + } + }, + NO_GENERICS, + ) +} + +/// A generator debuginfo node looks the same as a that of an enum type. +/// +/// See [build_enum_type_di_node] for more information. +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let generator_type = unique_type_id.expect_ty(); + let generator_type_and_layout = cx.layout_of(generator_type); + let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + type_map::Stub::Union, + unique_type_id, + &generator_type_name, + size_and_align_of(generator_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, generator_type_di_node| match generator_type_and_layout.variants { + Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => { + build_union_fields_for_direct_tag_generator( + cx, + generator_type_and_layout, + generator_type_di_node, + ) + } + Variants::Single { .. } + | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => { + bug!( + "Encountered generator with non-direct-tag layout: {:?}", + generator_type_and_layout + ) + } + }, + NO_GENERICS, + ) +} + +fn build_single_variant_union_fields<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, +) -> SmallVec<&'ll DIType> { + let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + variant_layout, + ); + + // NOTE: The field name of the union is the same as the variant name, not "variant0". + let variant_name = enum_adt_def.variant(variant_index).name.as_str(); + + smallvec![build_field_di_node( + cx, + enum_type_di_node, + variant_name, + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_di_node, + )] +} + +fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_indices: &mut dyn Iterator<Item = VariantIdx>, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices + .map(|variant_index| { + let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + + VariantFieldInfo { + variant_index, + variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + variant_layout, + ), + source_info: None, + } + }) + .collect(); + + let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + discr_type_name.as_str(), + tag_base_type, + &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + }), + enum_type_di_node, + ); + + build_union_fields_for_direct_tag_enum_or_generator( + cx, + enum_type_and_layout, + enum_type_di_node, + &variant_field_infos, + discr_type_di_node, + tag_field, + ) +} + +fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + dataful_variant_index: VariantIdx, + variant_indices: &mut dyn Iterator<Item = VariantIdx>, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + dataful_variant_index, + &enum_adt_def.variant(dataful_variant_index), + enum_type_and_layout.for_variant(cx, dataful_variant_index), + ); + + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + // Create an DW_TAG_enumerator for each variant except the dataful one. + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + "Discriminant$", + tag_base_type, + &mut variant_indices.filter_map(|variant_index| { + if let Some(discr_val) = + super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) + { + let discr = Discr { val: discr_val as u128, ty: tag_base_type }; + let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + Some((discr, variant_name)) + } else { + debug_assert_eq!(variant_index, dataful_variant_index); + None + } + }), + enum_type_di_node, + ); + + smallvec![ + build_field_di_node( + cx, + enum_type_di_node, + "dataful_variant", + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + dataful_variant_struct_type_di_node, + ), + build_field_di_node( + cx, + enum_type_di_node, + "discriminant", + cx.size_and_align_of(tag_base_type), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + discr_type_di_node, + ), + ] +} + +fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + generator_type_and_layout: TyAndLayout<'tcx>, + generator_type_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = generator_type_and_layout.variants else { + bug!("This function only supports layouts with directly encoded tags.") + }; + + let (generator_def_id, generator_substs) = match generator_type_and_layout.ty.kind() { + &ty::Generator(def_id, substs, _) => (def_id, substs.as_generator()), + _ => unreachable!(), + }; + + let (generator_layout, state_specific_upvar_names) = + generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + + let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + + // Build the type node for each field. + let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range + .map(|variant_index| { + let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( + cx, + variant_index, + generator_type_and_layout, + generator_type_di_node, + generator_layout, + &state_specific_upvar_names, + &common_upvar_names, + ); + + let span = generator_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line as c_uint)) + } else { + None + }; + + VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } + }) + .collect(); + + let tag_base_type = tag_base_type(cx, generator_type_and_layout); + let discr_type_name = "Discriminant$"; + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + discr_type_name, + tag_base_type, + &mut generator_substs + .discriminants(generator_def_id, cx.tcx) + .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), + generator_type_di_node, + ); + + build_union_fields_for_direct_tag_enum_or_generator( + cx, + generator_type_and_layout, + generator_type_di_node, + &variant_field_infos[..], + discr_type_di_node, + tag_field, + ) +} + +/// This is a helper function shared between enums and generators that makes sure fields have the +/// expect names. +fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_field_infos: &[VariantFieldInfo<'ll>], + discr_type_di_node: &'ll DIType, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); + + // We create a field in the union for each variant ... + unions_fields.extend(variant_field_infos.into_iter().map(|variant_member_info| { + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + let field_name = variant_union_field_name(variant_member_info.variant_index); + let (size, align) = size_and_align_of(enum_type_and_layout); + + // We use LLVMRustDIBuilderCreateMemberType() member type directly because + // the build_field_di_node() function does not support specifying a source location, + // which is something that we don't do anywhere else. + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + enum_type_di_node, + field_name.as_ptr().cast(), + field_name.len(), + file_di_node, + line_number, + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size.bits(), + align.bits() as u32, + // Union fields are always at offset zero + Size::ZERO.bits(), + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } + })); + + debug_assert_eq!( + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) + ); + + // ... and a field for the discriminant. + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + "discriminant", + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + discr_type_di_node, + )); + + unions_fields +} + +/// Information about a single field of the top-level DW_TAG_union_type. +struct VariantFieldInfo<'ll> { + variant_index: VariantIdx, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} + +fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { + const PRE_ALLOCATED: [&str; 16] = [ + "variant0", + "variant1", + "variant2", + "variant3", + "variant4", + "variant5", + "variant6", + "variant7", + "variant8", + "variant9", + "variant10", + "variant11", + "variant12", + "variant13", + "variant14", + "variant15", + ]; + + PRE_ALLOCATED + .get(variant_index.as_usize()) + .map(|&s| Cow::from(s)) + .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs new file mode 100644 index 00000000000..73e01d0453b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -0,0 +1,437 @@ +use rustc_codegen_ssa::debuginfo::{ + type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}, + wants_c_like_enum_debuginfo, +}; +use rustc_hir::def::CtorKind; +use rustc_index::vec::IndexVec; +use rustc_middle::{ + bug, + mir::{Field, GeneratorLayout, GeneratorSavedLocal}, + ty::{ + self, + layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, + util::Discr, + AdtDef, GeneratorSubsts, Ty, VariantDef, + }, +}; +use rustc_span::Symbol; +use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; +use std::borrow::Cow; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + build_field_di_node, build_generic_type_param_di_nodes, type_di_node, + type_map::{self, Stub}, + unknown_file_metadata, UNKNOWN_LINE_NUMBER, + }, + utils::{create_DIArray, get_namespace_for_item, DIB}, + }, + llvm::{ + self, + debuginfo::{DIFlags, DIType}, + }, +}; + +use super::{ + size_and_align_of, + type_map::{DINodeCreationResult, UniqueTypeId}, + SmallVec, +}; + +mod cpp_like; +mod native; + +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + + if wants_c_like_enum_debuginfo(enum_type_and_layout) { + return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); + } + + if cpp_like_debuginfo(cx.tcx) { + cpp_like::build_enum_type_di_node(cx, unique_type_id) + } else { + native::build_enum_type_di_node(cx, unique_type_id) + } +} + +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + if cpp_like_debuginfo(cx.tcx) { + cpp_like::build_generator_di_node(cx, unique_type_id) + } else { + native::build_generator_di_node(cx, unique_type_id) + } +} + +/// Build the debuginfo node for a C-style enum, i.e. an enum the variants of which have no fields. +/// +/// The resulting debuginfo will be a DW_TAG_enumeration_type. +fn build_c_style_enum_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> DINodeCreationResult<'ll> { + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + DINodeCreationResult { + di_node: build_enumeration_type_di_node( + cx, + &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), + tag_base_type(cx, enum_type_and_layout), + &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + }), + containing_scope, + ), + already_stored_in_typemap: false, + } +} + +/// Extract the type with which we want to describe the tag of the given enum or generator. +fn tag_base_type<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> Ty<'tcx> { + debug_assert!(match enum_type_and_layout.ty.kind() { + ty::Generator(..) => true, + ty::Adt(adt_def, _) => adt_def.is_enum(), + _ => false, + }); + + match enum_type_and_layout.layout.variants() { + // A single-variant enum has no discriminant. + Variants::Single { .. } => { + bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout) + } + + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => { + // Niche tags are always normalized to unsized integers of the correct size. + match tag.primitive() { + Primitive::Int(t, _) => t, + Primitive::F32 => Integer::I32, + Primitive::F64 => Integer::I64, + Primitive::Pointer => { + // If the niche is the NULL value of a reference, then `discr_enum_ty` will be + // a RawPtr. CodeView doesn't know what to do with enums whose base type is a + // pointer so we fix this up to just be `usize`. + // DWARF might be able to deal with this but with an integer type we are on + // the safe side there too. + cx.data_layout().ptr_sized_integer() + } + } + .to_ty(cx.tcx, false) + } + + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { + // Direct tags preserve the sign. + tag.primitive().to_ty(cx.tcx) + } + } +} + +/// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. +/// This is a helper function and does not register anything in the type map by itself. +/// +/// `variants` is an iterator of (discr-value, variant-name). +/// +// NOTE: Handling of discriminant values is somewhat inconsistent. They can appear as u128, +// u64, and i64. Here everything gets mapped to i64 because that's what LLVM's API expects. +fn build_enumeration_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + type_name: &str, + base_type: Ty<'tcx>, + variants: &mut dyn Iterator<Item = (Discr<'tcx>, Cow<'tcx, str>)>, + containing_scope: &'ll DIType, +) -> &'ll DIType { + let is_unsigned = match base_type.kind() { + ty::Int(_) => false, + ty::Uint(_) => true, + _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), + }; + + let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = variants + .map(|(discr, variant_name)| { + unsafe { + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + variant_name.as_ptr().cast(), + variant_name.len(), + // FIXME: what if enumeration has i128 discriminant? + discr.val as i64, + is_unsigned, + )) + } + }) + .collect(); + + let (size, align) = cx.size_and_align_of(base_type); + + unsafe { + llvm::LLVMRustDIBuilderCreateEnumerationType( + DIB(cx), + containing_scope, + type_name.as_ptr().cast(), + type_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + create_DIArray(DIB(cx), &enumerator_di_nodes[..]), + type_di_node(cx, base_type), + true, + ) + } +} + +/// Build the debuginfo node for the struct type describing a single variant of an enum. +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// In CPP-like mode, we have the exact same descriptions for each variant too: +/// +/// ```txt +/// DW_TAG_union_type (top-level type for enum) +/// DW_TAG_member (member for variant 1) +/// DW_TAG_member (member for variant 2) +/// DW_TAG_member (member for variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// DW_TAG_enumeration_type (type of tag) +/// ``` +/// +/// The node looks like: +/// +/// ```txt +/// DW_TAG_structure_type +/// DW_AT_name <name-of-variant> +/// DW_AT_byte_size 0x00000010 +/// DW_AT_alignment 0x00000008 +/// DW_TAG_member +/// DW_AT_name <name-of-field-0> +/// DW_AT_type <0x0000018e> +/// DW_AT_alignment 0x00000004 +/// DW_AT_data_member_location 4 +/// DW_TAG_member +/// DW_AT_name <name-of-field-1> +/// DW_AT_type <0x00000195> +/// DW_AT_alignment 0x00000008 +/// DW_AT_data_member_location 8 +/// ... +/// ``` +/// +/// The type of a variant is always a struct type with the name of the variant +/// and a DW_TAG_member for each field (but not the discriminant). +fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type: Ty<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, + variant_def: &VariantDef, + variant_layout: TyAndLayout<'tcx>, +) -> &'ll DIType { + debug_assert_eq!(variant_layout.ty, enum_type); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), + variant_def.name.as_str(), + // NOTE: We use size and align of enum_type, not from variant_layout: + cx.size_and_align_of(enum_type), + Some(enum_type_di_node), + DIFlags::FlagZero, + ), + |cx, struct_type_di_node| { + (0..variant_layout.fields.count()) + .map(|field_index| { + let field_name = if variant_def.ctor_kind != CtorKind::Fn { + // Fields have names + Cow::from(variant_def.fields[field_index].name.as_str()) + } else { + // Tuple-like + super::tuple_field_name(field_index) + }; + + let field_layout = variant_layout.field(cx, field_index); + + build_field_di_node( + cx, + struct_type_di_node, + &field_name, + (field_layout.size, field_layout.align.abi), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, enum_type), + ) + .di_node +} + +/// Build the struct type for describing a single generator state. +/// See [build_generator_variant_struct_type_di_node]. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + variant_index: VariantIdx, + generator_type_and_layout: TyAndLayout<'tcx>, + generator_type_di_node: &'ll DIType, + generator_layout: &GeneratorLayout<'tcx>, + state_specific_upvar_names: &IndexVec<GeneratorSavedLocal, Option<Symbol>>, + common_upvar_names: &[String], +) -> &'ll DIType { + let variant_name = GeneratorSubsts::variant_name(variant_index); + let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + generator_type_and_layout.ty, + variant_index, + ); + + let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); + + let generator_substs = match generator_type_and_layout.ty.kind() { + ty::Generator(_, substs, _) => substs.as_generator(), + _ => unreachable!(), + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &variant_name, + size_and_align_of(generator_type_and_layout), + Some(generator_type_di_node), + DIFlags::FlagZero, + ), + |cx, variant_struct_type_di_node| { + // Fields that just belong to this variant/state + let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + .map(|field_index| { + let generator_saved_local = generator_layout.variant_fields[variant_index] + [Field::from_usize(field_index)]; + let field_name_maybe = state_specific_upvar_names[generator_saved_local]; + let field_name = field_name_maybe + .as_ref() + .map(|s| Cow::from(s.as_str())) + .unwrap_or_else(|| super::tuple_field_name(field_index)); + + let field_type = variant_layout.field(cx, field_index).ty; + + build_field_di_node( + cx, + variant_struct_type_di_node, + &field_name, + cx.size_and_align_of(field_type), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_type), + ) + }) + .collect(); + + // Fields that are common to all states + let common_fields: SmallVec<_> = generator_substs + .prefix_tys() + .enumerate() + .map(|(index, upvar_ty)| { + build_field_di_node( + cx, + variant_struct_type_di_node, + &common_upvar_names[index], + cx.size_and_align_of(upvar_ty), + generator_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, upvar_ty), + ) + }) + .collect(); + + state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() + }, + |cx| build_generic_type_param_di_nodes(cx, generator_type_and_layout.ty), + ) + .di_node +} + +/// Returns the discriminant value corresponding to the variant index. +/// +/// Will return `None` if there is less than two variants (because then the enum won't have) +/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no +/// single discriminant value). +fn compute_discriminant_value<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_index: VariantIdx, +) -> Option<u64> { + match enum_type_and_layout.layout.variants() { + &Variants::Single { .. } => None, + &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( + enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val + as u64, + ), + &Variants::Multiple { + tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, + tag, + .. + } => { + if variant_index == dataful_variant { + None + } else { + let value = (variant_index.as_u32() as u128) + .wrapping_sub(niche_variants.start().as_u32() as u128) + .wrapping_add(niche_start); + let value = tag.size(cx).truncate(value); + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + Some(value as u64) + } + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs new file mode 100644 index 00000000000..f1935e0ec31 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -0,0 +1,441 @@ +use std::borrow::Cow; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + closure_saved_names_of_captured_variables, + enums::tag_base_type, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + type_map::{self, Stub, StubInfo, UniqueTypeId}, + unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, + UNKNOWN_LINE_NUMBER, + }, + utils::{create_DIArray, get_namespace_for_item, DIB}, + }, + llvm::{ + self, + debuginfo::{DIFile, DIFlags, DIType}, + }, +}; +use libc::c_uint; +use rustc_codegen_ssa::{ + debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, + traits::ConstMethods, +}; +use rustc_middle::{ + bug, + ty::{ + self, + layout::{LayoutOf, TyAndLayout}, + }, +}; +use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use smallvec::smallvec; + +/// Build the debuginfo node for an enum type. The listing below shows how such a +/// type looks like at the LLVM IR/DWARF level. It is a `DW_TAG_structure_type` +/// with a single `DW_TAG_variant_part` that in turn contains a `DW_TAG_variant` +/// for each variant of the enum. The variant-part also contains a single member +/// describing the discriminant, and a nested struct type for each of the variants. +/// +/// ```txt +/// ---> DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &enum_type_name, + size_and_align_of(enum_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, enum_type_di_node| { + // Build the struct type for each variant. These will be referenced by the + // DW_TAG_variant DIEs inside of the DW_TAG_variant_part DIE. + // We also called the names for the corresponding DW_TAG_variant DIEs here. + let variant_member_infos: SmallVec<_> = enum_adt_def + .variant_range() + .map(|variant_index| VariantMemberInfo { + variant_index, + variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( + cx, + enum_type, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + enum_type_and_layout.for_variant(cx, variant_index), + ), + source_info: None, + }) + .collect(); + + smallvec![build_enum_variant_part_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + &variant_member_infos[..], + )] + }, + // We don't seem to be emitting generic args on the enum type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a generator environment. It looks the same as the debuginfo for +/// an enum. See [build_enum_type_di_node] for more information. +/// +/// ```txt +/// +/// ---> DW_TAG_structure_type (top-level type for the generator) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let generator_type = unique_type_id.expect_ty(); + let &ty::Generator(generator_def_id, _, _ ) = generator_type.kind() else { + bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) + }; + + let containing_scope = get_namespace_for_item(cx, generator_def_id); + let generator_type_and_layout = cx.layout_of(generator_type); + + debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + + let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &generator_type_name, + size_and_align_of(generator_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, generator_type_di_node| { + let (generator_layout, state_specific_upvar_names) = + generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + + let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { + bug!( + "Encountered generator with non-direct-tag layout: {:?}", + generator_type_and_layout + ) + }; + + let common_upvar_names = + closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + + // Build variant struct types + let variant_struct_type_di_nodes: SmallVec<_> = variants + .indices() + .map(|variant_index| { + // FIXME: This is problematic because just a number is not a valid identifier. + // GeneratorSubsts::variant_name(variant_index), would be consistent + // with enums? + let variant_name = format!("{}", variant_index.as_usize()).into(); + + let span = generator_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line)) + } else { + None + }; + + VariantMemberInfo { + variant_index, + variant_name, + variant_struct_type_di_node: + super::build_generator_variant_struct_type_di_node( + cx, + variant_index, + generator_type_and_layout, + generator_type_di_node, + generator_layout, + &state_specific_upvar_names, + &common_upvar_names, + ), + source_info, + } + }) + .collect(); + + smallvec![build_enum_variant_part_di_node( + cx, + generator_type_and_layout, + generator_type_di_node, + &variant_struct_type_di_nodes[..], + )] + }, + // We don't seem to be emitting generic args on the generator type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Builds the DW_TAG_variant_part of an enum or generator debuginfo node: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// ---> DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +fn build_enum_variant_part_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_member_infos: &[VariantMemberInfo<'_, 'll>], +) -> &'ll DIType { + let tag_member_di_node = + build_discr_member_di_node(cx, enum_type_and_layout, enum_type_di_node); + + let variant_part_unique_type_id = + UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); + + let stub = StubInfo::new( + cx, + variant_part_unique_type_id, + |cx, variant_part_unique_type_id_str| unsafe { + let variant_part_name = ""; + llvm::LLVMRustDIBuilderCreateVariantPart( + DIB(cx), + enum_type_di_node, + variant_part_name.as_ptr().cast(), + variant_part_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + DIFlags::FlagZero, + tag_member_di_node, + create_DIArray(DIB(cx), &[]), + variant_part_unique_type_id_str.as_ptr().cast(), + variant_part_unique_type_id_str.len(), + ) + }, + ); + + type_map::build_type_with_children( + cx, + stub, + |cx, variant_part_di_node| { + variant_member_infos + .iter() + .map(|variant_member_info| { + build_enum_variant_member_di_node( + cx, + enum_type_and_layout, + variant_part_di_node, + variant_member_info, + ) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +/// Builds the DW_TAG_member describing where we can find the tag of an enum. +/// Returns `None` if the enum does not have a tag. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// ---> DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +fn build_discr_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_generator_type_and_layout: TyAndLayout<'tcx>, + enum_or_generator_type_di_node: &'ll DIType, +) -> Option<&'ll DIType> { + let tag_name = match enum_or_generator_type_and_layout.ty.kind() { + ty::Generator(..) => "__state", + _ => "", + }; + + // NOTE: This is actually wrong. This will become a member of + // of the DW_TAG_variant_part. But, due to LLVM's API, that + // can only be constructed with this DW_TAG_member already in created. + // In LLVM IR the wrong scope will be listed but when DWARF is + // generated from it, the DW_TAG_member will be a child the + // DW_TAG_variant_part. + let containing_scope = enum_or_generator_type_di_node; + + match enum_or_generator_type_and_layout.layout.variants() { + // A single-variant enum has no discriminant. + &Variants::Single { .. } => None, + + &Variants::Multiple { tag_field, .. } => { + let tag_base_type = tag_base_type(cx, enum_or_generator_type_and_layout); + let (size, align) = cx.size_and_align_of(tag_base_type); + + unsafe { + Some(llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + containing_scope, + tag_name.as_ptr().cast(), + tag_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + enum_or_generator_type_and_layout.fields.offset(tag_field).bits(), + DIFlags::FlagArtificial, + type_di_node(cx, tag_base_type), + )) + } + } + } +} + +/// Build the debuginfo node for `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// This node looks like: +/// +/// ```txt +/// DW_TAG_variant +/// DW_AT_discr_value 0 +/// DW_TAG_member +/// DW_AT_name None +/// DW_AT_type <0x000002a1> +/// DW_AT_alignment 0x00000002 +/// DW_AT_data_member_location 0 +/// ``` +/// +/// The DW_AT_discr_value is optional, and is omitted if +/// - This is the only variant of a univariant enum (i.e. their is no discriminant) +/// - This is the "dataful" variant of a niche-layout enum +/// (where only the other variants are identified by a single value) +/// +/// There is only ever a single member, the type of which is a struct that describes the +/// fields of the variant (excluding the discriminant). The name of the member is the name +/// of the variant as given in the source code. The DW_AT_data_member_location is always +/// zero. +/// +/// Note that the LLVM DIBuilder API is a bit unintuitive here. The DW_TAG_variant subtree +/// (including the DW_TAG_member) is built by a single call to +/// `LLVMRustDIBuilderCreateVariantMemberType()`. +fn build_enum_variant_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_part_di_node: &'ll DIType, + variant_member_info: &VariantMemberInfo<'_, 'll>, +) -> &'ll DIType { + let variant_index = variant_member_info.variant_index; + let discr_value = super::compute_discriminant_value(cx, enum_type_and_layout, variant_index); + + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + unsafe { + llvm::LLVMRustDIBuilderCreateVariantMemberType( + DIB(cx), + variant_part_di_node, + variant_member_info.variant_name.as_ptr().cast(), + variant_member_info.variant_name.len(), + file_di_node, + line_number, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + Size::ZERO.bits(), + discr_value.map(|v| cx.const_u64(v)), + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } +} + +/// Information needed for building a `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +struct VariantMemberInfo<'a, 'll> { + variant_index: VariantIdx, + variant_name: Cow<'a, str>, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs new file mode 100644 index 00000000000..87fbb737ea8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -0,0 +1,266 @@ +use std::cell::RefCell; + +use rustc_data_structures::{ + fingerprint::Fingerprint, + fx::FxHashMap, + stable_hasher::{HashStable, StableHasher}, +}; +use rustc_middle::{ + bug, + ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}, +}; +use rustc_target::abi::{Align, Size, VariantIdx}; + +use crate::{ + common::CodegenCx, + debuginfo::utils::{create_DIArray, debug_context, DIB}, + llvm::{ + self, + debuginfo::{DIFlags, DIScope, DIType}, + }, +}; + +use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER}; + +mod private { + // This type cannot be constructed outside of this module because + // it has a private field. We make use of this in order to prevent + // `UniqueTypeId` from being constructed directly, without asserting + // the preconditions. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] + pub struct HiddenZst; +} + +/// A unique identifier for anything that we create a debuginfo node for. +/// The types it contains are expected to already be normalized (which +/// is debug_asserted in the constructors). +/// +/// Note that there are some things that only show up in debuginfo, like +/// the separate type descriptions for each enum variant. These get an ID +/// too because they have their own debuginfo node in LLVM IR. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] +pub(super) enum UniqueTypeId<'tcx> { + /// The ID of a regular type as it shows up at the language level. + Ty(Ty<'tcx>, private::HiddenZst), + /// The ID for the single DW_TAG_variant_part nested inside the top-level + /// DW_TAG_structure_type that describes enums and generators. + VariantPart(Ty<'tcx>, private::HiddenZst), + /// The ID for the artificial struct type describing a single enum variant. + VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID of the artificial type we create for VTables. + VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst), +} + +impl<'tcx> UniqueTypeId<'tcx> { + pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { + debug_assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); + UniqueTypeId::Ty(t, private::HiddenZst) + } + + pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) + } + + pub fn for_enum_variant_struct_type( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) + } + + pub fn for_vtable_ty( + tcx: TyCtxt<'tcx>, + self_type: Ty<'tcx>, + implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, + ) -> Self { + debug_assert_eq!( + self_type, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type) + ); + debug_assert_eq!( + implemented_trait, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) + ); + UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst) + } + + /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` + /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. + /// + /// Right now this takes the form of a hex-encoded opaque hash value. + pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { + let mut hasher = StableHasher::new(); + let mut hcx = tcx.create_stable_hashing_context(); + hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)); + hasher.finish::<Fingerprint>().to_hex() + } + + pub fn expect_ty(self) -> Ty<'tcx> { + match self { + UniqueTypeId::Ty(ty, _) => ty, + _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self), + } + } +} + +/// The `TypeMap` is where the debug context holds the type metadata nodes +/// created so far. The debuginfo nodes are identified by `UniqueTypeId`. +#[derive(Default)] +pub(crate) struct TypeMap<'ll, 'tcx> { + pub(super) unique_id_to_di_node: RefCell<FxHashMap<UniqueTypeId<'tcx>, &'ll DIType>>, +} + +impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { + /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will + /// fail if the mapping already exists. + pub(super) fn insert(&self, unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType) { + if self.unique_id_to_di_node.borrow_mut().insert(unique_type_id, metadata).is_some() { + bug!("type metadata for unique ID '{:?}' is already in the `TypeMap`!", unique_type_id); + } + } + + pub(super) fn di_node_for_unique_id( + &self, + unique_type_id: UniqueTypeId<'tcx>, + ) -> Option<&'ll DIType> { + self.unique_id_to_di_node.borrow().get(&unique_type_id).cloned() + } +} + +pub struct DINodeCreationResult<'ll> { + pub di_node: &'ll DIType, + pub already_stored_in_typemap: bool, +} + +impl<'ll> DINodeCreationResult<'ll> { + pub fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { + DINodeCreationResult { di_node, already_stored_in_typemap } + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Stub<'ll> { + Struct, + Union, + VtableTy { vtable_holder: &'ll DIType }, +} + +pub struct StubInfo<'ll, 'tcx> { + metadata: &'ll DIType, + unique_type_id: UniqueTypeId<'tcx>, +} + +impl<'ll, 'tcx> StubInfo<'ll, 'tcx> { + pub(super) fn new( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + build: impl FnOnce(&CodegenCx<'ll, 'tcx>, /* unique_type_id_str: */ &str) -> &'ll DIType, + ) -> StubInfo<'ll, 'tcx> { + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + let di_node = build(cx, &unique_type_id_str); + StubInfo { metadata: di_node, unique_type_id } + } +} + +/// Create a stub debuginfo node onto which fields and nested types can be attached. +pub(super) fn stub<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + kind: Stub<'ll>, + unique_type_id: UniqueTypeId<'tcx>, + name: &str, + (size, align): (Size, Align), + containing_scope: Option<&'ll DIScope>, + flags: DIFlags, +) -> StubInfo<'ll, 'tcx> { + let empty_array = create_DIArray(DIB(cx), &[]); + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + + let metadata = match kind { + Stub::Struct | Stub::VtableTy { .. } => { + let vtable_holder = match kind { + Stub::VtableTy { vtable_holder } => Some(vtable_holder), + _ => None, + }; + unsafe { + llvm::LLVMRustDIBuilderCreateStructType( + DIB(cx), + containing_scope, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + None, + empty_array, + 0, + vtable_holder, + unique_type_id_str.as_ptr().cast(), + unique_type_id_str.len(), + ) + } + } + Stub::Union => unsafe { + llvm::LLVMRustDIBuilderCreateUnionType( + DIB(cx), + containing_scope, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + Some(empty_array), + 0, + unique_type_id_str.as_ptr().cast(), + unique_type_id_str.len(), + ) + }, + }; + StubInfo { metadata, unique_type_id } +} + +/// This function enables creating debuginfo nodes that can recursively refer to themselves. +/// It will first insert the given stub into the type map and only then execute the `members` +/// and `generics` closures passed in. These closures have access to the stub so they can +/// directly attach fields to them. If the type of a field transitively refers back +/// to the type currently being built, the stub will already be found in the type map, +/// which effectively breaks the recursion cycle. +pub(super) fn build_type_with_children<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + stub_info: StubInfo<'ll, 'tcx>, + members: impl FnOnce(&CodegenCx<'ll, 'tcx>, &'ll DIType) -> SmallVec<&'ll DIType>, + generics: impl FnOnce(&CodegenCx<'ll, 'tcx>) -> SmallVec<&'ll DIType>, +) -> DINodeCreationResult<'ll> { + debug_assert_eq!( + debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), + None + ); + + debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata); + + let members: SmallVec<_> = + members(cx, stub_info.metadata).into_iter().map(|node| Some(node)).collect(); + let generics: SmallVec<Option<&'ll DIType>> = + generics(cx).into_iter().map(|node| Some(node)).collect(); + + if !(members.is_empty() && generics.is_empty()) { + unsafe { + let members_array = create_DIArray(DIB(cx), &members[..]); + let generics_array = create_DIArray(DIB(cx), &generics[..]); + llvm::LLVMRustDICompositeTypeReplaceArrays( + DIB(cx), + stub_info.metadata, + Some(members_array), + Some(generics_array), + ); + } + } + + DINodeCreationResult { di_node: stub_info.metadata, already_stored_in_typemap: true } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 1b4a193dbf1..71699b5cf38 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -2,7 +2,7 @@ use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; -use self::metadata::{file_metadata, type_metadata, TypeMap}; +use self::metadata::{file_metadata, type_di_node}; use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; use self::namespace::mangled_name_of_instance; use self::utils::{create_DIArray, is_node_local_to_unit, DIB}; @@ -20,7 +20,7 @@ use crate::value::Value; use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::vec::IndexVec; @@ -31,14 +31,14 @@ use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, Span}; -use rustc_target::abi::{Primitive, Size}; +use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; +use rustc_target::abi::Size; use libc::c_uint; use smallvec::SmallVec; +use std::cell::OnceCell; use std::cell::RefCell; use std::iter; -use std::lazy::OnceCell; use tracing::debug; mod create_scope_map; @@ -48,7 +48,7 @@ mod namespace; mod utils; pub use self::create_scope_map::compute_mir_scopes; -pub use self::metadata::create_global_var_metadata; +pub use self::metadata::build_global_var_di_node; pub use self::metadata::extend_scope_to_file; #[allow(non_upper_case_globals)] @@ -57,24 +57,18 @@ const DW_TAG_auto_variable: c_uint = 0x100; const DW_TAG_arg_variable: c_uint = 0x101; /// A context object for maintaining all state needed by the debuginfo module. -pub struct CrateDebugContext<'a, 'tcx> { - llcontext: &'a llvm::Context, - llmod: &'a llvm::Module, - builder: &'a mut DIBuilder<'a>, - created_files: RefCell<FxHashMap<(Option<String>, Option<String>), &'a DIFile>>, - created_enum_disr_types: RefCell<FxHashMap<(DefId, Primitive), &'a DIType>>, - - type_map: TypeMap<'a, 'tcx>, - namespace_map: RefCell<DefIdMap<&'a DIScope>>, - - recursion_marker_type: OnceCell<&'a DIType>, - - // This collection is used to assert that composite types (structs, enums, - // ...) have their members only set once: - composite_types_completed: RefCell<FxHashSet<&'a DIType>>, +pub struct CodegenUnitDebugContext<'ll, 'tcx> { + llcontext: &'ll llvm::Context, + llmod: &'ll llvm::Module, + builder: &'ll mut DIBuilder<'ll>, + created_files: RefCell<FxHashMap<Option<(u128, SourceFileHash)>, &'ll DIFile>>, + + type_map: metadata::TypeMap<'ll, 'tcx>, + namespace_map: RefCell<DefIdMap<&'ll DIScope>>, + recursion_marker_type: OnceCell<&'ll DIType>, } -impl Drop for CrateDebugContext<'_, '_> { +impl Drop for CodegenUnitDebugContext<'_, '_> { fn drop(&mut self) { unsafe { llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _)); @@ -82,22 +76,20 @@ impl Drop for CrateDebugContext<'_, '_> { } } -impl<'a, 'tcx> CrateDebugContext<'a, 'tcx> { - pub fn new(llmod: &'a llvm::Module) -> Self { - debug!("CrateDebugContext::new"); +impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { + pub fn new(llmod: &'ll llvm::Module) -> Self { + debug!("CodegenUnitDebugContext::new"); let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; // DIBuilder inherits context from the module, so we'd better use the same one let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; - CrateDebugContext { + CodegenUnitDebugContext { llcontext, llmod, builder, created_files: Default::default(), - created_enum_disr_types: Default::default(), type_map: Default::default(), namespace_map: RefCell::new(Default::default()), recursion_marker_type: OnceCell::new(), - composite_types_completed: Default::default(), } } @@ -294,9 +286,8 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } // Initialize fn debug context (including scopes). - // FIXME(eddyb) figure out a way to not need `Option` for `dbg_scope`. let empty_scope = DebugScope { - dbg_scope: None, + dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), inlined_at: None, file_start_pos: BytePos(0), file_end_pos: BytePos(0), @@ -305,13 +296,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; // Fill in all the scopes, with the information from the MIR body. - compute_mir_scopes( - self, - instance, - mir, - self.dbg_scope_fn(instance, fn_abi, Some(llfn)), - &mut fn_debug_context, - ); + compute_mir_scopes(self, instance, mir, &mut fn_debug_context); Some(fn_debug_context) } @@ -415,7 +400,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { signature.push(if fn_abi.ret.is_ignore() { None } else { - Some(type_metadata(cx, fn_abi.ret.layout.ty)) + Some(type_di_node(cx, fn_abi.ret.layout.ty)) }); // Arguments types @@ -440,11 +425,11 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } _ => t, }; - Some(type_metadata(cx, t)) + Some(type_di_node(cx, t)) })); } else { signature - .extend(fn_abi.args.iter().map(|arg| Some(type_metadata(cx, arg.layout.ty)))); + .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); } create_DIArray(DIB(cx), &signature[..]) @@ -467,7 +452,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { if let GenericArgKind::Type(ty) = kind.unpack() { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); - let actual_type_metadata = type_metadata(cx, actual_type); + let actual_type_metadata = type_di_node(cx, actual_type); let name = name.as_str(); Some(unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( @@ -522,9 +507,9 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.needs_subst() { - Some(type_metadata(cx, impl_self_ty)) + Some(type_di_node(cx, impl_self_ty)) } else { - Some(namespace::item_namespace(cx, def.did)) + Some(namespace::item_namespace(cx, def.did())) } } _ => None, @@ -563,13 +548,13 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) } } - fn create_vtable_metadata( + fn create_vtable_debuginfo( &self, ty: Ty<'tcx>, trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, vtable: Self::Value, ) { - metadata::create_vtable_metadata(self, ty, trait_ref, vtable) + metadata::create_vtable_di_node(self, ty, trait_ref, vtable) } fn extend_scope_to_file( @@ -597,7 +582,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); - let type_metadata = type_metadata(self, variable_type); + let type_metadata = type_di_node(self, variable_type); let (argument_index, dwarf_tag) = match variable_kind { ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index fa75463067f..8f243673907 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -1,7 +1,7 @@ // Utility Functions. use super::namespace::item_namespace; -use super::CrateDebugContext; +use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; @@ -35,7 +35,7 @@ pub fn create_DIArray<'ll>( #[inline] pub fn debug_context<'a, 'll, 'tcx>( cx: &'a CodegenCx<'ll, 'tcx>, -) -> &'a CrateDebugContext<'ll, 'tcx> { +) -> &'a CodegenUnitDebugContext<'ll, 'tcx> { cx.dbg_cx.as_ref().unwrap() } @@ -46,7 +46,7 @@ pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { } pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { - item_namespace(cx, cx.tcx.parent(def_id).expect("get_namespace_for_item: missing parent?")) + item_namespace(cx, cx.tcx.parent(def_id)) } #[derive(Debug, PartialEq, Eq)] diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7f804ab5e63..9f364749287 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -134,7 +134,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::va_arg => { match fn_abi.ret.layout.abi { abi::Abi::Scalar(scalar) => { - match scalar.value { + match scalar.primitive() { Primitive::Int(..) => { if self.cx().size_of(ret_ty).bytes() < 4 { // `va_arg` should not be called on an integer type @@ -329,7 +329,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let b_ptr = self.bitcast(b, i8p_ty); let n = self.const_usize(layout.size().bytes()); let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); - self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)) + match self.cx.sess().target.arch.as_ref() { + "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), + _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), + } } } @@ -403,6 +406,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) } + fn type_checked_load( + &mut self, + llvtable: &'ll Value, + vtable_byte_offset: u64, + typeid: &'ll Value, + ) -> Self::Value { + let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + } + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { self.call_intrinsic("llvm.va_start", &[va_list]) } @@ -428,7 +441,7 @@ fn try_intrinsic<'ll>( bx.store(bx.const_i32(0), dest, ret_align); } else if wants_msvc_seh(bx.sess()) { codegen_msvc_try(bx, try_func, data, catch_func, dest); - } else if bx.sess().target.is_like_emscripten { + } else if bx.sess().target.os == "emscripten" { codegen_emcc_try(bx, try_func, data, catch_func, dest); } else { codegen_gnu_try(bx, try_func, data, catch_func, dest); @@ -813,6 +826,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, ) -> Result<&'ll Value, ()> { // macros for error handling: + #[allow(unused_macro_rules)] macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) @@ -1113,7 +1127,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) == Some(expected_bytes) => { - // Zero-extend iN to the array lengh: + // Zero-extend iN to the array length: let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); // Convert the integer to a byte array @@ -1141,6 +1155,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { + #[allow(unused_macro_rules)] macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) @@ -1836,6 +1851,27 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, simd_neg: Int => neg, Float => fneg; } + if name == sym::simd_arith_offset { + // This also checks that the first operand is a ptr type. + let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { + span_bug!(span, "must be called with a vector of pointer types as first argument") + }); + let layout = bx.layout_of(pointee.ty); + let ptrs = args[0].immediate(); + // The second argument must be a ptr-sized integer. + // (We don't care about the signedness, this is wrapping anyway.) + let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx()); + if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { + span_bug!( + span, + "must be called with a vector of pointer-sized integers as second argument" + ); + } + let offsets = args[1].immediate(); + + return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets])); + } + if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { let lhs = args[0].immediate(); let rhs = args[1].immediate(); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 875b4f033d1..6713a756735 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,12 +5,10 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(bool_to_option)] -#![feature(crate_visibility_modifier)] +#![feature(let_chains)] #![feature(let_else)] #![feature(extern_types)] #![feature(once_cell)] -#![feature(nll)] #![feature(iter_intersperse)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] @@ -103,19 +101,18 @@ impl Drop for TimeTraceProfiler { } impl ExtraBackendMethods for LlvmCodegenBackend { - fn new_metadata(&self, tcx: TyCtxt<'_>, mod_name: &str) -> ModuleLlvm { - ModuleLlvm::new_metadata(tcx, mod_name) - } - fn codegen_allocator<'tcx>( &self, tcx: TyCtxt<'tcx>, - module_llvm: &mut ModuleLlvm, module_name: &str, kind: AllocatorKind, has_alloc_error_handler: bool, - ) { - unsafe { allocator::codegen(tcx, module_llvm, module_name, kind, has_alloc_error_handler) } + ) -> ModuleLlvm { + let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); + unsafe { + allocator::codegen(tcx, &mut module_llvm, module_name, kind, has_alloc_error_handler); + } + module_llvm } fn compile_codegen_unit( &self, @@ -209,9 +206,16 @@ impl WriteBackendMethods for LlvmCodegenBackend { ) -> Result<(), FatalError> { back::write::optimize(cgcx, diag_handler, module, config) } + fn optimize_fat( + cgcx: &CodegenContext<Self>, + module: &mut ModuleCodegen<Self::Module>, + ) -> Result<(), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + back::lto::run_pass_manager(cgcx, &diag_handler, module, false) + } unsafe fn optimize_thin( cgcx: &CodegenContext<Self>, - thin: &mut ThinModule<Self>, + thin: ThinModule<Self>, ) -> Result<ModuleCodegen<Self::Module>, FatalError> { back::lto::optimize_thin_module(thin, cgcx) } @@ -229,15 +233,6 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer) { (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) } - fn run_lto_pass_manager( - cgcx: &CodegenContext<Self>, - module: &ModuleCodegen<Self::Module>, - config: &ModuleConfig, - thin: bool, - ) -> Result<(), FatalError> { - let diag_handler = cgcx.create_diag_handler(); - back::lto::run_pass_manager(cgcx, &diag_handler, module, config, thin) - } } unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis @@ -308,8 +303,8 @@ impl CodegenBackend for LlvmCodegenBackend { local stack variable in the ABI.) basic - Generate stack canaries in functions with: - - local variables of `[T; N]` type, where `T` is byte-sized and `N` > 8. + Generate stack canaries in functions with local variables of `[T; N]` + type, where `T` is byte-sized and `N` >= 8. none Do not generate stack canaries. diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 375b9927c86..d92d9d96fe2 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -381,9 +381,8 @@ pub enum AtomicOrdering { impl AtomicOrdering { pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { match ao { - rustc_codegen_ssa::common::AtomicOrdering::NotAtomic => AtomicOrdering::NotAtomic, rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered, - rustc_codegen_ssa::common::AtomicOrdering::Monotonic => AtomicOrdering::Monotonic, + rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic, rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire, rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release, rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => { @@ -443,6 +442,7 @@ pub enum MetadataType { MD_nonnull = 11, MD_align = 17, MD_type = 19, + MD_vcall_visibility = 28, MD_noundef = 29, } @@ -775,7 +775,7 @@ pub mod coverageinfo { } impl CounterMappingRegion { - crate fn code_region( + pub(crate) fn code_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -799,7 +799,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn branch_region( + pub(crate) fn branch_region( counter: coverage_map::Counter, false_counter: coverage_map::Counter, file_id: u32, @@ -824,7 +824,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn expansion_region( + pub(crate) fn expansion_region( file_id: u32, expanded_file_id: u32, start_line: u32, @@ -848,7 +848,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn skipped_region( + pub(crate) fn skipped_region( file_id: u32, start_line: u32, start_col: u32, @@ -871,7 +871,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn gap_region( + pub(crate) fn gap_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -1068,6 +1068,7 @@ extern "C" { pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1081,6 +1082,11 @@ extern "C" { Vals: *const &'a Value, Count: c_uint, ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); // Operations on scalar constants @@ -1128,11 +1134,7 @@ extern "C" { pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstExtractValue( - AggConstant: &Value, - IdxList: *const c_uint, - NumIdx: c_uint, - ) -> &Value; + pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1825,24 +1827,22 @@ extern "C" { pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - pub fn LLVMPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; - pub fn LLVMPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); - pub fn LLVMPassManagerBuilderSetSizeLevel(PMB: &PassManagerBuilder, Value: Bool); - pub fn LLVMPassManagerBuilderSetDisableUnrollLoops(PMB: &PassManagerBuilder, Value: Bool); - pub fn LLVMPassManagerBuilderUseInlinerWithThreshold( + pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; + pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); + pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( PMB: &PassManagerBuilder, threshold: c_uint, ); - pub fn LLVMPassManagerBuilderPopulateModulePassManager( + pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( PMB: &PassManagerBuilder, PM: &PassManager<'_>, ); - pub fn LLVMPassManagerBuilderPopulateFunctionPassManager( + pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( PMB: &PassManagerBuilder, PM: &PassManager<'_>, ); - pub fn LLVMPassManagerBuilderPopulateLTOPassManager( + pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( PMB: &PassManagerBuilder, PM: &PassManager<'_>, Internalize: Bool, @@ -1939,6 +1939,7 @@ extern "C" { name: *const c_char, value: u32, ); + pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; @@ -2308,6 +2309,7 @@ extern "C" { PGOGenPath: *const c_char, PGOUsePath: *const c_char, PGOSampleUsePath: *const c_char, + SizeLevel: c_int, ); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, @@ -2537,4 +2539,6 @@ extern "C" { remark_passes_len: usize, ); + #[allow(improper_ctypes)] + pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 4b324740a1f..ce6c6e3215c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -61,8 +61,8 @@ unsafe fn configure_llvm(sess: &Session) { full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("") } - let cg_opts = sess.opts.cg.llvm_args.iter(); - let tg_opts = sess.target.llvm_args.iter(); + let cg_opts = sess.opts.cg.llvm_args.iter().map(AsRef::as_ref); + let tg_opts = sess.target.llvm_args.iter().map(AsRef::as_ref); let sess_args = cg_opts.chain(tg_opts); let user_specified_args: FxHashSet<_> = @@ -187,9 +187,6 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2] ("x86", "avx512vaes") => smallvec!["vaes"], ("x86", "avx512gfni") => smallvec!["gfni"], ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], - ("aarch64", "fp") => smallvec!["fp-armv8"], - ("aarch64", "fp16") => smallvec!["fullfp16"], - ("aarch64", "fhm") => smallvec!["fp16fml"], ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], ("aarch64", "dpb") => smallvec!["ccpp"], ("aarch64", "dpb2") => smallvec!["ccdp"], @@ -198,6 +195,19 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2] ("aarch64", "pmuv3") => smallvec!["perfmon"], ("aarch64", "paca") => smallvec!["pauth"], ("aarch64", "pacg") => smallvec!["pauth"], + // Rust ties fp and neon together. In LLVM neon implicitly enables fp, + // but we manually enable neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], + ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], + ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], + ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], + ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], + ("aarch64", "sve") => smallvec!["sve", "neon"], + ("aarch64", "sve2") => smallvec!["sve2", "neon"], + ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], + ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], + ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], + ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], (_, s) => smallvec![s], } } @@ -208,17 +218,21 @@ pub fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, ) -> Option<&'static [&'static str]> { - for tied in tied_target_features(sess) { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); + if !features.is_empty() { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } } } - None + return None; } +// Used to generate cfg variables and apply features +// Must express features in the way Rust understands them pub fn target_features(sess: &Session) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); let mut features: Vec<Symbol> = @@ -228,13 +242,14 @@ pub fn target_features(sess: &Session) -> Vec<Symbol> { if sess.is_nightly_build() || gate.is_none() { Some(feature) } else { None } }) .filter(|feature| { + // check that all features in a given smallvec are enabled for llvm_feature in to_llvm_features(sess, feature) { let cstr = SmallCStr::new(llvm_feature); - if unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { - return true; + if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return false; } } - false + true }) .map(|feature| Symbol::intern(feature)) .collect(); @@ -362,8 +377,10 @@ fn handle_native(name: &str) -> &str { } pub fn target_cpu(sess: &Session) -> &str { - let name = sess.opts.cg.target_cpu.as_ref().unwrap_or(&sess.target.cpu); - handle_native(name) + match sess.opts.cg.target_cpu { + Some(ref name) => handle_native(name), + None => handle_native(sess.target.cpu.as_ref()), + } } /// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, @@ -425,6 +442,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // -Ctarget-features let supported_features = supported_target_features(sess); + let mut featsmap = FxHashMap::default(); let feats = sess .opts .cg @@ -470,35 +488,36 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } diag.emit(); } - Some((enable_disable, feature)) + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } + + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some( + to_llvm_features(sess, feature) + .into_iter() + .map(move |f| format!("{}{}", enable_disable, f)), + ) }) - .collect::<SmallVec<[(char, &str); 8]>>(); - - if diagnostics { - // FIXME(nagisa): figure out how to not allocate a full hashset here. - let featmap = feats.iter().map(|&(flag, feat)| (feat, flag == '+')).collect(); - if let Some(f) = check_tied_features(sess, &featmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); - } + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); } - features.extend(feats.into_iter().flat_map(|(enable_disable, feature)| { - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return SmallVec::<[_; 2]>::new(); - } - // ... otherwise though we run through `to_llvm_feature when - // passing requests down to LLVM. This means that all in-language - // features also work on the command line instead of having two - // different names when the LLVM name and the Rust name differ. - to_llvm_features(sess, feature) - .into_iter() - .map(|f| format!("{}{}", enable_disable, f)) - .collect() - })); features } @@ -527,6 +546,11 @@ pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_a // The new pass manager is enabled by default for LLVM >= 13. // This matches Clang, which also enables it since Clang 13. + // Since LLVM 15, the legacy pass manager is no longer supported. + if llvm_util::get_version() >= (15, 0, 0) { + return true; + } + // There are some perf issues with the new pass manager when targeting // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. // See https://github.com/rust-lang/rust/issues/89609. diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 21b77f7dea6..cf2d3c423c3 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -39,33 +39,33 @@ impl fmt::Debug for Type { } impl<'ll> CodegenCx<'ll, '_> { - crate fn type_named_struct(&self, name: &str) -> &'ll Type { + pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { let name = SmallCStr::new(name); unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } } - crate fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { + pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } } - crate fn type_void(&self) -> &'ll Type { + pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } } - crate fn type_metadata(&self) -> &'ll Type { + pub(crate) fn type_metadata(&self) -> &'ll Type { unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 - crate fn type_ix(&self, num_bits: u64) -> &'ll Type { + pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } } - crate fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { unsafe { llvm::LLVMVectorType(ty, len as c_uint) } } - crate fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { unsafe { let n_args = llvm::LLVMCountParamTypes(ty) as usize; let mut args = Vec::with_capacity(n_args); @@ -75,11 +75,11 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_bool(&self) -> &'ll Type { + pub(crate) fn type_bool(&self) -> &'ll Type { self.type_i8() } - crate fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { + pub(crate) fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { match t { ty::IntTy::Isize => self.type_isize(), ty::IntTy::I8 => self.type_i8(), @@ -90,7 +90,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { + pub(crate) fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { match t { ty::UintTy::Usize => self.type_isize(), ty::UintTy::U8 => self.type_i8(), @@ -101,14 +101,14 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { + pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { match t { ty::FloatTy::F32 => self.type_f32(), ty::FloatTy::F64 => self.type_f64(), } } - crate fn type_pointee_for_align(&self, align: Align) -> &'ll Type { + pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { // FIXME(eddyb) We could find a better approximation if ity.align < align. let ity = Integer::approximate_align(self, align); self.type_from_integer(ity) @@ -116,7 +116,7 @@ impl<'ll> CodegenCx<'ll, '_> { /// Return a LLVM type that has at most the required alignment, /// and exactly the required size, as a best-effort padding array. - crate fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { + pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { let unit = Integer::approximate_align(self, align); let size = size.bytes(); let unit_size = unit.size().bytes(); @@ -124,11 +124,11 @@ impl<'ll> CodegenCx<'ll, '_> { self.type_array(self.type_from_integer(unit), size / unit_size) } - crate fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } } - crate fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { unsafe { llvm::LLVMRustArrayType(ty, len) } } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index da378dc6493..86280523631 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -53,8 +53,8 @@ fn uncached_llvm_type<'a, 'tcx>( if let (&ty::Adt(def, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { - if def.is_enum() && !def.variants.is_empty() { - write!(&mut name, "::{}", def.variants[index].name).unwrap(); + if def.is_enum() && !def.variants().is_empty() { + write!(&mut name, "::{}", def.variant(index).name).unwrap(); } } if let (&ty::Generator(_, _, _), &Variants::Single { index }) = @@ -309,7 +309,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { scalar: Scalar, offset: Size, ) -> &'a Type { - match scalar.value { + match scalar.primitive() { Int(i, _) => cx.type_from_integer(i), F32 => cx.type_f32(), F64 => cx.type_f64(), @@ -362,8 +362,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { return cx.type_i1(); } - let offset = - if index == 0 { Size::ZERO } else { a.value.size(cx).align_to(b.value.align(cx).abi) }; + let offset = if index == 0 { Size::ZERO } else { a.size(cx).align_to(b.align(cx).abi) }; self.scalar_llvm_type_at(cx, scalar, offset) } |