diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
26 files changed, 597 insertions, 328 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index d9f8170a3cf..2a6ad1be763 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -236,14 +236,31 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::Nvptx64 => {} InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} InlineAsmArch::Hexagon => {} + InlineAsmArch::LoongArch64 => { + constraints.extend_from_slice(&[ + "~{$fcc0}".to_string(), + "~{$fcc1}".to_string(), + "~{$fcc2}".to_string(), + "~{$fcc3}".to_string(), + "~{$fcc4}".to_string(), + "~{$fcc5}".to_string(), + "~{$fcc6}".to_string(), + "~{$fcc7}".to_string(), + ]); + } InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} - InlineAsmArch::S390x => {} + InlineAsmArch::S390x => { + constraints.push("~{cc}".to_string()); + } InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} InlineAsmArch::Msp430 => { constraints.push("~{sr}".to_string()); } + InlineAsmArch::M68k => { + constraints.push("~{ccr}".to_string()); + } } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -381,7 +398,7 @@ impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { } unsafe { - llvm::LLVMRustAppendModuleInlineAsm( + llvm::LLVMAppendModuleInlineAsm( self.llmod, template_str.as_ptr().cast(), template_str.len(), @@ -439,9 +456,9 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, v, inputs, dest, catch, funclet) + bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) } else { - bx.call(fty, None, v, inputs, None) + bx.call(fty, None, None, v, inputs, None) }; // Store mark in a metadata node so we can map LLVM errors @@ -630,6 +647,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", @@ -671,6 +690,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -713,6 +735,7 @@ fn modifier_to_llvm( } } InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::LoongArch(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, InlineAsmRegClass::PowerPC(_) => None, @@ -768,6 +791,7 @@ fn modifier_to_llvm( InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::M68k(_) => None, InlineAsmRegClass::Err => unreachable!(), } } @@ -796,6 +820,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' cx.type_vector(cx.type_i64(), 2) } InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), @@ -839,6 +865,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 12da21dc477..a6416e9540c 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -198,7 +198,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { "arm" => ("arm", "--32"), _ => panic!("unsupported arch {}", sess.target.arch), }; - let result = std::process::Command::new(dlltool) + let result = std::process::Command::new(&dlltool) .args([ "-d", def_file_path.to_str().unwrap(), @@ -218,9 +218,13 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { match result { Err(e) => { - sess.emit_fatal(ErrorCallingDllTool { error: e }); + sess.emit_fatal(ErrorCallingDllTool { + dlltool_path: dlltool.to_string_lossy(), + error: e, + }); } - Ok(output) if !output.status.success() => { + // dlltool returns '0' on failure, so check for error output instead. + Ok(output) if !output.stderr.is_empty() => { sess.emit_fatal(DlltoolFailImportLibrary { stdout: String::from_utf8_lossy(&output.stdout), stderr: String::from_utf8_lossy(&output.stderr), @@ -431,7 +435,7 @@ fn string_to_io_error(s: String) -> io::Error { fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.cg.dlltool { return dlltool_path.clone().into_os_string(); } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index d2e01708a37..604f68eb6a4 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -25,7 +25,6 @@ use std::fs::File; use std::io; use std::iter; use std::path::Path; -use std::ptr; use std::slice; use std::sync::Arc; @@ -709,17 +708,6 @@ pub unsafe fn optimize_thin_module( let llmod = module.module_llvm.llmod(); save_temp_bitcode(cgcx, &module, "thin-lto-input"); - // Before we do much else find the "main" `DICompileUnit` that we'll be - // using below. If we find more than one though then rustc has changed - // in a way we're not ready for, so generate an ICE by returning - // an error. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); - if !cu2.is_null() { - return Err(write::llvm_err(&diag_handler, LlvmError::MultipleSourceDiCompileUnit)); - } - // Up next comes the per-module local analyses that we do for Thin LTO. // Each of these functions is basically copied from the LLVM // implementation and then tailored to suit this implementation. Ideally @@ -766,43 +754,6 @@ pub unsafe fn optimize_thin_module( save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } - // Ok now this is a bit unfortunate. This is also something you won't - // find upstream in LLVM's ThinLTO passes! This is a hack for now to - // work around bugs in LLVM. - // - // First discovered in #45511 it was found that as part of ThinLTO - // importing passes LLVM will import `DICompileUnit` metadata - // information across modules. This means that we'll be working with one - // LLVM module that has multiple `DICompileUnit` instances in it (a - // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of - // bugs in LLVM's backend which generates invalid DWARF in a situation - // like this: - // - // https://bugs.llvm.org/show_bug.cgi?id=35212 - // https://bugs.llvm.org/show_bug.cgi?id=35562 - // - // While the first bug there is fixed the second ended up causing #46346 - // which was basically a resurgence of #45511 after LLVM's bug 35212 was - // fixed. - // - // This function below is a huge hack around this problem. The function - // below is defined in `PassWrapper.cpp` and will basically "merge" - // all `DICompileUnit` instances in a module. Basically it'll take all - // the objects, rewrite all pointers of `DISubprogram` to point to the - // first `DICompileUnit`, and then delete all the other units. - // - // This is probably mangling to the debug info slightly (but hopefully - // not too much) but for now at least gets LLVM to emit valid DWARF (or - // so it appears). Hopefully we can remove this once upstream bugs are - // fixed in LLVM. - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); - save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); - } - // Alright now that we've done everything related to the ThinLTO // analysis it's time to run some optimizations! Here we use the same // `run_pass_manager` as the "fat" LTO above except that we tell it to diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index fb563f70ed0..ca2eab28f87 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -31,6 +31,7 @@ use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; @@ -363,6 +364,15 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void line: opt.line, column: opt.column, pass_name: &opt.pass_name, + kind: match opt.kind { + OptimizationDiagnosticKind::OptimizationRemark => "success", + OptimizationDiagnosticKind::OptimizationMissed + | OptimizationDiagnosticKind::OptimizationFailure => "missed", + OptimizationDiagnosticKind::OptimizationAnalysis + | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute + | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis", + OptimizationDiagnosticKind::OptimizationRemarkOther => "other", + }, message: &opt.message, }); } @@ -904,9 +914,9 @@ unsafe fn embed_bitcode( // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 63e8a67db53..4d0bcd53d15 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,7 +2,7 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; -use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock}; +use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -15,14 +15,15 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; -use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; -use rustc_target::spec::{HasTargetSpec, Target}; +use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use std::borrow::Cow; use std::ffi::CStr; use std::iter; @@ -216,6 +217,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -230,19 +232,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles = vec![funclet_bundle]; - // Set KCFI operand bundle - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = - if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; - if kcfi_bundle.is_some() { - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); - } + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); bundles.retain(|bundle| bundle.is_some()); let invoke = unsafe { @@ -841,7 +837,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + unsafe { + llvm::LLVMBuildIntCast2( + self.llbuilder, + val, + dest_ty, + if is_signed { True } else { False }, + UNNAMED, + ) + } } fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -981,13 +985,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); - let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { llvm::LLVMSetCleanup(landing_pad, llvm::True); } (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } + fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 1); + self.add_clause(landing_pad, self.const_array(self.type_i8p(), &[])); + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) + } + fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) { let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); let mut exn = self.const_poison(ty); @@ -1001,11 +1012,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { let name = cstr!("cleanuppad"); let ret = unsafe { - llvm::LLVMRustBuildCleanupPad( + llvm::LLVMBuildCleanupPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), + args.len() as c_uint, name.as_ptr(), ) }; @@ -1014,7 +1025,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { unsafe { - llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) .expect("LLVM does not have support for cleanupret"); } } @@ -1022,11 +1033,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { let name = cstr!("catchpad"); let ret = unsafe { - llvm::LLVMRustBuildCatchPad( + llvm::LLVMBuildCatchPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), + args.len() as c_uint, name.as_ptr(), ) }; @@ -1041,7 +1052,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) -> &'ll Value { let name = cstr!("catchswitch"); let ret = unsafe { - llvm::LLVMRustBuildCatchSwitch( + llvm::LLVMBuildCatchSwitch( self.llbuilder, parent, unwind, @@ -1052,7 +1063,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let ret = ret.expect("LLVM does not have support for catchswitch"); for handler in handlers { unsafe { - llvm::LLVMRustAddHandler(ret, handler); + llvm::LLVMAddHandler(ret, handler); } } ret @@ -1175,6 +1186,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1187,19 +1199,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles = vec![funclet_bundle]; - // Set KCFI operand bundle - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = - if let Some(fn_abi) = fn_abi && self.tcx.sess.is_sanitizer_kcfi_enabled() && is_indirect_call { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; - if kcfi_bundle.is_some() { - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); - } + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); bundles.retain(|bundle| bundle.is_some()); let call = unsafe { @@ -1376,8 +1382,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { - let ret = - unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; + let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; ret.expect("LLVM does not have support for catchret") } @@ -1449,7 +1454,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, None, f, args, None) + self.call(ty, None, None, f, args, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { @@ -1511,7 +1516,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) }; let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None) + self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None) } pub(crate) fn landing_pad( @@ -1528,4 +1533,71 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) } } + + // Emits CFI pointer type membership tests. + fn cfi_type_test( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + if is_indirect_call && fn_abi.is_some() && self.tcx.sess.is_sanitizer_cfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::CFI) { + return; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let typeid = typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + + // Test whether the function pointer is associated with the type identifier. + let cond = self.type_test(llfn, typeid_metadata); + let bb_pass = self.append_sibling_block("type_test.pass"); + let bb_fail = self.append_sibling_block("type_test.fail"); + self.cond_br(cond, bb_pass, bb_fail); + + self.switch_to_block(bb_fail); + self.abort(); + self.unreachable(); + + self.switch_to_block(bb_pass); + } + } + + // Emits KCFI operand bundles. + fn kcfi_operand_bundle( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) -> Option<llvm::OperandBundleDef<'ll>> { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + let kcfi_bundle = if is_indirect_call && self.tcx.sess.is_sanitizer_kcfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::KCFI) { + return None; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + kcfi_bundle + } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 6ee2a05ffd7..4b9ca2e7d19 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -27,7 +27,7 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> debug!("get_fn(instance={:?})", instance); - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); assert!(!instance.substs.has_escaping_bound_vars()); if let Some(&llfn) = cx.instances.borrow().get(&instance) { @@ -94,11 +94,11 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> // LLVM will prefix the name with `__imp_`. Ideally, we'd like the // existing logic below to set the Storage Class, but it has an // exemption for MinGW for backwards compatability. - let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi); + let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance)); unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } llfn } else { - cx.declare_fn(sym, fn_abi) + cx.declare_fn(sym, fn_abi, Some(instance)) }; debug!("get_fn: not casting pointer!"); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 4f8b5abd901..9127fba388b 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -10,7 +10,7 @@ use crate::value::Value; use rustc_ast::Mutability; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; @@ -261,7 +261,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { let hash = self.tcx.with_stable_hashing_context(|mut hcx| { let mut hasher = StableHasher::new(); alloc.hash_stable(&mut hcx, &mut hasher); - hasher.finish::<u128>() + hasher.finish::<Hash128>() }); llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes()); } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 9116e71beac..940358acde9 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -7,7 +7,6 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use cstr::cstr; -use libc::c_uint; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -486,10 +485,10 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // go into custom sections of the wasm executable. if self.tcx.sess.target.is_like_wasm { if let Some(section) = attrs.link_section { - let section = llvm::LLVMMDStringInContext( + let section = llvm::LLVMMDStringInContext2( self.llcx, section.as_str().as_ptr().cast(), - section.as_str().len() as c_uint, + section.as_str().len(), ); assert!(alloc.provenance().ptrs().is_empty()); @@ -498,17 +497,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // as part of the interpreter execution). let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); - let alloc = llvm::LLVMMDStringInContext( - self.llcx, - bytes.as_ptr().cast(), - bytes.len() as c_uint, - ); + let alloc = + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); let data = [section, alloc]; - let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); + let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); + let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, "wasm.custom_sections\0".as_ptr().cast(), - meta, + val, ); } } else { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f0d729d4779..83101a85435 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -228,18 +228,29 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. + // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) + if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Warning, + llvm::LLVMModFlagBehavior::Override, canonical_jump_tables, 1, ); } + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) + if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + enable_split_lto_unit, + 1, + ); + } + + // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { let kcfi = "kcfi\0".as_ptr().cast(); llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 240a9d2f371..21a1ac34844 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -163,7 +163,7 @@ impl CoverageMapGenerator { counter_regions.sort_unstable_by_key(|(_counter, region)| *region); for (counter, region) in counter_regions { let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; - let same_file = current_file_name.map_or(false, |p| p == file_name); + let same_file = current_file_name.is_some_and(|p| p == file_name); if !same_file { if current_file_name.is_some() { current_file_id += 1; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 3dc0ac03312..cd261293e9b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -207,6 +207,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< )), ty::List::empty(), ), + None, ); llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 99e4ded62f1..3fff112a020 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; +use rustc_index::Idx; /// Produces DIScope DIEs for each MIR Scope which has variables defined in it. // FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. @@ -93,7 +93,7 @@ fn make_mir_scope<'ll, 'tcx>( let callee = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - callee, + ty::EarlyBinder(callee), ); let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); cx.dbg_scope_fn(callee, callee_fn_abi, None) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index ff2b005d757..37f30917609 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -9,10 +9,9 @@ use rustc_ast::attr; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; +use rustc_middle::{bug, middle::debugger_visualizer::DebuggerVisualizerType}; use rustc_session::config::{CrateType, DebugInfo}; use rustc_span::symbol::sym; -use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -22,9 +21,9 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); // Load just the first byte as that's all that's necessary to force // LLVM to keep around the reference to the global. - let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); + let volatile_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); unsafe { - llvm::LLVMSetAlignment(volative_load_instruction, 1); + llvm::LLVMSetAlignment(volatile_load_instruction, 1); } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index e2a592d851a..bd2fba12602 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -29,7 +29,6 @@ use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; @@ -807,8 +806,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( name_in_debuginfo.push(codegen_unit_name); debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); - let rustc_producer = - format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); + let rustc_producer = format!("rustc version {}", tcx.sess.cfg_version); // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. let producer = format!("clang LLVM ({})", rustc_producer); @@ -830,24 +828,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( } .unwrap_or_default(); let split_name = split_name.to_str().unwrap(); - - // FIXME(#60020): - // - // This should actually be - // - // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); - // - // That is, we should set LLVM's emission kind to `LineTablesOnly` if - // we are compiling with "limited" debuginfo. However, some of the - // existing tools relied on slightly more debuginfo being generated than - // would be the case with `LineTablesOnly`, and we did not want to break - // these tools in a "drive-by fix", without a good idea or plan about - // what limited debuginfo should exactly look like. So for now we keep - // the emission kind as `FullDebug`. - // - // See https://github.com/rust-lang/rust/issues/60020 for details. - let kind = DebugEmissionKind::FullDebug; - assert!(tcx.sess.opts.debuginfo != DebugInfo::None); + let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( @@ -881,8 +862,6 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); if tcx.sess.opts.unstable_opts.profile { - let cu_desc_metadata = - llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); @@ -890,20 +869,17 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), path_to_mdstring(debug_context.llcontext, gcda_path), - cu_desc_metadata, + unit_metadata, ]; - let gcov_metadata = llvm::LLVMMDNodeInContext( + let gcov_metadata = llvm::LLVMMDNodeInContext2( debug_context.llcontext, gcov_cu_info.as_ptr(), - gcov_cu_info.len() as c_uint, + gcov_cu_info.len(), ); + let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); let llvm_gcov_ident = cstr!("llvm.gcov"); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - llvm_gcov_ident.as_ptr(), - gcov_metadata, - ); + llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, llvm_gcov_ident.as_ptr(), val); } // Insert `llvm.ident` metadata on the wasm targets since that will @@ -924,15 +900,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( return unit_metadata; }; - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { + fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata { let path_str = path_to_c_string(path); - unsafe { - llvm::LLVMMDStringInContext( - llcx, - path_str.as_ptr(), - path_str.as_bytes().len() as c_uint, - ) - } + unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) } } } @@ -1210,12 +1180,12 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( let names = get_parameter_names(cx, generics); let template_params: SmallVec<_> = iter::zip(substs, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_di_node = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -1223,10 +1193,8 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( name.len(), actual_type_di_node, ) - }) - } else { - None - } + } + }) }) .collect(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 69443b9b828..ecb0912d328 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -6,7 +6,7 @@ use rustc_codegen_ssa::{ traits::ConstMethods, }; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::{ bug, ty::{ @@ -62,7 +62,7 @@ const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; /// In CPP-like mode, we generate a union with a field for each variant and an /// explicit tag field. The field of each variant has a struct type -/// that encodes the discrimiant of the variant and it's data layout. +/// that encodes the discriminant of the variant and it's data layout. /// The union also has a nested enumeration type that is only used for encoding /// variant names in an efficient way. Its enumerator values do _not_ correspond /// to the enum's discriminant values. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index c3439591b92..9e0e847a155 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -3,7 +3,7 @@ use rustc_codegen_ssa::debuginfo::{ wants_c_like_enum_debuginfo, }; use rustc_hir::def::CtorKind; -use rustc_index::vec::IndexVec; +use rustc_index::IndexSlice; use rustc_middle::{ bug, mir::{GeneratorLayout, GeneratorSavedLocal}, @@ -323,7 +323,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, generator_layout: &GeneratorLayout<'tcx>, - state_specific_upvar_names: &IndexVec<GeneratorSavedLocal, Option<Symbol>>, + state_specific_upvar_names: &IndexSlice<GeneratorSavedLocal, Option<Symbol>>, common_upvar_names: &[String], ) -> &'ll DIType { let variant_name = GeneratorSubsts::variant_name(variant_index); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 5392534cfcb..c3f0a0033b0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -21,12 +21,13 @@ use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::Hash128; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; @@ -61,7 +62,7 @@ pub struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell<FxHashMap<Option<(u128, SourceFileHash)>, &'ll DIFile>>, + created_files: RefCell<FxHashMap<Option<(Hash128, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, @@ -209,8 +210,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { unsafe { - let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc); } } @@ -322,7 +322,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let def_id = instance.def_id(); - let containing_scope = get_containing_scope(self, instance); + let (containing_scope, is_method) = get_containing_scope(self, instance); let span = tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); @@ -378,8 +378,29 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - unsafe { - return llvm::LLVMRustDIBuilderCreateFunction( + // When we're adding a method to a type DIE, we only want a DW_AT_declaration there, because + // LLVM LTO can't unify type definitions when a child DIE is a full subprogram definition. + // When we use this `decl` below, the subprogram definition gets created at the CU level + // with a DW_AT_specification pointing back to the type's declaration. + let decl = is_method.then(|| unsafe { + llvm::LLVMRustDIBuilderCreateMethod( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + flags, + spflags & !DISPFlags::SPFlagDefinition, + template_parameters, + ) + }); + + return unsafe { + llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, name.as_ptr().cast(), @@ -394,15 +415,15 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { spflags, maybe_definition_llfn, template_parameters, - None, - ); - } + decl, + ) + }; fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> &'ll DIArray { - if cx.sess().opts.debuginfo == DebugInfo::Limited { + if cx.sess().opts.debuginfo != DebugInfo::Full { return create_DIArray(DIB(cx), &[]); } @@ -461,12 +482,12 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let names = get_parameter_names(cx, generics); iter::zip(substs, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_metadata = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -474,10 +495,8 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { name.len(), actual_type_metadata, )) - }) - } else { - None - } + } + }) }) .collect() } else { @@ -495,57 +514,53 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { names } + /// Returns a scope, plus `true` if that's a type scope for "class" methods, + /// otherwise `false` for plain namespace scopes. fn get_containing_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, - ) -> &'ll DIScope { + ) -> (&'ll DIScope, bool) { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - cx.tcx.type_of(impl_def_id).skip_binder(), + cx.tcx.type_of(impl_def_id), ); // Only "class" methods are generally understood by LLVM, // so avoid methods on other types (e.g., `<*mut T>::null`). - match impl_self_ty.kind() { - ty::Adt(def, ..) if !def.is_box() => { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.needs_subst() - { - Some(type_di_node(cx, impl_self_ty)) - } else { - Some(namespace::item_namespace(cx, def.did())) - } + if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() + { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - _ => None, } } else { // For trait method impls we still use the "parallel namespace" // strategy - None } - }); + } - self_type.unwrap_or_else(|| { - namespace::item_namespace( - cx, - DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }, - ) - }) + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); + (scope, false) } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 6a575095f7e..164b12cf8d4 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -19,8 +19,11 @@ use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; use rustc_codegen_ssa::traits::TypeMembershipMethods; -use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi}; +use rustc_middle::ty::{Instance, Ty}; +use rustc_symbol_mangling::typeid::{ + kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance, + TypeIdOptions, +}; use smallvec::SmallVec; /// Declare a function. @@ -116,7 +119,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + pub fn declare_fn( + &self, + name: &str, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + instance: Option<Instance<'tcx>>, + ) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); // Function addresses in Rust are never significant, allowing functions to @@ -132,13 +140,54 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.apply_attrs_llfn(self, llfn); if self.tcx.sess.is_sanitizer_cfi_enabled() { - let typeid = typeid_for_fnabi(self.tcx, fn_abi); - self.set_type_metadata(llfn, typeid); + if let Some(instance) = instance { + let typeid = typeid_for_instance(self.tcx, &instance, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_instance( + self.tcx, + &instance, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } else { + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi( + self.tcx, + fn_abi, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } } if self.tcx.sess.is_sanitizer_kcfi_enabled() { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - self.set_kcfi_type_metadata(llfn, kcfi_typeid); + // LLVM KCFI does not support multiple !kcfi_type attachments + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + if let Some(instance) = instance { + let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, &instance, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } else { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } } llfn diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index bae88d94293..6a9173ab450 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -67,7 +67,8 @@ pub(crate) struct ErrorWritingDEFFile { #[derive(Diagnostic)] #[diag(codegen_llvm_error_calling_dlltool)] -pub(crate) struct ErrorCallingDllTool { +pub(crate) struct ErrorCallingDllTool<'a> { + pub dlltool_path: Cow<'a, str>, pub error: std::io::Error, } @@ -195,6 +196,7 @@ pub(crate) struct FromLlvmOptimizationDiag<'a> { pub line: std::ffi::c_uint, pub column: std::ffi::c_uint, pub pass_name: &'a str, + pub kind: &'a str, pub message: &'a str, } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 012e25884ca..4e28034a850 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -110,6 +110,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call( simple_ty, None, + None, simple_fn, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -444,7 +445,7 @@ fn try_intrinsic<'ll>( ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, None, try_func, &[data], None); + bx.call(try_func_ty, None, None, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; @@ -543,7 +544,7 @@ fn codegen_msvc_try<'ll>( let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], normal, catchswitch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); bx.switch_to_block(normal); bx.ret(bx.const_i32(0)); @@ -587,7 +588,7 @@ fn codegen_msvc_try<'ll>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_i8p(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -595,7 +596,7 @@ fn codegen_msvc_try<'ll>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, catch_func, &[data, null], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet)); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -604,7 +605,7 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -647,7 +648,7 @@ fn codegen_gnu_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -665,13 +666,13 @@ fn codegen_gnu_try<'ll>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], None); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -711,7 +712,7 @@ fn codegen_emcc_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -750,13 +751,13 @@ fn codegen_emcc_try<'ll>( let catch_data = bx.bitcast(catch_data, bx.type_i8p()); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, catch_data], None); + bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -771,7 +772,7 @@ fn gen_fn<'ll, 'tcx>( ) -> (&'ll Type, &'ll Value) { let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, fn_abi); + let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. @@ -1205,6 +1206,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let c = bx.call( fn_ty, None, + None, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -1423,6 +1425,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None, @@ -1564,6 +1567,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None, @@ -2037,7 +2041,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, None, f, &[lhs, rhs], None); + let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 3f77ea77eff..805843e5863 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -10,6 +10,7 @@ #![feature(iter_intersperse)] #![feature(let_chains)] #![feature(never_type)] +#![feature(impl_trait_in_assoc_type)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] @@ -34,10 +35,10 @@ use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage}; -use rustc_macros::fluent_messages; +use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; use rustc_session::Session; @@ -69,7 +70,7 @@ mod declare; mod errors; mod intrinsic; -// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. +// The following is a workaround that replaces `pub mod llvm;` and that fixes issue 53912. #[path = "llvm/mod.rs"] mod llvm_; pub mod llvm { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 0d63e634ad8..de93a64c0d6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -552,6 +552,7 @@ pub enum ArchiveKind { K_BSD, K_DARWIN, K_COFF, + K_AIXBIG, } // LLVMRustThinLTOData @@ -679,7 +680,9 @@ pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_v pub mod coverageinfo { use super::coverage_map; - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) + /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegionKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum RegionKind { @@ -713,7 +716,9 @@ pub mod coverageinfo { /// array", encoded separately), and source location (start and end positions of the represented /// code region). /// - /// Matches LLVMRustCounterMappingRegion. + /// Corresponds to struct `llvm::coverage::CounterMappingRegion`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegion`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct CounterMappingRegion { @@ -946,15 +951,27 @@ pub mod debuginfo { NoDebug, FullDebug, LineTablesOnly, + DebugDirectivesOnly, } impl DebugEmissionKind { pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + // We should be setting LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we are + // instead adding a new debuginfo option "line-tables-only" so as to + // not break anything and to allow users to have 'limited' debug info. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. use rustc_session::config::DebugInfo; match kind { DebugInfo::None => DebugEmissionKind::NoDebug, - DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, - DebugInfo::Full => DebugEmissionKind::FullDebug, + DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly, + DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly, + DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug, } } } @@ -1006,7 +1023,7 @@ extern "C" { pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; @@ -1053,7 +1070,7 @@ extern "C" { // Operations on other types pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values pub fn LLVMTypeOf(Val: &Value) -> &Type; @@ -1072,7 +1089,12 @@ extern "C" { pub fn LLVMGetPoison(Ty: &Type) -> &Value; // Operations on metadata + // FIXME: deprecated, replace with LLVMMDStringInContext2 pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + + pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata; + + // FIXME: deprecated, replace with LLVMMDNodeInContext2 pub fn LLVMMDNodeInContext<'a>( C: &'a Context, Vals: *const &'a Value, @@ -1111,6 +1133,8 @@ extern "C" { Packed: Bool, ) -> &'a Value; + // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 + // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc pub fn LLVMConstArray<'a>( ElementTy: &'a Type, ConstantVals: *const &'a Value, @@ -1250,7 +1274,7 @@ extern "C" { pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); + pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata); // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; @@ -1290,38 +1314,38 @@ extern "C" { pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMRustBuildCleanupPad<'a>( + pub fn LLVMBuildCleanupPad<'a>( B: &Builder<'a>, ParentPad: Option<&'a Value>, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCleanupRet<'a>( + pub fn LLVMBuildCleanupRet<'a>( B: &Builder<'a>, CleanupPad: &'a Value, - UnwindBB: Option<&'a BasicBlock>, + BB: Option<&'a BasicBlock>, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchPad<'a>( + pub fn LLVMBuildCatchPad<'a>( B: &Builder<'a>, ParentPad: &'a Value, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchRet<'a>( + pub fn LLVMBuildCatchRet<'a>( B: &Builder<'a>, - Pad: &'a Value, + CatchPad: &'a Value, BB: &'a BasicBlock, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchSwitch<'a>( + pub fn LLVMBuildCatchSwitch<'a>( Builder: &Builder<'a>, ParentPad: Option<&'a Value>, - BB: Option<&'a BasicBlock>, + UnwindBB: Option<&'a BasicBlock>, NumHandlers: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); + pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); // Add a case to the switch instruction @@ -1615,11 +1639,12 @@ extern "C" { DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMRustBuildIntCast<'a>( + pub fn LLVMBuildIntCast2<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, - IsSigned: bool, + IsSigned: Bool, + Name: *const c_char, ) -> &'a Value; // Comparisons @@ -1908,7 +1933,7 @@ extern "C" { ); pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; @@ -1966,6 +1991,21 @@ extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateMethod<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + SPFlags: DISPFlags, + TParam: &'a DIArray, + ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2228,7 +2268,7 @@ extern "C" { pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine, cpu: *const c_char); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2444,12 +2484,6 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustThinLTOGetDICompileUnit( - M: &Module, - CU1: &mut *mut c_void, - CU2: &mut *mut c_void, - ); - pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index f820e752371..4f5cc575da6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -137,6 +137,7 @@ impl FromStr for ArchiveKind { "bsd" => Ok(ArchiveKind::K_BSD), "darwin" => Ok(ArchiveKind::K_DARWIN), "coff" => Ok(ArchiveKind::K_COFF), + "aix_big" => Ok(ArchiveKind::K_AIXBIG), _ => Err(()), } } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 46692fd5e8b..03be0654b50 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -16,7 +16,6 @@ use rustc_session::config::PrintRequest; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; -use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; use std::path::Path; @@ -132,6 +131,60 @@ pub fn time_trace_profiler_finish(file_name: &Path) { } } +pub enum TargetFeatureFoldStrength<'a> { + // The feature is only tied when enabling the feature, disabling + // this feature shouldn't disable the tied feature. + EnableOnly(&'a str), + // The feature is tied for both enabling and disabling this feature. + Both(&'a str), +} + +impl<'a> TargetFeatureFoldStrength<'a> { + fn as_str(&self) -> &'a str { + match self { + TargetFeatureFoldStrength::EnableOnly(feat) => feat, + TargetFeatureFoldStrength::Both(feat) => feat, + } + } +} + +pub struct LLVMFeature<'a> { + pub llvm_feature_name: &'a str, + pub dependency: Option<TargetFeatureFoldStrength<'a>>, +} + +impl<'a> LLVMFeature<'a> { + pub fn new(llvm_feature_name: &'a str) -> Self { + Self { llvm_feature_name, dependency: None } + } + + pub fn with_dependency( + llvm_feature_name: &'a str, + dependency: TargetFeatureFoldStrength<'a>, + ) -> Self { + Self { llvm_feature_name, dependency: Some(dependency) } + } + + pub fn contains(&self, feat: &str) -> bool { + self.iter().any(|dep| dep == feat) + } + + pub fn iter(&'a self) -> impl Iterator<Item = &'a str> { + let dependencies = self.dependency.iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + +impl<'a> IntoIterator for LLVMFeature<'a> { + type Item = &'a str; + type IntoIter = impl Iterator<Item = &'a str>; + + fn into_iter(self) -> Self::IntoIter { + let dependencies = self.dependency.into_iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + // WARNING: the features after applying `to_llvm_features` must be known // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. @@ -147,42 +200,65 @@ pub fn time_trace_profiler_finish(file_name: &Path) { // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { - ("x86", "sse4.2") => smallvec!["sse4.2", "crc32"], - ("x86", "pclmulqdq") => smallvec!["pclmul"], - ("x86", "rdrand") => smallvec!["rdrnd"], - ("x86", "bmi1") => smallvec!["bmi"], - ("x86", "cmpxchg16b") => smallvec!["cx16"], - // FIXME: These aliases are misleading, and should be removed before avx512_target_feature is - // stabilized. They must remain until std::arch switches off them. - // rust#100752 - ("x86", "avx512vaes") => smallvec!["vaes"], - ("x86", "avx512gfni") => smallvec!["gfni"], - ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], - ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], - ("aarch64", "dpb") => smallvec!["ccpp"], - ("aarch64", "dpb2") => smallvec!["ccdp"], - ("aarch64", "frintts") => smallvec!["fptoint"], - ("aarch64", "fcma") => smallvec!["complxnum"], - ("aarch64", "pmuv3") => smallvec!["perfmon"], - ("aarch64", "paca") => smallvec!["pauth"], - ("aarch64", "pacg") => smallvec!["pauth"], - // Rust ties fp and neon together. In LLVM neon implicitly enables fp, - // but we manually enable neon when a feature only implicitly enables fp - ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], - ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], - ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], - ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], - ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], - ("aarch64", "sve") => smallvec!["sve", "neon"], - ("aarch64", "sve2") => smallvec!["sve2", "neon"], - ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], - ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], - ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], - ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], - (_, s) => smallvec![s], + ("x86", "sse4.2") => { + LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32")) + } + ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"), + ("x86", "rdrand") => LLVMFeature::new("rdrnd"), + ("x86", "bmi1") => LLVMFeature::new("bmi"), + ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), + ("aarch64", "dpb") => LLVMFeature::new("ccpp"), + ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), + ("aarch64", "frintts") => LLVMFeature::new("fptoint"), + ("aarch64", "fcma") => LLVMFeature::new("complxnum"), + ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"), + ("aarch64", "paca") => LLVMFeature::new("pauth"), + ("aarch64", "pacg") => LLVMFeature::new("pauth"), + // Rust ties fp and neon together. + ("aarch64", "neon") => { + LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")) + } + // In LLVM neon implicitly enables fp, but we manually enable + // neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => { + LLVMFeature::with_dependency("f32mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "f64mm") => { + LLVMFeature::with_dependency("f64mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fhm") => { + LLVMFeature::with_dependency("fp16fml", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fp16") => { + LLVMFeature::with_dependency("fullfp16", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "jsconv") => { + LLVMFeature::with_dependency("jsconv", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve") => { + LLVMFeature::with_dependency("sve", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2") => { + LLVMFeature::with_dependency("sve2", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-aes") => { + LLVMFeature::with_dependency("sve2-aes", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sm4") => { + LLVMFeature::with_dependency("sve2-sm4", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sha3") => { + LLVMFeature::with_dependency("sve2-sha3", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-bitperm") => LLVMFeature::with_dependency( + "sve2-bitperm", + TargetFeatureFoldStrength::EnableOnly("neon"), + ), + (_, s) => LLVMFeature::new(s), } } @@ -280,18 +356,17 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { let mut rustc_target_features = supported_target_features(sess) .iter() .map(|(feature, _gate)| { - let desc = if let Some(llvm_feature) = to_llvm_features(sess, *feature).first() { - // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name; + let desc = match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() { Some(index) => { known_llvm_target_features.insert(llvm_feature); llvm_target_features[index].1 } None => "", - } - } else { - "" - }; + }; + (*feature, desc) }) .collect::<Vec<_>>(); @@ -329,7 +404,14 @@ pub(crate) fn print(req: PrintRequest, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess); match req { - PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, + PrintRequest::TargetCPUs => { + // SAFETY generate a C compatible string from a byte slice to pass + // the target CPU name into LLVM, the lifetime of the reference is + // at least as long as the C function + let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) + .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); + unsafe { llvm::LLVMRustPrintTargetCPUs(tm, cpu_cstring.as_ptr()) }; + } PrintRequest::TargetFeatures => print_target_features(sess, tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } @@ -468,10 +550,19 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. + let llvm_feature = to_llvm_features(sess, feature); + Some( - to_llvm_features(sess, feature) - .into_iter() - .map(move |f| format!("{}{}", enable_disable, f)), + std::iter::once(format!("{}{}", enable_disable, llvm_feature.llvm_feature_name)) + .chain(llvm_feature.dependency.into_iter().filter_map(move |feat| { + match (enable_disable, feat) { + ('-' | '+', TargetFeatureFoldStrength::Both(f)) + | ('+', TargetFeatureFoldStrength::EnableOnly(f)) => { + Some(format!("{}{}", enable_disable, f)) + } + _ => None, + } + })), ) }) .flatten(); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index d0ae36349df..c24854b27a0 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -48,10 +48,10 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { visibility: Visibility, symbol_name: &str, ) { - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let lldecl = self.declare_fn(symbol_name, fn_abi); + let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); @@ -125,8 +125,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) - .unwrap_or(false); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index ff111d96f84..d3fad5699c8 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -53,7 +53,7 @@ impl<'ll> CodegenCx<'ll, '_> { } pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 @@ -291,8 +291,24 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn add_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); let v = [self.const_usize(0), typeid_metadata]; unsafe { llvm::LLVMGlobalSetMetadata( @@ -307,13 +323,28 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - fn typeid_metadata(&self, typeid: String) -> &'ll Value { - unsafe { + fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> { + Some(unsafe { llvm::LLVMMDStringInContext( self.llcx, typeid.as_ptr() as *const c_char, typeid.len() as c_uint, ) + }) + } + + fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) } } |